2 # -*- coding: utf-8 -*-
4 from __future__ import unicode_literals
33 import xml.etree.ElementTree
43 compat_socket_create_connection,
47 compat_urllib_parse_urlparse,
48 compat_urllib_request,
54 # This is not clearly defined otherwise
55 compiled_regex_type = type(re.compile(''))
58 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20150101 Firefox/20.0 (Chrome)',
59 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
60 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
61 'Accept-Encoding': 'gzip, deflate',
62 'Accept-Language': 'en-us,en;q=0.5',
68 ENGLISH_MONTH_NAMES = [
69 'January', 'February', 'March', 'April', 'May', 'June',
70 'July', 'August', 'September', 'October', 'November', 'December']
73 def preferredencoding():
74 """Get preferred encoding.
76 Returns the best encoding scheme for the system, based on
77 locale.getpreferredencoding() and some further tweaks.
80 pref = locale.getpreferredencoding()
88 def write_json_file(obj, fn):
89 """ Encode obj as JSON and write it to fn, atomically if possible """
91 fn = encodeFilename(fn)
92 if sys.version_info < (3, 0) and sys.platform != 'win32':
93 encoding = get_filesystem_encoding()
94 # os.path.basename returns a bytes object, but NamedTemporaryFile
95 # will fail if the filename contains non ascii characters unless we
96 # use a unicode object
97 path_basename = lambda f: os.path.basename(fn).decode(encoding)
98 # the same for os.path.dirname
99 path_dirname = lambda f: os.path.dirname(fn).decode(encoding)
101 path_basename = os.path.basename
102 path_dirname = os.path.dirname
106 'prefix': path_basename(fn) + '.',
107 'dir': path_dirname(fn),
111 # In Python 2.x, json.dump expects a bytestream.
112 # In Python 3.x, it writes to a character stream
113 if sys.version_info < (3, 0):
121 tf = tempfile.NamedTemporaryFile(**compat_kwargs(args))
126 if sys.platform == 'win32':
127 # Need to remove existing file on Windows, else os.rename raises
128 # WindowsError or FileExistsError.
133 os.rename(tf.name, fn)
142 if sys.version_info >= (2, 7):
143 def find_xpath_attr(node, xpath, key, val=None):
144 """ Find the xpath xpath[@key=val] """
145 assert re.match(r'^[a-zA-Z_-]+$', key)
147 assert re.match(r'^[a-zA-Z0-9@\s:._-]*$', val)
148 expr = xpath + ('[@%s]' % key if val is None else "[@%s='%s']" % (key, val))
149 return node.find(expr)
151 def find_xpath_attr(node, xpath, key, val=None):
152 # Here comes the crazy part: In 2.6, if the xpath is a unicode,
153 # .//node does not match if a node is a direct child of . !
154 if isinstance(xpath, compat_str):
155 xpath = xpath.encode('ascii')
157 for f in node.findall(xpath):
158 if key not in f.attrib:
160 if val is None or f.attrib.get(key) == val:
164 # On python2.6 the xml.etree.ElementTree.Element methods don't support
165 # the namespace parameter
168 def xpath_with_ns(path, ns_map):
169 components = [c.split(':') for c in path.split('/')]
173 replaced.append(c[0])
176 replaced.append('{%s}%s' % (ns_map[ns], tag))
177 return '/'.join(replaced)
180 def xpath_element(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
181 if sys.version_info < (2, 7): # Crazy 2.6
182 xpath = xpath.encode('ascii')
186 if default is not NO_DEFAULT:
189 name = xpath if name is None else name
190 raise ExtractorError('Could not find XML element %s' % name)
196 def xpath_text(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
197 n = xpath_element(node, xpath, name, fatal=fatal, default=default)
198 if n is None or n == default:
201 if default is not NO_DEFAULT:
204 name = xpath if name is None else name
205 raise ExtractorError('Could not find XML element\'s text %s' % name)
211 def xpath_attr(node, xpath, key, name=None, fatal=False, default=NO_DEFAULT):
212 n = find_xpath_attr(node, xpath, key)
214 if default is not NO_DEFAULT:
217 name = '%s[@%s]' % (xpath, key) if name is None else name
218 raise ExtractorError('Could not find XML attribute %s' % name)
224 def get_element_by_id(id, html):
225 """Return the content of the tag with the specified ID in the passed HTML document"""
226 return get_element_by_attribute("id", id, html)
229 def get_element_by_attribute(attribute, value, html):
230 """Return the content of the tag with the specified attribute in the passed HTML document"""
232 m = re.search(r'''(?xs)
234 (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?
236 (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?
240 ''' % (re.escape(attribute), re.escape(value)), html)
244 res = m.group('content')
246 if res.startswith('"') or res.startswith("'"):
249 return unescapeHTML(res)
252 def clean_html(html):
253 """Clean an HTML snippet into a readable string"""
255 if html is None: # Convenience for sanitizing descriptions etc.
259 html = html.replace('\n', ' ')
260 html = re.sub(r'\s*<\s*br\s*/?\s*>\s*', '\n', html)
261 html = re.sub(r'<\s*/\s*p\s*>\s*<\s*p[^>]*>', '\n', html)
263 html = re.sub('<.*?>', '', html)
264 # Replace html entities
265 html = unescapeHTML(html)
269 def sanitize_open(filename, open_mode):
270 """Try to open the given filename, and slightly tweak it if this fails.
272 Attempts to open the given filename. If this fails, it tries to change
273 the filename slightly, step by step, until it's either able to open it
274 or it fails and raises a final exception, like the standard open()
277 It returns the tuple (stream, definitive_file_name).
281 if sys.platform == 'win32':
283 msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
284 return (sys.stdout.buffer if hasattr(sys.stdout, 'buffer') else sys.stdout, filename)
285 stream = open(encodeFilename(filename), open_mode)
286 return (stream, filename)
287 except (IOError, OSError) as err:
288 if err.errno in (errno.EACCES,):
291 # In case of error, try to remove win32 forbidden chars
292 alt_filename = sanitize_path(filename)
293 if alt_filename == filename:
296 # An exception here should be caught in the caller
297 stream = open(encodeFilename(alt_filename), open_mode)
298 return (stream, alt_filename)
301 def timeconvert(timestr):
302 """Convert RFC 2822 defined time string into system timestamp"""
304 timetuple = email.utils.parsedate_tz(timestr)
305 if timetuple is not None:
306 timestamp = email.utils.mktime_tz(timetuple)
310 def sanitize_filename(s, restricted=False, is_id=False):
311 """Sanitizes a string so it could be used as part of a filename.
312 If restricted is set, use a stricter subset of allowed characters.
313 Set is_id if this is not an arbitrary string, but an ID that should be kept if possible
315 def replace_insane(char):
316 if char == '?' or ord(char) < 32 or ord(char) == 127:
319 return '' if restricted else '\''
321 return '_-' if restricted else ' -'
322 elif char in '\\/|*<>':
324 if restricted and (char in '!&\'()[]{}$;`^,#' or char.isspace()):
326 if restricted and ord(char) > 127:
331 s = re.sub(r'[0-9]+(?::[0-9]+)+', lambda m: m.group(0).replace(':', '_'), s)
332 result = ''.join(map(replace_insane, s))
334 while '__' in result:
335 result = result.replace('__', '_')
336 result = result.strip('_')
337 # Common case of "Foreign band name - English song title"
338 if restricted and result.startswith('-_'):
340 if result.startswith('-'):
341 result = '_' + result[len('-'):]
342 result = result.lstrip('.')
348 def sanitize_path(s):
349 """Sanitizes and normalizes path on Windows"""
350 if sys.platform != 'win32':
352 drive_or_unc, _ = os.path.splitdrive(s)
353 if sys.version_info < (2, 7) and not drive_or_unc:
354 drive_or_unc, _ = os.path.splitunc(s)
355 norm_path = os.path.normpath(remove_start(s, drive_or_unc)).split(os.path.sep)
359 path_part if path_part in ['.', '..'] else re.sub('(?:[/<>:"\\|\\\\?\\*]|\.$)', '#', path_part)
360 for path_part in norm_path]
362 sanitized_path.insert(0, drive_or_unc + os.path.sep)
363 return os.path.join(*sanitized_path)
366 def orderedSet(iterable):
367 """ Remove all duplicates from the input iterable """
375 def _htmlentity_transform(entity):
376 """Transforms an HTML entity to a character."""
377 # Known non-numeric HTML entity
378 if entity in compat_html_entities.name2codepoint:
379 return compat_chr(compat_html_entities.name2codepoint[entity])
381 mobj = re.match(r'#(x[0-9a-fA-F]+|[0-9]+)', entity)
383 numstr = mobj.group(1)
384 if numstr.startswith('x'):
386 numstr = '0%s' % numstr
389 return compat_chr(int(numstr, base))
391 # Unknown entity in name, return its literal representation
392 return ('&%s;' % entity)
398 assert type(s) == compat_str
401 r'&([^;]+);', lambda m: _htmlentity_transform(m.group(1)), s)
404 def get_subprocess_encoding():
405 if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
406 # For subprocess calls, encode with locale encoding
407 # Refer to http://stackoverflow.com/a/9951851/35070
408 encoding = preferredencoding()
410 encoding = sys.getfilesystemencoding()
416 def encodeFilename(s, for_subprocess=False):
418 @param s The name of the file
421 assert type(s) == compat_str
423 # Python 3 has a Unicode API
424 if sys.version_info >= (3, 0):
427 # Pass '' directly to use Unicode APIs on Windows 2000 and up
428 # (Detecting Windows NT 4 is tricky because 'major >= 4' would
429 # match Windows 9x series as well. Besides, NT 4 is obsolete.)
430 if not for_subprocess and sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
433 return s.encode(get_subprocess_encoding(), 'ignore')
436 def decodeFilename(b, for_subprocess=False):
438 if sys.version_info >= (3, 0):
441 if not isinstance(b, bytes):
444 return b.decode(get_subprocess_encoding(), 'ignore')
447 def encodeArgument(s):
448 if not isinstance(s, compat_str):
449 # Legacy code that uses byte strings
450 # Uncomment the following line after fixing all post processors
451 # assert False, 'Internal error: %r should be of type %r, is %r' % (s, compat_str, type(s))
452 s = s.decode('ascii')
453 return encodeFilename(s, True)
456 def decodeArgument(b):
457 return decodeFilename(b, True)
460 def decodeOption(optval):
463 if isinstance(optval, bytes):
464 optval = optval.decode(preferredencoding())
466 assert isinstance(optval, compat_str)
470 def formatSeconds(secs):
472 return '%d:%02d:%02d' % (secs // 3600, (secs % 3600) // 60, secs % 60)
474 return '%d:%02d' % (secs // 60, secs % 60)
479 def make_HTTPS_handler(params, **kwargs):
480 opts_no_check_certificate = params.get('nocheckcertificate', False)
481 if hasattr(ssl, 'create_default_context'): # Python >= 3.4 or 2.7.9
482 context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
483 if opts_no_check_certificate:
484 context.check_hostname = False
485 context.verify_mode = ssl.CERT_NONE
487 return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
490 # (create_default_context present but HTTPSHandler has no context=)
493 if sys.version_info < (3, 2):
494 return YoutubeDLHTTPSHandler(params, **kwargs)
496 context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
497 context.verify_mode = (ssl.CERT_NONE
498 if opts_no_check_certificate
499 else ssl.CERT_REQUIRED)
500 context.set_default_verify_paths()
501 return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
504 def bug_reports_message():
505 if ytdl_is_updateable():
506 update_cmd = 'type youtube-dl -U to update'
508 update_cmd = 'see https://yt-dl.org/update on how to update'
509 msg = '; please report this issue on https://yt-dl.org/bug .'
510 msg += ' Make sure you are using the latest version; %s.' % update_cmd
511 msg += ' Be sure to call youtube-dl with the --verbose flag and include its complete output.'
515 class ExtractorError(Exception):
516 """Error during info extraction."""
518 def __init__(self, msg, tb=None, expected=False, cause=None, video_id=None):
519 """ tb, if given, is the original traceback (so that it can be printed out).
520 If expected is set, this is a normal error message and most likely not a bug in youtube-dl.
523 if sys.exc_info()[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError):
525 if video_id is not None:
526 msg = video_id + ': ' + msg
528 msg += ' (caused by %r)' % cause
530 msg += bug_reports_message()
531 super(ExtractorError, self).__init__(msg)
534 self.exc_info = sys.exc_info() # preserve original exception
536 self.video_id = video_id
538 def format_traceback(self):
539 if self.traceback is None:
541 return ''.join(traceback.format_tb(self.traceback))
544 class UnsupportedError(ExtractorError):
545 def __init__(self, url):
546 super(UnsupportedError, self).__init__(
547 'Unsupported URL: %s' % url, expected=True)
551 class RegexNotFoundError(ExtractorError):
552 """Error when a regex didn't match"""
556 class DownloadError(Exception):
557 """Download Error exception.
559 This exception may be thrown by FileDownloader objects if they are not
560 configured to continue on errors. They will contain the appropriate
564 def __init__(self, msg, exc_info=None):
565 """ exc_info, if given, is the original exception that caused the trouble (as returned by sys.exc_info()). """
566 super(DownloadError, self).__init__(msg)
567 self.exc_info = exc_info
570 class SameFileError(Exception):
571 """Same File exception.
573 This exception will be thrown by FileDownloader objects if they detect
574 multiple files would have to be downloaded to the same file on disk.
579 class PostProcessingError(Exception):
580 """Post Processing exception.
582 This exception may be raised by PostProcessor's .run() method to
583 indicate an error in the postprocessing task.
586 def __init__(self, msg):
590 class MaxDownloadsReached(Exception):
591 """ --max-downloads limit has been reached. """
595 class UnavailableVideoError(Exception):
596 """Unavailable Format exception.
598 This exception will be thrown when a video is requested
599 in a format that is not available for that video.
604 class ContentTooShortError(Exception):
605 """Content Too Short exception.
607 This exception may be raised by FileDownloader objects when a file they
608 download is too small for what the server announced first, indicating
609 the connection was probably interrupted.
612 def __init__(self, downloaded, expected):
614 self.downloaded = downloaded
615 self.expected = expected
618 def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs):
619 # Working around python 2 bug (see http://bugs.python.org/issue17849) by limiting
620 # expected HTTP responses to meet HTTP/1.0 or later (see also
621 # https://github.com/rg3/youtube-dl/issues/6727)
622 if sys.version_info < (3, 0):
623 kwargs[b'strict'] = True
624 hc = http_class(*args, **kwargs)
625 source_address = ydl_handler._params.get('source_address')
626 if source_address is not None:
627 sa = (source_address, 0)
628 if hasattr(hc, 'source_address'): # Python 2.7+
629 hc.source_address = sa
631 def _hc_connect(self, *args, **kwargs):
632 sock = compat_socket_create_connection(
633 (self.host, self.port), self.timeout, sa)
635 self.sock = ssl.wrap_socket(
636 sock, self.key_file, self.cert_file,
637 ssl_version=ssl.PROTOCOL_TLSv1)
640 hc.connect = functools.partial(_hc_connect, hc)
645 class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
646 """Handler for HTTP requests and responses.
648 This class, when installed with an OpenerDirector, automatically adds
649 the standard headers to every HTTP request and handles gzipped and
650 deflated responses from web servers. If compression is to be avoided in
651 a particular request, the original request in the program code only has
652 to include the HTTP header "Youtubedl-No-Compression", which will be
653 removed before making the real request.
655 Part of this code was copied from:
657 http://techknack.net/python-urllib2-handlers/
659 Andrew Rowls, the author of that code, agreed to release it to the
663 def __init__(self, params, *args, **kwargs):
664 compat_urllib_request.HTTPHandler.__init__(self, *args, **kwargs)
665 self._params = params
667 def http_open(self, req):
668 return self.do_open(functools.partial(
669 _create_http_connection, self, compat_http_client.HTTPConnection, False),
675 return zlib.decompress(data, -zlib.MAX_WBITS)
677 return zlib.decompress(data)
680 def addinfourl_wrapper(stream, headers, url, code):
681 if hasattr(compat_urllib_request.addinfourl, 'getcode'):
682 return compat_urllib_request.addinfourl(stream, headers, url, code)
683 ret = compat_urllib_request.addinfourl(stream, headers, url)
687 def http_request(self, req):
688 # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
689 # always respected by websites, some tend to give out URLs with non percent-encoded
690 # non-ASCII characters (see telemb.py, ard.py [#3412])
691 # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
692 # To work around aforementioned issue we will replace request's original URL with
693 # percent-encoded one
694 # Since redirects are also affected (e.g. http://www.southpark.de/alle-episoden/s18e09)
695 # the code of this workaround has been moved here from YoutubeDL.urlopen()
696 url = req.get_full_url()
697 url_escaped = escape_url(url)
699 # Substitute URL if any change after escaping
700 if url != url_escaped:
701 req_type = HEADRequest if req.get_method() == 'HEAD' else compat_urllib_request.Request
703 url_escaped, data=req.data, headers=req.headers,
704 origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
705 new_req.timeout = req.timeout
708 for h, v in std_headers.items():
709 # Capitalize is needed because of Python bug 2275: http://bugs.python.org/issue2275
710 # The dict keys are capitalized because of this bug by urllib
711 if h.capitalize() not in req.headers:
713 if 'Youtubedl-no-compression' in req.headers:
714 if 'Accept-encoding' in req.headers:
715 del req.headers['Accept-encoding']
716 del req.headers['Youtubedl-no-compression']
718 if sys.version_info < (2, 7) and '#' in req.get_full_url():
719 # Python 2.6 is brain-dead when it comes to fragments
720 req._Request__original = req._Request__original.partition('#')[0]
721 req._Request__r_type = req._Request__r_type.partition('#')[0]
725 def http_response(self, req, resp):
728 if resp.headers.get('Content-encoding', '') == 'gzip':
729 content = resp.read()
730 gz = gzip.GzipFile(fileobj=io.BytesIO(content), mode='rb')
732 uncompressed = io.BytesIO(gz.read())
733 except IOError as original_ioerror:
734 # There may be junk add the end of the file
735 # See http://stackoverflow.com/q/4928560/35070 for details
736 for i in range(1, 1024):
738 gz = gzip.GzipFile(fileobj=io.BytesIO(content[:-i]), mode='rb')
739 uncompressed = io.BytesIO(gz.read())
744 raise original_ioerror
745 resp = self.addinfourl_wrapper(uncompressed, old_resp.headers, old_resp.url, old_resp.code)
746 resp.msg = old_resp.msg
748 if resp.headers.get('Content-encoding', '') == 'deflate':
749 gz = io.BytesIO(self.deflate(resp.read()))
750 resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
751 resp.msg = old_resp.msg
752 # Percent-encode redirect URL of Location HTTP header to satisfy RFC 3986 (see
753 # https://github.com/rg3/youtube-dl/issues/6457).
754 if 300 <= resp.code < 400:
755 location = resp.headers.get('Location')
757 # As of RFC 2616 default charset is iso-8859-1 that is respected by python 3
758 if sys.version_info >= (3, 0):
759 location = location.encode('iso-8859-1').decode('utf-8')
760 location_escaped = escape_url(location)
761 if location != location_escaped:
762 del resp.headers['Location']
763 resp.headers['Location'] = location_escaped
766 https_request = http_request
767 https_response = http_response
770 class YoutubeDLHTTPSHandler(compat_urllib_request.HTTPSHandler):
771 def __init__(self, params, https_conn_class=None, *args, **kwargs):
772 compat_urllib_request.HTTPSHandler.__init__(self, *args, **kwargs)
773 self._https_conn_class = https_conn_class or compat_http_client.HTTPSConnection
774 self._params = params
776 def https_open(self, req):
778 if hasattr(self, '_context'): # python > 2.6
779 kwargs['context'] = self._context
780 if hasattr(self, '_check_hostname'): # python 3.x
781 kwargs['check_hostname'] = self._check_hostname
782 return self.do_open(functools.partial(
783 _create_http_connection, self, self._https_conn_class, True),
787 class YoutubeDLCookieProcessor(compat_urllib_request.HTTPCookieProcessor):
788 def __init__(self, cookiejar=None):
789 compat_urllib_request.HTTPCookieProcessor.__init__(self, cookiejar)
791 def http_response(self, request, response):
792 # Python 2 will choke on next HTTP request in row if there are non-ASCII
793 # characters in Set-Cookie HTTP header of last response (see
794 # https://github.com/rg3/youtube-dl/issues/6769).
795 # In order to at least prevent crashing we will percent encode Set-Cookie
796 # header before HTTPCookieProcessor starts processing it.
797 # if sys.version_info < (3, 0) and response.headers:
798 # for set_cookie_header in ('Set-Cookie', 'Set-Cookie2'):
799 # set_cookie = response.headers.get(set_cookie_header)
801 # set_cookie_escaped = compat_urllib_parse.quote(set_cookie, b"%/;:@&=+$,!~*'()?#[] ")
802 # if set_cookie != set_cookie_escaped:
803 # del response.headers[set_cookie_header]
804 # response.headers[set_cookie_header] = set_cookie_escaped
805 return compat_urllib_request.HTTPCookieProcessor.http_response(self, request, response)
807 https_request = compat_urllib_request.HTTPCookieProcessor.http_request
808 https_response = http_response
811 def parse_iso8601(date_str, delimiter='T', timezone=None):
812 """ Return a UNIX timestamp from the given date """
819 r'(\.[0-9]+)?(?:Z$| ?(?P<sign>\+|-)(?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})$)',
822 timezone = datetime.timedelta()
824 date_str = date_str[:-len(m.group(0))]
825 if not m.group('sign'):
826 timezone = datetime.timedelta()
828 sign = 1 if m.group('sign') == '+' else -1
829 timezone = datetime.timedelta(
830 hours=sign * int(m.group('hours')),
831 minutes=sign * int(m.group('minutes')))
832 date_format = '%Y-%m-%d{0}%H:%M:%S'.format(delimiter)
833 dt = datetime.datetime.strptime(date_str, date_format) - timezone
834 return calendar.timegm(dt.timetuple())
837 def unified_strdate(date_str, day_first=True):
838 """Return a string with the date in the format YYYYMMDD"""
844 date_str = date_str.replace(',', ' ')
845 # %z (UTC offset) is only supported in python>=3.2
846 if not re.match(r'^[0-9]{1,2}-[0-9]{1,2}-[0-9]{4}$', date_str):
847 date_str = re.sub(r' ?(\+|-)[0-9]{2}:?[0-9]{2}$', '', date_str)
848 # Remove AM/PM + timezone
849 date_str = re.sub(r'(?i)\s*(?:AM|PM)(?:\s+[A-Z]+)?', '', date_str)
851 format_expressions = [
856 '%b %dst %Y %I:%M%p',
857 '%b %dnd %Y %I:%M%p',
858 '%b %dth %Y %I:%M%p',
864 '%Y-%m-%d %H:%M:%S.%f',
867 '%Y-%m-%dT%H:%M:%SZ',
868 '%Y-%m-%dT%H:%M:%S.%fZ',
869 '%Y-%m-%dT%H:%M:%S.%f0Z',
871 '%Y-%m-%dT%H:%M:%S.%f',
875 format_expressions.extend([
883 format_expressions.extend([
890 for expression in format_expressions:
892 upload_date = datetime.datetime.strptime(date_str, expression).strftime('%Y%m%d')
895 if upload_date is None:
896 timetuple = email.utils.parsedate_tz(date_str)
898 upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d')
902 def determine_ext(url, default_ext='unknown_video'):
905 guess = url.partition('?')[0].rpartition('.')[2]
906 if re.match(r'^[A-Za-z0-9]+$', guess):
912 def subtitles_filename(filename, sub_lang, sub_format):
913 return filename.rsplit('.', 1)[0] + '.' + sub_lang + '.' + sub_format
916 def date_from_str(date_str):
918 Return a datetime object from a string in the format YYYYMMDD or
919 (now|today)[+-][0-9](day|week|month|year)(s)?"""
920 today = datetime.date.today()
921 if date_str in ('now', 'today'):
923 if date_str == 'yesterday':
924 return today - datetime.timedelta(days=1)
925 match = re.match('(now|today)(?P<sign>[+-])(?P<time>\d+)(?P<unit>day|week|month|year)(s)?', date_str)
926 if match is not None:
927 sign = match.group('sign')
928 time = int(match.group('time'))
931 unit = match.group('unit')
932 # A bad aproximation?
940 delta = datetime.timedelta(**{unit: time})
942 return datetime.datetime.strptime(date_str, "%Y%m%d").date()
945 def hyphenate_date(date_str):
947 Convert a date in 'YYYYMMDD' format to 'YYYY-MM-DD' format"""
948 match = re.match(r'^(\d\d\d\d)(\d\d)(\d\d)$', date_str)
949 if match is not None:
950 return '-'.join(match.groups())
955 class DateRange(object):
956 """Represents a time interval between two dates"""
958 def __init__(self, start=None, end=None):
959 """start and end must be strings in the format accepted by date"""
960 if start is not None:
961 self.start = date_from_str(start)
963 self.start = datetime.datetime.min.date()
965 self.end = date_from_str(end)
967 self.end = datetime.datetime.max.date()
968 if self.start > self.end:
969 raise ValueError('Date range: "%s" , the start date must be before the end date' % self)
973 """Returns a range that only contains the given day"""
976 def __contains__(self, date):
977 """Check if the date is in the range"""
978 if not isinstance(date, datetime.date):
979 date = date_from_str(date)
980 return self.start <= date <= self.end
983 return '%s - %s' % (self.start.isoformat(), self.end.isoformat())
987 """ Returns the platform name as a compat_str """
988 res = platform.platform()
989 if isinstance(res, bytes):
990 res = res.decode(preferredencoding())
992 assert isinstance(res, compat_str)
996 def _windows_write_string(s, out):
997 """ Returns True if the string was written using special methods,
998 False if it has yet to be written out."""
999 # Adapted from http://stackoverflow.com/a/3259271/35070
1002 import ctypes.wintypes
1010 fileno = out.fileno()
1011 except AttributeError:
1012 # If the output stream doesn't have a fileno, it's virtual
1014 except io.UnsupportedOperation:
1015 # Some strange Windows pseudo files?
1017 if fileno not in WIN_OUTPUT_IDS:
1020 GetStdHandle = ctypes.WINFUNCTYPE(
1021 ctypes.wintypes.HANDLE, ctypes.wintypes.DWORD)(
1022 (b"GetStdHandle", ctypes.windll.kernel32))
1023 h = GetStdHandle(WIN_OUTPUT_IDS[fileno])
1025 WriteConsoleW = ctypes.WINFUNCTYPE(
1026 ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE, ctypes.wintypes.LPWSTR,
1027 ctypes.wintypes.DWORD, ctypes.POINTER(ctypes.wintypes.DWORD),
1028 ctypes.wintypes.LPVOID)((b"WriteConsoleW", ctypes.windll.kernel32))
1029 written = ctypes.wintypes.DWORD(0)
1031 GetFileType = ctypes.WINFUNCTYPE(ctypes.wintypes.DWORD, ctypes.wintypes.DWORD)((b"GetFileType", ctypes.windll.kernel32))
1032 FILE_TYPE_CHAR = 0x0002
1033 FILE_TYPE_REMOTE = 0x8000
1034 GetConsoleMode = ctypes.WINFUNCTYPE(
1035 ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE,
1036 ctypes.POINTER(ctypes.wintypes.DWORD))(
1037 (b"GetConsoleMode", ctypes.windll.kernel32))
1038 INVALID_HANDLE_VALUE = ctypes.wintypes.DWORD(-1).value
1040 def not_a_console(handle):
1041 if handle == INVALID_HANDLE_VALUE or handle is None:
1043 return ((GetFileType(handle) & ~FILE_TYPE_REMOTE) != FILE_TYPE_CHAR or
1044 GetConsoleMode(handle, ctypes.byref(ctypes.wintypes.DWORD())) == 0)
1046 if not_a_console(h):
1049 def next_nonbmp_pos(s):
1051 return next(i for i, c in enumerate(s) if ord(c) > 0xffff)
1052 except StopIteration:
1056 count = min(next_nonbmp_pos(s), 1024)
1058 ret = WriteConsoleW(
1059 h, s, count if count else 2, ctypes.byref(written), None)
1061 raise OSError('Failed to write string')
1062 if not count: # We just wrote a non-BMP character
1063 assert written.value == 2
1066 assert written.value > 0
1067 s = s[written.value:]
1071 def write_string(s, out=None, encoding=None):
1074 assert type(s) == compat_str
1076 if sys.platform == 'win32' and encoding is None and hasattr(out, 'fileno'):
1077 if _windows_write_string(s, out):
1080 if ('b' in getattr(out, 'mode', '') or
1081 sys.version_info[0] < 3): # Python 2 lies about mode of sys.stderr
1082 byt = s.encode(encoding or preferredencoding(), 'ignore')
1084 elif hasattr(out, 'buffer'):
1085 enc = encoding or getattr(out, 'encoding', None) or preferredencoding()
1086 byt = s.encode(enc, 'ignore')
1087 out.buffer.write(byt)
1093 def bytes_to_intlist(bs):
1096 if isinstance(bs[0], int): # Python 3
1099 return [ord(c) for c in bs]
1102 def intlist_to_bytes(xs):
1105 return struct_pack('%dB' % len(xs), *xs)
1108 # Cross-platform file locking
1109 if sys.platform == 'win32':
1110 import ctypes.wintypes
1113 class OVERLAPPED(ctypes.Structure):
1115 ('Internal', ctypes.wintypes.LPVOID),
1116 ('InternalHigh', ctypes.wintypes.LPVOID),
1117 ('Offset', ctypes.wintypes.DWORD),
1118 ('OffsetHigh', ctypes.wintypes.DWORD),
1119 ('hEvent', ctypes.wintypes.HANDLE),
1122 kernel32 = ctypes.windll.kernel32
1123 LockFileEx = kernel32.LockFileEx
1124 LockFileEx.argtypes = [
1125 ctypes.wintypes.HANDLE, # hFile
1126 ctypes.wintypes.DWORD, # dwFlags
1127 ctypes.wintypes.DWORD, # dwReserved
1128 ctypes.wintypes.DWORD, # nNumberOfBytesToLockLow
1129 ctypes.wintypes.DWORD, # nNumberOfBytesToLockHigh
1130 ctypes.POINTER(OVERLAPPED) # Overlapped
1132 LockFileEx.restype = ctypes.wintypes.BOOL
1133 UnlockFileEx = kernel32.UnlockFileEx
1134 UnlockFileEx.argtypes = [
1135 ctypes.wintypes.HANDLE, # hFile
1136 ctypes.wintypes.DWORD, # dwReserved
1137 ctypes.wintypes.DWORD, # nNumberOfBytesToLockLow
1138 ctypes.wintypes.DWORD, # nNumberOfBytesToLockHigh
1139 ctypes.POINTER(OVERLAPPED) # Overlapped
1141 UnlockFileEx.restype = ctypes.wintypes.BOOL
1142 whole_low = 0xffffffff
1143 whole_high = 0x7fffffff
1145 def _lock_file(f, exclusive):
1146 overlapped = OVERLAPPED()
1147 overlapped.Offset = 0
1148 overlapped.OffsetHigh = 0
1149 overlapped.hEvent = 0
1150 f._lock_file_overlapped_p = ctypes.pointer(overlapped)
1151 handle = msvcrt.get_osfhandle(f.fileno())
1152 if not LockFileEx(handle, 0x2 if exclusive else 0x0, 0,
1153 whole_low, whole_high, f._lock_file_overlapped_p):
1154 raise OSError('Locking file failed: %r' % ctypes.FormatError())
1156 def _unlock_file(f):
1157 assert f._lock_file_overlapped_p
1158 handle = msvcrt.get_osfhandle(f.fileno())
1159 if not UnlockFileEx(handle, 0,
1160 whole_low, whole_high, f._lock_file_overlapped_p):
1161 raise OSError('Unlocking file failed: %r' % ctypes.FormatError())
1166 def _lock_file(f, exclusive):
1167 fcntl.flock(f, fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH)
1169 def _unlock_file(f):
1170 fcntl.flock(f, fcntl.LOCK_UN)
1173 class locked_file(object):
1174 def __init__(self, filename, mode, encoding=None):
1175 assert mode in ['r', 'a', 'w']
1176 self.f = io.open(filename, mode, encoding=encoding)
1179 def __enter__(self):
1180 exclusive = self.mode != 'r'
1182 _lock_file(self.f, exclusive)
1188 def __exit__(self, etype, value, traceback):
1190 _unlock_file(self.f)
1197 def write(self, *args):
1198 return self.f.write(*args)
1200 def read(self, *args):
1201 return self.f.read(*args)
1204 def get_filesystem_encoding():
1205 encoding = sys.getfilesystemencoding()
1206 return encoding if encoding is not None else 'utf-8'
1209 def shell_quote(args):
1211 encoding = get_filesystem_encoding()
1213 if isinstance(a, bytes):
1214 # We may get a filename encoded with 'encodeFilename'
1215 a = a.decode(encoding)
1216 quoted_args.append(pipes.quote(a))
1217 return ' '.join(quoted_args)
1220 def smuggle_url(url, data):
1221 """ Pass additional data in a URL for internal use. """
1223 sdata = compat_urllib_parse.urlencode(
1224 {'__youtubedl_smuggle': json.dumps(data)})
1225 return url + '#' + sdata
1228 def unsmuggle_url(smug_url, default=None):
1229 if '#__youtubedl_smuggle' not in smug_url:
1230 return smug_url, default
1231 url, _, sdata = smug_url.rpartition('#')
1232 jsond = compat_parse_qs(sdata)['__youtubedl_smuggle'][0]
1233 data = json.loads(jsond)
1237 def format_bytes(bytes):
1240 if type(bytes) is str:
1241 bytes = float(bytes)
1245 exponent = int(math.log(bytes, 1024.0))
1246 suffix = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'][exponent]
1247 converted = float(bytes) / float(1024 ** exponent)
1248 return '%.2f%s' % (converted, suffix)
1251 def parse_filesize(s):
1255 # The lower-case forms are of course incorrect and inofficial,
1256 # but we support those too
1294 units_re = '|'.join(re.escape(u) for u in _UNIT_TABLE)
1296 r'(?P<num>[0-9]+(?:[,.][0-9]*)?)\s*(?P<unit>%s)' % units_re, s)
1300 num_str = m.group('num').replace(',', '.')
1301 mult = _UNIT_TABLE[m.group('unit')]
1302 return int(float(num_str) * mult)
1305 def month_by_name(name):
1306 """ Return the number of a month by (locale-independently) English name """
1309 return ENGLISH_MONTH_NAMES.index(name) + 1
1314 def month_by_abbreviation(abbrev):
1315 """ Return the number of a month by (locale-independently) English
1319 return [s[:3] for s in ENGLISH_MONTH_NAMES].index(abbrev) + 1
1324 def fix_xml_ampersands(xml_str):
1325 """Replace all the '&' by '&' in XML"""
1327 r'&(?!amp;|lt;|gt;|apos;|quot;|#x[0-9a-fA-F]{,4};|#[0-9]{,4};)',
1332 def setproctitle(title):
1333 assert isinstance(title, compat_str)
1335 libc = ctypes.cdll.LoadLibrary("libc.so.6")
1338 title_bytes = title.encode('utf-8')
1339 buf = ctypes.create_string_buffer(len(title_bytes))
1340 buf.value = title_bytes
1342 libc.prctl(15, buf, 0, 0, 0)
1343 except AttributeError:
1344 return # Strange libc, just skip this
1347 def remove_start(s, start):
1348 if s.startswith(start):
1349 return s[len(start):]
1353 def remove_end(s, end):
1355 return s[:-len(end)]
1359 def url_basename(url):
1360 path = compat_urlparse.urlparse(url).path
1361 return path.strip('/').split('/')[-1]
1364 class HEADRequest(compat_urllib_request.Request):
1365 def get_method(self):
1369 def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1):
1372 v = getattr(v, get_attr, None)
1378 return int(v) * invscale // scale
1383 def str_or_none(v, default=None):
1384 return default if v is None else compat_str(v)
1387 def str_to_int(int_str):
1388 """ A more relaxed version of int_or_none """
1391 int_str = re.sub(r'[,\.\+]', '', int_str)
1395 def float_or_none(v, scale=1, invscale=1, default=None):
1399 return float(v) * invscale / scale
1404 def parse_duration(s):
1405 if not isinstance(s, compat_basestring):
1413 (?P<only_mins>[0-9.]+)\s*(?:mins?\.?|minutes?)\s*|
1414 (?P<only_hours>[0-9.]+)\s*(?:hours?)|
1416 \s*(?P<hours_reversed>[0-9]+)\s*(?:[:h]|hours?)\s*(?P<mins_reversed>[0-9]+)\s*(?:[:m]|mins?\.?|minutes?)\s*|
1419 (?:(?P<days>[0-9]+)\s*(?:[:d]|days?)\s*)?
1420 (?P<hours>[0-9]+)\s*(?:[:h]|hours?)\s*
1422 (?P<mins>[0-9]+)\s*(?:[:m]|mins?|minutes?)\s*
1424 (?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?\s*(?:s|secs?|seconds?)?
1429 if m.group('only_mins'):
1430 return float_or_none(m.group('only_mins'), invscale=60)
1431 if m.group('only_hours'):
1432 return float_or_none(m.group('only_hours'), invscale=60 * 60)
1434 res += int(m.group('secs'))
1435 if m.group('mins_reversed'):
1436 res += int(m.group('mins_reversed')) * 60
1438 res += int(m.group('mins')) * 60
1439 if m.group('hours'):
1440 res += int(m.group('hours')) * 60 * 60
1441 if m.group('hours_reversed'):
1442 res += int(m.group('hours_reversed')) * 60 * 60
1444 res += int(m.group('days')) * 24 * 60 * 60
1446 res += float(m.group('ms'))
1450 def prepend_extension(filename, ext, expected_real_ext=None):
1451 name, real_ext = os.path.splitext(filename)
1453 '{0}.{1}{2}'.format(name, ext, real_ext)
1454 if not expected_real_ext or real_ext[1:] == expected_real_ext
1455 else '{0}.{1}'.format(filename, ext))
1458 def replace_extension(filename, ext, expected_real_ext=None):
1459 name, real_ext = os.path.splitext(filename)
1460 return '{0}.{1}'.format(
1461 name if not expected_real_ext or real_ext[1:] == expected_real_ext else filename,
1465 def check_executable(exe, args=[]):
1466 """ Checks if the given binary is installed somewhere in PATH, and returns its name.
1467 args can be a list of arguments for a short output (like -version) """
1469 subprocess.Popen([exe] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
1475 def get_exe_version(exe, args=['--version'],
1476 version_re=None, unrecognized='present'):
1477 """ Returns the version of the specified executable,
1478 or False if the executable is not present """
1480 out, _ = subprocess.Popen(
1481 [encodeArgument(exe)] + args,
1482 stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()
1485 if isinstance(out, bytes): # Python 2.x
1486 out = out.decode('ascii', 'ignore')
1487 return detect_exe_version(out, version_re, unrecognized)
1490 def detect_exe_version(output, version_re=None, unrecognized='present'):
1491 assert isinstance(output, compat_str)
1492 if version_re is None:
1493 version_re = r'version\s+([-0-9._a-zA-Z]+)'
1494 m = re.search(version_re, output)
1501 class PagedList(object):
1503 # This is only useful for tests
1504 return len(self.getslice())
1507 class OnDemandPagedList(PagedList):
1508 def __init__(self, pagefunc, pagesize):
1509 self._pagefunc = pagefunc
1510 self._pagesize = pagesize
1512 def getslice(self, start=0, end=None):
1514 for pagenum in itertools.count(start // self._pagesize):
1515 firstid = pagenum * self._pagesize
1516 nextfirstid = pagenum * self._pagesize + self._pagesize
1517 if start >= nextfirstid:
1520 page_results = list(self._pagefunc(pagenum))
1523 start % self._pagesize
1524 if firstid <= start < nextfirstid
1528 ((end - 1) % self._pagesize) + 1
1529 if (end is not None and firstid <= end <= nextfirstid)
1532 if startv != 0 or endv is not None:
1533 page_results = page_results[startv:endv]
1534 res.extend(page_results)
1536 # A little optimization - if current page is not "full", ie. does
1537 # not contain page_size videos then we can assume that this page
1538 # is the last one - there are no more ids on further pages -
1539 # i.e. no need to query again.
1540 if len(page_results) + startv < self._pagesize:
1543 # If we got the whole page, but the next page is not interesting,
1544 # break out early as well
1545 if end == nextfirstid:
1550 class InAdvancePagedList(PagedList):
1551 def __init__(self, pagefunc, pagecount, pagesize):
1552 self._pagefunc = pagefunc
1553 self._pagecount = pagecount
1554 self._pagesize = pagesize
1556 def getslice(self, start=0, end=None):
1558 start_page = start // self._pagesize
1560 self._pagecount if end is None else (end // self._pagesize + 1))
1561 skip_elems = start - start_page * self._pagesize
1562 only_more = None if end is None else end - start
1563 for pagenum in range(start_page, end_page):
1564 page = list(self._pagefunc(pagenum))
1566 page = page[skip_elems:]
1568 if only_more is not None:
1569 if len(page) < only_more:
1570 only_more -= len(page)
1572 page = page[:only_more]
1579 def uppercase_escape(s):
1580 unicode_escape = codecs.getdecoder('unicode_escape')
1582 r'\\U[0-9a-fA-F]{8}',
1583 lambda m: unicode_escape(m.group(0))[0],
1587 def lowercase_escape(s):
1588 unicode_escape = codecs.getdecoder('unicode_escape')
1590 r'\\u[0-9a-fA-F]{4}',
1591 lambda m: unicode_escape(m.group(0))[0],
1595 def escape_rfc3986(s):
1596 """Escape non-ASCII characters as suggested by RFC 3986"""
1597 if sys.version_info < (3, 0) and isinstance(s, compat_str):
1598 s = s.encode('utf-8')
1599 return compat_urllib_parse.quote(s, b"%/;:@&=+$,!~*'()?#[]")
1602 def escape_url(url):
1603 """Escape URL as suggested by RFC 3986"""
1604 url_parsed = compat_urllib_parse_urlparse(url)
1605 return url_parsed._replace(
1606 path=escape_rfc3986(url_parsed.path),
1607 params=escape_rfc3986(url_parsed.params),
1608 query=escape_rfc3986(url_parsed.query),
1609 fragment=escape_rfc3986(url_parsed.fragment)
1613 struct.pack('!I', 0)
1615 # In Python 2.6 (and some 2.7 versions), struct requires a bytes argument
1616 def struct_pack(spec, *args):
1617 if isinstance(spec, compat_str):
1618 spec = spec.encode('ascii')
1619 return struct.pack(spec, *args)
1621 def struct_unpack(spec, *args):
1622 if isinstance(spec, compat_str):
1623 spec = spec.encode('ascii')
1624 return struct.unpack(spec, *args)
1626 struct_pack = struct.pack
1627 struct_unpack = struct.unpack
1630 def read_batch_urls(batch_fd):
1632 if not isinstance(url, compat_str):
1633 url = url.decode('utf-8', 'replace')
1634 BOM_UTF8 = '\xef\xbb\xbf'
1635 if url.startswith(BOM_UTF8):
1636 url = url[len(BOM_UTF8):]
1638 if url.startswith(('#', ';', ']')):
1642 with contextlib.closing(batch_fd) as fd:
1643 return [url for url in map(fixup, fd) if url]
1646 def urlencode_postdata(*args, **kargs):
1647 return compat_urllib_parse.urlencode(*args, **kargs).encode('ascii')
1650 def encode_dict(d, encoding='utf-8'):
1651 return dict((k.encode(encoding), v.encode(encoding)) for k, v in d.items())
1655 etree_iter = xml.etree.ElementTree.Element.iter
1656 except AttributeError: # Python <=2.6
1657 etree_iter = lambda n: n.findall('.//*')
1661 class TreeBuilder(xml.etree.ElementTree.TreeBuilder):
1662 def doctype(self, name, pubid, system):
1663 pass # Ignore doctypes
1665 parser = xml.etree.ElementTree.XMLParser(target=TreeBuilder())
1666 kwargs = {'parser': parser} if sys.version_info >= (2, 7) else {}
1667 tree = xml.etree.ElementTree.XML(s.encode('utf-8'), **kwargs)
1668 # Fix up XML parser in Python 2.x
1669 if sys.version_info < (3, 0):
1670 for n in etree_iter(tree):
1671 if n.text is not None:
1672 if not isinstance(n.text, compat_str):
1673 n.text = n.text.decode('utf-8')
1686 def parse_age_limit(s):
1689 m = re.match(r'^(?P<age>\d{1,2})\+?$', s)
1690 return int(m.group('age')) if m else US_RATINGS.get(s, None)
1693 def strip_jsonp(code):
1695 r'(?s)^[a-zA-Z0-9_]+\s*\(\s*(.*)\);?\s*?(?://[^\n]*)*$', r'\1', code)
1698 def js_to_json(code):
1701 if v in ('true', 'false', 'null'):
1703 if v.startswith('"'):
1704 v = re.sub(r"\\'", "'", v[1:-1])
1705 elif v.startswith("'"):
1707 v = re.sub(r"\\\\|\\'|\"", lambda m: {
1714 res = re.sub(r'''(?x)
1715 "(?:[^"\\]*(?:\\\\|\\['"nu]))*[^"\\]*"|
1716 '(?:[^'\\]*(?:\\\\|\\['"nu]))*[^'\\]*'|
1717 [a-zA-Z_][.a-zA-Z_0-9]*
1719 res = re.sub(r',(\s*[\]}])', lambda m: m.group(1), res)
1723 def qualities(quality_ids):
1724 """ Get a numeric quality value out of a list of possible values """
1727 return quality_ids.index(qid)
1733 DEFAULT_OUTTMPL = '%(title)s-%(id)s.%(ext)s'
1736 def limit_length(s, length):
1737 """ Add ellipses to overly long strings """
1742 return s[:length - len(ELLIPSES)] + ELLIPSES
1746 def version_tuple(v):
1747 return tuple(int(e) for e in re.split(r'[-.]', v))
1750 def is_outdated_version(version, limit, assume_new=True):
1752 return not assume_new
1754 return version_tuple(version) < version_tuple(limit)
1756 return not assume_new
1759 def ytdl_is_updateable():
1760 """ Returns if youtube-dl can be updated with -U """
1761 from zipimport import zipimporter
1763 return isinstance(globals().get('__loader__'), zipimporter) or hasattr(sys, 'frozen')
1766 def args_to_str(args):
1767 # Get a short string representation for a subprocess command
1768 return ' '.join(shlex_quote(a) for a in args)
1771 def mimetype2ext(mt):
1772 _, _, res = mt.rpartition('/')
1776 'x-mp4-fragmented': 'mp4',
1781 def urlhandle_detect_ext(url_handle):
1784 getheader = lambda h: url_handle.headers[h]
1785 except AttributeError: # Python < 3
1786 getheader = url_handle.info().getheader
1788 cd = getheader('Content-Disposition')
1790 m = re.match(r'attachment;\s*filename="(?P<filename>[^"]+)"', cd)
1792 e = determine_ext(m.group('filename'), default_ext=None)
1796 return mimetype2ext(getheader('Content-Type'))
1799 def encode_data_uri(data, mime_type):
1800 return 'data:%s;base64,%s' % (mime_type, base64.b64encode(data).decode('ascii'))
1803 def age_restricted(content_limit, age_limit):
1804 """ Returns True iff the content should be blocked """
1806 if age_limit is None: # No limit set
1808 if content_limit is None:
1809 return False # Content available for everyone
1810 return age_limit < content_limit
1813 def is_html(first_bytes):
1814 """ Detect whether a file contains HTML by examining its first bytes. """
1817 (b'\xef\xbb\xbf', 'utf-8'),
1818 (b'\x00\x00\xfe\xff', 'utf-32-be'),
1819 (b'\xff\xfe\x00\x00', 'utf-32-le'),
1820 (b'\xff\xfe', 'utf-16-le'),
1821 (b'\xfe\xff', 'utf-16-be'),
1823 for bom, enc in BOMS:
1824 if first_bytes.startswith(bom):
1825 s = first_bytes[len(bom):].decode(enc, 'replace')
1828 s = first_bytes.decode('utf-8', 'replace')
1830 return re.match(r'^\s*<', s)
1833 def determine_protocol(info_dict):
1834 protocol = info_dict.get('protocol')
1835 if protocol is not None:
1838 url = info_dict['url']
1839 if url.startswith('rtmp'):
1841 elif url.startswith('mms'):
1843 elif url.startswith('rtsp'):
1846 ext = determine_ext(url)
1852 return compat_urllib_parse_urlparse(url).scheme
1855 def render_table(header_row, data):
1856 """ Render a list of rows, each as a list of values """
1857 table = [header_row] + data
1858 max_lens = [max(len(compat_str(v)) for v in col) for col in zip(*table)]
1859 format_str = ' '.join('%-' + compat_str(ml + 1) + 's' for ml in max_lens[:-1]) + '%s'
1860 return '\n'.join(format_str % tuple(row) for row in table)
1863 def _match_one(filter_part, dct):
1864 COMPARISON_OPERATORS = {
1872 operator_rex = re.compile(r'''(?x)\s*
1874 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
1876 (?P<intval>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)|
1877 (?P<strval>(?![0-9.])[a-z0-9A-Z]*)
1880 ''' % '|'.join(map(re.escape, COMPARISON_OPERATORS.keys())))
1881 m = operator_rex.search(filter_part)
1883 op = COMPARISON_OPERATORS[m.group('op')]
1884 if m.group('strval') is not None:
1885 if m.group('op') not in ('=', '!='):
1887 'Operator %s does not support string values!' % m.group('op'))
1888 comparison_value = m.group('strval')
1891 comparison_value = int(m.group('intval'))
1893 comparison_value = parse_filesize(m.group('intval'))
1894 if comparison_value is None:
1895 comparison_value = parse_filesize(m.group('intval') + 'B')
1896 if comparison_value is None:
1898 'Invalid integer value %r in filter part %r' % (
1899 m.group('intval'), filter_part))
1900 actual_value = dct.get(m.group('key'))
1901 if actual_value is None:
1902 return m.group('none_inclusive')
1903 return op(actual_value, comparison_value)
1906 '': lambda v: v is not None,
1907 '!': lambda v: v is None,
1909 operator_rex = re.compile(r'''(?x)\s*
1910 (?P<op>%s)\s*(?P<key>[a-z_]+)
1912 ''' % '|'.join(map(re.escape, UNARY_OPERATORS.keys())))
1913 m = operator_rex.search(filter_part)
1915 op = UNARY_OPERATORS[m.group('op')]
1916 actual_value = dct.get(m.group('key'))
1917 return op(actual_value)
1919 raise ValueError('Invalid filter part %r' % filter_part)
1922 def match_str(filter_str, dct):
1923 """ Filter a dictionary with a simple string syntax. Returns True (=passes filter) or false """
1926 _match_one(filter_part, dct) for filter_part in filter_str.split('&'))
1929 def match_filter_func(filter_str):
1930 def _match_func(info_dict):
1931 if match_str(filter_str, info_dict):
1934 video_title = info_dict.get('title', info_dict.get('id', 'video'))
1935 return '%s does not pass filter %s, skipping ..' % (video_title, filter_str)
1939 def parse_dfxp_time_expr(time_expr):
1943 mobj = re.match(r'^(?P<time_offset>\d+(?:\.\d+)?)s?$', time_expr)
1945 return float(mobj.group('time_offset'))
1947 mobj = re.match(r'^(\d+):(\d\d):(\d\d(?:\.\d+)?)$', time_expr)
1949 return 3600 * int(mobj.group(1)) + 60 * int(mobj.group(2)) + float(mobj.group(3))
1952 def srt_subtitles_timecode(seconds):
1953 return '%02d:%02d:%02d,%03d' % (seconds / 3600, (seconds % 3600) / 60, seconds % 60, (seconds % 1) * 1000)
1956 def dfxp2srt(dfxp_data):
1957 _x = functools.partial(xpath_with_ns, ns_map={
1958 'ttml': 'http://www.w3.org/ns/ttml',
1959 'ttaf1': 'http://www.w3.org/2006/10/ttaf1',
1962 def parse_node(node):
1963 str_or_empty = functools.partial(str_or_none, default='')
1965 out = str_or_empty(node.text)
1968 if child.tag in (_x('ttml:br'), _x('ttaf1:br'), 'br'):
1969 out += '\n' + str_or_empty(child.tail)
1970 elif child.tag in (_x('ttml:span'), _x('ttaf1:span'), 'span'):
1971 out += str_or_empty(parse_node(child))
1973 out += str_or_empty(xml.etree.ElementTree.tostring(child))
1977 dfxp = xml.etree.ElementTree.fromstring(dfxp_data.encode('utf-8'))
1979 paras = dfxp.findall(_x('.//ttml:p')) or dfxp.findall(_x('.//ttaf1:p')) or dfxp.findall('.//p')
1982 raise ValueError('Invalid dfxp/TTML subtitle')
1984 for para, index in zip(paras, itertools.count(1)):
1985 begin_time = parse_dfxp_time_expr(para.attrib['begin'])
1986 end_time = parse_dfxp_time_expr(para.attrib.get('end'))
1988 end_time = begin_time + parse_dfxp_time_expr(para.attrib['dur'])
1989 out.append('%d\n%s --> %s\n%s\n\n' % (
1991 srt_subtitles_timecode(begin_time),
1992 srt_subtitles_timecode(end_time),
1998 def cli_option(params, command_option, param):
1999 param = params.get(param)
2000 return [command_option, param] if param is not None else []
2003 def cli_bool_option(params, command_option, param, true_value='true', false_value='false', separator=None):
2004 param = params.get(param)
2005 assert isinstance(param, bool)
2007 return [command_option + separator + (true_value if param else false_value)]
2008 return [command_option, true_value if param else false_value]
2011 def cli_valueless_option(params, command_option, param, expected_value=True):
2012 param = params.get(param)
2013 return [command_option] if param == expected_value else []
2016 def cli_configuration_args(params, param, default=[]):
2017 ex_args = params.get(param)
2020 assert isinstance(ex_args, list)
2024 class ISO639Utils(object):
2025 # See http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
2214 def short2long(cls, code):
2215 """Convert language code from ISO 639-1 to ISO 639-2/T"""
2216 return cls._lang_map.get(code[:2])
2219 def long2short(cls, code):
2220 """Convert language code from ISO 639-2/T to ISO 639-1"""
2221 for short_name, long_name in cls._lang_map.items():
2222 if long_name == code:
2226 class ISO3166Utils(object):
2227 # From http://data.okfn.org/data/core/country-list
2229 'AF': 'Afghanistan',
2230 'AX': 'Ã…land Islands',
2233 'AS': 'American Samoa',
2238 'AG': 'Antigua and Barbuda',
2255 'BO': 'Bolivia, Plurinational State of',
2256 'BQ': 'Bonaire, Sint Eustatius and Saba',
2257 'BA': 'Bosnia and Herzegovina',
2259 'BV': 'Bouvet Island',
2261 'IO': 'British Indian Ocean Territory',
2262 'BN': 'Brunei Darussalam',
2264 'BF': 'Burkina Faso',
2270 'KY': 'Cayman Islands',
2271 'CF': 'Central African Republic',
2275 'CX': 'Christmas Island',
2276 'CC': 'Cocos (Keeling) Islands',
2280 'CD': 'Congo, the Democratic Republic of the',
2281 'CK': 'Cook Islands',
2283 'CI': 'Côte d\'Ivoire',
2288 'CZ': 'Czech Republic',
2292 'DO': 'Dominican Republic',
2295 'SV': 'El Salvador',
2296 'GQ': 'Equatorial Guinea',
2300 'FK': 'Falkland Islands (Malvinas)',
2301 'FO': 'Faroe Islands',
2305 'GF': 'French Guiana',
2306 'PF': 'French Polynesia',
2307 'TF': 'French Southern Territories',
2322 'GW': 'Guinea-Bissau',
2325 'HM': 'Heard Island and McDonald Islands',
2326 'VA': 'Holy See (Vatican City State)',
2333 'IR': 'Iran, Islamic Republic of',
2336 'IM': 'Isle of Man',
2346 'KP': 'Korea, Democratic People\'s Republic of',
2347 'KR': 'Korea, Republic of',
2350 'LA': 'Lao People\'s Democratic Republic',
2356 'LI': 'Liechtenstein',
2360 'MK': 'Macedonia, the Former Yugoslav Republic of',
2367 'MH': 'Marshall Islands',
2373 'FM': 'Micronesia, Federated States of',
2374 'MD': 'Moldova, Republic of',
2385 'NL': 'Netherlands',
2386 'NC': 'New Caledonia',
2387 'NZ': 'New Zealand',
2392 'NF': 'Norfolk Island',
2393 'MP': 'Northern Mariana Islands',
2398 'PS': 'Palestine, State of',
2400 'PG': 'Papua New Guinea',
2403 'PH': 'Philippines',
2407 'PR': 'Puerto Rico',
2411 'RU': 'Russian Federation',
2413 'BL': 'Saint Barthélemy',
2414 'SH': 'Saint Helena, Ascension and Tristan da Cunha',
2415 'KN': 'Saint Kitts and Nevis',
2416 'LC': 'Saint Lucia',
2417 'MF': 'Saint Martin (French part)',
2418 'PM': 'Saint Pierre and Miquelon',
2419 'VC': 'Saint Vincent and the Grenadines',
2422 'ST': 'Sao Tome and Principe',
2423 'SA': 'Saudi Arabia',
2427 'SL': 'Sierra Leone',
2429 'SX': 'Sint Maarten (Dutch part)',
2432 'SB': 'Solomon Islands',
2434 'ZA': 'South Africa',
2435 'GS': 'South Georgia and the South Sandwich Islands',
2436 'SS': 'South Sudan',
2441 'SJ': 'Svalbard and Jan Mayen',
2444 'CH': 'Switzerland',
2445 'SY': 'Syrian Arab Republic',
2446 'TW': 'Taiwan, Province of China',
2448 'TZ': 'Tanzania, United Republic of',
2450 'TL': 'Timor-Leste',
2454 'TT': 'Trinidad and Tobago',
2457 'TM': 'Turkmenistan',
2458 'TC': 'Turks and Caicos Islands',
2462 'AE': 'United Arab Emirates',
2463 'GB': 'United Kingdom',
2464 'US': 'United States',
2465 'UM': 'United States Minor Outlying Islands',
2469 'VE': 'Venezuela, Bolivarian Republic of',
2471 'VG': 'Virgin Islands, British',
2472 'VI': 'Virgin Islands, U.S.',
2473 'WF': 'Wallis and Futuna',
2474 'EH': 'Western Sahara',
2481 def short2full(cls, code):
2482 """Convert an ISO 3166-2 country code to the corresponding full name"""
2483 return cls._country_map.get(code.upper())
2486 class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
2487 def __init__(self, proxies=None):
2488 # Set default handlers
2489 for type in ('http', 'https'):
2490 setattr(self, '%s_open' % type,
2491 lambda r, proxy='__noproxy__', type=type, meth=self.proxy_open:
2492 meth(r, proxy, type))
2493 return compat_urllib_request.ProxyHandler.__init__(self, proxies)
2495 def proxy_open(self, req, proxy, type):
2496 req_proxy = req.headers.get('Ytdl-request-proxy')
2497 if req_proxy is not None:
2499 del req.headers['Ytdl-request-proxy']
2501 if proxy == '__noproxy__':
2502 return None # No Proxy
2503 return compat_urllib_request.ProxyHandler.proxy_open(
2504 self, req, proxy, type)