2 # -*- coding: utf-8 -*-
4 from __future__ import unicode_literals
33 import xml.etree.ElementTree
43 compat_socket_create_connection,
47 compat_urllib_parse_urlparse,
48 compat_urllib_request,
54 # This is not clearly defined otherwise
55 compiled_regex_type = type(re.compile(''))
58 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20150101 Firefox/20.0 (Chrome)',
59 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
60 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
61 'Accept-Encoding': 'gzip, deflate',
62 'Accept-Language': 'en-us,en;q=0.5',
68 ENGLISH_MONTH_NAMES = [
69 'January', 'February', 'March', 'April', 'May', 'June',
70 'July', 'August', 'September', 'October', 'November', 'December']
73 def preferredencoding():
74 """Get preferred encoding.
76 Returns the best encoding scheme for the system, based on
77 locale.getpreferredencoding() and some further tweaks.
80 pref = locale.getpreferredencoding()
88 def write_json_file(obj, fn):
89 """ Encode obj as JSON and write it to fn, atomically if possible """
91 fn = encodeFilename(fn)
92 if sys.version_info < (3, 0) and sys.platform != 'win32':
93 encoding = get_filesystem_encoding()
94 # os.path.basename returns a bytes object, but NamedTemporaryFile
95 # will fail if the filename contains non ascii characters unless we
96 # use a unicode object
97 path_basename = lambda f: os.path.basename(fn).decode(encoding)
98 # the same for os.path.dirname
99 path_dirname = lambda f: os.path.dirname(fn).decode(encoding)
101 path_basename = os.path.basename
102 path_dirname = os.path.dirname
106 'prefix': path_basename(fn) + '.',
107 'dir': path_dirname(fn),
111 # In Python 2.x, json.dump expects a bytestream.
112 # In Python 3.x, it writes to a character stream
113 if sys.version_info < (3, 0):
121 tf = tempfile.NamedTemporaryFile(**compat_kwargs(args))
126 if sys.platform == 'win32':
127 # Need to remove existing file on Windows, else os.rename raises
128 # WindowsError or FileExistsError.
133 os.rename(tf.name, fn)
142 if sys.version_info >= (2, 7):
143 def find_xpath_attr(node, xpath, key, val=None):
144 """ Find the xpath xpath[@key=val] """
145 assert re.match(r'^[a-zA-Z_-]+$', key)
147 assert re.match(r'^[a-zA-Z0-9@\s:._-]*$', val)
148 expr = xpath + ('[@%s]' % key if val is None else "[@%s='%s']" % (key, val))
149 return node.find(expr)
151 def find_xpath_attr(node, xpath, key, val=None):
152 # Here comes the crazy part: In 2.6, if the xpath is a unicode,
153 # .//node does not match if a node is a direct child of . !
154 if isinstance(xpath, compat_str):
155 xpath = xpath.encode('ascii')
157 for f in node.findall(xpath):
158 if key not in f.attrib:
160 if val is None or f.attrib.get(key) == val:
164 # On python2.6 the xml.etree.ElementTree.Element methods don't support
165 # the namespace parameter
168 def xpath_with_ns(path, ns_map):
169 components = [c.split(':') for c in path.split('/')]
173 replaced.append(c[0])
176 replaced.append('{%s}%s' % (ns_map[ns], tag))
177 return '/'.join(replaced)
180 def xpath_element(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
181 def _find_xpath(xpath):
182 if sys.version_info < (2, 7): # Crazy 2.6
183 xpath = xpath.encode('ascii')
184 return node.find(xpath)
186 if isinstance(xpath, (str, compat_str)):
187 n = _find_xpath(xpath)
195 if default is not NO_DEFAULT:
198 name = xpath if name is None else name
199 raise ExtractorError('Could not find XML element %s' % name)
205 def xpath_text(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
206 n = xpath_element(node, xpath, name, fatal=fatal, default=default)
207 if n is None or n == default:
210 if default is not NO_DEFAULT:
213 name = xpath if name is None else name
214 raise ExtractorError('Could not find XML element\'s text %s' % name)
220 def xpath_attr(node, xpath, key, name=None, fatal=False, default=NO_DEFAULT):
221 n = find_xpath_attr(node, xpath, key)
223 if default is not NO_DEFAULT:
226 name = '%s[@%s]' % (xpath, key) if name is None else name
227 raise ExtractorError('Could not find XML attribute %s' % name)
233 def get_element_by_id(id, html):
234 """Return the content of the tag with the specified ID in the passed HTML document"""
235 return get_element_by_attribute("id", id, html)
238 def get_element_by_attribute(attribute, value, html):
239 """Return the content of the tag with the specified attribute in the passed HTML document"""
241 m = re.search(r'''(?xs)
243 (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?
245 (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?
249 ''' % (re.escape(attribute), re.escape(value)), html)
253 res = m.group('content')
255 if res.startswith('"') or res.startswith("'"):
258 return unescapeHTML(res)
261 def clean_html(html):
262 """Clean an HTML snippet into a readable string"""
264 if html is None: # Convenience for sanitizing descriptions etc.
268 html = html.replace('\n', ' ')
269 html = re.sub(r'\s*<\s*br\s*/?\s*>\s*', '\n', html)
270 html = re.sub(r'<\s*/\s*p\s*>\s*<\s*p[^>]*>', '\n', html)
272 html = re.sub('<.*?>', '', html)
273 # Replace html entities
274 html = unescapeHTML(html)
278 def sanitize_open(filename, open_mode):
279 """Try to open the given filename, and slightly tweak it if this fails.
281 Attempts to open the given filename. If this fails, it tries to change
282 the filename slightly, step by step, until it's either able to open it
283 or it fails and raises a final exception, like the standard open()
286 It returns the tuple (stream, definitive_file_name).
290 if sys.platform == 'win32':
292 msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
293 return (sys.stdout.buffer if hasattr(sys.stdout, 'buffer') else sys.stdout, filename)
294 stream = open(encodeFilename(filename), open_mode)
295 return (stream, filename)
296 except (IOError, OSError) as err:
297 if err.errno in (errno.EACCES,):
300 # In case of error, try to remove win32 forbidden chars
301 alt_filename = sanitize_path(filename)
302 if alt_filename == filename:
305 # An exception here should be caught in the caller
306 stream = open(encodeFilename(alt_filename), open_mode)
307 return (stream, alt_filename)
310 def timeconvert(timestr):
311 """Convert RFC 2822 defined time string into system timestamp"""
313 timetuple = email.utils.parsedate_tz(timestr)
314 if timetuple is not None:
315 timestamp = email.utils.mktime_tz(timetuple)
319 def sanitize_filename(s, restricted=False, is_id=False):
320 """Sanitizes a string so it could be used as part of a filename.
321 If restricted is set, use a stricter subset of allowed characters.
322 Set is_id if this is not an arbitrary string, but an ID that should be kept if possible
324 def replace_insane(char):
325 if char == '?' or ord(char) < 32 or ord(char) == 127:
328 return '' if restricted else '\''
330 return '_-' if restricted else ' -'
331 elif char in '\\/|*<>':
333 if restricted and (char in '!&\'()[]{}$;`^,#' or char.isspace()):
335 if restricted and ord(char) > 127:
340 s = re.sub(r'[0-9]+(?::[0-9]+)+', lambda m: m.group(0).replace(':', '_'), s)
341 result = ''.join(map(replace_insane, s))
343 while '__' in result:
344 result = result.replace('__', '_')
345 result = result.strip('_')
346 # Common case of "Foreign band name - English song title"
347 if restricted and result.startswith('-_'):
349 if result.startswith('-'):
350 result = '_' + result[len('-'):]
351 result = result.lstrip('.')
357 def sanitize_path(s):
358 """Sanitizes and normalizes path on Windows"""
359 if sys.platform != 'win32':
361 drive_or_unc, _ = os.path.splitdrive(s)
362 if sys.version_info < (2, 7) and not drive_or_unc:
363 drive_or_unc, _ = os.path.splitunc(s)
364 norm_path = os.path.normpath(remove_start(s, drive_or_unc)).split(os.path.sep)
368 path_part if path_part in ['.', '..'] else re.sub('(?:[/<>:"\\|\\\\?\\*]|\.$)', '#', path_part)
369 for path_part in norm_path]
371 sanitized_path.insert(0, drive_or_unc + os.path.sep)
372 return os.path.join(*sanitized_path)
375 def orderedSet(iterable):
376 """ Remove all duplicates from the input iterable """
384 def _htmlentity_transform(entity):
385 """Transforms an HTML entity to a character."""
386 # Known non-numeric HTML entity
387 if entity in compat_html_entities.name2codepoint:
388 return compat_chr(compat_html_entities.name2codepoint[entity])
390 mobj = re.match(r'#(x[0-9a-fA-F]+|[0-9]+)', entity)
392 numstr = mobj.group(1)
393 if numstr.startswith('x'):
395 numstr = '0%s' % numstr
398 return compat_chr(int(numstr, base))
400 # Unknown entity in name, return its literal representation
401 return ('&%s;' % entity)
407 assert type(s) == compat_str
410 r'&([^;]+);', lambda m: _htmlentity_transform(m.group(1)), s)
413 def get_subprocess_encoding():
414 if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
415 # For subprocess calls, encode with locale encoding
416 # Refer to http://stackoverflow.com/a/9951851/35070
417 encoding = preferredencoding()
419 encoding = sys.getfilesystemencoding()
425 def encodeFilename(s, for_subprocess=False):
427 @param s The name of the file
430 assert type(s) == compat_str
432 # Python 3 has a Unicode API
433 if sys.version_info >= (3, 0):
436 # Pass '' directly to use Unicode APIs on Windows 2000 and up
437 # (Detecting Windows NT 4 is tricky because 'major >= 4' would
438 # match Windows 9x series as well. Besides, NT 4 is obsolete.)
439 if not for_subprocess and sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
442 return s.encode(get_subprocess_encoding(), 'ignore')
445 def decodeFilename(b, for_subprocess=False):
447 if sys.version_info >= (3, 0):
450 if not isinstance(b, bytes):
453 return b.decode(get_subprocess_encoding(), 'ignore')
456 def encodeArgument(s):
457 if not isinstance(s, compat_str):
458 # Legacy code that uses byte strings
459 # Uncomment the following line after fixing all post processors
460 # assert False, 'Internal error: %r should be of type %r, is %r' % (s, compat_str, type(s))
461 s = s.decode('ascii')
462 return encodeFilename(s, True)
465 def decodeArgument(b):
466 return decodeFilename(b, True)
469 def decodeOption(optval):
472 if isinstance(optval, bytes):
473 optval = optval.decode(preferredencoding())
475 assert isinstance(optval, compat_str)
479 def formatSeconds(secs):
481 return '%d:%02d:%02d' % (secs // 3600, (secs % 3600) // 60, secs % 60)
483 return '%d:%02d' % (secs // 60, secs % 60)
488 def make_HTTPS_handler(params, **kwargs):
489 opts_no_check_certificate = params.get('nocheckcertificate', False)
490 if hasattr(ssl, 'create_default_context'): # Python >= 3.4 or 2.7.9
491 context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
492 if opts_no_check_certificate:
493 context.check_hostname = False
494 context.verify_mode = ssl.CERT_NONE
496 return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
499 # (create_default_context present but HTTPSHandler has no context=)
502 if sys.version_info < (3, 2):
503 return YoutubeDLHTTPSHandler(params, **kwargs)
505 context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
506 context.verify_mode = (ssl.CERT_NONE
507 if opts_no_check_certificate
508 else ssl.CERT_REQUIRED)
509 context.set_default_verify_paths()
510 return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
513 def bug_reports_message():
514 if ytdl_is_updateable():
515 update_cmd = 'type youtube-dl -U to update'
517 update_cmd = 'see https://yt-dl.org/update on how to update'
518 msg = '; please report this issue on https://yt-dl.org/bug .'
519 msg += ' Make sure you are using the latest version; %s.' % update_cmd
520 msg += ' Be sure to call youtube-dl with the --verbose flag and include its complete output.'
524 class ExtractorError(Exception):
525 """Error during info extraction."""
527 def __init__(self, msg, tb=None, expected=False, cause=None, video_id=None):
528 """ tb, if given, is the original traceback (so that it can be printed out).
529 If expected is set, this is a normal error message and most likely not a bug in youtube-dl.
532 if sys.exc_info()[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError):
534 if video_id is not None:
535 msg = video_id + ': ' + msg
537 msg += ' (caused by %r)' % cause
539 msg += bug_reports_message()
540 super(ExtractorError, self).__init__(msg)
543 self.exc_info = sys.exc_info() # preserve original exception
545 self.video_id = video_id
547 def format_traceback(self):
548 if self.traceback is None:
550 return ''.join(traceback.format_tb(self.traceback))
553 class UnsupportedError(ExtractorError):
554 def __init__(self, url):
555 super(UnsupportedError, self).__init__(
556 'Unsupported URL: %s' % url, expected=True)
560 class RegexNotFoundError(ExtractorError):
561 """Error when a regex didn't match"""
565 class DownloadError(Exception):
566 """Download Error exception.
568 This exception may be thrown by FileDownloader objects if they are not
569 configured to continue on errors. They will contain the appropriate
573 def __init__(self, msg, exc_info=None):
574 """ exc_info, if given, is the original exception that caused the trouble (as returned by sys.exc_info()). """
575 super(DownloadError, self).__init__(msg)
576 self.exc_info = exc_info
579 class SameFileError(Exception):
580 """Same File exception.
582 This exception will be thrown by FileDownloader objects if they detect
583 multiple files would have to be downloaded to the same file on disk.
588 class PostProcessingError(Exception):
589 """Post Processing exception.
591 This exception may be raised by PostProcessor's .run() method to
592 indicate an error in the postprocessing task.
595 def __init__(self, msg):
599 class MaxDownloadsReached(Exception):
600 """ --max-downloads limit has been reached. """
604 class UnavailableVideoError(Exception):
605 """Unavailable Format exception.
607 This exception will be thrown when a video is requested
608 in a format that is not available for that video.
613 class ContentTooShortError(Exception):
614 """Content Too Short exception.
616 This exception may be raised by FileDownloader objects when a file they
617 download is too small for what the server announced first, indicating
618 the connection was probably interrupted.
621 def __init__(self, downloaded, expected):
623 self.downloaded = downloaded
624 self.expected = expected
627 def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs):
628 # Working around python 2 bug (see http://bugs.python.org/issue17849) by limiting
629 # expected HTTP responses to meet HTTP/1.0 or later (see also
630 # https://github.com/rg3/youtube-dl/issues/6727)
631 if sys.version_info < (3, 0):
632 kwargs[b'strict'] = True
633 hc = http_class(*args, **kwargs)
634 source_address = ydl_handler._params.get('source_address')
635 if source_address is not None:
636 sa = (source_address, 0)
637 if hasattr(hc, 'source_address'): # Python 2.7+
638 hc.source_address = sa
640 def _hc_connect(self, *args, **kwargs):
641 sock = compat_socket_create_connection(
642 (self.host, self.port), self.timeout, sa)
644 self.sock = ssl.wrap_socket(
645 sock, self.key_file, self.cert_file,
646 ssl_version=ssl.PROTOCOL_TLSv1)
649 hc.connect = functools.partial(_hc_connect, hc)
654 class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
655 """Handler for HTTP requests and responses.
657 This class, when installed with an OpenerDirector, automatically adds
658 the standard headers to every HTTP request and handles gzipped and
659 deflated responses from web servers. If compression is to be avoided in
660 a particular request, the original request in the program code only has
661 to include the HTTP header "Youtubedl-No-Compression", which will be
662 removed before making the real request.
664 Part of this code was copied from:
666 http://techknack.net/python-urllib2-handlers/
668 Andrew Rowls, the author of that code, agreed to release it to the
672 def __init__(self, params, *args, **kwargs):
673 compat_urllib_request.HTTPHandler.__init__(self, *args, **kwargs)
674 self._params = params
676 def http_open(self, req):
677 return self.do_open(functools.partial(
678 _create_http_connection, self, compat_http_client.HTTPConnection, False),
684 return zlib.decompress(data, -zlib.MAX_WBITS)
686 return zlib.decompress(data)
689 def addinfourl_wrapper(stream, headers, url, code):
690 if hasattr(compat_urllib_request.addinfourl, 'getcode'):
691 return compat_urllib_request.addinfourl(stream, headers, url, code)
692 ret = compat_urllib_request.addinfourl(stream, headers, url)
696 def http_request(self, req):
697 # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
698 # always respected by websites, some tend to give out URLs with non percent-encoded
699 # non-ASCII characters (see telemb.py, ard.py [#3412])
700 # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
701 # To work around aforementioned issue we will replace request's original URL with
702 # percent-encoded one
703 # Since redirects are also affected (e.g. http://www.southpark.de/alle-episoden/s18e09)
704 # the code of this workaround has been moved here from YoutubeDL.urlopen()
705 url = req.get_full_url()
706 url_escaped = escape_url(url)
708 # Substitute URL if any change after escaping
709 if url != url_escaped:
710 req_type = HEADRequest if req.get_method() == 'HEAD' else compat_urllib_request.Request
712 url_escaped, data=req.data, headers=req.headers,
713 origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
714 new_req.timeout = req.timeout
717 for h, v in std_headers.items():
718 # Capitalize is needed because of Python bug 2275: http://bugs.python.org/issue2275
719 # The dict keys are capitalized because of this bug by urllib
720 if h.capitalize() not in req.headers:
722 if 'Youtubedl-no-compression' in req.headers:
723 if 'Accept-encoding' in req.headers:
724 del req.headers['Accept-encoding']
725 del req.headers['Youtubedl-no-compression']
727 if sys.version_info < (2, 7) and '#' in req.get_full_url():
728 # Python 2.6 is brain-dead when it comes to fragments
729 req._Request__original = req._Request__original.partition('#')[0]
730 req._Request__r_type = req._Request__r_type.partition('#')[0]
734 def http_response(self, req, resp):
737 if resp.headers.get('Content-encoding', '') == 'gzip':
738 content = resp.read()
739 gz = gzip.GzipFile(fileobj=io.BytesIO(content), mode='rb')
741 uncompressed = io.BytesIO(gz.read())
742 except IOError as original_ioerror:
743 # There may be junk add the end of the file
744 # See http://stackoverflow.com/q/4928560/35070 for details
745 for i in range(1, 1024):
747 gz = gzip.GzipFile(fileobj=io.BytesIO(content[:-i]), mode='rb')
748 uncompressed = io.BytesIO(gz.read())
753 raise original_ioerror
754 resp = self.addinfourl_wrapper(uncompressed, old_resp.headers, old_resp.url, old_resp.code)
755 resp.msg = old_resp.msg
757 if resp.headers.get('Content-encoding', '') == 'deflate':
758 gz = io.BytesIO(self.deflate(resp.read()))
759 resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
760 resp.msg = old_resp.msg
761 # Percent-encode redirect URL of Location HTTP header to satisfy RFC 3986 (see
762 # https://github.com/rg3/youtube-dl/issues/6457).
763 if 300 <= resp.code < 400:
764 location = resp.headers.get('Location')
766 # As of RFC 2616 default charset is iso-8859-1 that is respected by python 3
767 if sys.version_info >= (3, 0):
768 location = location.encode('iso-8859-1').decode('utf-8')
769 location_escaped = escape_url(location)
770 if location != location_escaped:
771 del resp.headers['Location']
772 resp.headers['Location'] = location_escaped
775 https_request = http_request
776 https_response = http_response
779 class YoutubeDLHTTPSHandler(compat_urllib_request.HTTPSHandler):
780 def __init__(self, params, https_conn_class=None, *args, **kwargs):
781 compat_urllib_request.HTTPSHandler.__init__(self, *args, **kwargs)
782 self._https_conn_class = https_conn_class or compat_http_client.HTTPSConnection
783 self._params = params
785 def https_open(self, req):
787 if hasattr(self, '_context'): # python > 2.6
788 kwargs['context'] = self._context
789 if hasattr(self, '_check_hostname'): # python 3.x
790 kwargs['check_hostname'] = self._check_hostname
791 return self.do_open(functools.partial(
792 _create_http_connection, self, self._https_conn_class, True),
796 class YoutubeDLCookieProcessor(compat_urllib_request.HTTPCookieProcessor):
797 def __init__(self, cookiejar=None):
798 compat_urllib_request.HTTPCookieProcessor.__init__(self, cookiejar)
800 def http_response(self, request, response):
801 # Python 2 will choke on next HTTP request in row if there are non-ASCII
802 # characters in Set-Cookie HTTP header of last response (see
803 # https://github.com/rg3/youtube-dl/issues/6769).
804 # In order to at least prevent crashing we will percent encode Set-Cookie
805 # header before HTTPCookieProcessor starts processing it.
806 # if sys.version_info < (3, 0) and response.headers:
807 # for set_cookie_header in ('Set-Cookie', 'Set-Cookie2'):
808 # set_cookie = response.headers.get(set_cookie_header)
810 # set_cookie_escaped = compat_urllib_parse.quote(set_cookie, b"%/;:@&=+$,!~*'()?#[] ")
811 # if set_cookie != set_cookie_escaped:
812 # del response.headers[set_cookie_header]
813 # response.headers[set_cookie_header] = set_cookie_escaped
814 return compat_urllib_request.HTTPCookieProcessor.http_response(self, request, response)
816 https_request = compat_urllib_request.HTTPCookieProcessor.http_request
817 https_response = http_response
820 def parse_iso8601(date_str, delimiter='T', timezone=None):
821 """ Return a UNIX timestamp from the given date """
826 date_str = re.sub(r'\.[0-9]+', '', date_str)
830 r'(?:Z$| ?(?P<sign>\+|-)(?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})$)',
833 timezone = datetime.timedelta()
835 date_str = date_str[:-len(m.group(0))]
836 if not m.group('sign'):
837 timezone = datetime.timedelta()
839 sign = 1 if m.group('sign') == '+' else -1
840 timezone = datetime.timedelta(
841 hours=sign * int(m.group('hours')),
842 minutes=sign * int(m.group('minutes')))
844 date_format = '%Y-%m-%d{0}%H:%M:%S'.format(delimiter)
845 dt = datetime.datetime.strptime(date_str, date_format) - timezone
846 return calendar.timegm(dt.timetuple())
851 def unified_strdate(date_str, day_first=True):
852 """Return a string with the date in the format YYYYMMDD"""
858 date_str = date_str.replace(',', ' ')
859 # %z (UTC offset) is only supported in python>=3.2
860 if not re.match(r'^[0-9]{1,2}-[0-9]{1,2}-[0-9]{4}$', date_str):
861 date_str = re.sub(r' ?(\+|-)[0-9]{2}:?[0-9]{2}$', '', date_str)
862 # Remove AM/PM + timezone
863 date_str = re.sub(r'(?i)\s*(?:AM|PM)(?:\s+[A-Z]+)?', '', date_str)
865 format_expressions = [
870 '%b %dst %Y %I:%M%p',
871 '%b %dnd %Y %I:%M%p',
872 '%b %dth %Y %I:%M%p',
878 '%Y-%m-%d %H:%M:%S.%f',
881 '%Y-%m-%dT%H:%M:%SZ',
882 '%Y-%m-%dT%H:%M:%S.%fZ',
883 '%Y-%m-%dT%H:%M:%S.%f0Z',
885 '%Y-%m-%dT%H:%M:%S.%f',
889 format_expressions.extend([
897 format_expressions.extend([
904 for expression in format_expressions:
906 upload_date = datetime.datetime.strptime(date_str, expression).strftime('%Y%m%d')
909 if upload_date is None:
910 timetuple = email.utils.parsedate_tz(date_str)
912 upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d')
916 def determine_ext(url, default_ext='unknown_video'):
919 guess = url.partition('?')[0].rpartition('.')[2]
920 if re.match(r'^[A-Za-z0-9]+$', guess):
926 def subtitles_filename(filename, sub_lang, sub_format):
927 return filename.rsplit('.', 1)[0] + '.' + sub_lang + '.' + sub_format
930 def date_from_str(date_str):
932 Return a datetime object from a string in the format YYYYMMDD or
933 (now|today)[+-][0-9](day|week|month|year)(s)?"""
934 today = datetime.date.today()
935 if date_str in ('now', 'today'):
937 if date_str == 'yesterday':
938 return today - datetime.timedelta(days=1)
939 match = re.match('(now|today)(?P<sign>[+-])(?P<time>\d+)(?P<unit>day|week|month|year)(s)?', date_str)
940 if match is not None:
941 sign = match.group('sign')
942 time = int(match.group('time'))
945 unit = match.group('unit')
946 # A bad aproximation?
954 delta = datetime.timedelta(**{unit: time})
956 return datetime.datetime.strptime(date_str, "%Y%m%d").date()
959 def hyphenate_date(date_str):
961 Convert a date in 'YYYYMMDD' format to 'YYYY-MM-DD' format"""
962 match = re.match(r'^(\d\d\d\d)(\d\d)(\d\d)$', date_str)
963 if match is not None:
964 return '-'.join(match.groups())
969 class DateRange(object):
970 """Represents a time interval between two dates"""
972 def __init__(self, start=None, end=None):
973 """start and end must be strings in the format accepted by date"""
974 if start is not None:
975 self.start = date_from_str(start)
977 self.start = datetime.datetime.min.date()
979 self.end = date_from_str(end)
981 self.end = datetime.datetime.max.date()
982 if self.start > self.end:
983 raise ValueError('Date range: "%s" , the start date must be before the end date' % self)
987 """Returns a range that only contains the given day"""
990 def __contains__(self, date):
991 """Check if the date is in the range"""
992 if not isinstance(date, datetime.date):
993 date = date_from_str(date)
994 return self.start <= date <= self.end
997 return '%s - %s' % (self.start.isoformat(), self.end.isoformat())
1000 def platform_name():
1001 """ Returns the platform name as a compat_str """
1002 res = platform.platform()
1003 if isinstance(res, bytes):
1004 res = res.decode(preferredencoding())
1006 assert isinstance(res, compat_str)
1010 def _windows_write_string(s, out):
1011 """ Returns True if the string was written using special methods,
1012 False if it has yet to be written out."""
1013 # Adapted from http://stackoverflow.com/a/3259271/35070
1016 import ctypes.wintypes
1024 fileno = out.fileno()
1025 except AttributeError:
1026 # If the output stream doesn't have a fileno, it's virtual
1028 except io.UnsupportedOperation:
1029 # Some strange Windows pseudo files?
1031 if fileno not in WIN_OUTPUT_IDS:
1034 GetStdHandle = ctypes.WINFUNCTYPE(
1035 ctypes.wintypes.HANDLE, ctypes.wintypes.DWORD)(
1036 (b"GetStdHandle", ctypes.windll.kernel32))
1037 h = GetStdHandle(WIN_OUTPUT_IDS[fileno])
1039 WriteConsoleW = ctypes.WINFUNCTYPE(
1040 ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE, ctypes.wintypes.LPWSTR,
1041 ctypes.wintypes.DWORD, ctypes.POINTER(ctypes.wintypes.DWORD),
1042 ctypes.wintypes.LPVOID)((b"WriteConsoleW", ctypes.windll.kernel32))
1043 written = ctypes.wintypes.DWORD(0)
1045 GetFileType = ctypes.WINFUNCTYPE(ctypes.wintypes.DWORD, ctypes.wintypes.DWORD)((b"GetFileType", ctypes.windll.kernel32))
1046 FILE_TYPE_CHAR = 0x0002
1047 FILE_TYPE_REMOTE = 0x8000
1048 GetConsoleMode = ctypes.WINFUNCTYPE(
1049 ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE,
1050 ctypes.POINTER(ctypes.wintypes.DWORD))(
1051 (b"GetConsoleMode", ctypes.windll.kernel32))
1052 INVALID_HANDLE_VALUE = ctypes.wintypes.DWORD(-1).value
1054 def not_a_console(handle):
1055 if handle == INVALID_HANDLE_VALUE or handle is None:
1057 return ((GetFileType(handle) & ~FILE_TYPE_REMOTE) != FILE_TYPE_CHAR or
1058 GetConsoleMode(handle, ctypes.byref(ctypes.wintypes.DWORD())) == 0)
1060 if not_a_console(h):
1063 def next_nonbmp_pos(s):
1065 return next(i for i, c in enumerate(s) if ord(c) > 0xffff)
1066 except StopIteration:
1070 count = min(next_nonbmp_pos(s), 1024)
1072 ret = WriteConsoleW(
1073 h, s, count if count else 2, ctypes.byref(written), None)
1075 raise OSError('Failed to write string')
1076 if not count: # We just wrote a non-BMP character
1077 assert written.value == 2
1080 assert written.value > 0
1081 s = s[written.value:]
1085 def write_string(s, out=None, encoding=None):
1088 assert type(s) == compat_str
1090 if sys.platform == 'win32' and encoding is None and hasattr(out, 'fileno'):
1091 if _windows_write_string(s, out):
1094 if ('b' in getattr(out, 'mode', '') or
1095 sys.version_info[0] < 3): # Python 2 lies about mode of sys.stderr
1096 byt = s.encode(encoding or preferredencoding(), 'ignore')
1098 elif hasattr(out, 'buffer'):
1099 enc = encoding or getattr(out, 'encoding', None) or preferredencoding()
1100 byt = s.encode(enc, 'ignore')
1101 out.buffer.write(byt)
1107 def bytes_to_intlist(bs):
1110 if isinstance(bs[0], int): # Python 3
1113 return [ord(c) for c in bs]
1116 def intlist_to_bytes(xs):
1119 return struct_pack('%dB' % len(xs), *xs)
1122 # Cross-platform file locking
1123 if sys.platform == 'win32':
1124 import ctypes.wintypes
1127 class OVERLAPPED(ctypes.Structure):
1129 ('Internal', ctypes.wintypes.LPVOID),
1130 ('InternalHigh', ctypes.wintypes.LPVOID),
1131 ('Offset', ctypes.wintypes.DWORD),
1132 ('OffsetHigh', ctypes.wintypes.DWORD),
1133 ('hEvent', ctypes.wintypes.HANDLE),
1136 kernel32 = ctypes.windll.kernel32
1137 LockFileEx = kernel32.LockFileEx
1138 LockFileEx.argtypes = [
1139 ctypes.wintypes.HANDLE, # hFile
1140 ctypes.wintypes.DWORD, # dwFlags
1141 ctypes.wintypes.DWORD, # dwReserved
1142 ctypes.wintypes.DWORD, # nNumberOfBytesToLockLow
1143 ctypes.wintypes.DWORD, # nNumberOfBytesToLockHigh
1144 ctypes.POINTER(OVERLAPPED) # Overlapped
1146 LockFileEx.restype = ctypes.wintypes.BOOL
1147 UnlockFileEx = kernel32.UnlockFileEx
1148 UnlockFileEx.argtypes = [
1149 ctypes.wintypes.HANDLE, # hFile
1150 ctypes.wintypes.DWORD, # dwReserved
1151 ctypes.wintypes.DWORD, # nNumberOfBytesToLockLow
1152 ctypes.wintypes.DWORD, # nNumberOfBytesToLockHigh
1153 ctypes.POINTER(OVERLAPPED) # Overlapped
1155 UnlockFileEx.restype = ctypes.wintypes.BOOL
1156 whole_low = 0xffffffff
1157 whole_high = 0x7fffffff
1159 def _lock_file(f, exclusive):
1160 overlapped = OVERLAPPED()
1161 overlapped.Offset = 0
1162 overlapped.OffsetHigh = 0
1163 overlapped.hEvent = 0
1164 f._lock_file_overlapped_p = ctypes.pointer(overlapped)
1165 handle = msvcrt.get_osfhandle(f.fileno())
1166 if not LockFileEx(handle, 0x2 if exclusive else 0x0, 0,
1167 whole_low, whole_high, f._lock_file_overlapped_p):
1168 raise OSError('Locking file failed: %r' % ctypes.FormatError())
1170 def _unlock_file(f):
1171 assert f._lock_file_overlapped_p
1172 handle = msvcrt.get_osfhandle(f.fileno())
1173 if not UnlockFileEx(handle, 0,
1174 whole_low, whole_high, f._lock_file_overlapped_p):
1175 raise OSError('Unlocking file failed: %r' % ctypes.FormatError())
1180 def _lock_file(f, exclusive):
1181 fcntl.flock(f, fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH)
1183 def _unlock_file(f):
1184 fcntl.flock(f, fcntl.LOCK_UN)
1187 class locked_file(object):
1188 def __init__(self, filename, mode, encoding=None):
1189 assert mode in ['r', 'a', 'w']
1190 self.f = io.open(filename, mode, encoding=encoding)
1193 def __enter__(self):
1194 exclusive = self.mode != 'r'
1196 _lock_file(self.f, exclusive)
1202 def __exit__(self, etype, value, traceback):
1204 _unlock_file(self.f)
1211 def write(self, *args):
1212 return self.f.write(*args)
1214 def read(self, *args):
1215 return self.f.read(*args)
1218 def get_filesystem_encoding():
1219 encoding = sys.getfilesystemencoding()
1220 return encoding if encoding is not None else 'utf-8'
1223 def shell_quote(args):
1225 encoding = get_filesystem_encoding()
1227 if isinstance(a, bytes):
1228 # We may get a filename encoded with 'encodeFilename'
1229 a = a.decode(encoding)
1230 quoted_args.append(pipes.quote(a))
1231 return ' '.join(quoted_args)
1234 def smuggle_url(url, data):
1235 """ Pass additional data in a URL for internal use. """
1237 sdata = compat_urllib_parse.urlencode(
1238 {'__youtubedl_smuggle': json.dumps(data)})
1239 return url + '#' + sdata
1242 def unsmuggle_url(smug_url, default=None):
1243 if '#__youtubedl_smuggle' not in smug_url:
1244 return smug_url, default
1245 url, _, sdata = smug_url.rpartition('#')
1246 jsond = compat_parse_qs(sdata)['__youtubedl_smuggle'][0]
1247 data = json.loads(jsond)
1251 def format_bytes(bytes):
1254 if type(bytes) is str:
1255 bytes = float(bytes)
1259 exponent = int(math.log(bytes, 1024.0))
1260 suffix = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'][exponent]
1261 converted = float(bytes) / float(1024 ** exponent)
1262 return '%.2f%s' % (converted, suffix)
1265 def parse_filesize(s):
1269 # The lower-case forms are of course incorrect and inofficial,
1270 # but we support those too
1308 units_re = '|'.join(re.escape(u) for u in _UNIT_TABLE)
1310 r'(?P<num>[0-9]+(?:[,.][0-9]*)?)\s*(?P<unit>%s)' % units_re, s)
1314 num_str = m.group('num').replace(',', '.')
1315 mult = _UNIT_TABLE[m.group('unit')]
1316 return int(float(num_str) * mult)
1319 def month_by_name(name):
1320 """ Return the number of a month by (locale-independently) English name """
1323 return ENGLISH_MONTH_NAMES.index(name) + 1
1328 def month_by_abbreviation(abbrev):
1329 """ Return the number of a month by (locale-independently) English
1333 return [s[:3] for s in ENGLISH_MONTH_NAMES].index(abbrev) + 1
1338 def fix_xml_ampersands(xml_str):
1339 """Replace all the '&' by '&' in XML"""
1341 r'&(?!amp;|lt;|gt;|apos;|quot;|#x[0-9a-fA-F]{,4};|#[0-9]{,4};)',
1346 def setproctitle(title):
1347 assert isinstance(title, compat_str)
1349 libc = ctypes.cdll.LoadLibrary("libc.so.6")
1352 title_bytes = title.encode('utf-8')
1353 buf = ctypes.create_string_buffer(len(title_bytes))
1354 buf.value = title_bytes
1356 libc.prctl(15, buf, 0, 0, 0)
1357 except AttributeError:
1358 return # Strange libc, just skip this
1361 def remove_start(s, start):
1362 if s.startswith(start):
1363 return s[len(start):]
1367 def remove_end(s, end):
1369 return s[:-len(end)]
1373 def url_basename(url):
1374 path = compat_urlparse.urlparse(url).path
1375 return path.strip('/').split('/')[-1]
1378 class HEADRequest(compat_urllib_request.Request):
1379 def get_method(self):
1383 def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1):
1386 v = getattr(v, get_attr, None)
1392 return int(v) * invscale // scale
1397 def str_or_none(v, default=None):
1398 return default if v is None else compat_str(v)
1401 def str_to_int(int_str):
1402 """ A more relaxed version of int_or_none """
1405 int_str = re.sub(r'[,\.\+]', '', int_str)
1409 def float_or_none(v, scale=1, invscale=1, default=None):
1413 return float(v) * invscale / scale
1418 def parse_duration(s):
1419 if not isinstance(s, compat_basestring):
1427 (?P<only_mins>[0-9.]+)\s*(?:mins?\.?|minutes?)\s*|
1428 (?P<only_hours>[0-9.]+)\s*(?:hours?)|
1430 \s*(?P<hours_reversed>[0-9]+)\s*(?:[:h]|hours?)\s*(?P<mins_reversed>[0-9]+)\s*(?:[:m]|mins?\.?|minutes?)\s*|
1433 (?:(?P<days>[0-9]+)\s*(?:[:d]|days?)\s*)?
1434 (?P<hours>[0-9]+)\s*(?:[:h]|hours?)\s*
1436 (?P<mins>[0-9]+)\s*(?:[:m]|mins?|minutes?)\s*
1438 (?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?\s*(?:s|secs?|seconds?)?
1443 if m.group('only_mins'):
1444 return float_or_none(m.group('only_mins'), invscale=60)
1445 if m.group('only_hours'):
1446 return float_or_none(m.group('only_hours'), invscale=60 * 60)
1448 res += int(m.group('secs'))
1449 if m.group('mins_reversed'):
1450 res += int(m.group('mins_reversed')) * 60
1452 res += int(m.group('mins')) * 60
1453 if m.group('hours'):
1454 res += int(m.group('hours')) * 60 * 60
1455 if m.group('hours_reversed'):
1456 res += int(m.group('hours_reversed')) * 60 * 60
1458 res += int(m.group('days')) * 24 * 60 * 60
1460 res += float(m.group('ms'))
1464 def prepend_extension(filename, ext, expected_real_ext=None):
1465 name, real_ext = os.path.splitext(filename)
1467 '{0}.{1}{2}'.format(name, ext, real_ext)
1468 if not expected_real_ext or real_ext[1:] == expected_real_ext
1469 else '{0}.{1}'.format(filename, ext))
1472 def replace_extension(filename, ext, expected_real_ext=None):
1473 name, real_ext = os.path.splitext(filename)
1474 return '{0}.{1}'.format(
1475 name if not expected_real_ext or real_ext[1:] == expected_real_ext else filename,
1479 def check_executable(exe, args=[]):
1480 """ Checks if the given binary is installed somewhere in PATH, and returns its name.
1481 args can be a list of arguments for a short output (like -version) """
1483 subprocess.Popen([exe] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
1489 def get_exe_version(exe, args=['--version'],
1490 version_re=None, unrecognized='present'):
1491 """ Returns the version of the specified executable,
1492 or False if the executable is not present """
1494 out, _ = subprocess.Popen(
1495 [encodeArgument(exe)] + args,
1496 stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()
1499 if isinstance(out, bytes): # Python 2.x
1500 out = out.decode('ascii', 'ignore')
1501 return detect_exe_version(out, version_re, unrecognized)
1504 def detect_exe_version(output, version_re=None, unrecognized='present'):
1505 assert isinstance(output, compat_str)
1506 if version_re is None:
1507 version_re = r'version\s+([-0-9._a-zA-Z]+)'
1508 m = re.search(version_re, output)
1515 class PagedList(object):
1517 # This is only useful for tests
1518 return len(self.getslice())
1521 class OnDemandPagedList(PagedList):
1522 def __init__(self, pagefunc, pagesize):
1523 self._pagefunc = pagefunc
1524 self._pagesize = pagesize
1526 def getslice(self, start=0, end=None):
1528 for pagenum in itertools.count(start // self._pagesize):
1529 firstid = pagenum * self._pagesize
1530 nextfirstid = pagenum * self._pagesize + self._pagesize
1531 if start >= nextfirstid:
1534 page_results = list(self._pagefunc(pagenum))
1537 start % self._pagesize
1538 if firstid <= start < nextfirstid
1542 ((end - 1) % self._pagesize) + 1
1543 if (end is not None and firstid <= end <= nextfirstid)
1546 if startv != 0 or endv is not None:
1547 page_results = page_results[startv:endv]
1548 res.extend(page_results)
1550 # A little optimization - if current page is not "full", ie. does
1551 # not contain page_size videos then we can assume that this page
1552 # is the last one - there are no more ids on further pages -
1553 # i.e. no need to query again.
1554 if len(page_results) + startv < self._pagesize:
1557 # If we got the whole page, but the next page is not interesting,
1558 # break out early as well
1559 if end == nextfirstid:
1564 class InAdvancePagedList(PagedList):
1565 def __init__(self, pagefunc, pagecount, pagesize):
1566 self._pagefunc = pagefunc
1567 self._pagecount = pagecount
1568 self._pagesize = pagesize
1570 def getslice(self, start=0, end=None):
1572 start_page = start // self._pagesize
1574 self._pagecount if end is None else (end // self._pagesize + 1))
1575 skip_elems = start - start_page * self._pagesize
1576 only_more = None if end is None else end - start
1577 for pagenum in range(start_page, end_page):
1578 page = list(self._pagefunc(pagenum))
1580 page = page[skip_elems:]
1582 if only_more is not None:
1583 if len(page) < only_more:
1584 only_more -= len(page)
1586 page = page[:only_more]
1593 def uppercase_escape(s):
1594 unicode_escape = codecs.getdecoder('unicode_escape')
1596 r'\\U[0-9a-fA-F]{8}',
1597 lambda m: unicode_escape(m.group(0))[0],
1601 def lowercase_escape(s):
1602 unicode_escape = codecs.getdecoder('unicode_escape')
1604 r'\\u[0-9a-fA-F]{4}',
1605 lambda m: unicode_escape(m.group(0))[0],
1609 def escape_rfc3986(s):
1610 """Escape non-ASCII characters as suggested by RFC 3986"""
1611 if sys.version_info < (3, 0) and isinstance(s, compat_str):
1612 s = s.encode('utf-8')
1613 return compat_urllib_parse.quote(s, b"%/;:@&=+$,!~*'()?#[]")
1616 def escape_url(url):
1617 """Escape URL as suggested by RFC 3986"""
1618 url_parsed = compat_urllib_parse_urlparse(url)
1619 return url_parsed._replace(
1620 path=escape_rfc3986(url_parsed.path),
1621 params=escape_rfc3986(url_parsed.params),
1622 query=escape_rfc3986(url_parsed.query),
1623 fragment=escape_rfc3986(url_parsed.fragment)
1627 struct.pack('!I', 0)
1629 # In Python 2.6 (and some 2.7 versions), struct requires a bytes argument
1630 def struct_pack(spec, *args):
1631 if isinstance(spec, compat_str):
1632 spec = spec.encode('ascii')
1633 return struct.pack(spec, *args)
1635 def struct_unpack(spec, *args):
1636 if isinstance(spec, compat_str):
1637 spec = spec.encode('ascii')
1638 return struct.unpack(spec, *args)
1640 struct_pack = struct.pack
1641 struct_unpack = struct.unpack
1644 def read_batch_urls(batch_fd):
1646 if not isinstance(url, compat_str):
1647 url = url.decode('utf-8', 'replace')
1648 BOM_UTF8 = '\xef\xbb\xbf'
1649 if url.startswith(BOM_UTF8):
1650 url = url[len(BOM_UTF8):]
1652 if url.startswith(('#', ';', ']')):
1656 with contextlib.closing(batch_fd) as fd:
1657 return [url for url in map(fixup, fd) if url]
1660 def urlencode_postdata(*args, **kargs):
1661 return compat_urllib_parse.urlencode(*args, **kargs).encode('ascii')
1664 def encode_dict(d, encoding='utf-8'):
1665 return dict((k.encode(encoding), v.encode(encoding)) for k, v in d.items())
1669 etree_iter = xml.etree.ElementTree.Element.iter
1670 except AttributeError: # Python <=2.6
1671 etree_iter = lambda n: n.findall('.//*')
1675 class TreeBuilder(xml.etree.ElementTree.TreeBuilder):
1676 def doctype(self, name, pubid, system):
1677 pass # Ignore doctypes
1679 parser = xml.etree.ElementTree.XMLParser(target=TreeBuilder())
1680 kwargs = {'parser': parser} if sys.version_info >= (2, 7) else {}
1681 tree = xml.etree.ElementTree.XML(s.encode('utf-8'), **kwargs)
1682 # Fix up XML parser in Python 2.x
1683 if sys.version_info < (3, 0):
1684 for n in etree_iter(tree):
1685 if n.text is not None:
1686 if not isinstance(n.text, compat_str):
1687 n.text = n.text.decode('utf-8')
1700 def parse_age_limit(s):
1703 m = re.match(r'^(?P<age>\d{1,2})\+?$', s)
1704 return int(m.group('age')) if m else US_RATINGS.get(s, None)
1707 def strip_jsonp(code):
1709 r'(?s)^[a-zA-Z0-9_]+\s*\(\s*(.*)\);?\s*?(?://[^\n]*)*$', r'\1', code)
1712 def js_to_json(code):
1715 if v in ('true', 'false', 'null'):
1717 if v.startswith('"'):
1718 v = re.sub(r"\\'", "'", v[1:-1])
1719 elif v.startswith("'"):
1721 v = re.sub(r"\\\\|\\'|\"", lambda m: {
1728 res = re.sub(r'''(?x)
1729 "(?:[^"\\]*(?:\\\\|\\['"nu]))*[^"\\]*"|
1730 '(?:[^'\\]*(?:\\\\|\\['"nu]))*[^'\\]*'|
1731 [a-zA-Z_][.a-zA-Z_0-9]*
1733 res = re.sub(r',(\s*[\]}])', lambda m: m.group(1), res)
1737 def qualities(quality_ids):
1738 """ Get a numeric quality value out of a list of possible values """
1741 return quality_ids.index(qid)
1747 DEFAULT_OUTTMPL = '%(title)s-%(id)s.%(ext)s'
1750 def limit_length(s, length):
1751 """ Add ellipses to overly long strings """
1756 return s[:length - len(ELLIPSES)] + ELLIPSES
1760 def version_tuple(v):
1761 return tuple(int(e) for e in re.split(r'[-.]', v))
1764 def is_outdated_version(version, limit, assume_new=True):
1766 return not assume_new
1768 return version_tuple(version) < version_tuple(limit)
1770 return not assume_new
1773 def ytdl_is_updateable():
1774 """ Returns if youtube-dl can be updated with -U """
1775 from zipimport import zipimporter
1777 return isinstance(globals().get('__loader__'), zipimporter) or hasattr(sys, 'frozen')
1780 def args_to_str(args):
1781 # Get a short string representation for a subprocess command
1782 return ' '.join(shlex_quote(a) for a in args)
1785 def mimetype2ext(mt):
1786 _, _, res = mt.rpartition('/')
1790 'x-mp4-fragmented': 'mp4',
1795 def urlhandle_detect_ext(url_handle):
1798 getheader = lambda h: url_handle.headers[h]
1799 except AttributeError: # Python < 3
1800 getheader = url_handle.info().getheader
1802 cd = getheader('Content-Disposition')
1804 m = re.match(r'attachment;\s*filename="(?P<filename>[^"]+)"', cd)
1806 e = determine_ext(m.group('filename'), default_ext=None)
1810 return mimetype2ext(getheader('Content-Type'))
1813 def encode_data_uri(data, mime_type):
1814 return 'data:%s;base64,%s' % (mime_type, base64.b64encode(data).decode('ascii'))
1817 def age_restricted(content_limit, age_limit):
1818 """ Returns True iff the content should be blocked """
1820 if age_limit is None: # No limit set
1822 if content_limit is None:
1823 return False # Content available for everyone
1824 return age_limit < content_limit
1827 def is_html(first_bytes):
1828 """ Detect whether a file contains HTML by examining its first bytes. """
1831 (b'\xef\xbb\xbf', 'utf-8'),
1832 (b'\x00\x00\xfe\xff', 'utf-32-be'),
1833 (b'\xff\xfe\x00\x00', 'utf-32-le'),
1834 (b'\xff\xfe', 'utf-16-le'),
1835 (b'\xfe\xff', 'utf-16-be'),
1837 for bom, enc in BOMS:
1838 if first_bytes.startswith(bom):
1839 s = first_bytes[len(bom):].decode(enc, 'replace')
1842 s = first_bytes.decode('utf-8', 'replace')
1844 return re.match(r'^\s*<', s)
1847 def determine_protocol(info_dict):
1848 protocol = info_dict.get('protocol')
1849 if protocol is not None:
1852 url = info_dict['url']
1853 if url.startswith('rtmp'):
1855 elif url.startswith('mms'):
1857 elif url.startswith('rtsp'):
1860 ext = determine_ext(url)
1866 return compat_urllib_parse_urlparse(url).scheme
1869 def render_table(header_row, data):
1870 """ Render a list of rows, each as a list of values """
1871 table = [header_row] + data
1872 max_lens = [max(len(compat_str(v)) for v in col) for col in zip(*table)]
1873 format_str = ' '.join('%-' + compat_str(ml + 1) + 's' for ml in max_lens[:-1]) + '%s'
1874 return '\n'.join(format_str % tuple(row) for row in table)
1877 def _match_one(filter_part, dct):
1878 COMPARISON_OPERATORS = {
1886 operator_rex = re.compile(r'''(?x)\s*
1888 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
1890 (?P<intval>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)|
1891 (?P<strval>(?![0-9.])[a-z0-9A-Z]*)
1894 ''' % '|'.join(map(re.escape, COMPARISON_OPERATORS.keys())))
1895 m = operator_rex.search(filter_part)
1897 op = COMPARISON_OPERATORS[m.group('op')]
1898 if m.group('strval') is not None:
1899 if m.group('op') not in ('=', '!='):
1901 'Operator %s does not support string values!' % m.group('op'))
1902 comparison_value = m.group('strval')
1905 comparison_value = int(m.group('intval'))
1907 comparison_value = parse_filesize(m.group('intval'))
1908 if comparison_value is None:
1909 comparison_value = parse_filesize(m.group('intval') + 'B')
1910 if comparison_value is None:
1912 'Invalid integer value %r in filter part %r' % (
1913 m.group('intval'), filter_part))
1914 actual_value = dct.get(m.group('key'))
1915 if actual_value is None:
1916 return m.group('none_inclusive')
1917 return op(actual_value, comparison_value)
1920 '': lambda v: v is not None,
1921 '!': lambda v: v is None,
1923 operator_rex = re.compile(r'''(?x)\s*
1924 (?P<op>%s)\s*(?P<key>[a-z_]+)
1926 ''' % '|'.join(map(re.escape, UNARY_OPERATORS.keys())))
1927 m = operator_rex.search(filter_part)
1929 op = UNARY_OPERATORS[m.group('op')]
1930 actual_value = dct.get(m.group('key'))
1931 return op(actual_value)
1933 raise ValueError('Invalid filter part %r' % filter_part)
1936 def match_str(filter_str, dct):
1937 """ Filter a dictionary with a simple string syntax. Returns True (=passes filter) or false """
1940 _match_one(filter_part, dct) for filter_part in filter_str.split('&'))
1943 def match_filter_func(filter_str):
1944 def _match_func(info_dict):
1945 if match_str(filter_str, info_dict):
1948 video_title = info_dict.get('title', info_dict.get('id', 'video'))
1949 return '%s does not pass filter %s, skipping ..' % (video_title, filter_str)
1953 def parse_dfxp_time_expr(time_expr):
1957 mobj = re.match(r'^(?P<time_offset>\d+(?:\.\d+)?)s?$', time_expr)
1959 return float(mobj.group('time_offset'))
1961 mobj = re.match(r'^(\d+):(\d\d):(\d\d(?:\.\d+)?)$', time_expr)
1963 return 3600 * int(mobj.group(1)) + 60 * int(mobj.group(2)) + float(mobj.group(3))
1966 def srt_subtitles_timecode(seconds):
1967 return '%02d:%02d:%02d,%03d' % (seconds / 3600, (seconds % 3600) / 60, seconds % 60, (seconds % 1) * 1000)
1970 def dfxp2srt(dfxp_data):
1971 _x = functools.partial(xpath_with_ns, ns_map={
1972 'ttml': 'http://www.w3.org/ns/ttml',
1973 'ttaf1': 'http://www.w3.org/2006/10/ttaf1',
1976 def parse_node(node):
1977 str_or_empty = functools.partial(str_or_none, default='')
1979 out = str_or_empty(node.text)
1982 if child.tag in (_x('ttml:br'), _x('ttaf1:br'), 'br'):
1983 out += '\n' + str_or_empty(child.tail)
1984 elif child.tag in (_x('ttml:span'), _x('ttaf1:span'), 'span'):
1985 out += str_or_empty(parse_node(child))
1987 out += str_or_empty(xml.etree.ElementTree.tostring(child))
1991 dfxp = xml.etree.ElementTree.fromstring(dfxp_data.encode('utf-8'))
1993 paras = dfxp.findall(_x('.//ttml:p')) or dfxp.findall(_x('.//ttaf1:p')) or dfxp.findall('.//p')
1996 raise ValueError('Invalid dfxp/TTML subtitle')
1998 for para, index in zip(paras, itertools.count(1)):
1999 begin_time = parse_dfxp_time_expr(para.attrib['begin'])
2000 end_time = parse_dfxp_time_expr(para.attrib.get('end'))
2002 end_time = begin_time + parse_dfxp_time_expr(para.attrib['dur'])
2003 out.append('%d\n%s --> %s\n%s\n\n' % (
2005 srt_subtitles_timecode(begin_time),
2006 srt_subtitles_timecode(end_time),
2012 def cli_option(params, command_option, param):
2013 param = params.get(param)
2014 return [command_option, param] if param is not None else []
2017 def cli_bool_option(params, command_option, param, true_value='true', false_value='false', separator=None):
2018 param = params.get(param)
2019 assert isinstance(param, bool)
2021 return [command_option + separator + (true_value if param else false_value)]
2022 return [command_option, true_value if param else false_value]
2025 def cli_valueless_option(params, command_option, param, expected_value=True):
2026 param = params.get(param)
2027 return [command_option] if param == expected_value else []
2030 def cli_configuration_args(params, param, default=[]):
2031 ex_args = params.get(param)
2034 assert isinstance(ex_args, list)
2038 class ISO639Utils(object):
2039 # See http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
2228 def short2long(cls, code):
2229 """Convert language code from ISO 639-1 to ISO 639-2/T"""
2230 return cls._lang_map.get(code[:2])
2233 def long2short(cls, code):
2234 """Convert language code from ISO 639-2/T to ISO 639-1"""
2235 for short_name, long_name in cls._lang_map.items():
2236 if long_name == code:
2240 class ISO3166Utils(object):
2241 # From http://data.okfn.org/data/core/country-list
2243 'AF': 'Afghanistan',
2244 'AX': 'Ã…land Islands',
2247 'AS': 'American Samoa',
2252 'AG': 'Antigua and Barbuda',
2269 'BO': 'Bolivia, Plurinational State of',
2270 'BQ': 'Bonaire, Sint Eustatius and Saba',
2271 'BA': 'Bosnia and Herzegovina',
2273 'BV': 'Bouvet Island',
2275 'IO': 'British Indian Ocean Territory',
2276 'BN': 'Brunei Darussalam',
2278 'BF': 'Burkina Faso',
2284 'KY': 'Cayman Islands',
2285 'CF': 'Central African Republic',
2289 'CX': 'Christmas Island',
2290 'CC': 'Cocos (Keeling) Islands',
2294 'CD': 'Congo, the Democratic Republic of the',
2295 'CK': 'Cook Islands',
2297 'CI': 'Côte d\'Ivoire',
2302 'CZ': 'Czech Republic',
2306 'DO': 'Dominican Republic',
2309 'SV': 'El Salvador',
2310 'GQ': 'Equatorial Guinea',
2314 'FK': 'Falkland Islands (Malvinas)',
2315 'FO': 'Faroe Islands',
2319 'GF': 'French Guiana',
2320 'PF': 'French Polynesia',
2321 'TF': 'French Southern Territories',
2336 'GW': 'Guinea-Bissau',
2339 'HM': 'Heard Island and McDonald Islands',
2340 'VA': 'Holy See (Vatican City State)',
2347 'IR': 'Iran, Islamic Republic of',
2350 'IM': 'Isle of Man',
2360 'KP': 'Korea, Democratic People\'s Republic of',
2361 'KR': 'Korea, Republic of',
2364 'LA': 'Lao People\'s Democratic Republic',
2370 'LI': 'Liechtenstein',
2374 'MK': 'Macedonia, the Former Yugoslav Republic of',
2381 'MH': 'Marshall Islands',
2387 'FM': 'Micronesia, Federated States of',
2388 'MD': 'Moldova, Republic of',
2399 'NL': 'Netherlands',
2400 'NC': 'New Caledonia',
2401 'NZ': 'New Zealand',
2406 'NF': 'Norfolk Island',
2407 'MP': 'Northern Mariana Islands',
2412 'PS': 'Palestine, State of',
2414 'PG': 'Papua New Guinea',
2417 'PH': 'Philippines',
2421 'PR': 'Puerto Rico',
2425 'RU': 'Russian Federation',
2427 'BL': 'Saint Barthélemy',
2428 'SH': 'Saint Helena, Ascension and Tristan da Cunha',
2429 'KN': 'Saint Kitts and Nevis',
2430 'LC': 'Saint Lucia',
2431 'MF': 'Saint Martin (French part)',
2432 'PM': 'Saint Pierre and Miquelon',
2433 'VC': 'Saint Vincent and the Grenadines',
2436 'ST': 'Sao Tome and Principe',
2437 'SA': 'Saudi Arabia',
2441 'SL': 'Sierra Leone',
2443 'SX': 'Sint Maarten (Dutch part)',
2446 'SB': 'Solomon Islands',
2448 'ZA': 'South Africa',
2449 'GS': 'South Georgia and the South Sandwich Islands',
2450 'SS': 'South Sudan',
2455 'SJ': 'Svalbard and Jan Mayen',
2458 'CH': 'Switzerland',
2459 'SY': 'Syrian Arab Republic',
2460 'TW': 'Taiwan, Province of China',
2462 'TZ': 'Tanzania, United Republic of',
2464 'TL': 'Timor-Leste',
2468 'TT': 'Trinidad and Tobago',
2471 'TM': 'Turkmenistan',
2472 'TC': 'Turks and Caicos Islands',
2476 'AE': 'United Arab Emirates',
2477 'GB': 'United Kingdom',
2478 'US': 'United States',
2479 'UM': 'United States Minor Outlying Islands',
2483 'VE': 'Venezuela, Bolivarian Republic of',
2485 'VG': 'Virgin Islands, British',
2486 'VI': 'Virgin Islands, U.S.',
2487 'WF': 'Wallis and Futuna',
2488 'EH': 'Western Sahara',
2495 def short2full(cls, code):
2496 """Convert an ISO 3166-2 country code to the corresponding full name"""
2497 return cls._country_map.get(code.upper())
2500 class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
2501 def __init__(self, proxies=None):
2502 # Set default handlers
2503 for type in ('http', 'https'):
2504 setattr(self, '%s_open' % type,
2505 lambda r, proxy='__noproxy__', type=type, meth=self.proxy_open:
2506 meth(r, proxy, type))
2507 return compat_urllib_request.ProxyHandler.__init__(self, proxies)
2509 def proxy_open(self, req, proxy, type):
2510 req_proxy = req.headers.get('Ytdl-request-proxy')
2511 if req_proxy is not None:
2513 del req.headers['Ytdl-request-proxy']
2515 if proxy == '__noproxy__':
2516 return None # No Proxy
2517 return compat_urllib_request.ProxyHandler.proxy_open(
2518 self, req, proxy, type)