4 from __future__ import unicode_literals
34 import xml.etree.ElementTree
38 compat_HTMLParseError,
42 compat_ctypes_WINFUNCTYPE,
43 compat_etree_fromstring,
46 compat_html_entities_html5,
52 compat_socket_create_connection,
58 compat_urllib_parse_urlencode,
59 compat_urllib_parse_urlparse,
60 compat_urllib_parse_unquote_plus,
61 compat_urllib_request,
72 def register_socks_protocols():
73 # "Register" SOCKS protocols
74 # In Python < 2.6.5, urlsplit() suffers from bug https://bugs.python.org/issue7904
75 # URLs with protocols not in urlparse.uses_netloc are not handled correctly
76 for scheme in ('socks', 'socks4', 'socks4a', 'socks5'):
77 if scheme not in compat_urlparse.uses_netloc:
78 compat_urlparse.uses_netloc.append(scheme)
81 # This is not clearly defined otherwise
82 compiled_regex_type = type(re.compile(''))
85 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:59.0) Gecko/20100101 Firefox/59.0',
86 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
87 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
88 'Accept-Encoding': 'gzip, deflate',
89 'Accept-Language': 'en-us,en;q=0.5',
94 'Safari': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27',
100 ENGLISH_MONTH_NAMES = [
101 'January', 'February', 'March', 'April', 'May', 'June',
102 'July', 'August', 'September', 'October', 'November', 'December']
105 'en': ENGLISH_MONTH_NAMES,
107 'janvier', 'février', 'mars', 'avril', 'mai', 'juin',
108 'juillet', 'août', 'septembre', 'octobre', 'novembre', 'décembre'],
112 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'aac',
113 'flv', 'f4v', 'f4a', 'f4b',
114 'webm', 'ogg', 'ogv', 'oga', 'ogx', 'spx', 'opus',
115 'mkv', 'mka', 'mk3d',
124 'f4f', 'f4m', 'm3u8', 'smil')
126 # needed for sanitizing filenames in restricted mode
127 ACCENT_CHARS = dict(zip('ÂÃÄÀÁÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖŐØŒÙÚÛÜŰÝÞßàáâãäåæçèéêëìíîïðñòóôõöőøœùúûüűýþÿ',
128 itertools.chain('AAAAAA', ['AE'], 'CEEEEIIIIDNOOOOOOO', ['OE'], 'UUUUUYP', ['ss'],
129 'aaaaaa', ['ae'], 'ceeeeiiiionooooooo', ['oe'], 'uuuuuypy')))
152 '%Y-%m-%d %H:%M:%S.%f',
155 '%Y-%m-%dT%H:%M:%SZ',
156 '%Y-%m-%dT%H:%M:%S.%fZ',
157 '%Y-%m-%dT%H:%M:%S.%f0Z',
159 '%Y-%m-%dT%H:%M:%S.%f',
162 '%b %d %Y at %H:%M:%S',
164 '%B %d %Y at %H:%M:%S',
167 DATE_FORMATS_DAY_FIRST = list(DATE_FORMATS)
168 DATE_FORMATS_DAY_FIRST.extend([
177 DATE_FORMATS_MONTH_FIRST = list(DATE_FORMATS)
178 DATE_FORMATS_MONTH_FIRST.extend([
186 PACKED_CODES_RE = r"}\('(.+)',(\d+),(\d+),'([^']+)'\.split\('\|'\)"
187 JSON_LD_RE = r'(?is)<script[^>]+type=(["\'])application/ld\+json\1[^>]*>(?P<json_ld>.+?)</script>'
190 def preferredencoding():
191 """Get preferred encoding.
193 Returns the best encoding scheme for the system, based on
194 locale.getpreferredencoding() and some further tweaks.
197 pref = locale.getpreferredencoding()
205 def write_json_file(obj, fn):
206 """ Encode obj as JSON and write it to fn, atomically if possible """
208 fn = encodeFilename(fn)
209 if sys.version_info < (3, 0) and sys.platform != 'win32':
210 encoding = get_filesystem_encoding()
211 # os.path.basename returns a bytes object, but NamedTemporaryFile
212 # will fail if the filename contains non ascii characters unless we
213 # use a unicode object
214 path_basename = lambda f: os.path.basename(fn).decode(encoding)
215 # the same for os.path.dirname
216 path_dirname = lambda f: os.path.dirname(fn).decode(encoding)
218 path_basename = os.path.basename
219 path_dirname = os.path.dirname
223 'prefix': path_basename(fn) + '.',
224 'dir': path_dirname(fn),
228 # In Python 2.x, json.dump expects a bytestream.
229 # In Python 3.x, it writes to a character stream
230 if sys.version_info < (3, 0):
238 tf = tempfile.NamedTemporaryFile(**compat_kwargs(args))
243 if sys.platform == 'win32':
244 # Need to remove existing file on Windows, else os.rename raises
245 # WindowsError or FileExistsError.
250 os.rename(tf.name, fn)
259 if sys.version_info >= (2, 7):
260 def find_xpath_attr(node, xpath, key, val=None):
261 """ Find the xpath xpath[@key=val] """
262 assert re.match(r'^[a-zA-Z_-]+$', key)
263 expr = xpath + ('[@%s]' % key if val is None else "[@%s='%s']" % (key, val))
264 return node.find(expr)
266 def find_xpath_attr(node, xpath, key, val=None):
267 for f in node.findall(compat_xpath(xpath)):
268 if key not in f.attrib:
270 if val is None or f.attrib.get(key) == val:
274 # On python2.6 the xml.etree.ElementTree.Element methods don't support
275 # the namespace parameter
278 def xpath_with_ns(path, ns_map):
279 components = [c.split(':') for c in path.split('/')]
283 replaced.append(c[0])
286 replaced.append('{%s}%s' % (ns_map[ns], tag))
287 return '/'.join(replaced)
290 def xpath_element(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
291 def _find_xpath(xpath):
292 return node.find(compat_xpath(xpath))
294 if isinstance(xpath, (str, compat_str)):
295 n = _find_xpath(xpath)
303 if default is not NO_DEFAULT:
306 name = xpath if name is None else name
307 raise ExtractorError('Could not find XML element %s' % name)
313 def xpath_text(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
314 n = xpath_element(node, xpath, name, fatal=fatal, default=default)
315 if n is None or n == default:
318 if default is not NO_DEFAULT:
321 name = xpath if name is None else name
322 raise ExtractorError('Could not find XML element\'s text %s' % name)
328 def xpath_attr(node, xpath, key, name=None, fatal=False, default=NO_DEFAULT):
329 n = find_xpath_attr(node, xpath, key)
331 if default is not NO_DEFAULT:
334 name = '%s[@%s]' % (xpath, key) if name is None else name
335 raise ExtractorError('Could not find XML attribute %s' % name)
341 def get_element_by_id(id, html):
342 """Return the content of the tag with the specified ID in the passed HTML document"""
343 return get_element_by_attribute('id', id, html)
346 def get_element_by_class(class_name, html):
347 """Return the content of the first tag with the specified class in the passed HTML document"""
348 retval = get_elements_by_class(class_name, html)
349 return retval[0] if retval else None
352 def get_element_by_attribute(attribute, value, html, escape_value=True):
353 retval = get_elements_by_attribute(attribute, value, html, escape_value)
354 return retval[0] if retval else None
357 def get_elements_by_class(class_name, html):
358 """Return the content of all tags with the specified class in the passed HTML document as a list"""
359 return get_elements_by_attribute(
360 'class', r'[^\'"]*\b%s\b[^\'"]*' % re.escape(class_name),
361 html, escape_value=False)
364 def get_elements_by_attribute(attribute, value, html, escape_value=True):
365 """Return the content of the tag with the specified attribute in the passed HTML document"""
367 value = re.escape(value) if escape_value else value
370 for m in re.finditer(r'''(?xs)
372 (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^"]*"|='[^']*'|))*?
374 (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^"]*"|='[^']*'|))*?
378 ''' % (re.escape(attribute), value), html):
379 res = m.group('content')
381 if res.startswith('"') or res.startswith("'"):
384 retlist.append(unescapeHTML(res))
389 class HTMLAttributeParser(compat_HTMLParser):
390 """Trivial HTML parser to gather the attributes for a single element"""
393 compat_HTMLParser.__init__(self)
395 def handle_starttag(self, tag, attrs):
396 self.attrs = dict(attrs)
399 def extract_attributes(html_element):
400 """Given a string for an HTML element such as
402 a="foo" B="bar" c="&98;az" d=boz
403 empty= noval entity="&"
406 Decode and return a dictionary of attributes.
408 'a': 'foo', 'b': 'bar', c: 'baz', d: 'boz',
409 'empty': '', 'noval': None, 'entity': '&',
410 'sq': '"', 'dq': '\''
412 NB HTMLParser is stricter in Python 2.6 & 3.2 than in later versions,
413 but the cases in the unit test will work for all of 2.6, 2.7, 3.2-3.5.
415 parser = HTMLAttributeParser()
417 parser.feed(html_element)
419 # Older Python may throw HTMLParseError in case of malformed HTML
420 except compat_HTMLParseError:
425 def clean_html(html):
426 """Clean an HTML snippet into a readable string"""
428 if html is None: # Convenience for sanitizing descriptions etc.
432 html = html.replace('\n', ' ')
433 html = re.sub(r'(?u)\s*<\s*br\s*/?\s*>\s*', '\n', html)
434 html = re.sub(r'(?u)<\s*/\s*p\s*>\s*<\s*p[^>]*>', '\n', html)
436 html = re.sub('<.*?>', '', html)
437 # Replace html entities
438 html = unescapeHTML(html)
442 def sanitize_open(filename, open_mode):
443 """Try to open the given filename, and slightly tweak it if this fails.
445 Attempts to open the given filename. If this fails, it tries to change
446 the filename slightly, step by step, until it's either able to open it
447 or it fails and raises a final exception, like the standard open()
450 It returns the tuple (stream, definitive_file_name).
454 if sys.platform == 'win32':
456 msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
457 return (sys.stdout.buffer if hasattr(sys.stdout, 'buffer') else sys.stdout, filename)
458 stream = open(encodeFilename(filename), open_mode)
459 return (stream, filename)
460 except (IOError, OSError) as err:
461 if err.errno in (errno.EACCES,):
464 # In case of error, try to remove win32 forbidden chars
465 alt_filename = sanitize_path(filename)
466 if alt_filename == filename:
469 # An exception here should be caught in the caller
470 stream = open(encodeFilename(alt_filename), open_mode)
471 return (stream, alt_filename)
474 def timeconvert(timestr):
475 """Convert RFC 2822 defined time string into system timestamp"""
477 timetuple = email.utils.parsedate_tz(timestr)
478 if timetuple is not None:
479 timestamp = email.utils.mktime_tz(timetuple)
483 def sanitize_filename(s, restricted=False, is_id=False):
484 """Sanitizes a string so it could be used as part of a filename.
485 If restricted is set, use a stricter subset of allowed characters.
486 Set is_id if this is not an arbitrary string, but an ID that should be kept
489 def replace_insane(char):
490 if restricted and char in ACCENT_CHARS:
491 return ACCENT_CHARS[char]
492 if char == '?' or ord(char) < 32 or ord(char) == 127:
495 return '' if restricted else '\''
497 return '_-' if restricted else ' -'
498 elif char in '\\/|*<>':
500 if restricted and (char in '!&\'()[]{}$;`^,#' or char.isspace()):
502 if restricted and ord(char) > 127:
507 s = re.sub(r'[0-9]+(?::[0-9]+)+', lambda m: m.group(0).replace(':', '_'), s)
508 result = ''.join(map(replace_insane, s))
510 while '__' in result:
511 result = result.replace('__', '_')
512 result = result.strip('_')
513 # Common case of "Foreign band name - English song title"
514 if restricted and result.startswith('-_'):
516 if result.startswith('-'):
517 result = '_' + result[len('-'):]
518 result = result.lstrip('.')
524 def sanitize_path(s):
525 """Sanitizes and normalizes path on Windows"""
526 if sys.platform != 'win32':
528 drive_or_unc, _ = os.path.splitdrive(s)
529 if sys.version_info < (2, 7) and not drive_or_unc:
530 drive_or_unc, _ = os.path.splitunc(s)
531 norm_path = os.path.normpath(remove_start(s, drive_or_unc)).split(os.path.sep)
535 path_part if path_part in ['.', '..'] else re.sub(r'(?:[/<>:"\|\\?\*]|[\s.]$)', '#', path_part)
536 for path_part in norm_path]
538 sanitized_path.insert(0, drive_or_unc + os.path.sep)
539 return os.path.join(*sanitized_path)
542 def sanitize_url(url):
543 # Prepend protocol-less URLs with `http:` scheme in order to mitigate
544 # the number of unwanted failures due to missing protocol
545 if url.startswith('//'):
546 return 'http:%s' % url
547 # Fix some common typos seen so far
549 # https://github.com/rg3/youtube-dl/issues/15649
550 (r'^httpss://', r'https://'),
551 # https://bx1.be/lives/direct-tv/
552 (r'^rmtp([es]?)://', r'rtmp\1://'),
554 for mistake, fixup in COMMON_TYPOS:
555 if re.match(mistake, url):
556 return re.sub(mistake, fixup, url)
560 def sanitized_Request(url, *args, **kwargs):
561 return compat_urllib_request.Request(sanitize_url(url), *args, **kwargs)
565 """Expand shell variables and ~"""
566 return os.path.expandvars(compat_expanduser(s))
569 def orderedSet(iterable):
570 """ Remove all duplicates from the input iterable """
578 def _htmlentity_transform(entity_with_semicolon):
579 """Transforms an HTML entity to a character."""
580 entity = entity_with_semicolon[:-1]
582 # Known non-numeric HTML entity
583 if entity in compat_html_entities.name2codepoint:
584 return compat_chr(compat_html_entities.name2codepoint[entity])
586 # TODO: HTML5 allows entities without a semicolon. For example,
587 # 'Éric' should be decoded as 'Éric'.
588 if entity_with_semicolon in compat_html_entities_html5:
589 return compat_html_entities_html5[entity_with_semicolon]
591 mobj = re.match(r'#(x[0-9a-fA-F]+|[0-9]+)', entity)
593 numstr = mobj.group(1)
594 if numstr.startswith('x'):
596 numstr = '0%s' % numstr
599 # See https://github.com/rg3/youtube-dl/issues/7518
601 return compat_chr(int(numstr, base))
605 # Unknown entity in name, return its literal representation
606 return '&%s;' % entity
612 assert type(s) == compat_str
615 r'&([^&;]+;)', lambda m: _htmlentity_transform(m.group(1)), s)
618 def get_subprocess_encoding():
619 if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
620 # For subprocess calls, encode with locale encoding
621 # Refer to http://stackoverflow.com/a/9951851/35070
622 encoding = preferredencoding()
624 encoding = sys.getfilesystemencoding()
630 def encodeFilename(s, for_subprocess=False):
632 @param s The name of the file
635 assert type(s) == compat_str
637 # Python 3 has a Unicode API
638 if sys.version_info >= (3, 0):
641 # Pass '' directly to use Unicode APIs on Windows 2000 and up
642 # (Detecting Windows NT 4 is tricky because 'major >= 4' would
643 # match Windows 9x series as well. Besides, NT 4 is obsolete.)
644 if not for_subprocess and sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
647 # Jython assumes filenames are Unicode strings though reported as Python 2.x compatible
648 if sys.platform.startswith('java'):
651 return s.encode(get_subprocess_encoding(), 'ignore')
654 def decodeFilename(b, for_subprocess=False):
656 if sys.version_info >= (3, 0):
659 if not isinstance(b, bytes):
662 return b.decode(get_subprocess_encoding(), 'ignore')
665 def encodeArgument(s):
666 if not isinstance(s, compat_str):
667 # Legacy code that uses byte strings
668 # Uncomment the following line after fixing all post processors
669 # assert False, 'Internal error: %r should be of type %r, is %r' % (s, compat_str, type(s))
670 s = s.decode('ascii')
671 return encodeFilename(s, True)
674 def decodeArgument(b):
675 return decodeFilename(b, True)
678 def decodeOption(optval):
681 if isinstance(optval, bytes):
682 optval = optval.decode(preferredencoding())
684 assert isinstance(optval, compat_str)
688 def formatSeconds(secs):
690 return '%d:%02d:%02d' % (secs // 3600, (secs % 3600) // 60, secs % 60)
692 return '%d:%02d' % (secs // 60, secs % 60)
697 def make_HTTPS_handler(params, **kwargs):
698 opts_no_check_certificate = params.get('nocheckcertificate', False)
699 if hasattr(ssl, 'create_default_context'): # Python >= 3.4 or 2.7.9
700 context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
701 if opts_no_check_certificate:
702 context.check_hostname = False
703 context.verify_mode = ssl.CERT_NONE
705 return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
708 # (create_default_context present but HTTPSHandler has no context=)
711 if sys.version_info < (3, 2):
712 return YoutubeDLHTTPSHandler(params, **kwargs)
714 context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
715 context.verify_mode = (ssl.CERT_NONE
716 if opts_no_check_certificate
717 else ssl.CERT_REQUIRED)
718 context.set_default_verify_paths()
719 return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
722 def bug_reports_message():
723 if ytdl_is_updateable():
724 update_cmd = 'type youtube-dl -U to update'
726 update_cmd = 'see https://yt-dl.org/update on how to update'
727 msg = '; please report this issue on https://yt-dl.org/bug .'
728 msg += ' Make sure you are using the latest version; %s.' % update_cmd
729 msg += ' Be sure to call youtube-dl with the --verbose flag and include its complete output.'
733 class YoutubeDLError(Exception):
734 """Base exception for YoutubeDL errors."""
738 class ExtractorError(YoutubeDLError):
739 """Error during info extraction."""
741 def __init__(self, msg, tb=None, expected=False, cause=None, video_id=None):
742 """ tb, if given, is the original traceback (so that it can be printed out).
743 If expected is set, this is a normal error message and most likely not a bug in youtube-dl.
746 if sys.exc_info()[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError):
748 if video_id is not None:
749 msg = video_id + ': ' + msg
751 msg += ' (caused by %r)' % cause
753 msg += bug_reports_message()
754 super(ExtractorError, self).__init__(msg)
757 self.exc_info = sys.exc_info() # preserve original exception
759 self.video_id = video_id
761 def format_traceback(self):
762 if self.traceback is None:
764 return ''.join(traceback.format_tb(self.traceback))
767 class UnsupportedError(ExtractorError):
768 def __init__(self, url):
769 super(UnsupportedError, self).__init__(
770 'Unsupported URL: %s' % url, expected=True)
774 class RegexNotFoundError(ExtractorError):
775 """Error when a regex didn't match"""
779 class GeoRestrictedError(ExtractorError):
780 """Geographic restriction Error exception.
782 This exception may be thrown when a video is not available from your
783 geographic location due to geographic restrictions imposed by a website.
785 def __init__(self, msg, countries=None):
786 super(GeoRestrictedError, self).__init__(msg, expected=True)
788 self.countries = countries
791 class DownloadError(YoutubeDLError):
792 """Download Error exception.
794 This exception may be thrown by FileDownloader objects if they are not
795 configured to continue on errors. They will contain the appropriate
799 def __init__(self, msg, exc_info=None):
800 """ exc_info, if given, is the original exception that caused the trouble (as returned by sys.exc_info()). """
801 super(DownloadError, self).__init__(msg)
802 self.exc_info = exc_info
805 class SameFileError(YoutubeDLError):
806 """Same File exception.
808 This exception will be thrown by FileDownloader objects if they detect
809 multiple files would have to be downloaded to the same file on disk.
814 class PostProcessingError(YoutubeDLError):
815 """Post Processing exception.
817 This exception may be raised by PostProcessor's .run() method to
818 indicate an error in the postprocessing task.
821 def __init__(self, msg):
822 super(PostProcessingError, self).__init__(msg)
826 class MaxDownloadsReached(YoutubeDLError):
827 """ --max-downloads limit has been reached. """
831 class UnavailableVideoError(YoutubeDLError):
832 """Unavailable Format exception.
834 This exception will be thrown when a video is requested
835 in a format that is not available for that video.
840 class ContentTooShortError(YoutubeDLError):
841 """Content Too Short exception.
843 This exception may be raised by FileDownloader objects when a file they
844 download is too small for what the server announced first, indicating
845 the connection was probably interrupted.
848 def __init__(self, downloaded, expected):
849 super(ContentTooShortError, self).__init__(
850 'Downloaded {0} bytes, expected {1} bytes'.format(downloaded, expected)
853 self.downloaded = downloaded
854 self.expected = expected
857 class XAttrMetadataError(YoutubeDLError):
858 def __init__(self, code=None, msg='Unknown error'):
859 super(XAttrMetadataError, self).__init__(msg)
863 # Parsing code and msg
864 if (self.code in (errno.ENOSPC, errno.EDQUOT) or
865 'No space left' in self.msg or 'Disk quota excedded' in self.msg):
866 self.reason = 'NO_SPACE'
867 elif self.code == errno.E2BIG or 'Argument list too long' in self.msg:
868 self.reason = 'VALUE_TOO_LONG'
870 self.reason = 'NOT_SUPPORTED'
873 class XAttrUnavailableError(YoutubeDLError):
877 def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs):
878 # Working around python 2 bug (see http://bugs.python.org/issue17849) by limiting
879 # expected HTTP responses to meet HTTP/1.0 or later (see also
880 # https://github.com/rg3/youtube-dl/issues/6727)
881 if sys.version_info < (3, 0):
882 kwargs['strict'] = True
883 hc = http_class(*args, **compat_kwargs(kwargs))
884 source_address = ydl_handler._params.get('source_address')
886 if source_address is not None:
887 filter_for = socket.AF_INET if '.' in source_address else socket.AF_INET6
888 # This is to workaround _create_connection() from socket where it will try all
889 # address data from getaddrinfo() including IPv6. This filters the result from
890 # getaddrinfo() based on the source_address value.
891 # This is based on the cpython socket.create_connection() function.
892 # https://github.com/python/cpython/blob/master/Lib/socket.py#L691
893 def _create_connection(address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, source_address=None):
896 addrs = socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM)
897 ip_addrs = [addr for addr in addrs if addr[0] == filter_for]
899 af, socktype, proto, canonname, sa = res
902 sock = socket.socket(af, socktype, proto)
903 if timeout is not socket._GLOBAL_DEFAULT_TIMEOUT:
904 sock.settimeout(timeout)
905 sock.bind(source_address)
907 err = None # Explicitly break reference cycle
909 except socket.error as _:
916 raise socket.error('Unknown error occurred')
917 hc._create_connection = _create_connection
919 sa = (source_address, 0)
920 if hasattr(hc, 'source_address'): # Python 2.7+
921 hc.source_address = sa
923 def _hc_connect(self, *args, **kwargs):
924 sock = compat_socket_create_connection(
925 (self.host, self.port), self.timeout, sa)
927 self.sock = ssl.wrap_socket(
928 sock, self.key_file, self.cert_file,
929 ssl_version=ssl.PROTOCOL_TLSv1)
932 hc.connect = functools.partial(_hc_connect, hc)
937 def handle_youtubedl_headers(headers):
938 filtered_headers = headers
940 if 'Youtubedl-no-compression' in filtered_headers:
941 filtered_headers = dict((k, v) for k, v in filtered_headers.items() if k.lower() != 'accept-encoding')
942 del filtered_headers['Youtubedl-no-compression']
944 return filtered_headers
947 class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
948 """Handler for HTTP requests and responses.
950 This class, when installed with an OpenerDirector, automatically adds
951 the standard headers to every HTTP request and handles gzipped and
952 deflated responses from web servers. If compression is to be avoided in
953 a particular request, the original request in the program code only has
954 to include the HTTP header "Youtubedl-no-compression", which will be
955 removed before making the real request.
957 Part of this code was copied from:
959 http://techknack.net/python-urllib2-handlers/
961 Andrew Rowls, the author of that code, agreed to release it to the
965 def __init__(self, params, *args, **kwargs):
966 compat_urllib_request.HTTPHandler.__init__(self, *args, **kwargs)
967 self._params = params
969 def http_open(self, req):
970 conn_class = compat_http_client.HTTPConnection
972 socks_proxy = req.headers.get('Ytdl-socks-proxy')
974 conn_class = make_socks_conn_class(conn_class, socks_proxy)
975 del req.headers['Ytdl-socks-proxy']
977 return self.do_open(functools.partial(
978 _create_http_connection, self, conn_class, False),
984 return zlib.decompress(data, -zlib.MAX_WBITS)
986 return zlib.decompress(data)
988 def http_request(self, req):
989 # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
990 # always respected by websites, some tend to give out URLs with non percent-encoded
991 # non-ASCII characters (see telemb.py, ard.py [#3412])
992 # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
993 # To work around aforementioned issue we will replace request's original URL with
994 # percent-encoded one
995 # Since redirects are also affected (e.g. http://www.southpark.de/alle-episoden/s18e09)
996 # the code of this workaround has been moved here from YoutubeDL.urlopen()
997 url = req.get_full_url()
998 url_escaped = escape_url(url)
1000 # Substitute URL if any change after escaping
1001 if url != url_escaped:
1002 req = update_Request(req, url=url_escaped)
1004 for h, v in std_headers.items():
1005 # Capitalize is needed because of Python bug 2275: http://bugs.python.org/issue2275
1006 # The dict keys are capitalized because of this bug by urllib
1007 if h.capitalize() not in req.headers:
1008 req.add_header(h, v)
1010 req.headers = handle_youtubedl_headers(req.headers)
1012 if sys.version_info < (2, 7) and '#' in req.get_full_url():
1013 # Python 2.6 is brain-dead when it comes to fragments
1014 req._Request__original = req._Request__original.partition('#')[0]
1015 req._Request__r_type = req._Request__r_type.partition('#')[0]
1019 def http_response(self, req, resp):
1022 if resp.headers.get('Content-encoding', '') == 'gzip':
1023 content = resp.read()
1024 gz = gzip.GzipFile(fileobj=io.BytesIO(content), mode='rb')
1026 uncompressed = io.BytesIO(gz.read())
1027 except IOError as original_ioerror:
1028 # There may be junk add the end of the file
1029 # See http://stackoverflow.com/q/4928560/35070 for details
1030 for i in range(1, 1024):
1032 gz = gzip.GzipFile(fileobj=io.BytesIO(content[:-i]), mode='rb')
1033 uncompressed = io.BytesIO(gz.read())
1038 raise original_ioerror
1039 resp = compat_urllib_request.addinfourl(uncompressed, old_resp.headers, old_resp.url, old_resp.code)
1040 resp.msg = old_resp.msg
1041 del resp.headers['Content-encoding']
1043 if resp.headers.get('Content-encoding', '') == 'deflate':
1044 gz = io.BytesIO(self.deflate(resp.read()))
1045 resp = compat_urllib_request.addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code)
1046 resp.msg = old_resp.msg
1047 del resp.headers['Content-encoding']
1048 # Percent-encode redirect URL of Location HTTP header to satisfy RFC 3986 (see
1049 # https://github.com/rg3/youtube-dl/issues/6457).
1050 if 300 <= resp.code < 400:
1051 location = resp.headers.get('Location')
1053 # As of RFC 2616 default charset is iso-8859-1 that is respected by python 3
1054 if sys.version_info >= (3, 0):
1055 location = location.encode('iso-8859-1').decode('utf-8')
1057 location = location.decode('utf-8')
1058 location_escaped = escape_url(location)
1059 if location != location_escaped:
1060 del resp.headers['Location']
1061 if sys.version_info < (3, 0):
1062 location_escaped = location_escaped.encode('utf-8')
1063 resp.headers['Location'] = location_escaped
1066 https_request = http_request
1067 https_response = http_response
1070 def make_socks_conn_class(base_class, socks_proxy):
1071 assert issubclass(base_class, (
1072 compat_http_client.HTTPConnection, compat_http_client.HTTPSConnection))
1074 url_components = compat_urlparse.urlparse(socks_proxy)
1075 if url_components.scheme.lower() == 'socks5':
1076 socks_type = ProxyType.SOCKS5
1077 elif url_components.scheme.lower() in ('socks', 'socks4'):
1078 socks_type = ProxyType.SOCKS4
1079 elif url_components.scheme.lower() == 'socks4a':
1080 socks_type = ProxyType.SOCKS4A
1082 def unquote_if_non_empty(s):
1085 return compat_urllib_parse_unquote_plus(s)
1089 url_components.hostname, url_components.port or 1080,
1091 unquote_if_non_empty(url_components.username),
1092 unquote_if_non_empty(url_components.password),
1095 class SocksConnection(base_class):
1097 self.sock = sockssocket()
1098 self.sock.setproxy(*proxy_args)
1099 if type(self.timeout) in (int, float):
1100 self.sock.settimeout(self.timeout)
1101 self.sock.connect((self.host, self.port))
1103 if isinstance(self, compat_http_client.HTTPSConnection):
1104 if hasattr(self, '_context'): # Python > 2.6
1105 self.sock = self._context.wrap_socket(
1106 self.sock, server_hostname=self.host)
1108 self.sock = ssl.wrap_socket(self.sock)
1110 return SocksConnection
1113 class YoutubeDLHTTPSHandler(compat_urllib_request.HTTPSHandler):
1114 def __init__(self, params, https_conn_class=None, *args, **kwargs):
1115 compat_urllib_request.HTTPSHandler.__init__(self, *args, **kwargs)
1116 self._https_conn_class = https_conn_class or compat_http_client.HTTPSConnection
1117 self._params = params
1119 def https_open(self, req):
1121 conn_class = self._https_conn_class
1123 if hasattr(self, '_context'): # python > 2.6
1124 kwargs['context'] = self._context
1125 if hasattr(self, '_check_hostname'): # python 3.x
1126 kwargs['check_hostname'] = self._check_hostname
1128 socks_proxy = req.headers.get('Ytdl-socks-proxy')
1130 conn_class = make_socks_conn_class(conn_class, socks_proxy)
1131 del req.headers['Ytdl-socks-proxy']
1133 return self.do_open(functools.partial(
1134 _create_http_connection, self, conn_class, True),
1138 class YoutubeDLCookieProcessor(compat_urllib_request.HTTPCookieProcessor):
1139 def __init__(self, cookiejar=None):
1140 compat_urllib_request.HTTPCookieProcessor.__init__(self, cookiejar)
1142 def http_response(self, request, response):
1143 # Python 2 will choke on next HTTP request in row if there are non-ASCII
1144 # characters in Set-Cookie HTTP header of last response (see
1145 # https://github.com/rg3/youtube-dl/issues/6769).
1146 # In order to at least prevent crashing we will percent encode Set-Cookie
1147 # header before HTTPCookieProcessor starts processing it.
1148 # if sys.version_info < (3, 0) and response.headers:
1149 # for set_cookie_header in ('Set-Cookie', 'Set-Cookie2'):
1150 # set_cookie = response.headers.get(set_cookie_header)
1152 # set_cookie_escaped = compat_urllib_parse.quote(set_cookie, b"%/;:@&=+$,!~*'()?#[] ")
1153 # if set_cookie != set_cookie_escaped:
1154 # del response.headers[set_cookie_header]
1155 # response.headers[set_cookie_header] = set_cookie_escaped
1156 return compat_urllib_request.HTTPCookieProcessor.http_response(self, request, response)
1158 https_request = compat_urllib_request.HTTPCookieProcessor.http_request
1159 https_response = http_response
1162 def extract_timezone(date_str):
1164 r'^.{8,}?(?P<tz>Z$| ?(?P<sign>\+|-)(?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})$)',
1167 timezone = datetime.timedelta()
1169 date_str = date_str[:-len(m.group('tz'))]
1170 if not m.group('sign'):
1171 timezone = datetime.timedelta()
1173 sign = 1 if m.group('sign') == '+' else -1
1174 timezone = datetime.timedelta(
1175 hours=sign * int(m.group('hours')),
1176 minutes=sign * int(m.group('minutes')))
1177 return timezone, date_str
1180 def parse_iso8601(date_str, delimiter='T', timezone=None):
1181 """ Return a UNIX timestamp from the given date """
1183 if date_str is None:
1186 date_str = re.sub(r'\.[0-9]+', '', date_str)
1188 if timezone is None:
1189 timezone, date_str = extract_timezone(date_str)
1192 date_format = '%Y-%m-%d{0}%H:%M:%S'.format(delimiter)
1193 dt = datetime.datetime.strptime(date_str, date_format) - timezone
1194 return calendar.timegm(dt.timetuple())
1199 def date_formats(day_first=True):
1200 return DATE_FORMATS_DAY_FIRST if day_first else DATE_FORMATS_MONTH_FIRST
1203 def unified_strdate(date_str, day_first=True):
1204 """Return a string with the date in the format YYYYMMDD"""
1206 if date_str is None:
1210 date_str = date_str.replace(',', ' ')
1211 # Remove AM/PM + timezone
1212 date_str = re.sub(r'(?i)\s*(?:AM|PM)(?:\s+[A-Z]+)?', '', date_str)
1213 _, date_str = extract_timezone(date_str)
1215 for expression in date_formats(day_first):
1217 upload_date = datetime.datetime.strptime(date_str, expression).strftime('%Y%m%d')
1220 if upload_date is None:
1221 timetuple = email.utils.parsedate_tz(date_str)
1224 upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d')
1227 if upload_date is not None:
1228 return compat_str(upload_date)
1231 def unified_timestamp(date_str, day_first=True):
1232 if date_str is None:
1235 date_str = re.sub(r'[,|]', '', date_str)
1237 pm_delta = 12 if re.search(r'(?i)PM', date_str) else 0
1238 timezone, date_str = extract_timezone(date_str)
1240 # Remove AM/PM + timezone
1241 date_str = re.sub(r'(?i)\s*(?:AM|PM)(?:\s+[A-Z]+)?', '', date_str)
1243 # Remove unrecognized timezones from ISO 8601 alike timestamps
1244 m = re.search(r'\d{1,2}:\d{1,2}(?:\.\d+)?(?P<tz>\s*[A-Z]+)$', date_str)
1246 date_str = date_str[:-len(m.group('tz'))]
1248 # Python only supports microseconds, so remove nanoseconds
1249 m = re.search(r'^([0-9]{4,}-[0-9]{1,2}-[0-9]{1,2}T[0-9]{1,2}:[0-9]{1,2}:[0-9]{1,2}\.[0-9]{6})[0-9]+$', date_str)
1251 date_str = m.group(1)
1253 for expression in date_formats(day_first):
1255 dt = datetime.datetime.strptime(date_str, expression) - timezone + datetime.timedelta(hours=pm_delta)
1256 return calendar.timegm(dt.timetuple())
1259 timetuple = email.utils.parsedate_tz(date_str)
1261 return calendar.timegm(timetuple) + pm_delta * 3600
1264 def determine_ext(url, default_ext='unknown_video'):
1265 if url is None or '.' not in url:
1267 guess = url.partition('?')[0].rpartition('.')[2]
1268 if re.match(r'^[A-Za-z0-9]+$', guess):
1270 # Try extract ext from URLs like http://example.com/foo/bar.mp4/?download
1271 elif guess.rstrip('/') in KNOWN_EXTENSIONS:
1272 return guess.rstrip('/')
1277 def subtitles_filename(filename, sub_lang, sub_format):
1278 return filename.rsplit('.', 1)[0] + '.' + sub_lang + '.' + sub_format
1281 def date_from_str(date_str):
1283 Return a datetime object from a string in the format YYYYMMDD or
1284 (now|today)[+-][0-9](day|week|month|year)(s)?"""
1285 today = datetime.date.today()
1286 if date_str in ('now', 'today'):
1288 if date_str == 'yesterday':
1289 return today - datetime.timedelta(days=1)
1290 match = re.match(r'(now|today)(?P<sign>[+-])(?P<time>\d+)(?P<unit>day|week|month|year)(s)?', date_str)
1291 if match is not None:
1292 sign = match.group('sign')
1293 time = int(match.group('time'))
1296 unit = match.group('unit')
1297 # A bad approximation?
1301 elif unit == 'year':
1305 delta = datetime.timedelta(**{unit: time})
1306 return today + delta
1307 return datetime.datetime.strptime(date_str, '%Y%m%d').date()
1310 def hyphenate_date(date_str):
1312 Convert a date in 'YYYYMMDD' format to 'YYYY-MM-DD' format"""
1313 match = re.match(r'^(\d\d\d\d)(\d\d)(\d\d)$', date_str)
1314 if match is not None:
1315 return '-'.join(match.groups())
1320 class DateRange(object):
1321 """Represents a time interval between two dates"""
1323 def __init__(self, start=None, end=None):
1324 """start and end must be strings in the format accepted by date"""
1325 if start is not None:
1326 self.start = date_from_str(start)
1328 self.start = datetime.datetime.min.date()
1330 self.end = date_from_str(end)
1332 self.end = datetime.datetime.max.date()
1333 if self.start > self.end:
1334 raise ValueError('Date range: "%s" , the start date must be before the end date' % self)
1338 """Returns a range that only contains the given day"""
1339 return cls(day, day)
1341 def __contains__(self, date):
1342 """Check if the date is in the range"""
1343 if not isinstance(date, datetime.date):
1344 date = date_from_str(date)
1345 return self.start <= date <= self.end
1348 return '%s - %s' % (self.start.isoformat(), self.end.isoformat())
1351 def platform_name():
1352 """ Returns the platform name as a compat_str """
1353 res = platform.platform()
1354 if isinstance(res, bytes):
1355 res = res.decode(preferredencoding())
1357 assert isinstance(res, compat_str)
1361 def _windows_write_string(s, out):
1362 """ Returns True if the string was written using special methods,
1363 False if it has yet to be written out."""
1364 # Adapted from http://stackoverflow.com/a/3259271/35070
1367 import ctypes.wintypes
1375 fileno = out.fileno()
1376 except AttributeError:
1377 # If the output stream doesn't have a fileno, it's virtual
1379 except io.UnsupportedOperation:
1380 # Some strange Windows pseudo files?
1382 if fileno not in WIN_OUTPUT_IDS:
1385 GetStdHandle = compat_ctypes_WINFUNCTYPE(
1386 ctypes.wintypes.HANDLE, ctypes.wintypes.DWORD)(
1387 ('GetStdHandle', ctypes.windll.kernel32))
1388 h = GetStdHandle(WIN_OUTPUT_IDS[fileno])
1390 WriteConsoleW = compat_ctypes_WINFUNCTYPE(
1391 ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE, ctypes.wintypes.LPWSTR,
1392 ctypes.wintypes.DWORD, ctypes.POINTER(ctypes.wintypes.DWORD),
1393 ctypes.wintypes.LPVOID)(('WriteConsoleW', ctypes.windll.kernel32))
1394 written = ctypes.wintypes.DWORD(0)
1396 GetFileType = compat_ctypes_WINFUNCTYPE(ctypes.wintypes.DWORD, ctypes.wintypes.DWORD)(('GetFileType', ctypes.windll.kernel32))
1397 FILE_TYPE_CHAR = 0x0002
1398 FILE_TYPE_REMOTE = 0x8000
1399 GetConsoleMode = compat_ctypes_WINFUNCTYPE(
1400 ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE,
1401 ctypes.POINTER(ctypes.wintypes.DWORD))(
1402 ('GetConsoleMode', ctypes.windll.kernel32))
1403 INVALID_HANDLE_VALUE = ctypes.wintypes.DWORD(-1).value
1405 def not_a_console(handle):
1406 if handle == INVALID_HANDLE_VALUE or handle is None:
1408 return ((GetFileType(handle) & ~FILE_TYPE_REMOTE) != FILE_TYPE_CHAR or
1409 GetConsoleMode(handle, ctypes.byref(ctypes.wintypes.DWORD())) == 0)
1411 if not_a_console(h):
1414 def next_nonbmp_pos(s):
1416 return next(i for i, c in enumerate(s) if ord(c) > 0xffff)
1417 except StopIteration:
1421 count = min(next_nonbmp_pos(s), 1024)
1423 ret = WriteConsoleW(
1424 h, s, count if count else 2, ctypes.byref(written), None)
1426 raise OSError('Failed to write string')
1427 if not count: # We just wrote a non-BMP character
1428 assert written.value == 2
1431 assert written.value > 0
1432 s = s[written.value:]
1436 def write_string(s, out=None, encoding=None):
1439 assert type(s) == compat_str
1441 if sys.platform == 'win32' and encoding is None and hasattr(out, 'fileno'):
1442 if _windows_write_string(s, out):
1445 if ('b' in getattr(out, 'mode', '') or
1446 sys.version_info[0] < 3): # Python 2 lies about mode of sys.stderr
1447 byt = s.encode(encoding or preferredencoding(), 'ignore')
1449 elif hasattr(out, 'buffer'):
1450 enc = encoding or getattr(out, 'encoding', None) or preferredencoding()
1451 byt = s.encode(enc, 'ignore')
1452 out.buffer.write(byt)
1458 def bytes_to_intlist(bs):
1461 if isinstance(bs[0], int): # Python 3
1464 return [ord(c) for c in bs]
1467 def intlist_to_bytes(xs):
1470 return compat_struct_pack('%dB' % len(xs), *xs)
1473 # Cross-platform file locking
1474 if sys.platform == 'win32':
1475 import ctypes.wintypes
1478 class OVERLAPPED(ctypes.Structure):
1480 ('Internal', ctypes.wintypes.LPVOID),
1481 ('InternalHigh', ctypes.wintypes.LPVOID),
1482 ('Offset', ctypes.wintypes.DWORD),
1483 ('OffsetHigh', ctypes.wintypes.DWORD),
1484 ('hEvent', ctypes.wintypes.HANDLE),
1487 kernel32 = ctypes.windll.kernel32
1488 LockFileEx = kernel32.LockFileEx
1489 LockFileEx.argtypes = [
1490 ctypes.wintypes.HANDLE, # hFile
1491 ctypes.wintypes.DWORD, # dwFlags
1492 ctypes.wintypes.DWORD, # dwReserved
1493 ctypes.wintypes.DWORD, # nNumberOfBytesToLockLow
1494 ctypes.wintypes.DWORD, # nNumberOfBytesToLockHigh
1495 ctypes.POINTER(OVERLAPPED) # Overlapped
1497 LockFileEx.restype = ctypes.wintypes.BOOL
1498 UnlockFileEx = kernel32.UnlockFileEx
1499 UnlockFileEx.argtypes = [
1500 ctypes.wintypes.HANDLE, # hFile
1501 ctypes.wintypes.DWORD, # dwReserved
1502 ctypes.wintypes.DWORD, # nNumberOfBytesToLockLow
1503 ctypes.wintypes.DWORD, # nNumberOfBytesToLockHigh
1504 ctypes.POINTER(OVERLAPPED) # Overlapped
1506 UnlockFileEx.restype = ctypes.wintypes.BOOL
1507 whole_low = 0xffffffff
1508 whole_high = 0x7fffffff
1510 def _lock_file(f, exclusive):
1511 overlapped = OVERLAPPED()
1512 overlapped.Offset = 0
1513 overlapped.OffsetHigh = 0
1514 overlapped.hEvent = 0
1515 f._lock_file_overlapped_p = ctypes.pointer(overlapped)
1516 handle = msvcrt.get_osfhandle(f.fileno())
1517 if not LockFileEx(handle, 0x2 if exclusive else 0x0, 0,
1518 whole_low, whole_high, f._lock_file_overlapped_p):
1519 raise OSError('Locking file failed: %r' % ctypes.FormatError())
1521 def _unlock_file(f):
1522 assert f._lock_file_overlapped_p
1523 handle = msvcrt.get_osfhandle(f.fileno())
1524 if not UnlockFileEx(handle, 0,
1525 whole_low, whole_high, f._lock_file_overlapped_p):
1526 raise OSError('Unlocking file failed: %r' % ctypes.FormatError())
1529 # Some platforms, such as Jython, is missing fcntl
1533 def _lock_file(f, exclusive):
1534 fcntl.flock(f, fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH)
1536 def _unlock_file(f):
1537 fcntl.flock(f, fcntl.LOCK_UN)
1539 UNSUPPORTED_MSG = 'file locking is not supported on this platform'
1541 def _lock_file(f, exclusive):
1542 raise IOError(UNSUPPORTED_MSG)
1544 def _unlock_file(f):
1545 raise IOError(UNSUPPORTED_MSG)
1548 class locked_file(object):
1549 def __init__(self, filename, mode, encoding=None):
1550 assert mode in ['r', 'a', 'w']
1551 self.f = io.open(filename, mode, encoding=encoding)
1554 def __enter__(self):
1555 exclusive = self.mode != 'r'
1557 _lock_file(self.f, exclusive)
1563 def __exit__(self, etype, value, traceback):
1565 _unlock_file(self.f)
1572 def write(self, *args):
1573 return self.f.write(*args)
1575 def read(self, *args):
1576 return self.f.read(*args)
1579 def get_filesystem_encoding():
1580 encoding = sys.getfilesystemencoding()
1581 return encoding if encoding is not None else 'utf-8'
1584 def shell_quote(args):
1586 encoding = get_filesystem_encoding()
1588 if isinstance(a, bytes):
1589 # We may get a filename encoded with 'encodeFilename'
1590 a = a.decode(encoding)
1591 quoted_args.append(compat_shlex_quote(a))
1592 return ' '.join(quoted_args)
1595 def smuggle_url(url, data):
1596 """ Pass additional data in a URL for internal use. """
1598 url, idata = unsmuggle_url(url, {})
1600 sdata = compat_urllib_parse_urlencode(
1601 {'__youtubedl_smuggle': json.dumps(data)})
1602 return url + '#' + sdata
1605 def unsmuggle_url(smug_url, default=None):
1606 if '#__youtubedl_smuggle' not in smug_url:
1607 return smug_url, default
1608 url, _, sdata = smug_url.rpartition('#')
1609 jsond = compat_parse_qs(sdata)['__youtubedl_smuggle'][0]
1610 data = json.loads(jsond)
1614 def format_bytes(bytes):
1617 if type(bytes) is str:
1618 bytes = float(bytes)
1622 exponent = int(math.log(bytes, 1024.0))
1623 suffix = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'][exponent]
1624 converted = float(bytes) / float(1024 ** exponent)
1625 return '%.2f%s' % (converted, suffix)
1628 def lookup_unit_table(unit_table, s):
1629 units_re = '|'.join(re.escape(u) for u in unit_table)
1631 r'(?P<num>[0-9]+(?:[,.][0-9]*)?)\s*(?P<unit>%s)\b' % units_re, s)
1634 num_str = m.group('num').replace(',', '.')
1635 mult = unit_table[m.group('unit')]
1636 return int(float(num_str) * mult)
1639 def parse_filesize(s):
1643 # The lower-case forms are of course incorrect and unofficial,
1644 # but we support those too
1661 'megabytes': 1000 ** 2,
1662 'mebibytes': 1024 ** 2,
1668 'gigabytes': 1000 ** 3,
1669 'gibibytes': 1024 ** 3,
1675 'terabytes': 1000 ** 4,
1676 'tebibytes': 1024 ** 4,
1682 'petabytes': 1000 ** 5,
1683 'pebibytes': 1024 ** 5,
1689 'exabytes': 1000 ** 6,
1690 'exbibytes': 1024 ** 6,
1696 'zettabytes': 1000 ** 7,
1697 'zebibytes': 1024 ** 7,
1703 'yottabytes': 1000 ** 8,
1704 'yobibytes': 1024 ** 8,
1707 return lookup_unit_table(_UNIT_TABLE, s)
1716 if re.match(r'^[\d,.]+$', s):
1717 return str_to_int(s)
1728 return lookup_unit_table(_UNIT_TABLE, s)
1731 def parse_resolution(s):
1735 mobj = re.search(r'\b(?P<w>\d+)\s*[xX×]\s*(?P<h>\d+)\b', s)
1738 'width': int(mobj.group('w')),
1739 'height': int(mobj.group('h')),
1742 mobj = re.search(r'\b(\d+)[pPiI]\b', s)
1744 return {'height': int(mobj.group(1))}
1746 mobj = re.search(r'\b([48])[kK]\b', s)
1748 return {'height': int(mobj.group(1)) * 540}
1753 def month_by_name(name, lang='en'):
1754 """ Return the number of a month by (locale-independently) English name """
1756 month_names = MONTH_NAMES.get(lang, MONTH_NAMES['en'])
1759 return month_names.index(name) + 1
1764 def month_by_abbreviation(abbrev):
1765 """ Return the number of a month by (locale-independently) English
1769 return [s[:3] for s in ENGLISH_MONTH_NAMES].index(abbrev) + 1
1774 def fix_xml_ampersands(xml_str):
1775 """Replace all the '&' by '&' in XML"""
1777 r'&(?!amp;|lt;|gt;|apos;|quot;|#x[0-9a-fA-F]{,4};|#[0-9]{,4};)',
1782 def setproctitle(title):
1783 assert isinstance(title, compat_str)
1785 # ctypes in Jython is not complete
1786 # http://bugs.jython.org/issue2148
1787 if sys.platform.startswith('java'):
1791 libc = ctypes.cdll.LoadLibrary('libc.so.6')
1795 # LoadLibrary in Windows Python 2.7.13 only expects
1796 # a bytestring, but since unicode_literals turns
1797 # every string into a unicode string, it fails.
1799 title_bytes = title.encode('utf-8')
1800 buf = ctypes.create_string_buffer(len(title_bytes))
1801 buf.value = title_bytes
1803 libc.prctl(15, buf, 0, 0, 0)
1804 except AttributeError:
1805 return # Strange libc, just skip this
1808 def remove_start(s, start):
1809 return s[len(start):] if s is not None and s.startswith(start) else s
1812 def remove_end(s, end):
1813 return s[:-len(end)] if s is not None and s.endswith(end) else s
1816 def remove_quotes(s):
1817 if s is None or len(s) < 2:
1819 for quote in ('"', "'", ):
1820 if s[0] == quote and s[-1] == quote:
1825 def url_basename(url):
1826 path = compat_urlparse.urlparse(url).path
1827 return path.strip('/').split('/')[-1]
1831 return re.match(r'https?://[^?#&]+/', url).group()
1834 def urljoin(base, path):
1835 if isinstance(path, bytes):
1836 path = path.decode('utf-8')
1837 if not isinstance(path, compat_str) or not path:
1839 if re.match(r'^(?:https?:)?//', path):
1841 if isinstance(base, bytes):
1842 base = base.decode('utf-8')
1843 if not isinstance(base, compat_str) or not re.match(
1844 r'^(?:https?:)?//', base):
1846 return compat_urlparse.urljoin(base, path)
1849 class HEADRequest(compat_urllib_request.Request):
1850 def get_method(self):
1854 class PUTRequest(compat_urllib_request.Request):
1855 def get_method(self):
1859 def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1):
1862 v = getattr(v, get_attr, None)
1868 return int(v) * invscale // scale
1873 def str_or_none(v, default=None):
1874 return default if v is None else compat_str(v)
1877 def str_to_int(int_str):
1878 """ A more relaxed version of int_or_none """
1881 int_str = re.sub(r'[,\.\+]', '', int_str)
1885 def float_or_none(v, scale=1, invscale=1, default=None):
1889 return float(v) * invscale / scale
1894 def bool_or_none(v, default=None):
1895 return v if isinstance(v, bool) else default
1898 def strip_or_none(v):
1899 return None if v is None else v.strip()
1902 def url_or_none(url):
1903 if not url or not isinstance(url, compat_str):
1906 return url if re.match(r'^(?:[a-zA-Z][\da-zA-Z.+-]*:)?//', url) else None
1909 def parse_duration(s):
1910 if not isinstance(s, compat_basestring):
1915 days, hours, mins, secs, ms = [None] * 5
1916 m = re.match(r'(?:(?:(?:(?P<days>[0-9]+):)?(?P<hours>[0-9]+):)?(?P<mins>[0-9]+):)?(?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?Z?$', s)
1918 days, hours, mins, secs, ms = m.groups()
1923 [0-9]+\s*y(?:ears?)?\s*
1926 [0-9]+\s*m(?:onths?)?\s*
1929 [0-9]+\s*w(?:eeks?)?\s*
1932 (?P<days>[0-9]+)\s*d(?:ays?)?\s*
1936 (?P<hours>[0-9]+)\s*h(?:ours?)?\s*
1939 (?P<mins>[0-9]+)\s*m(?:in(?:ute)?s?)?\s*
1942 (?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?\s*s(?:ec(?:ond)?s?)?\s*
1945 days, hours, mins, secs, ms = m.groups()
1947 m = re.match(r'(?i)(?:(?P<hours>[0-9.]+)\s*(?:hours?)|(?P<mins>[0-9.]+)\s*(?:mins?\.?|minutes?)\s*)Z?$', s)
1949 hours, mins = m.groups()
1955 duration += float(secs)
1957 duration += float(mins) * 60
1959 duration += float(hours) * 60 * 60
1961 duration += float(days) * 24 * 60 * 60
1963 duration += float(ms)
1967 def prepend_extension(filename, ext, expected_real_ext=None):
1968 name, real_ext = os.path.splitext(filename)
1970 '{0}.{1}{2}'.format(name, ext, real_ext)
1971 if not expected_real_ext or real_ext[1:] == expected_real_ext
1972 else '{0}.{1}'.format(filename, ext))
1975 def replace_extension(filename, ext, expected_real_ext=None):
1976 name, real_ext = os.path.splitext(filename)
1977 return '{0}.{1}'.format(
1978 name if not expected_real_ext or real_ext[1:] == expected_real_ext else filename,
1982 def check_executable(exe, args=[]):
1983 """ Checks if the given binary is installed somewhere in PATH, and returns its name.
1984 args can be a list of arguments for a short output (like -version) """
1986 subprocess.Popen([exe] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
1992 def get_exe_version(exe, args=['--version'],
1993 version_re=None, unrecognized='present'):
1994 """ Returns the version of the specified executable,
1995 or False if the executable is not present """
1997 # STDIN should be redirected too. On UNIX-like systems, ffmpeg triggers
1998 # SIGTTOU if youtube-dl is run in the background.
1999 # See https://github.com/rg3/youtube-dl/issues/955#issuecomment-209789656
2000 out, _ = subprocess.Popen(
2001 [encodeArgument(exe)] + args,
2002 stdin=subprocess.PIPE,
2003 stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()
2006 if isinstance(out, bytes): # Python 2.x
2007 out = out.decode('ascii', 'ignore')
2008 return detect_exe_version(out, version_re, unrecognized)
2011 def detect_exe_version(output, version_re=None, unrecognized='present'):
2012 assert isinstance(output, compat_str)
2013 if version_re is None:
2014 version_re = r'version\s+([-0-9._a-zA-Z]+)'
2015 m = re.search(version_re, output)
2022 class PagedList(object):
2024 # This is only useful for tests
2025 return len(self.getslice())
2028 class OnDemandPagedList(PagedList):
2029 def __init__(self, pagefunc, pagesize, use_cache=True):
2030 self._pagefunc = pagefunc
2031 self._pagesize = pagesize
2032 self._use_cache = use_cache
2036 def getslice(self, start=0, end=None):
2038 for pagenum in itertools.count(start // self._pagesize):
2039 firstid = pagenum * self._pagesize
2040 nextfirstid = pagenum * self._pagesize + self._pagesize
2041 if start >= nextfirstid:
2046 page_results = self._cache.get(pagenum)
2047 if page_results is None:
2048 page_results = list(self._pagefunc(pagenum))
2050 self._cache[pagenum] = page_results
2053 start % self._pagesize
2054 if firstid <= start < nextfirstid
2058 ((end - 1) % self._pagesize) + 1
2059 if (end is not None and firstid <= end <= nextfirstid)
2062 if startv != 0 or endv is not None:
2063 page_results = page_results[startv:endv]
2064 res.extend(page_results)
2066 # A little optimization - if current page is not "full", ie. does
2067 # not contain page_size videos then we can assume that this page
2068 # is the last one - there are no more ids on further pages -
2069 # i.e. no need to query again.
2070 if len(page_results) + startv < self._pagesize:
2073 # If we got the whole page, but the next page is not interesting,
2074 # break out early as well
2075 if end == nextfirstid:
2080 class InAdvancePagedList(PagedList):
2081 def __init__(self, pagefunc, pagecount, pagesize):
2082 self._pagefunc = pagefunc
2083 self._pagecount = pagecount
2084 self._pagesize = pagesize
2086 def getslice(self, start=0, end=None):
2088 start_page = start // self._pagesize
2090 self._pagecount if end is None else (end // self._pagesize + 1))
2091 skip_elems = start - start_page * self._pagesize
2092 only_more = None if end is None else end - start
2093 for pagenum in range(start_page, end_page):
2094 page = list(self._pagefunc(pagenum))
2096 page = page[skip_elems:]
2098 if only_more is not None:
2099 if len(page) < only_more:
2100 only_more -= len(page)
2102 page = page[:only_more]
2109 def uppercase_escape(s):
2110 unicode_escape = codecs.getdecoder('unicode_escape')
2112 r'\\U[0-9a-fA-F]{8}',
2113 lambda m: unicode_escape(m.group(0))[0],
2117 def lowercase_escape(s):
2118 unicode_escape = codecs.getdecoder('unicode_escape')
2120 r'\\u[0-9a-fA-F]{4}',
2121 lambda m: unicode_escape(m.group(0))[0],
2125 def escape_rfc3986(s):
2126 """Escape non-ASCII characters as suggested by RFC 3986"""
2127 if sys.version_info < (3, 0) and isinstance(s, compat_str):
2128 s = s.encode('utf-8')
2129 return compat_urllib_parse.quote(s, b"%/;:@&=+$,!~*'()?#[]")
2132 def escape_url(url):
2133 """Escape URL as suggested by RFC 3986"""
2134 url_parsed = compat_urllib_parse_urlparse(url)
2135 return url_parsed._replace(
2136 netloc=url_parsed.netloc.encode('idna').decode('ascii'),
2137 path=escape_rfc3986(url_parsed.path),
2138 params=escape_rfc3986(url_parsed.params),
2139 query=escape_rfc3986(url_parsed.query),
2140 fragment=escape_rfc3986(url_parsed.fragment)
2144 def read_batch_urls(batch_fd):
2146 if not isinstance(url, compat_str):
2147 url = url.decode('utf-8', 'replace')
2148 BOM_UTF8 = '\xef\xbb\xbf'
2149 if url.startswith(BOM_UTF8):
2150 url = url[len(BOM_UTF8):]
2152 if url.startswith(('#', ';', ']')):
2156 with contextlib.closing(batch_fd) as fd:
2157 return [url for url in map(fixup, fd) if url]
2160 def urlencode_postdata(*args, **kargs):
2161 return compat_urllib_parse_urlencode(*args, **kargs).encode('ascii')
2164 def update_url_query(url, query):
2167 parsed_url = compat_urlparse.urlparse(url)
2168 qs = compat_parse_qs(parsed_url.query)
2170 return compat_urlparse.urlunparse(parsed_url._replace(
2171 query=compat_urllib_parse_urlencode(qs, True)))
2174 def update_Request(req, url=None, data=None, headers={}, query={}):
2175 req_headers = req.headers.copy()
2176 req_headers.update(headers)
2177 req_data = data or req.data
2178 req_url = update_url_query(url or req.get_full_url(), query)
2179 req_get_method = req.get_method()
2180 if req_get_method == 'HEAD':
2181 req_type = HEADRequest
2182 elif req_get_method == 'PUT':
2183 req_type = PUTRequest
2185 req_type = compat_urllib_request.Request
2187 req_url, data=req_data, headers=req_headers,
2188 origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
2189 if hasattr(req, 'timeout'):
2190 new_req.timeout = req.timeout
2194 def _multipart_encode_impl(data, boundary):
2195 content_type = 'multipart/form-data; boundary=%s' % boundary
2198 for k, v in data.items():
2199 out += b'--' + boundary.encode('ascii') + b'\r\n'
2200 if isinstance(k, compat_str):
2201 k = k.encode('utf-8')
2202 if isinstance(v, compat_str):
2203 v = v.encode('utf-8')
2204 # RFC 2047 requires non-ASCII field names to be encoded, while RFC 7578
2205 # suggests sending UTF-8 directly. Firefox sends UTF-8, too
2206 content = b'Content-Disposition: form-data; name="' + k + b'"\r\n\r\n' + v + b'\r\n'
2207 if boundary.encode('ascii') in content:
2208 raise ValueError('Boundary overlaps with data')
2211 out += b'--' + boundary.encode('ascii') + b'--\r\n'
2213 return out, content_type
2216 def multipart_encode(data, boundary=None):
2218 Encode a dict to RFC 7578-compliant form-data
2221 A dict where keys and values can be either Unicode or bytes-like
2224 If specified a Unicode object, it's used as the boundary. Otherwise
2225 a random boundary is generated.
2227 Reference: https://tools.ietf.org/html/rfc7578
2229 has_specified_boundary = boundary is not None
2232 if boundary is None:
2233 boundary = '---------------' + str(random.randrange(0x0fffffff, 0xffffffff))
2236 out, content_type = _multipart_encode_impl(data, boundary)
2239 if has_specified_boundary:
2243 return out, content_type
2246 def dict_get(d, key_or_keys, default=None, skip_false_values=True):
2247 if isinstance(key_or_keys, (list, tuple)):
2248 for key in key_or_keys:
2249 if key not in d or d[key] is None or skip_false_values and not d[key]:
2253 return d.get(key_or_keys, default)
2256 def try_get(src, getter, expected_type=None):
2257 if not isinstance(getter, (list, tuple)):
2262 except (AttributeError, KeyError, TypeError, IndexError):
2265 if expected_type is None or isinstance(v, expected_type):
2269 def merge_dicts(*dicts):
2271 for a_dict in dicts:
2272 for k, v in a_dict.items():
2275 if (k not in merged or
2276 (isinstance(v, compat_str) and v and
2277 isinstance(merged[k], compat_str) and
2283 def encode_compat_str(string, encoding=preferredencoding(), errors='strict'):
2284 return string if isinstance(string, compat_str) else compat_str(string, encoding, errors)
2296 TV_PARENTAL_GUIDELINES = {
2306 def parse_age_limit(s):
2308 return s if 0 <= s <= 21 else None
2309 if not isinstance(s, compat_basestring):
2311 m = re.match(r'^(?P<age>\d{1,2})\+?$', s)
2313 return int(m.group('age'))
2315 return US_RATINGS[s]
2316 m = re.match(r'^TV[_-]?(%s)$' % '|'.join(k[3:] for k in TV_PARENTAL_GUIDELINES), s)
2318 return TV_PARENTAL_GUIDELINES['TV-' + m.group(1)]
2322 def strip_jsonp(code):
2325 (?:window\.)?(?P<func_name>[a-zA-Z0-9_.$]*)
2326 (?:\s*&&\s*(?P=func_name))?
2327 \s*\(\s*(?P<callback_data>.*)\);?
2328 \s*?(?://[^\n]*)*$''',
2329 r'\g<callback_data>', code)
2332 def js_to_json(code):
2333 COMMENT_RE = r'/\*(?:(?!\*/).)*?\*/|//[^\n]*'
2334 SKIP_RE = r'\s*(?:{comment})?\s*'.format(comment=COMMENT_RE)
2336 (r'(?s)^(0[xX][0-9a-fA-F]+){skip}:?$'.format(skip=SKIP_RE), 16),
2337 (r'(?s)^(0+[0-7]+){skip}:?$'.format(skip=SKIP_RE), 8),
2342 if v in ('true', 'false', 'null'):
2344 elif v.startswith('/*') or v.startswith('//') or v == ',':
2347 if v[0] in ("'", '"'):
2348 v = re.sub(r'(?s)\\.|"', lambda m: {
2353 }.get(m.group(0), m.group(0)), v[1:-1])
2355 for regex, base in INTEGER_TABLE:
2356 im = re.match(regex, v)
2358 i = int(im.group(1), base)
2359 return '"%d":' % i if v.endswith(':') else '%d' % i
2363 return re.sub(r'''(?sx)
2364 "(?:[^"\\]*(?:\\\\|\\['"nurtbfx/\n]))*[^"\\]*"|
2365 '(?:[^'\\]*(?:\\\\|\\['"nurtbfx/\n]))*[^'\\]*'|
2366 {comment}|,(?={skip}[\]}}])|
2367 (?:(?<![0-9])[eE]|[a-df-zA-DF-Z_])[.a-zA-Z_0-9]*|
2368 \b(?:0[xX][0-9a-fA-F]+|0+[0-7]+)(?:{skip}:)?|
2370 '''.format(comment=COMMENT_RE, skip=SKIP_RE), fix_kv, code)
2373 def qualities(quality_ids):
2374 """ Get a numeric quality value out of a list of possible values """
2377 return quality_ids.index(qid)
2383 DEFAULT_OUTTMPL = '%(title)s-%(id)s.%(ext)s'
2386 def limit_length(s, length):
2387 """ Add ellipses to overly long strings """
2392 return s[:length - len(ELLIPSES)] + ELLIPSES
2396 def version_tuple(v):
2397 return tuple(int(e) for e in re.split(r'[-.]', v))
2400 def is_outdated_version(version, limit, assume_new=True):
2402 return not assume_new
2404 return version_tuple(version) < version_tuple(limit)
2406 return not assume_new
2409 def ytdl_is_updateable():
2410 """ Returns if youtube-dl can be updated with -U """
2411 from zipimport import zipimporter
2413 return isinstance(globals().get('__loader__'), zipimporter) or hasattr(sys, 'frozen')
2416 def args_to_str(args):
2417 # Get a short string representation for a subprocess command
2418 return ' '.join(compat_shlex_quote(a) for a in args)
2421 def error_to_compat_str(err):
2423 # On python 2 error byte string must be decoded with proper
2424 # encoding rather than ascii
2425 if sys.version_info[0] < 3:
2426 err_str = err_str.decode(preferredencoding())
2430 def mimetype2ext(mt):
2436 # Per RFC 3003, audio/mpeg can be .mp1, .mp2 or .mp3. Here use .mp3 as
2437 # it's the most popular one
2438 'audio/mpeg': 'mp3',
2443 _, _, res = mt.rpartition('/')
2444 res = res.split(';')[0].strip().lower()
2448 'smptett+xml': 'tt',
2452 'x-mp4-fragmented': 'mp4',
2453 'x-ms-sami': 'sami',
2456 'x-mpegurl': 'm3u8',
2457 'vnd.apple.mpegurl': 'm3u8',
2461 'vnd.ms-sstr+xml': 'ism',
2467 def parse_codecs(codecs_str):
2468 # http://tools.ietf.org/html/rfc6381
2471 splited_codecs = list(filter(None, map(
2472 lambda str: str.strip(), codecs_str.strip().strip(',').split(','))))
2473 vcodec, acodec = None, None
2474 for full_codec in splited_codecs:
2475 codec = full_codec.split('.')[0]
2476 if codec in ('avc1', 'avc2', 'avc3', 'avc4', 'vp9', 'vp8', 'hev1', 'hev2', 'h263', 'h264', 'mp4v', 'hvc1'):
2479 elif codec in ('mp4a', 'opus', 'vorbis', 'mp3', 'aac', 'ac-3', 'ec-3', 'eac3', 'dtsc', 'dtse', 'dtsh', 'dtsl'):
2483 write_string('WARNING: Unknown codec %s\n' % full_codec, sys.stderr)
2484 if not vcodec and not acodec:
2485 if len(splited_codecs) == 2:
2490 elif len(splited_codecs) == 1:
2497 'vcodec': vcodec or 'none',
2498 'acodec': acodec or 'none',
2503 def urlhandle_detect_ext(url_handle):
2504 getheader = url_handle.headers.get
2506 cd = getheader('Content-Disposition')
2508 m = re.match(r'attachment;\s*filename="(?P<filename>[^"]+)"', cd)
2510 e = determine_ext(m.group('filename'), default_ext=None)
2514 return mimetype2ext(getheader('Content-Type'))
2517 def encode_data_uri(data, mime_type):
2518 return 'data:%s;base64,%s' % (mime_type, base64.b64encode(data).decode('ascii'))
2521 def age_restricted(content_limit, age_limit):
2522 """ Returns True iff the content should be blocked """
2524 if age_limit is None: # No limit set
2526 if content_limit is None:
2527 return False # Content available for everyone
2528 return age_limit < content_limit
2531 def is_html(first_bytes):
2532 """ Detect whether a file contains HTML by examining its first bytes. """
2535 (b'\xef\xbb\xbf', 'utf-8'),
2536 (b'\x00\x00\xfe\xff', 'utf-32-be'),
2537 (b'\xff\xfe\x00\x00', 'utf-32-le'),
2538 (b'\xff\xfe', 'utf-16-le'),
2539 (b'\xfe\xff', 'utf-16-be'),
2541 for bom, enc in BOMS:
2542 if first_bytes.startswith(bom):
2543 s = first_bytes[len(bom):].decode(enc, 'replace')
2546 s = first_bytes.decode('utf-8', 'replace')
2548 return re.match(r'^\s*<', s)
2551 def determine_protocol(info_dict):
2552 protocol = info_dict.get('protocol')
2553 if protocol is not None:
2556 url = info_dict['url']
2557 if url.startswith('rtmp'):
2559 elif url.startswith('mms'):
2561 elif url.startswith('rtsp'):
2564 ext = determine_ext(url)
2570 return compat_urllib_parse_urlparse(url).scheme
2573 def render_table(header_row, data):
2574 """ Render a list of rows, each as a list of values """
2575 table = [header_row] + data
2576 max_lens = [max(len(compat_str(v)) for v in col) for col in zip(*table)]
2577 format_str = ' '.join('%-' + compat_str(ml + 1) + 's' for ml in max_lens[:-1]) + '%s'
2578 return '\n'.join(format_str % tuple(row) for row in table)
2581 def _match_one(filter_part, dct):
2582 COMPARISON_OPERATORS = {
2590 operator_rex = re.compile(r'''(?x)\s*
2592 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
2594 (?P<intval>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)|
2595 (?P<quote>["\'])(?P<quotedstrval>(?:\\.|(?!(?P=quote)|\\).)+?)(?P=quote)|
2596 (?P<strval>(?![0-9.])[a-z0-9A-Z]*)
2599 ''' % '|'.join(map(re.escape, COMPARISON_OPERATORS.keys())))
2600 m = operator_rex.search(filter_part)
2602 op = COMPARISON_OPERATORS[m.group('op')]
2603 actual_value = dct.get(m.group('key'))
2604 if (m.group('quotedstrval') is not None or
2605 m.group('strval') is not None or
2606 # If the original field is a string and matching comparisonvalue is
2607 # a number we should respect the origin of the original field
2608 # and process comparison value as a string (see
2609 # https://github.com/rg3/youtube-dl/issues/11082).
2610 actual_value is not None and m.group('intval') is not None and
2611 isinstance(actual_value, compat_str)):
2612 if m.group('op') not in ('=', '!='):
2614 'Operator %s does not support string values!' % m.group('op'))
2615 comparison_value = m.group('quotedstrval') or m.group('strval') or m.group('intval')
2616 quote = m.group('quote')
2617 if quote is not None:
2618 comparison_value = comparison_value.replace(r'\%s' % quote, quote)
2621 comparison_value = int(m.group('intval'))
2623 comparison_value = parse_filesize(m.group('intval'))
2624 if comparison_value is None:
2625 comparison_value = parse_filesize(m.group('intval') + 'B')
2626 if comparison_value is None:
2628 'Invalid integer value %r in filter part %r' % (
2629 m.group('intval'), filter_part))
2630 if actual_value is None:
2631 return m.group('none_inclusive')
2632 return op(actual_value, comparison_value)
2635 '': lambda v: (v is True) if isinstance(v, bool) else (v is not None),
2636 '!': lambda v: (v is False) if isinstance(v, bool) else (v is None),
2638 operator_rex = re.compile(r'''(?x)\s*
2639 (?P<op>%s)\s*(?P<key>[a-z_]+)
2641 ''' % '|'.join(map(re.escape, UNARY_OPERATORS.keys())))
2642 m = operator_rex.search(filter_part)
2644 op = UNARY_OPERATORS[m.group('op')]
2645 actual_value = dct.get(m.group('key'))
2646 return op(actual_value)
2648 raise ValueError('Invalid filter part %r' % filter_part)
2651 def match_str(filter_str, dct):
2652 """ Filter a dictionary with a simple string syntax. Returns True (=passes filter) or false """
2655 _match_one(filter_part, dct) for filter_part in filter_str.split('&'))
2658 def match_filter_func(filter_str):
2659 def _match_func(info_dict):
2660 if match_str(filter_str, info_dict):
2663 video_title = info_dict.get('title', info_dict.get('id', 'video'))
2664 return '%s does not pass filter %s, skipping ..' % (video_title, filter_str)
2668 def parse_dfxp_time_expr(time_expr):
2672 mobj = re.match(r'^(?P<time_offset>\d+(?:\.\d+)?)s?$', time_expr)
2674 return float(mobj.group('time_offset'))
2676 mobj = re.match(r'^(\d+):(\d\d):(\d\d(?:(?:\.|:)\d+)?)$', time_expr)
2678 return 3600 * int(mobj.group(1)) + 60 * int(mobj.group(2)) + float(mobj.group(3).replace(':', '.'))
2681 def srt_subtitles_timecode(seconds):
2682 return '%02d:%02d:%02d,%03d' % (seconds / 3600, (seconds % 3600) / 60, seconds % 60, (seconds % 1) * 1000)
2685 def dfxp2srt(dfxp_data):
2687 @param dfxp_data A bytes-like object containing DFXP data
2688 @returns A unicode object containing converted SRT data
2690 LEGACY_NAMESPACES = (
2691 (b'http://www.w3.org/ns/ttml', [
2692 b'http://www.w3.org/2004/11/ttaf1',
2693 b'http://www.w3.org/2006/04/ttaf1',
2694 b'http://www.w3.org/2006/10/ttaf1',
2696 (b'http://www.w3.org/ns/ttml#styling', [
2697 b'http://www.w3.org/ns/ttml#style',
2701 SUPPORTED_STYLING = [
2710 _x = functools.partial(xpath_with_ns, ns_map={
2711 'xml': 'http://www.w3.org/XML/1998/namespace',
2712 'ttml': 'http://www.w3.org/ns/ttml',
2713 'tts': 'http://www.w3.org/ns/ttml#styling',
2719 class TTMLPElementParser(object):
2721 _unclosed_elements = []
2722 _applied_styles = []
2724 def start(self, tag, attrib):
2725 if tag in (_x('ttml:br'), 'br'):
2728 unclosed_elements = []
2730 element_style_id = attrib.get('style')
2732 style.update(default_style)
2733 if element_style_id:
2734 style.update(styles.get(element_style_id, {}))
2735 for prop in SUPPORTED_STYLING:
2736 prop_val = attrib.get(_x('tts:' + prop))
2738 style[prop] = prop_val
2741 for k, v in sorted(style.items()):
2742 if self._applied_styles and self._applied_styles[-1].get(k) == v:
2745 font += ' color="%s"' % v
2746 elif k == 'fontSize':
2747 font += ' size="%s"' % v
2748 elif k == 'fontFamily':
2749 font += ' face="%s"' % v
2750 elif k == 'fontWeight' and v == 'bold':
2752 unclosed_elements.append('b')
2753 elif k == 'fontStyle' and v == 'italic':
2755 unclosed_elements.append('i')
2756 elif k == 'textDecoration' and v == 'underline':
2758 unclosed_elements.append('u')
2760 self._out += '<font' + font + '>'
2761 unclosed_elements.append('font')
2763 if self._applied_styles:
2764 applied_style.update(self._applied_styles[-1])
2765 applied_style.update(style)
2766 self._applied_styles.append(applied_style)
2767 self._unclosed_elements.append(unclosed_elements)
2770 if tag not in (_x('ttml:br'), 'br'):
2771 unclosed_elements = self._unclosed_elements.pop()
2772 for element in reversed(unclosed_elements):
2773 self._out += '</%s>' % element
2774 if unclosed_elements and self._applied_styles:
2775 self._applied_styles.pop()
2777 def data(self, data):
2781 return self._out.strip()
2783 def parse_node(node):
2784 target = TTMLPElementParser()
2785 parser = xml.etree.ElementTree.XMLParser(target=target)
2786 parser.feed(xml.etree.ElementTree.tostring(node))
2787 return parser.close()
2789 for k, v in LEGACY_NAMESPACES:
2791 dfxp_data = dfxp_data.replace(ns, k)
2793 dfxp = compat_etree_fromstring(dfxp_data)
2795 paras = dfxp.findall(_x('.//ttml:p')) or dfxp.findall('.//p')
2798 raise ValueError('Invalid dfxp/TTML subtitle')
2802 for style in dfxp.findall(_x('.//ttml:style')):
2803 style_id = style.get('id') or style.get(_x('xml:id'))
2806 parent_style_id = style.get('style')
2808 if parent_style_id not in styles:
2811 styles[style_id] = styles[parent_style_id].copy()
2812 for prop in SUPPORTED_STYLING:
2813 prop_val = style.get(_x('tts:' + prop))
2815 styles.setdefault(style_id, {})[prop] = prop_val
2821 for p in ('body', 'div'):
2822 ele = xpath_element(dfxp, [_x('.//ttml:' + p), './/' + p])
2825 style = styles.get(ele.get('style'))
2828 default_style.update(style)
2830 for para, index in zip(paras, itertools.count(1)):
2831 begin_time = parse_dfxp_time_expr(para.attrib.get('begin'))
2832 end_time = parse_dfxp_time_expr(para.attrib.get('end'))
2833 dur = parse_dfxp_time_expr(para.attrib.get('dur'))
2834 if begin_time is None:
2839 end_time = begin_time + dur
2840 out.append('%d\n%s --> %s\n%s\n\n' % (
2842 srt_subtitles_timecode(begin_time),
2843 srt_subtitles_timecode(end_time),
2849 def cli_option(params, command_option, param):
2850 param = params.get(param)
2852 param = compat_str(param)
2853 return [command_option, param] if param is not None else []
2856 def cli_bool_option(params, command_option, param, true_value='true', false_value='false', separator=None):
2857 param = params.get(param)
2860 assert isinstance(param, bool)
2862 return [command_option + separator + (true_value if param else false_value)]
2863 return [command_option, true_value if param else false_value]
2866 def cli_valueless_option(params, command_option, param, expected_value=True):
2867 param = params.get(param)
2868 return [command_option] if param == expected_value else []
2871 def cli_configuration_args(params, param, default=[]):
2872 ex_args = params.get(param)
2875 assert isinstance(ex_args, list)
2879 class ISO639Utils(object):
2880 # See http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
3069 def short2long(cls, code):
3070 """Convert language code from ISO 639-1 to ISO 639-2/T"""
3071 return cls._lang_map.get(code[:2])
3074 def long2short(cls, code):
3075 """Convert language code from ISO 639-2/T to ISO 639-1"""
3076 for short_name, long_name in cls._lang_map.items():
3077 if long_name == code:
3081 class ISO3166Utils(object):
3082 # From http://data.okfn.org/data/core/country-list
3084 'AF': 'Afghanistan',
3085 'AX': 'Åland Islands',
3088 'AS': 'American Samoa',
3093 'AG': 'Antigua and Barbuda',
3110 'BO': 'Bolivia, Plurinational State of',
3111 'BQ': 'Bonaire, Sint Eustatius and Saba',
3112 'BA': 'Bosnia and Herzegovina',
3114 'BV': 'Bouvet Island',
3116 'IO': 'British Indian Ocean Territory',
3117 'BN': 'Brunei Darussalam',
3119 'BF': 'Burkina Faso',
3125 'KY': 'Cayman Islands',
3126 'CF': 'Central African Republic',
3130 'CX': 'Christmas Island',
3131 'CC': 'Cocos (Keeling) Islands',
3135 'CD': 'Congo, the Democratic Republic of the',
3136 'CK': 'Cook Islands',
3138 'CI': 'Côte d\'Ivoire',
3143 'CZ': 'Czech Republic',
3147 'DO': 'Dominican Republic',
3150 'SV': 'El Salvador',
3151 'GQ': 'Equatorial Guinea',
3155 'FK': 'Falkland Islands (Malvinas)',
3156 'FO': 'Faroe Islands',
3160 'GF': 'French Guiana',
3161 'PF': 'French Polynesia',
3162 'TF': 'French Southern Territories',
3177 'GW': 'Guinea-Bissau',
3180 'HM': 'Heard Island and McDonald Islands',
3181 'VA': 'Holy See (Vatican City State)',
3188 'IR': 'Iran, Islamic Republic of',
3191 'IM': 'Isle of Man',
3201 'KP': 'Korea, Democratic People\'s Republic of',
3202 'KR': 'Korea, Republic of',
3205 'LA': 'Lao People\'s Democratic Republic',
3211 'LI': 'Liechtenstein',
3215 'MK': 'Macedonia, the Former Yugoslav Republic of',
3222 'MH': 'Marshall Islands',
3228 'FM': 'Micronesia, Federated States of',
3229 'MD': 'Moldova, Republic of',
3240 'NL': 'Netherlands',
3241 'NC': 'New Caledonia',
3242 'NZ': 'New Zealand',
3247 'NF': 'Norfolk Island',
3248 'MP': 'Northern Mariana Islands',
3253 'PS': 'Palestine, State of',
3255 'PG': 'Papua New Guinea',
3258 'PH': 'Philippines',
3262 'PR': 'Puerto Rico',
3266 'RU': 'Russian Federation',
3268 'BL': 'Saint Barthélemy',
3269 'SH': 'Saint Helena, Ascension and Tristan da Cunha',
3270 'KN': 'Saint Kitts and Nevis',
3271 'LC': 'Saint Lucia',
3272 'MF': 'Saint Martin (French part)',
3273 'PM': 'Saint Pierre and Miquelon',
3274 'VC': 'Saint Vincent and the Grenadines',
3277 'ST': 'Sao Tome and Principe',
3278 'SA': 'Saudi Arabia',
3282 'SL': 'Sierra Leone',
3284 'SX': 'Sint Maarten (Dutch part)',
3287 'SB': 'Solomon Islands',
3289 'ZA': 'South Africa',
3290 'GS': 'South Georgia and the South Sandwich Islands',
3291 'SS': 'South Sudan',
3296 'SJ': 'Svalbard and Jan Mayen',
3299 'CH': 'Switzerland',
3300 'SY': 'Syrian Arab Republic',
3301 'TW': 'Taiwan, Province of China',
3303 'TZ': 'Tanzania, United Republic of',
3305 'TL': 'Timor-Leste',
3309 'TT': 'Trinidad and Tobago',
3312 'TM': 'Turkmenistan',
3313 'TC': 'Turks and Caicos Islands',
3317 'AE': 'United Arab Emirates',
3318 'GB': 'United Kingdom',
3319 'US': 'United States',
3320 'UM': 'United States Minor Outlying Islands',
3324 'VE': 'Venezuela, Bolivarian Republic of',
3326 'VG': 'Virgin Islands, British',
3327 'VI': 'Virgin Islands, U.S.',
3328 'WF': 'Wallis and Futuna',
3329 'EH': 'Western Sahara',
3336 def short2full(cls, code):
3337 """Convert an ISO 3166-2 country code to the corresponding full name"""
3338 return cls._country_map.get(code.upper())
3341 class GeoUtils(object):
3342 # Major IPv4 address blocks per country
3344 'AD': '85.94.160.0/19',
3345 'AE': '94.200.0.0/13',
3346 'AF': '149.54.0.0/17',
3347 'AG': '209.59.64.0/18',
3348 'AI': '204.14.248.0/21',
3349 'AL': '46.99.0.0/16',
3350 'AM': '46.70.0.0/15',
3351 'AO': '105.168.0.0/13',
3352 'AP': '159.117.192.0/21',
3353 'AR': '181.0.0.0/12',
3354 'AS': '202.70.112.0/20',
3355 'AT': '84.112.0.0/13',
3356 'AU': '1.128.0.0/11',
3357 'AW': '181.41.0.0/18',
3358 'AZ': '5.191.0.0/16',
3359 'BA': '31.176.128.0/17',
3360 'BB': '65.48.128.0/17',
3361 'BD': '114.130.0.0/16',
3363 'BF': '129.45.128.0/17',
3364 'BG': '95.42.0.0/15',
3365 'BH': '37.131.0.0/17',
3366 'BI': '154.117.192.0/18',
3367 'BJ': '137.255.0.0/16',
3368 'BL': '192.131.134.0/24',
3369 'BM': '196.12.64.0/18',
3370 'BN': '156.31.0.0/16',
3371 'BO': '161.56.0.0/16',
3372 'BQ': '161.0.80.0/20',
3373 'BR': '152.240.0.0/12',
3374 'BS': '24.51.64.0/18',
3375 'BT': '119.2.96.0/19',
3376 'BW': '168.167.0.0/16',
3377 'BY': '178.120.0.0/13',
3378 'BZ': '179.42.192.0/18',
3379 'CA': '99.224.0.0/11',
3380 'CD': '41.243.0.0/16',
3381 'CF': '196.32.200.0/21',
3382 'CG': '197.214.128.0/17',
3383 'CH': '85.0.0.0/13',
3384 'CI': '154.232.0.0/14',
3385 'CK': '202.65.32.0/19',
3386 'CL': '152.172.0.0/14',
3387 'CM': '165.210.0.0/15',
3388 'CN': '36.128.0.0/10',
3389 'CO': '181.240.0.0/12',
3390 'CR': '201.192.0.0/12',
3391 'CU': '152.206.0.0/15',
3392 'CV': '165.90.96.0/19',
3393 'CW': '190.88.128.0/17',
3394 'CY': '46.198.0.0/15',
3395 'CZ': '88.100.0.0/14',
3397 'DJ': '197.241.0.0/17',
3398 'DK': '87.48.0.0/12',
3399 'DM': '192.243.48.0/20',
3400 'DO': '152.166.0.0/15',
3401 'DZ': '41.96.0.0/12',
3402 'EC': '186.68.0.0/15',
3403 'EE': '90.190.0.0/15',
3404 'EG': '156.160.0.0/11',
3405 'ER': '196.200.96.0/20',
3406 'ES': '88.0.0.0/11',
3407 'ET': '196.188.0.0/14',
3408 'EU': '2.16.0.0/13',
3409 'FI': '91.152.0.0/13',
3410 'FJ': '144.120.0.0/16',
3411 'FM': '119.252.112.0/20',
3412 'FO': '88.85.32.0/19',
3414 'GA': '41.158.0.0/15',
3416 'GD': '74.122.88.0/21',
3417 'GE': '31.146.0.0/16',
3418 'GF': '161.22.64.0/18',
3419 'GG': '62.68.160.0/19',
3420 'GH': '45.208.0.0/14',
3421 'GI': '85.115.128.0/19',
3422 'GL': '88.83.0.0/19',
3423 'GM': '160.182.0.0/15',
3424 'GN': '197.149.192.0/18',
3425 'GP': '104.250.0.0/19',
3426 'GQ': '105.235.224.0/20',
3427 'GR': '94.64.0.0/13',
3428 'GT': '168.234.0.0/16',
3429 'GU': '168.123.0.0/16',
3430 'GW': '197.214.80.0/20',
3431 'GY': '181.41.64.0/18',
3432 'HK': '113.252.0.0/14',
3433 'HN': '181.210.0.0/16',
3434 'HR': '93.136.0.0/13',
3435 'HT': '148.102.128.0/17',
3436 'HU': '84.0.0.0/14',
3437 'ID': '39.192.0.0/10',
3438 'IE': '87.32.0.0/12',
3439 'IL': '79.176.0.0/13',
3440 'IM': '5.62.80.0/20',
3441 'IN': '117.192.0.0/10',
3442 'IO': '203.83.48.0/21',
3443 'IQ': '37.236.0.0/14',
3444 'IR': '2.176.0.0/12',
3445 'IS': '82.221.0.0/16',
3446 'IT': '79.0.0.0/10',
3447 'JE': '87.244.64.0/18',
3448 'JM': '72.27.0.0/17',
3449 'JO': '176.29.0.0/16',
3450 'JP': '126.0.0.0/8',
3451 'KE': '105.48.0.0/12',
3452 'KG': '158.181.128.0/17',
3453 'KH': '36.37.128.0/17',
3454 'KI': '103.25.140.0/22',
3455 'KM': '197.255.224.0/20',
3456 'KN': '198.32.32.0/19',
3457 'KP': '175.45.176.0/22',
3458 'KR': '175.192.0.0/10',
3459 'KW': '37.36.0.0/14',
3460 'KY': '64.96.0.0/15',
3461 'KZ': '2.72.0.0/13',
3462 'LA': '115.84.64.0/18',
3463 'LB': '178.135.0.0/16',
3464 'LC': '192.147.231.0/24',
3465 'LI': '82.117.0.0/19',
3466 'LK': '112.134.0.0/15',
3467 'LR': '41.86.0.0/19',
3468 'LS': '129.232.0.0/17',
3469 'LT': '78.56.0.0/13',
3470 'LU': '188.42.0.0/16',
3471 'LV': '46.109.0.0/16',
3472 'LY': '41.252.0.0/14',
3473 'MA': '105.128.0.0/11',
3474 'MC': '88.209.64.0/18',
3475 'MD': '37.246.0.0/16',
3476 'ME': '178.175.0.0/17',
3477 'MF': '74.112.232.0/21',
3478 'MG': '154.126.0.0/17',
3479 'MH': '117.103.88.0/21',
3480 'MK': '77.28.0.0/15',
3481 'ML': '154.118.128.0/18',
3482 'MM': '37.111.0.0/17',
3483 'MN': '49.0.128.0/17',
3484 'MO': '60.246.0.0/16',
3485 'MP': '202.88.64.0/20',
3486 'MQ': '109.203.224.0/19',
3487 'MR': '41.188.64.0/18',
3488 'MS': '208.90.112.0/22',
3489 'MT': '46.11.0.0/16',
3490 'MU': '105.16.0.0/12',
3491 'MV': '27.114.128.0/18',
3492 'MW': '105.234.0.0/16',
3493 'MX': '187.192.0.0/11',
3494 'MY': '175.136.0.0/13',
3495 'MZ': '197.218.0.0/15',
3496 'NA': '41.182.0.0/16',
3497 'NC': '101.101.0.0/18',
3498 'NE': '197.214.0.0/18',
3499 'NF': '203.17.240.0/22',
3500 'NG': '105.112.0.0/12',
3501 'NI': '186.76.0.0/15',
3502 'NL': '145.96.0.0/11',
3503 'NO': '84.208.0.0/13',
3504 'NP': '36.252.0.0/15',
3505 'NR': '203.98.224.0/19',
3506 'NU': '49.156.48.0/22',
3507 'NZ': '49.224.0.0/14',
3508 'OM': '5.36.0.0/15',
3509 'PA': '186.72.0.0/15',
3510 'PE': '186.160.0.0/14',
3511 'PF': '123.50.64.0/18',
3512 'PG': '124.240.192.0/19',
3513 'PH': '49.144.0.0/13',
3514 'PK': '39.32.0.0/11',
3515 'PL': '83.0.0.0/11',
3516 'PM': '70.36.0.0/20',
3517 'PR': '66.50.0.0/16',
3518 'PS': '188.161.0.0/16',
3519 'PT': '85.240.0.0/13',
3520 'PW': '202.124.224.0/20',
3521 'PY': '181.120.0.0/14',
3522 'QA': '37.210.0.0/15',
3523 'RE': '139.26.0.0/16',
3524 'RO': '79.112.0.0/13',
3525 'RS': '178.220.0.0/14',
3526 'RU': '5.136.0.0/13',
3527 'RW': '105.178.0.0/15',
3528 'SA': '188.48.0.0/13',
3529 'SB': '202.1.160.0/19',
3530 'SC': '154.192.0.0/11',
3531 'SD': '154.96.0.0/13',
3532 'SE': '78.64.0.0/12',
3533 'SG': '152.56.0.0/14',
3534 'SI': '188.196.0.0/14',
3535 'SK': '78.98.0.0/15',
3536 'SL': '197.215.0.0/17',
3537 'SM': '89.186.32.0/19',
3538 'SN': '41.82.0.0/15',
3539 'SO': '197.220.64.0/19',
3540 'SR': '186.179.128.0/17',
3541 'SS': '105.235.208.0/21',
3542 'ST': '197.159.160.0/19',
3543 'SV': '168.243.0.0/16',
3544 'SX': '190.102.0.0/20',
3546 'SZ': '41.84.224.0/19',
3547 'TC': '65.255.48.0/20',
3548 'TD': '154.68.128.0/19',
3549 'TG': '196.168.0.0/14',
3550 'TH': '171.96.0.0/13',
3551 'TJ': '85.9.128.0/18',
3552 'TK': '27.96.24.0/21',
3553 'TL': '180.189.160.0/20',
3554 'TM': '95.85.96.0/19',
3555 'TN': '197.0.0.0/11',
3556 'TO': '175.176.144.0/21',
3557 'TR': '78.160.0.0/11',
3558 'TT': '186.44.0.0/15',
3559 'TV': '202.2.96.0/19',
3560 'TW': '120.96.0.0/11',
3561 'TZ': '156.156.0.0/14',
3562 'UA': '93.72.0.0/13',
3563 'UG': '154.224.0.0/13',
3565 'UY': '167.56.0.0/13',
3566 'UZ': '82.215.64.0/18',
3567 'VA': '212.77.0.0/19',
3568 'VC': '24.92.144.0/20',
3569 'VE': '186.88.0.0/13',
3570 'VG': '172.103.64.0/18',
3571 'VI': '146.226.0.0/16',
3572 'VN': '14.160.0.0/11',
3573 'VU': '202.80.32.0/20',
3574 'WF': '117.20.32.0/21',
3575 'WS': '202.4.32.0/19',
3576 'YE': '134.35.0.0/16',
3577 'YT': '41.242.116.0/22',
3578 'ZA': '41.0.0.0/11',
3579 'ZM': '165.56.0.0/13',
3580 'ZW': '41.85.192.0/19',
3584 def random_ipv4(cls, code_or_block):
3585 if len(code_or_block) == 2:
3586 block = cls._country_ip_map.get(code_or_block.upper())
3590 block = code_or_block
3591 addr, preflen = block.split('/')
3592 addr_min = compat_struct_unpack('!L', socket.inet_aton(addr))[0]
3593 addr_max = addr_min | (0xffffffff >> int(preflen))
3594 return compat_str(socket.inet_ntoa(
3595 compat_struct_pack('!L', random.randint(addr_min, addr_max))))
3598 class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
3599 def __init__(self, proxies=None):
3600 # Set default handlers
3601 for type in ('http', 'https'):
3602 setattr(self, '%s_open' % type,
3603 lambda r, proxy='__noproxy__', type=type, meth=self.proxy_open:
3604 meth(r, proxy, type))
3605 compat_urllib_request.ProxyHandler.__init__(self, proxies)
3607 def proxy_open(self, req, proxy, type):
3608 req_proxy = req.headers.get('Ytdl-request-proxy')
3609 if req_proxy is not None:
3611 del req.headers['Ytdl-request-proxy']
3613 if proxy == '__noproxy__':
3614 return None # No Proxy
3615 if compat_urlparse.urlparse(proxy).scheme.lower() in ('socks', 'socks4', 'socks4a', 'socks5'):
3616 req.add_header('Ytdl-socks-proxy', proxy)
3617 # youtube-dl's http/https handlers do wrapping the socket with socks
3619 return compat_urllib_request.ProxyHandler.proxy_open(
3620 self, req, proxy, type)
3623 # Both long_to_bytes and bytes_to_long are adapted from PyCrypto, which is
3624 # released into Public Domain
3625 # https://github.com/dlitz/pycrypto/blob/master/lib/Crypto/Util/number.py#L387
3627 def long_to_bytes(n, blocksize=0):
3628 """long_to_bytes(n:long, blocksize:int) : string
3629 Convert a long integer to a byte string.
3631 If optional blocksize is given and greater than zero, pad the front of the
3632 byte string with binary zeros so that the length is a multiple of
3635 # after much testing, this algorithm was deemed to be the fastest
3639 s = compat_struct_pack('>I', n & 0xffffffff) + s
3641 # strip off leading zeros
3642 for i in range(len(s)):
3643 if s[i] != b'\000'[0]:
3646 # only happens when n == 0
3650 # add back some pad bytes. this could be done more efficiently w.r.t. the
3651 # de-padding being done above, but sigh...
3652 if blocksize > 0 and len(s) % blocksize:
3653 s = (blocksize - len(s) % blocksize) * b'\000' + s
3657 def bytes_to_long(s):
3658 """bytes_to_long(string) : long
3659 Convert a byte string to a long integer.
3661 This is (essentially) the inverse of long_to_bytes().
3666 extra = (4 - length % 4)
3667 s = b'\000' * extra + s
3668 length = length + extra
3669 for i in range(0, length, 4):
3670 acc = (acc << 32) + compat_struct_unpack('>I', s[i:i + 4])[0]
3674 def ohdave_rsa_encrypt(data, exponent, modulus):
3676 Implement OHDave's RSA algorithm. See http://www.ohdave.com/rsa/
3679 data: data to encrypt, bytes-like object
3680 exponent, modulus: parameter e and N of RSA algorithm, both integer
3681 Output: hex string of encrypted data
3683 Limitation: supports one block encryption only
3686 payload = int(binascii.hexlify(data[::-1]), 16)
3687 encrypted = pow(payload, exponent, modulus)
3688 return '%x' % encrypted
3691 def pkcs1pad(data, length):
3693 Padding input data with PKCS#1 scheme
3695 @param {int[]} data input data
3696 @param {int} length target length
3697 @returns {int[]} padded data
3699 if len(data) > length - 11:
3700 raise ValueError('Input data too long for PKCS#1 padding')
3702 pseudo_random = [random.randint(0, 254) for _ in range(length - len(data) - 3)]
3703 return [0, 2] + pseudo_random + [0] + data
3706 def encode_base_n(num, n, table=None):
3707 FULL_TABLE = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
3709 table = FULL_TABLE[:n]
3712 raise ValueError('base %d exceeds table length %d' % (n, len(table)))
3719 ret = table[num % n] + ret
3724 def decode_packed_codes(code):
3725 mobj = re.search(PACKED_CODES_RE, code)
3726 obfucasted_code, base, count, symbols = mobj.groups()
3729 symbols = symbols.split('|')
3734 base_n_count = encode_base_n(count, base)
3735 symbol_table[base_n_count] = symbols[count] or base_n_count
3738 r'\b(\w+)\b', lambda mobj: symbol_table[mobj.group(0)],
3742 def parse_m3u8_attributes(attrib):
3744 for (key, val) in re.findall(r'(?P<key>[A-Z0-9-]+)=(?P<val>"[^"]+"|[^",]+)(?:,|$)', attrib):
3745 if val.startswith('"'):
3751 def urshift(val, n):
3752 return val >> n if val >= 0 else (val + 0x100000000) >> n
3755 # Based on png2str() written by @gdkchan and improved by @yokrysty
3756 # Originally posted at https://github.com/rg3/youtube-dl/issues/9706
3757 def decode_png(png_data):
3758 # Reference: https://www.w3.org/TR/PNG/
3759 header = png_data[8:]
3761 if png_data[:8] != b'\x89PNG\x0d\x0a\x1a\x0a' or header[4:8] != b'IHDR':
3762 raise IOError('Not a valid PNG file.')
3764 int_map = {1: '>B', 2: '>H', 4: '>I'}
3765 unpack_integer = lambda x: compat_struct_unpack(int_map[len(x)], x)[0]
3770 length = unpack_integer(header[:4])
3773 chunk_type = header[:4]
3776 chunk_data = header[:length]
3777 header = header[length:]
3779 header = header[4:] # Skip CRC
3787 ihdr = chunks[0]['data']
3789 width = unpack_integer(ihdr[:4])
3790 height = unpack_integer(ihdr[4:8])
3794 for chunk in chunks:
3795 if chunk['type'] == b'IDAT':
3796 idat += chunk['data']
3799 raise IOError('Unable to read PNG data.')
3801 decompressed_data = bytearray(zlib.decompress(idat))
3806 def _get_pixel(idx):
3811 for y in range(height):
3812 basePos = y * (1 + stride)
3813 filter_type = decompressed_data[basePos]
3817 pixels.append(current_row)
3819 for x in range(stride):
3820 color = decompressed_data[1 + basePos + x]
3821 basex = y * stride + x
3826 left = _get_pixel(basex - 3)
3828 up = _get_pixel(basex - stride)
3830 if filter_type == 1: # Sub
3831 color = (color + left) & 0xff
3832 elif filter_type == 2: # Up
3833 color = (color + up) & 0xff
3834 elif filter_type == 3: # Average
3835 color = (color + ((left + up) >> 1)) & 0xff
3836 elif filter_type == 4: # Paeth
3842 c = _get_pixel(basex - stride - 3)
3850 if pa <= pb and pa <= pc:
3851 color = (color + a) & 0xff
3853 color = (color + b) & 0xff
3855 color = (color + c) & 0xff
3857 current_row.append(color)
3859 return width, height, pixels
3862 def write_xattr(path, key, value):
3863 # This mess below finds the best xattr tool for the job
3865 # try the pyxattr module...
3868 if hasattr(xattr, 'set'): # pyxattr
3869 # Unicode arguments are not supported in python-pyxattr until
3871 # See https://github.com/rg3/youtube-dl/issues/5498
3872 pyxattr_required_version = '0.5.0'
3873 if version_tuple(xattr.__version__) < version_tuple(pyxattr_required_version):
3874 # TODO: fallback to CLI tools
3875 raise XAttrUnavailableError(
3876 'python-pyxattr is detected but is too old. '
3877 'youtube-dl requires %s or above while your version is %s. '
3878 'Falling back to other xattr implementations' % (
3879 pyxattr_required_version, xattr.__version__))
3881 setxattr = xattr.set
3883 setxattr = xattr.setxattr
3886 setxattr(path, key, value)
3887 except EnvironmentError as e:
3888 raise XAttrMetadataError(e.errno, e.strerror)
3891 if compat_os_name == 'nt':
3892 # Write xattrs to NTFS Alternate Data Streams:
3893 # http://en.wikipedia.org/wiki/NTFS#Alternate_data_streams_.28ADS.29
3894 assert ':' not in key
3895 assert os.path.exists(path)
3897 ads_fn = path + ':' + key
3899 with open(ads_fn, 'wb') as f:
3901 except EnvironmentError as e:
3902 raise XAttrMetadataError(e.errno, e.strerror)
3904 user_has_setfattr = check_executable('setfattr', ['--version'])
3905 user_has_xattr = check_executable('xattr', ['-h'])
3907 if user_has_setfattr or user_has_xattr:
3909 value = value.decode('utf-8')
3910 if user_has_setfattr:
3911 executable = 'setfattr'
3912 opts = ['-n', key, '-v', value]
3913 elif user_has_xattr:
3914 executable = 'xattr'
3915 opts = ['-w', key, value]
3917 cmd = ([encodeFilename(executable, True)] +
3918 [encodeArgument(o) for o in opts] +
3919 [encodeFilename(path, True)])
3922 p = subprocess.Popen(
3923 cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
3924 except EnvironmentError as e:
3925 raise XAttrMetadataError(e.errno, e.strerror)
3926 stdout, stderr = p.communicate()
3927 stderr = stderr.decode('utf-8', 'replace')
3928 if p.returncode != 0:
3929 raise XAttrMetadataError(p.returncode, stderr)
3932 # On Unix, and can't find pyxattr, setfattr, or xattr.
3933 if sys.platform.startswith('linux'):
3934 raise XAttrUnavailableError(
3935 "Couldn't find a tool to set the xattrs. "
3936 "Install either the python 'pyxattr' or 'xattr' "
3937 "modules, or the GNU 'attr' package "
3938 "(which contains the 'setfattr' tool).")
3940 raise XAttrUnavailableError(
3941 "Couldn't find a tool to set the xattrs. "
3942 "Install either the python 'xattr' module, "
3943 "or the 'xattr' binary.")
3946 def random_birthday(year_field, month_field, day_field):
3948 year_field: str(random.randint(1950, 1995)),
3949 month_field: str(random.randint(1, 12)),
3950 day_field: str(random.randint(1, 31)),