2 # -*- coding: utf-8 -*-
4 from __future__ import unicode_literals
33 import xml.etree.ElementTree
40 compat_etree_fromstring,
42 compat_html_entities_html5,
47 compat_socket_create_connection,
53 compat_urllib_parse_urlencode,
54 compat_urllib_parse_urlparse,
55 compat_urllib_parse_unquote_plus,
56 compat_urllib_request,
67 def register_socks_protocols():
68 # "Register" SOCKS protocols
69 # In Python < 2.6.5, urlsplit() suffers from bug https://bugs.python.org/issue7904
70 # URLs with protocols not in urlparse.uses_netloc are not handled correctly
71 for scheme in ('socks', 'socks4', 'socks4a', 'socks5'):
72 if scheme not in compat_urlparse.uses_netloc:
73 compat_urlparse.uses_netloc.append(scheme)
76 # This is not clearly defined otherwise
77 compiled_regex_type = type(re.compile(''))
80 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20150101 Firefox/47.0 (Chrome)',
81 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
82 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
83 'Accept-Encoding': 'gzip, deflate',
84 'Accept-Language': 'en-us,en;q=0.5',
90 ENGLISH_MONTH_NAMES = [
91 'January', 'February', 'March', 'April', 'May', 'June',
92 'July', 'August', 'September', 'October', 'November', 'December']
94 FRENCH_MONTH_NAMES = [
95 'janvier', 'fevrier', 'mars', 'avril', 'mai', 'juin',
96 'juillet', 'aout', 'septembre', 'octobre', 'novembre', 'decembre']
99 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'aac',
100 'flv', 'f4v', 'f4a', 'f4b',
101 'webm', 'ogg', 'ogv', 'oga', 'ogx', 'spx', 'opus',
102 'mkv', 'mka', 'mk3d',
111 'f4f', 'f4m', 'm3u8', 'smil')
113 # needed for sanitizing filenames in restricted mode
114 ACCENT_CHARS = dict(zip('ÂÃÄÀÁÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖŐØŒÙÚÛÜŰÝÞßàáâãäåæçèéêëìíîïðñòóôõöőøœùúûüűýþÿ',
115 itertools.chain('AAAAAA', ['AE'], 'CEEEEIIIIDNOOOOOOO', ['OE'], 'UUUUUYP', ['ss'],
116 'aaaaaa', ['ae'], 'ceeeeiiiionooooooo', ['oe'], 'uuuuuypy')))
132 '%Y-%m-%d %H:%M:%S.%f',
135 '%Y-%m-%dT%H:%M:%SZ',
136 '%Y-%m-%dT%H:%M:%S.%fZ',
137 '%Y-%m-%dT%H:%M:%S.%f0Z',
139 '%Y-%m-%dT%H:%M:%S.%f',
143 DATE_FORMATS_DAY_FIRST = list(DATE_FORMATS)
144 DATE_FORMATS_DAY_FIRST.extend([
153 DATE_FORMATS_MONTH_FIRST = list(DATE_FORMATS)
154 DATE_FORMATS_MONTH_FIRST.extend([
163 def preferredencoding():
164 """Get preferred encoding.
166 Returns the best encoding scheme for the system, based on
167 locale.getpreferredencoding() and some further tweaks.
170 pref = locale.getpreferredencoding()
178 def write_json_file(obj, fn):
179 """ Encode obj as JSON and write it to fn, atomically if possible """
181 fn = encodeFilename(fn)
182 if sys.version_info < (3, 0) and sys.platform != 'win32':
183 encoding = get_filesystem_encoding()
184 # os.path.basename returns a bytes object, but NamedTemporaryFile
185 # will fail if the filename contains non ascii characters unless we
186 # use a unicode object
187 path_basename = lambda f: os.path.basename(fn).decode(encoding)
188 # the same for os.path.dirname
189 path_dirname = lambda f: os.path.dirname(fn).decode(encoding)
191 path_basename = os.path.basename
192 path_dirname = os.path.dirname
196 'prefix': path_basename(fn) + '.',
197 'dir': path_dirname(fn),
201 # In Python 2.x, json.dump expects a bytestream.
202 # In Python 3.x, it writes to a character stream
203 if sys.version_info < (3, 0):
211 tf = tempfile.NamedTemporaryFile(**compat_kwargs(args))
216 if sys.platform == 'win32':
217 # Need to remove existing file on Windows, else os.rename raises
218 # WindowsError or FileExistsError.
223 os.rename(tf.name, fn)
232 if sys.version_info >= (2, 7):
233 def find_xpath_attr(node, xpath, key, val=None):
234 """ Find the xpath xpath[@key=val] """
235 assert re.match(r'^[a-zA-Z_-]+$', key)
236 expr = xpath + ('[@%s]' % key if val is None else "[@%s='%s']" % (key, val))
237 return node.find(expr)
239 def find_xpath_attr(node, xpath, key, val=None):
240 for f in node.findall(compat_xpath(xpath)):
241 if key not in f.attrib:
243 if val is None or f.attrib.get(key) == val:
247 # On python2.6 the xml.etree.ElementTree.Element methods don't support
248 # the namespace parameter
251 def xpath_with_ns(path, ns_map):
252 components = [c.split(':') for c in path.split('/')]
256 replaced.append(c[0])
259 replaced.append('{%s}%s' % (ns_map[ns], tag))
260 return '/'.join(replaced)
263 def xpath_element(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
264 def _find_xpath(xpath):
265 return node.find(compat_xpath(xpath))
267 if isinstance(xpath, (str, compat_str)):
268 n = _find_xpath(xpath)
276 if default is not NO_DEFAULT:
279 name = xpath if name is None else name
280 raise ExtractorError('Could not find XML element %s' % name)
286 def xpath_text(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
287 n = xpath_element(node, xpath, name, fatal=fatal, default=default)
288 if n is None or n == default:
291 if default is not NO_DEFAULT:
294 name = xpath if name is None else name
295 raise ExtractorError('Could not find XML element\'s text %s' % name)
301 def xpath_attr(node, xpath, key, name=None, fatal=False, default=NO_DEFAULT):
302 n = find_xpath_attr(node, xpath, key)
304 if default is not NO_DEFAULT:
307 name = '%s[@%s]' % (xpath, key) if name is None else name
308 raise ExtractorError('Could not find XML attribute %s' % name)
314 def get_element_by_id(id, html):
315 """Return the content of the tag with the specified ID in the passed HTML document"""
316 return get_element_by_attribute('id', id, html)
319 def get_element_by_class(class_name, html):
320 return get_element_by_attribute(
321 'class', r'[^\'"]*\b%s\b[^\'"]*' % re.escape(class_name),
322 html, escape_value=False)
325 def get_element_by_attribute(attribute, value, html, escape_value=True):
326 """Return the content of the tag with the specified attribute in the passed HTML document"""
328 value = re.escape(value) if escape_value else value
330 m = re.search(r'''(?xs)
332 (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^"]*"|='[^']*'))*?
334 (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^"]*"|='[^']*'))*?
338 ''' % (re.escape(attribute), value), html)
342 res = m.group('content')
344 if res.startswith('"') or res.startswith("'"):
347 return unescapeHTML(res)
350 class HTMLAttributeParser(compat_HTMLParser):
351 """Trivial HTML parser to gather the attributes for a single element"""
354 compat_HTMLParser.__init__(self)
356 def handle_starttag(self, tag, attrs):
357 self.attrs = dict(attrs)
360 def extract_attributes(html_element):
361 """Given a string for an HTML element such as
363 a="foo" B="bar" c="&98;az" d=boz
364 empty= noval entity="&"
367 Decode and return a dictionary of attributes.
369 'a': 'foo', 'b': 'bar', c: 'baz', d: 'boz',
370 'empty': '', 'noval': None, 'entity': '&',
371 'sq': '"', 'dq': '\''
373 NB HTMLParser is stricter in Python 2.6 & 3.2 than in later versions,
374 but the cases in the unit test will work for all of 2.6, 2.7, 3.2-3.5.
376 parser = HTMLAttributeParser()
377 parser.feed(html_element)
382 def clean_html(html):
383 """Clean an HTML snippet into a readable string"""
385 if html is None: # Convenience for sanitizing descriptions etc.
389 html = html.replace('\n', ' ')
390 html = re.sub(r'\s*<\s*br\s*/?\s*>\s*', '\n', html)
391 html = re.sub(r'<\s*/\s*p\s*>\s*<\s*p[^>]*>', '\n', html)
393 html = re.sub('<.*?>', '', html)
394 # Replace html entities
395 html = unescapeHTML(html)
399 def sanitize_open(filename, open_mode):
400 """Try to open the given filename, and slightly tweak it if this fails.
402 Attempts to open the given filename. If this fails, it tries to change
403 the filename slightly, step by step, until it's either able to open it
404 or it fails and raises a final exception, like the standard open()
407 It returns the tuple (stream, definitive_file_name).
411 if sys.platform == 'win32':
413 msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
414 return (sys.stdout.buffer if hasattr(sys.stdout, 'buffer') else sys.stdout, filename)
415 stream = open(encodeFilename(filename), open_mode)
416 return (stream, filename)
417 except (IOError, OSError) as err:
418 if err.errno in (errno.EACCES,):
421 # In case of error, try to remove win32 forbidden chars
422 alt_filename = sanitize_path(filename)
423 if alt_filename == filename:
426 # An exception here should be caught in the caller
427 stream = open(encodeFilename(alt_filename), open_mode)
428 return (stream, alt_filename)
431 def timeconvert(timestr):
432 """Convert RFC 2822 defined time string into system timestamp"""
434 timetuple = email.utils.parsedate_tz(timestr)
435 if timetuple is not None:
436 timestamp = email.utils.mktime_tz(timetuple)
440 def sanitize_filename(s, restricted=False, is_id=False):
441 """Sanitizes a string so it could be used as part of a filename.
442 If restricted is set, use a stricter subset of allowed characters.
443 Set is_id if this is not an arbitrary string, but an ID that should be kept if possible
445 def replace_insane(char):
446 if restricted and char in ACCENT_CHARS:
447 return ACCENT_CHARS[char]
448 if char == '?' or ord(char) < 32 or ord(char) == 127:
451 return '' if restricted else '\''
453 return '_-' if restricted else ' -'
454 elif char in '\\/|*<>':
456 if restricted and (char in '!&\'()[]{}$;`^,#' or char.isspace()):
458 if restricted and ord(char) > 127:
463 s = re.sub(r'[0-9]+(?::[0-9]+)+', lambda m: m.group(0).replace(':', '_'), s)
464 result = ''.join(map(replace_insane, s))
466 while '__' in result:
467 result = result.replace('__', '_')
468 result = result.strip('_')
469 # Common case of "Foreign band name - English song title"
470 if restricted and result.startswith('-_'):
472 if result.startswith('-'):
473 result = '_' + result[len('-'):]
474 result = result.lstrip('.')
480 def sanitize_path(s):
481 """Sanitizes and normalizes path on Windows"""
482 if sys.platform != 'win32':
484 drive_or_unc, _ = os.path.splitdrive(s)
485 if sys.version_info < (2, 7) and not drive_or_unc:
486 drive_or_unc, _ = os.path.splitunc(s)
487 norm_path = os.path.normpath(remove_start(s, drive_or_unc)).split(os.path.sep)
491 path_part if path_part in ['.', '..'] else re.sub('(?:[/<>:"\\|\\\\?\\*]|[\s.]$)', '#', path_part)
492 for path_part in norm_path]
494 sanitized_path.insert(0, drive_or_unc + os.path.sep)
495 return os.path.join(*sanitized_path)
498 # Prepend protocol-less URLs with `http:` scheme in order to mitigate the number of
499 # unwanted failures due to missing protocol
500 def sanitize_url(url):
501 return 'http:%s' % url if url.startswith('//') else url
504 def sanitized_Request(url, *args, **kwargs):
505 return compat_urllib_request.Request(sanitize_url(url), *args, **kwargs)
508 def orderedSet(iterable):
509 """ Remove all duplicates from the input iterable """
517 def _htmlentity_transform(entity_with_semicolon):
518 """Transforms an HTML entity to a character."""
519 entity = entity_with_semicolon[:-1]
521 # Known non-numeric HTML entity
522 if entity in compat_html_entities.name2codepoint:
523 return compat_chr(compat_html_entities.name2codepoint[entity])
525 # TODO: HTML5 allows entities without a semicolon. For example,
526 # 'Éric' should be decoded as 'Éric'.
527 if entity_with_semicolon in compat_html_entities_html5:
528 return compat_html_entities_html5[entity_with_semicolon]
530 mobj = re.match(r'#(x[0-9a-fA-F]+|[0-9]+)', entity)
532 numstr = mobj.group(1)
533 if numstr.startswith('x'):
535 numstr = '0%s' % numstr
538 # See https://github.com/rg3/youtube-dl/issues/7518
540 return compat_chr(int(numstr, base))
544 # Unknown entity in name, return its literal representation
545 return '&%s;' % entity
551 assert type(s) == compat_str
554 r'&([^;]+;)', lambda m: _htmlentity_transform(m.group(1)), s)
557 def get_subprocess_encoding():
558 if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
559 # For subprocess calls, encode with locale encoding
560 # Refer to http://stackoverflow.com/a/9951851/35070
561 encoding = preferredencoding()
563 encoding = sys.getfilesystemencoding()
569 def encodeFilename(s, for_subprocess=False):
571 @param s The name of the file
574 assert type(s) == compat_str
576 # Python 3 has a Unicode API
577 if sys.version_info >= (3, 0):
580 # Pass '' directly to use Unicode APIs on Windows 2000 and up
581 # (Detecting Windows NT 4 is tricky because 'major >= 4' would
582 # match Windows 9x series as well. Besides, NT 4 is obsolete.)
583 if not for_subprocess and sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
586 # Jython assumes filenames are Unicode strings though reported as Python 2.x compatible
587 if sys.platform.startswith('java'):
590 return s.encode(get_subprocess_encoding(), 'ignore')
593 def decodeFilename(b, for_subprocess=False):
595 if sys.version_info >= (3, 0):
598 if not isinstance(b, bytes):
601 return b.decode(get_subprocess_encoding(), 'ignore')
604 def encodeArgument(s):
605 if not isinstance(s, compat_str):
606 # Legacy code that uses byte strings
607 # Uncomment the following line after fixing all post processors
608 # assert False, 'Internal error: %r should be of type %r, is %r' % (s, compat_str, type(s))
609 s = s.decode('ascii')
610 return encodeFilename(s, True)
613 def decodeArgument(b):
614 return decodeFilename(b, True)
617 def decodeOption(optval):
620 if isinstance(optval, bytes):
621 optval = optval.decode(preferredencoding())
623 assert isinstance(optval, compat_str)
627 def formatSeconds(secs):
629 return '%d:%02d:%02d' % (secs // 3600, (secs % 3600) // 60, secs % 60)
631 return '%d:%02d' % (secs // 60, secs % 60)
636 def make_HTTPS_handler(params, **kwargs):
637 opts_no_check_certificate = params.get('nocheckcertificate', False)
638 if hasattr(ssl, 'create_default_context'): # Python >= 3.4 or 2.7.9
639 context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
640 if opts_no_check_certificate:
641 context.check_hostname = False
642 context.verify_mode = ssl.CERT_NONE
644 return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
647 # (create_default_context present but HTTPSHandler has no context=)
650 if sys.version_info < (3, 2):
651 return YoutubeDLHTTPSHandler(params, **kwargs)
653 context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
654 context.verify_mode = (ssl.CERT_NONE
655 if opts_no_check_certificate
656 else ssl.CERT_REQUIRED)
657 context.set_default_verify_paths()
658 return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
661 def bug_reports_message():
662 if ytdl_is_updateable():
663 update_cmd = 'type youtube-dl -U to update'
665 update_cmd = 'see https://yt-dl.org/update on how to update'
666 msg = '; please report this issue on https://yt-dl.org/bug .'
667 msg += ' Make sure you are using the latest version; %s.' % update_cmd
668 msg += ' Be sure to call youtube-dl with the --verbose flag and include its complete output.'
672 class ExtractorError(Exception):
673 """Error during info extraction."""
675 def __init__(self, msg, tb=None, expected=False, cause=None, video_id=None):
676 """ tb, if given, is the original traceback (so that it can be printed out).
677 If expected is set, this is a normal error message and most likely not a bug in youtube-dl.
680 if sys.exc_info()[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError):
682 if video_id is not None:
683 msg = video_id + ': ' + msg
685 msg += ' (caused by %r)' % cause
687 msg += bug_reports_message()
688 super(ExtractorError, self).__init__(msg)
691 self.exc_info = sys.exc_info() # preserve original exception
693 self.video_id = video_id
695 def format_traceback(self):
696 if self.traceback is None:
698 return ''.join(traceback.format_tb(self.traceback))
701 class UnsupportedError(ExtractorError):
702 def __init__(self, url):
703 super(UnsupportedError, self).__init__(
704 'Unsupported URL: %s' % url, expected=True)
708 class RegexNotFoundError(ExtractorError):
709 """Error when a regex didn't match"""
713 class DownloadError(Exception):
714 """Download Error exception.
716 This exception may be thrown by FileDownloader objects if they are not
717 configured to continue on errors. They will contain the appropriate
721 def __init__(self, msg, exc_info=None):
722 """ exc_info, if given, is the original exception that caused the trouble (as returned by sys.exc_info()). """
723 super(DownloadError, self).__init__(msg)
724 self.exc_info = exc_info
727 class SameFileError(Exception):
728 """Same File exception.
730 This exception will be thrown by FileDownloader objects if they detect
731 multiple files would have to be downloaded to the same file on disk.
736 class PostProcessingError(Exception):
737 """Post Processing exception.
739 This exception may be raised by PostProcessor's .run() method to
740 indicate an error in the postprocessing task.
743 def __init__(self, msg):
747 class MaxDownloadsReached(Exception):
748 """ --max-downloads limit has been reached. """
752 class UnavailableVideoError(Exception):
753 """Unavailable Format exception.
755 This exception will be thrown when a video is requested
756 in a format that is not available for that video.
761 class ContentTooShortError(Exception):
762 """Content Too Short exception.
764 This exception may be raised by FileDownloader objects when a file they
765 download is too small for what the server announced first, indicating
766 the connection was probably interrupted.
769 def __init__(self, downloaded, expected):
771 self.downloaded = downloaded
772 self.expected = expected
775 def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs):
776 # Working around python 2 bug (see http://bugs.python.org/issue17849) by limiting
777 # expected HTTP responses to meet HTTP/1.0 or later (see also
778 # https://github.com/rg3/youtube-dl/issues/6727)
779 if sys.version_info < (3, 0):
780 kwargs[b'strict'] = True
781 hc = http_class(*args, **kwargs)
782 source_address = ydl_handler._params.get('source_address')
783 if source_address is not None:
784 sa = (source_address, 0)
785 if hasattr(hc, 'source_address'): # Python 2.7+
786 hc.source_address = sa
788 def _hc_connect(self, *args, **kwargs):
789 sock = compat_socket_create_connection(
790 (self.host, self.port), self.timeout, sa)
792 self.sock = ssl.wrap_socket(
793 sock, self.key_file, self.cert_file,
794 ssl_version=ssl.PROTOCOL_TLSv1)
797 hc.connect = functools.partial(_hc_connect, hc)
802 def handle_youtubedl_headers(headers):
803 filtered_headers = headers
805 if 'Youtubedl-no-compression' in filtered_headers:
806 filtered_headers = dict((k, v) for k, v in filtered_headers.items() if k.lower() != 'accept-encoding')
807 del filtered_headers['Youtubedl-no-compression']
809 return filtered_headers
812 class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
813 """Handler for HTTP requests and responses.
815 This class, when installed with an OpenerDirector, automatically adds
816 the standard headers to every HTTP request and handles gzipped and
817 deflated responses from web servers. If compression is to be avoided in
818 a particular request, the original request in the program code only has
819 to include the HTTP header "Youtubedl-no-compression", which will be
820 removed before making the real request.
822 Part of this code was copied from:
824 http://techknack.net/python-urllib2-handlers/
826 Andrew Rowls, the author of that code, agreed to release it to the
830 def __init__(self, params, *args, **kwargs):
831 compat_urllib_request.HTTPHandler.__init__(self, *args, **kwargs)
832 self._params = params
834 def http_open(self, req):
835 conn_class = compat_http_client.HTTPConnection
837 socks_proxy = req.headers.get('Ytdl-socks-proxy')
839 conn_class = make_socks_conn_class(conn_class, socks_proxy)
840 del req.headers['Ytdl-socks-proxy']
842 return self.do_open(functools.partial(
843 _create_http_connection, self, conn_class, False),
849 return zlib.decompress(data, -zlib.MAX_WBITS)
851 return zlib.decompress(data)
854 def addinfourl_wrapper(stream, headers, url, code):
855 if hasattr(compat_urllib_request.addinfourl, 'getcode'):
856 return compat_urllib_request.addinfourl(stream, headers, url, code)
857 ret = compat_urllib_request.addinfourl(stream, headers, url)
861 def http_request(self, req):
862 # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
863 # always respected by websites, some tend to give out URLs with non percent-encoded
864 # non-ASCII characters (see telemb.py, ard.py [#3412])
865 # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
866 # To work around aforementioned issue we will replace request's original URL with
867 # percent-encoded one
868 # Since redirects are also affected (e.g. http://www.southpark.de/alle-episoden/s18e09)
869 # the code of this workaround has been moved here from YoutubeDL.urlopen()
870 url = req.get_full_url()
871 url_escaped = escape_url(url)
873 # Substitute URL if any change after escaping
874 if url != url_escaped:
875 req = update_Request(req, url=url_escaped)
877 for h, v in std_headers.items():
878 # Capitalize is needed because of Python bug 2275: http://bugs.python.org/issue2275
879 # The dict keys are capitalized because of this bug by urllib
880 if h.capitalize() not in req.headers:
883 req.headers = handle_youtubedl_headers(req.headers)
885 if sys.version_info < (2, 7) and '#' in req.get_full_url():
886 # Python 2.6 is brain-dead when it comes to fragments
887 req._Request__original = req._Request__original.partition('#')[0]
888 req._Request__r_type = req._Request__r_type.partition('#')[0]
892 def http_response(self, req, resp):
895 if resp.headers.get('Content-encoding', '') == 'gzip':
896 content = resp.read()
897 gz = gzip.GzipFile(fileobj=io.BytesIO(content), mode='rb')
899 uncompressed = io.BytesIO(gz.read())
900 except IOError as original_ioerror:
901 # There may be junk add the end of the file
902 # See http://stackoverflow.com/q/4928560/35070 for details
903 for i in range(1, 1024):
905 gz = gzip.GzipFile(fileobj=io.BytesIO(content[:-i]), mode='rb')
906 uncompressed = io.BytesIO(gz.read())
911 raise original_ioerror
912 resp = self.addinfourl_wrapper(uncompressed, old_resp.headers, old_resp.url, old_resp.code)
913 resp.msg = old_resp.msg
914 del resp.headers['Content-encoding']
916 if resp.headers.get('Content-encoding', '') == 'deflate':
917 gz = io.BytesIO(self.deflate(resp.read()))
918 resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
919 resp.msg = old_resp.msg
920 del resp.headers['Content-encoding']
921 # Percent-encode redirect URL of Location HTTP header to satisfy RFC 3986 (see
922 # https://github.com/rg3/youtube-dl/issues/6457).
923 if 300 <= resp.code < 400:
924 location = resp.headers.get('Location')
926 # As of RFC 2616 default charset is iso-8859-1 that is respected by python 3
927 if sys.version_info >= (3, 0):
928 location = location.encode('iso-8859-1').decode('utf-8')
930 location = location.decode('utf-8')
931 location_escaped = escape_url(location)
932 if location != location_escaped:
933 del resp.headers['Location']
934 if sys.version_info < (3, 0):
935 location_escaped = location_escaped.encode('utf-8')
936 resp.headers['Location'] = location_escaped
939 https_request = http_request
940 https_response = http_response
943 def make_socks_conn_class(base_class, socks_proxy):
944 assert issubclass(base_class, (
945 compat_http_client.HTTPConnection, compat_http_client.HTTPSConnection))
947 url_components = compat_urlparse.urlparse(socks_proxy)
948 if url_components.scheme.lower() == 'socks5':
949 socks_type = ProxyType.SOCKS5
950 elif url_components.scheme.lower() in ('socks', 'socks4'):
951 socks_type = ProxyType.SOCKS4
952 elif url_components.scheme.lower() == 'socks4a':
953 socks_type = ProxyType.SOCKS4A
955 def unquote_if_non_empty(s):
958 return compat_urllib_parse_unquote_plus(s)
962 url_components.hostname, url_components.port or 1080,
964 unquote_if_non_empty(url_components.username),
965 unquote_if_non_empty(url_components.password),
968 class SocksConnection(base_class):
970 self.sock = sockssocket()
971 self.sock.setproxy(*proxy_args)
972 if type(self.timeout) in (int, float):
973 self.sock.settimeout(self.timeout)
974 self.sock.connect((self.host, self.port))
976 if isinstance(self, compat_http_client.HTTPSConnection):
977 if hasattr(self, '_context'): # Python > 2.6
978 self.sock = self._context.wrap_socket(
979 self.sock, server_hostname=self.host)
981 self.sock = ssl.wrap_socket(self.sock)
983 return SocksConnection
986 class YoutubeDLHTTPSHandler(compat_urllib_request.HTTPSHandler):
987 def __init__(self, params, https_conn_class=None, *args, **kwargs):
988 compat_urllib_request.HTTPSHandler.__init__(self, *args, **kwargs)
989 self._https_conn_class = https_conn_class or compat_http_client.HTTPSConnection
990 self._params = params
992 def https_open(self, req):
994 conn_class = self._https_conn_class
996 if hasattr(self, '_context'): # python > 2.6
997 kwargs['context'] = self._context
998 if hasattr(self, '_check_hostname'): # python 3.x
999 kwargs['check_hostname'] = self._check_hostname
1001 socks_proxy = req.headers.get('Ytdl-socks-proxy')
1003 conn_class = make_socks_conn_class(conn_class, socks_proxy)
1004 del req.headers['Ytdl-socks-proxy']
1006 return self.do_open(functools.partial(
1007 _create_http_connection, self, conn_class, True),
1011 class YoutubeDLCookieProcessor(compat_urllib_request.HTTPCookieProcessor):
1012 def __init__(self, cookiejar=None):
1013 compat_urllib_request.HTTPCookieProcessor.__init__(self, cookiejar)
1015 def http_response(self, request, response):
1016 # Python 2 will choke on next HTTP request in row if there are non-ASCII
1017 # characters in Set-Cookie HTTP header of last response (see
1018 # https://github.com/rg3/youtube-dl/issues/6769).
1019 # In order to at least prevent crashing we will percent encode Set-Cookie
1020 # header before HTTPCookieProcessor starts processing it.
1021 # if sys.version_info < (3, 0) and response.headers:
1022 # for set_cookie_header in ('Set-Cookie', 'Set-Cookie2'):
1023 # set_cookie = response.headers.get(set_cookie_header)
1025 # set_cookie_escaped = compat_urllib_parse.quote(set_cookie, b"%/;:@&=+$,!~*'()?#[] ")
1026 # if set_cookie != set_cookie_escaped:
1027 # del response.headers[set_cookie_header]
1028 # response.headers[set_cookie_header] = set_cookie_escaped
1029 return compat_urllib_request.HTTPCookieProcessor.http_response(self, request, response)
1031 https_request = compat_urllib_request.HTTPCookieProcessor.http_request
1032 https_response = http_response
1035 def extract_timezone(date_str):
1037 r'^.{8,}?(?P<tz>Z$| ?(?P<sign>\+|-)(?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})$)',
1040 timezone = datetime.timedelta()
1042 date_str = date_str[:-len(m.group('tz'))]
1043 if not m.group('sign'):
1044 timezone = datetime.timedelta()
1046 sign = 1 if m.group('sign') == '+' else -1
1047 timezone = datetime.timedelta(
1048 hours=sign * int(m.group('hours')),
1049 minutes=sign * int(m.group('minutes')))
1050 return timezone, date_str
1053 def parse_iso8601(date_str, delimiter='T', timezone=None):
1054 """ Return a UNIX timestamp from the given date """
1056 if date_str is None:
1059 date_str = re.sub(r'\.[0-9]+', '', date_str)
1061 if timezone is None:
1062 timezone, date_str = extract_timezone(date_str)
1065 date_format = '%Y-%m-%d{0}%H:%M:%S'.format(delimiter)
1066 dt = datetime.datetime.strptime(date_str, date_format) - timezone
1067 return calendar.timegm(dt.timetuple())
1072 def date_formats(day_first=True):
1073 return DATE_FORMATS_DAY_FIRST if day_first else DATE_FORMATS_MONTH_FIRST
1076 def unified_strdate(date_str, day_first=True):
1077 """Return a string with the date in the format YYYYMMDD"""
1079 if date_str is None:
1083 date_str = date_str.replace(',', ' ')
1084 # Remove AM/PM + timezone
1085 date_str = re.sub(r'(?i)\s*(?:AM|PM)(?:\s+[A-Z]+)?', '', date_str)
1086 _, date_str = extract_timezone(date_str)
1088 for expression in date_formats(day_first):
1090 upload_date = datetime.datetime.strptime(date_str, expression).strftime('%Y%m%d')
1093 if upload_date is None:
1094 timetuple = email.utils.parsedate_tz(date_str)
1097 upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d')
1100 if upload_date is not None:
1101 return compat_str(upload_date)
1104 def unified_timestamp(date_str, day_first=True):
1105 if date_str is None:
1108 date_str = date_str.replace(',', ' ')
1110 pm_delta = 12 if re.search(r'(?i)PM', date_str) else 0
1111 timezone, date_str = extract_timezone(date_str)
1113 # Remove AM/PM + timezone
1114 date_str = re.sub(r'(?i)\s*(?:AM|PM)(?:\s+[A-Z]+)?', '', date_str)
1116 for expression in date_formats(day_first):
1118 dt = datetime.datetime.strptime(date_str, expression) - timezone + datetime.timedelta(hours=pm_delta)
1119 return calendar.timegm(dt.timetuple())
1122 timetuple = email.utils.parsedate_tz(date_str)
1124 return calendar.timegm(timetuple) + pm_delta * 3600
1127 def determine_ext(url, default_ext='unknown_video'):
1130 guess = url.partition('?')[0].rpartition('.')[2]
1131 if re.match(r'^[A-Za-z0-9]+$', guess):
1133 # Try extract ext from URLs like http://example.com/foo/bar.mp4/?download
1134 elif guess.rstrip('/') in KNOWN_EXTENSIONS:
1135 return guess.rstrip('/')
1140 def subtitles_filename(filename, sub_lang, sub_format):
1141 return filename.rsplit('.', 1)[0] + '.' + sub_lang + '.' + sub_format
1144 def date_from_str(date_str):
1146 Return a datetime object from a string in the format YYYYMMDD or
1147 (now|today)[+-][0-9](day|week|month|year)(s)?"""
1148 today = datetime.date.today()
1149 if date_str in ('now', 'today'):
1151 if date_str == 'yesterday':
1152 return today - datetime.timedelta(days=1)
1153 match = re.match('(now|today)(?P<sign>[+-])(?P<time>\d+)(?P<unit>day|week|month|year)(s)?', date_str)
1154 if match is not None:
1155 sign = match.group('sign')
1156 time = int(match.group('time'))
1159 unit = match.group('unit')
1160 # A bad approximation?
1164 elif unit == 'year':
1168 delta = datetime.timedelta(**{unit: time})
1169 return today + delta
1170 return datetime.datetime.strptime(date_str, '%Y%m%d').date()
1173 def hyphenate_date(date_str):
1175 Convert a date in 'YYYYMMDD' format to 'YYYY-MM-DD' format"""
1176 match = re.match(r'^(\d\d\d\d)(\d\d)(\d\d)$', date_str)
1177 if match is not None:
1178 return '-'.join(match.groups())
1183 class DateRange(object):
1184 """Represents a time interval between two dates"""
1186 def __init__(self, start=None, end=None):
1187 """start and end must be strings in the format accepted by date"""
1188 if start is not None:
1189 self.start = date_from_str(start)
1191 self.start = datetime.datetime.min.date()
1193 self.end = date_from_str(end)
1195 self.end = datetime.datetime.max.date()
1196 if self.start > self.end:
1197 raise ValueError('Date range: "%s" , the start date must be before the end date' % self)
1201 """Returns a range that only contains the given day"""
1202 return cls(day, day)
1204 def __contains__(self, date):
1205 """Check if the date is in the range"""
1206 if not isinstance(date, datetime.date):
1207 date = date_from_str(date)
1208 return self.start <= date <= self.end
1211 return '%s - %s' % (self.start.isoformat(), self.end.isoformat())
1214 def platform_name():
1215 """ Returns the platform name as a compat_str """
1216 res = platform.platform()
1217 if isinstance(res, bytes):
1218 res = res.decode(preferredencoding())
1220 assert isinstance(res, compat_str)
1224 def _windows_write_string(s, out):
1225 """ Returns True if the string was written using special methods,
1226 False if it has yet to be written out."""
1227 # Adapted from http://stackoverflow.com/a/3259271/35070
1230 import ctypes.wintypes
1238 fileno = out.fileno()
1239 except AttributeError:
1240 # If the output stream doesn't have a fileno, it's virtual
1242 except io.UnsupportedOperation:
1243 # Some strange Windows pseudo files?
1245 if fileno not in WIN_OUTPUT_IDS:
1248 GetStdHandle = ctypes.WINFUNCTYPE(
1249 ctypes.wintypes.HANDLE, ctypes.wintypes.DWORD)(
1250 (b'GetStdHandle', ctypes.windll.kernel32))
1251 h = GetStdHandle(WIN_OUTPUT_IDS[fileno])
1253 WriteConsoleW = ctypes.WINFUNCTYPE(
1254 ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE, ctypes.wintypes.LPWSTR,
1255 ctypes.wintypes.DWORD, ctypes.POINTER(ctypes.wintypes.DWORD),
1256 ctypes.wintypes.LPVOID)((b'WriteConsoleW', ctypes.windll.kernel32))
1257 written = ctypes.wintypes.DWORD(0)
1259 GetFileType = ctypes.WINFUNCTYPE(ctypes.wintypes.DWORD, ctypes.wintypes.DWORD)((b'GetFileType', ctypes.windll.kernel32))
1260 FILE_TYPE_CHAR = 0x0002
1261 FILE_TYPE_REMOTE = 0x8000
1262 GetConsoleMode = ctypes.WINFUNCTYPE(
1263 ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE,
1264 ctypes.POINTER(ctypes.wintypes.DWORD))(
1265 (b'GetConsoleMode', ctypes.windll.kernel32))
1266 INVALID_HANDLE_VALUE = ctypes.wintypes.DWORD(-1).value
1268 def not_a_console(handle):
1269 if handle == INVALID_HANDLE_VALUE or handle is None:
1271 return ((GetFileType(handle) & ~FILE_TYPE_REMOTE) != FILE_TYPE_CHAR or
1272 GetConsoleMode(handle, ctypes.byref(ctypes.wintypes.DWORD())) == 0)
1274 if not_a_console(h):
1277 def next_nonbmp_pos(s):
1279 return next(i for i, c in enumerate(s) if ord(c) > 0xffff)
1280 except StopIteration:
1284 count = min(next_nonbmp_pos(s), 1024)
1286 ret = WriteConsoleW(
1287 h, s, count if count else 2, ctypes.byref(written), None)
1289 raise OSError('Failed to write string')
1290 if not count: # We just wrote a non-BMP character
1291 assert written.value == 2
1294 assert written.value > 0
1295 s = s[written.value:]
1299 def write_string(s, out=None, encoding=None):
1302 assert type(s) == compat_str
1304 if sys.platform == 'win32' and encoding is None and hasattr(out, 'fileno'):
1305 if _windows_write_string(s, out):
1308 if ('b' in getattr(out, 'mode', '') or
1309 sys.version_info[0] < 3): # Python 2 lies about mode of sys.stderr
1310 byt = s.encode(encoding or preferredencoding(), 'ignore')
1312 elif hasattr(out, 'buffer'):
1313 enc = encoding or getattr(out, 'encoding', None) or preferredencoding()
1314 byt = s.encode(enc, 'ignore')
1315 out.buffer.write(byt)
1321 def bytes_to_intlist(bs):
1324 if isinstance(bs[0], int): # Python 3
1327 return [ord(c) for c in bs]
1330 def intlist_to_bytes(xs):
1333 return compat_struct_pack('%dB' % len(xs), *xs)
1336 # Cross-platform file locking
1337 if sys.platform == 'win32':
1338 import ctypes.wintypes
1341 class OVERLAPPED(ctypes.Structure):
1343 ('Internal', ctypes.wintypes.LPVOID),
1344 ('InternalHigh', ctypes.wintypes.LPVOID),
1345 ('Offset', ctypes.wintypes.DWORD),
1346 ('OffsetHigh', ctypes.wintypes.DWORD),
1347 ('hEvent', ctypes.wintypes.HANDLE),
1350 kernel32 = ctypes.windll.kernel32
1351 LockFileEx = kernel32.LockFileEx
1352 LockFileEx.argtypes = [
1353 ctypes.wintypes.HANDLE, # hFile
1354 ctypes.wintypes.DWORD, # dwFlags
1355 ctypes.wintypes.DWORD, # dwReserved
1356 ctypes.wintypes.DWORD, # nNumberOfBytesToLockLow
1357 ctypes.wintypes.DWORD, # nNumberOfBytesToLockHigh
1358 ctypes.POINTER(OVERLAPPED) # Overlapped
1360 LockFileEx.restype = ctypes.wintypes.BOOL
1361 UnlockFileEx = kernel32.UnlockFileEx
1362 UnlockFileEx.argtypes = [
1363 ctypes.wintypes.HANDLE, # hFile
1364 ctypes.wintypes.DWORD, # dwReserved
1365 ctypes.wintypes.DWORD, # nNumberOfBytesToLockLow
1366 ctypes.wintypes.DWORD, # nNumberOfBytesToLockHigh
1367 ctypes.POINTER(OVERLAPPED) # Overlapped
1369 UnlockFileEx.restype = ctypes.wintypes.BOOL
1370 whole_low = 0xffffffff
1371 whole_high = 0x7fffffff
1373 def _lock_file(f, exclusive):
1374 overlapped = OVERLAPPED()
1375 overlapped.Offset = 0
1376 overlapped.OffsetHigh = 0
1377 overlapped.hEvent = 0
1378 f._lock_file_overlapped_p = ctypes.pointer(overlapped)
1379 handle = msvcrt.get_osfhandle(f.fileno())
1380 if not LockFileEx(handle, 0x2 if exclusive else 0x0, 0,
1381 whole_low, whole_high, f._lock_file_overlapped_p):
1382 raise OSError('Locking file failed: %r' % ctypes.FormatError())
1384 def _unlock_file(f):
1385 assert f._lock_file_overlapped_p
1386 handle = msvcrt.get_osfhandle(f.fileno())
1387 if not UnlockFileEx(handle, 0,
1388 whole_low, whole_high, f._lock_file_overlapped_p):
1389 raise OSError('Unlocking file failed: %r' % ctypes.FormatError())
1392 # Some platforms, such as Jython, is missing fcntl
1396 def _lock_file(f, exclusive):
1397 fcntl.flock(f, fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH)
1399 def _unlock_file(f):
1400 fcntl.flock(f, fcntl.LOCK_UN)
1402 UNSUPPORTED_MSG = 'file locking is not supported on this platform'
1404 def _lock_file(f, exclusive):
1405 raise IOError(UNSUPPORTED_MSG)
1407 def _unlock_file(f):
1408 raise IOError(UNSUPPORTED_MSG)
1411 class locked_file(object):
1412 def __init__(self, filename, mode, encoding=None):
1413 assert mode in ['r', 'a', 'w']
1414 self.f = io.open(filename, mode, encoding=encoding)
1417 def __enter__(self):
1418 exclusive = self.mode != 'r'
1420 _lock_file(self.f, exclusive)
1426 def __exit__(self, etype, value, traceback):
1428 _unlock_file(self.f)
1435 def write(self, *args):
1436 return self.f.write(*args)
1438 def read(self, *args):
1439 return self.f.read(*args)
1442 def get_filesystem_encoding():
1443 encoding = sys.getfilesystemencoding()
1444 return encoding if encoding is not None else 'utf-8'
1447 def shell_quote(args):
1449 encoding = get_filesystem_encoding()
1451 if isinstance(a, bytes):
1452 # We may get a filename encoded with 'encodeFilename'
1453 a = a.decode(encoding)
1454 quoted_args.append(pipes.quote(a))
1455 return ' '.join(quoted_args)
1458 def smuggle_url(url, data):
1459 """ Pass additional data in a URL for internal use. """
1461 url, idata = unsmuggle_url(url, {})
1463 sdata = compat_urllib_parse_urlencode(
1464 {'__youtubedl_smuggle': json.dumps(data)})
1465 return url + '#' + sdata
1468 def unsmuggle_url(smug_url, default=None):
1469 if '#__youtubedl_smuggle' not in smug_url:
1470 return smug_url, default
1471 url, _, sdata = smug_url.rpartition('#')
1472 jsond = compat_parse_qs(sdata)['__youtubedl_smuggle'][0]
1473 data = json.loads(jsond)
1477 def format_bytes(bytes):
1480 if type(bytes) is str:
1481 bytes = float(bytes)
1485 exponent = int(math.log(bytes, 1024.0))
1486 suffix = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'][exponent]
1487 converted = float(bytes) / float(1024 ** exponent)
1488 return '%.2f%s' % (converted, suffix)
1491 def lookup_unit_table(unit_table, s):
1492 units_re = '|'.join(re.escape(u) for u in unit_table)
1494 r'(?P<num>[0-9]+(?:[,.][0-9]*)?)\s*(?P<unit>%s)\b' % units_re, s)
1497 num_str = m.group('num').replace(',', '.')
1498 mult = unit_table[m.group('unit')]
1499 return int(float(num_str) * mult)
1502 def parse_filesize(s):
1506 # The lower-case forms are of course incorrect and unofficial,
1507 # but we support those too
1524 'megabytes': 1000 ** 2,
1525 'mebibytes': 1024 ** 2,
1531 'gigabytes': 1000 ** 3,
1532 'gibibytes': 1024 ** 3,
1538 'terabytes': 1000 ** 4,
1539 'tebibytes': 1024 ** 4,
1545 'petabytes': 1000 ** 5,
1546 'pebibytes': 1024 ** 5,
1552 'exabytes': 1000 ** 6,
1553 'exbibytes': 1024 ** 6,
1559 'zettabytes': 1000 ** 7,
1560 'zebibytes': 1024 ** 7,
1566 'yottabytes': 1000 ** 8,
1567 'yobibytes': 1024 ** 8,
1570 return lookup_unit_table(_UNIT_TABLE, s)
1579 if re.match(r'^[\d,.]+$', s):
1580 return str_to_int(s)
1591 return lookup_unit_table(_UNIT_TABLE, s)
1594 def month_by_name(name, lang='en'):
1595 """ Return the number of a month by (locale-independently) English name """
1597 name_list = ENGLISH_MONTH_NAMES
1600 name_list = FRENCH_MONTH_NAMES
1603 return name_list.index(name) + 1
1608 def month_by_abbreviation(abbrev):
1609 """ Return the number of a month by (locale-independently) English
1613 return [s[:3] for s in ENGLISH_MONTH_NAMES].index(abbrev) + 1
1618 def fix_xml_ampersands(xml_str):
1619 """Replace all the '&' by '&' in XML"""
1621 r'&(?!amp;|lt;|gt;|apos;|quot;|#x[0-9a-fA-F]{,4};|#[0-9]{,4};)',
1626 def setproctitle(title):
1627 assert isinstance(title, compat_str)
1629 # ctypes in Jython is not complete
1630 # http://bugs.jython.org/issue2148
1631 if sys.platform.startswith('java'):
1635 libc = ctypes.cdll.LoadLibrary('libc.so.6')
1638 title_bytes = title.encode('utf-8')
1639 buf = ctypes.create_string_buffer(len(title_bytes))
1640 buf.value = title_bytes
1642 libc.prctl(15, buf, 0, 0, 0)
1643 except AttributeError:
1644 return # Strange libc, just skip this
1647 def remove_start(s, start):
1648 return s[len(start):] if s is not None and s.startswith(start) else s
1651 def remove_end(s, end):
1652 return s[:-len(end)] if s is not None and s.endswith(end) else s
1655 def remove_quotes(s):
1656 if s is None or len(s) < 2:
1658 for quote in ('"', "'", ):
1659 if s[0] == quote and s[-1] == quote:
1664 def url_basename(url):
1665 path = compat_urlparse.urlparse(url).path
1666 return path.strip('/').split('/')[-1]
1669 class HEADRequest(compat_urllib_request.Request):
1670 def get_method(self):
1674 class PUTRequest(compat_urllib_request.Request):
1675 def get_method(self):
1679 def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1):
1682 v = getattr(v, get_attr, None)
1688 return int(v) * invscale // scale
1693 def str_or_none(v, default=None):
1694 return default if v is None else compat_str(v)
1697 def str_to_int(int_str):
1698 """ A more relaxed version of int_or_none """
1701 int_str = re.sub(r'[,\.\+]', '', int_str)
1705 def float_or_none(v, scale=1, invscale=1, default=None):
1709 return float(v) * invscale / scale
1714 def strip_or_none(v):
1715 return None if v is None else v.strip()
1718 def parse_duration(s):
1719 if not isinstance(s, compat_basestring):
1724 days, hours, mins, secs, ms = [None] * 5
1725 m = re.match(r'(?:(?:(?:(?P<days>[0-9]+):)?(?P<hours>[0-9]+):)?(?P<mins>[0-9]+):)?(?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?$', s)
1727 days, hours, mins, secs, ms = m.groups()
1732 (?P<days>[0-9]+)\s*d(?:ays?)?\s*
1735 (?P<hours>[0-9]+)\s*h(?:ours?)?\s*
1738 (?P<mins>[0-9]+)\s*m(?:in(?:ute)?s?)?\s*
1741 (?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?\s*s(?:ec(?:ond)?s?)?\s*
1744 days, hours, mins, secs, ms = m.groups()
1746 m = re.match(r'(?i)(?:(?P<hours>[0-9.]+)\s*(?:hours?)|(?P<mins>[0-9.]+)\s*(?:mins?\.?|minutes?)\s*)$', s)
1748 hours, mins = m.groups()
1754 duration += float(secs)
1756 duration += float(mins) * 60
1758 duration += float(hours) * 60 * 60
1760 duration += float(days) * 24 * 60 * 60
1762 duration += float(ms)
1766 def prepend_extension(filename, ext, expected_real_ext=None):
1767 name, real_ext = os.path.splitext(filename)
1769 '{0}.{1}{2}'.format(name, ext, real_ext)
1770 if not expected_real_ext or real_ext[1:] == expected_real_ext
1771 else '{0}.{1}'.format(filename, ext))
1774 def replace_extension(filename, ext, expected_real_ext=None):
1775 name, real_ext = os.path.splitext(filename)
1776 return '{0}.{1}'.format(
1777 name if not expected_real_ext or real_ext[1:] == expected_real_ext else filename,
1781 def check_executable(exe, args=[]):
1782 """ Checks if the given binary is installed somewhere in PATH, and returns its name.
1783 args can be a list of arguments for a short output (like -version) """
1785 subprocess.Popen([exe] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
1791 def get_exe_version(exe, args=['--version'],
1792 version_re=None, unrecognized='present'):
1793 """ Returns the version of the specified executable,
1794 or False if the executable is not present """
1796 out, _ = subprocess.Popen(
1797 [encodeArgument(exe)] + args,
1798 stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()
1801 if isinstance(out, bytes): # Python 2.x
1802 out = out.decode('ascii', 'ignore')
1803 return detect_exe_version(out, version_re, unrecognized)
1806 def detect_exe_version(output, version_re=None, unrecognized='present'):
1807 assert isinstance(output, compat_str)
1808 if version_re is None:
1809 version_re = r'version\s+([-0-9._a-zA-Z]+)'
1810 m = re.search(version_re, output)
1817 class PagedList(object):
1819 # This is only useful for tests
1820 return len(self.getslice())
1823 class OnDemandPagedList(PagedList):
1824 def __init__(self, pagefunc, pagesize, use_cache=False):
1825 self._pagefunc = pagefunc
1826 self._pagesize = pagesize
1827 self._use_cache = use_cache
1831 def getslice(self, start=0, end=None):
1833 for pagenum in itertools.count(start // self._pagesize):
1834 firstid = pagenum * self._pagesize
1835 nextfirstid = pagenum * self._pagesize + self._pagesize
1836 if start >= nextfirstid:
1841 page_results = self._cache.get(pagenum)
1842 if page_results is None:
1843 page_results = list(self._pagefunc(pagenum))
1845 self._cache[pagenum] = page_results
1848 start % self._pagesize
1849 if firstid <= start < nextfirstid
1853 ((end - 1) % self._pagesize) + 1
1854 if (end is not None and firstid <= end <= nextfirstid)
1857 if startv != 0 or endv is not None:
1858 page_results = page_results[startv:endv]
1859 res.extend(page_results)
1861 # A little optimization - if current page is not "full", ie. does
1862 # not contain page_size videos then we can assume that this page
1863 # is the last one - there are no more ids on further pages -
1864 # i.e. no need to query again.
1865 if len(page_results) + startv < self._pagesize:
1868 # If we got the whole page, but the next page is not interesting,
1869 # break out early as well
1870 if end == nextfirstid:
1875 class InAdvancePagedList(PagedList):
1876 def __init__(self, pagefunc, pagecount, pagesize):
1877 self._pagefunc = pagefunc
1878 self._pagecount = pagecount
1879 self._pagesize = pagesize
1881 def getslice(self, start=0, end=None):
1883 start_page = start // self._pagesize
1885 self._pagecount if end is None else (end // self._pagesize + 1))
1886 skip_elems = start - start_page * self._pagesize
1887 only_more = None if end is None else end - start
1888 for pagenum in range(start_page, end_page):
1889 page = list(self._pagefunc(pagenum))
1891 page = page[skip_elems:]
1893 if only_more is not None:
1894 if len(page) < only_more:
1895 only_more -= len(page)
1897 page = page[:only_more]
1904 def uppercase_escape(s):
1905 unicode_escape = codecs.getdecoder('unicode_escape')
1907 r'\\U[0-9a-fA-F]{8}',
1908 lambda m: unicode_escape(m.group(0))[0],
1912 def lowercase_escape(s):
1913 unicode_escape = codecs.getdecoder('unicode_escape')
1915 r'\\u[0-9a-fA-F]{4}',
1916 lambda m: unicode_escape(m.group(0))[0],
1920 def escape_rfc3986(s):
1921 """Escape non-ASCII characters as suggested by RFC 3986"""
1922 if sys.version_info < (3, 0) and isinstance(s, compat_str):
1923 s = s.encode('utf-8')
1924 return compat_urllib_parse.quote(s, b"%/;:@&=+$,!~*'()?#[]")
1927 def escape_url(url):
1928 """Escape URL as suggested by RFC 3986"""
1929 url_parsed = compat_urllib_parse_urlparse(url)
1930 return url_parsed._replace(
1931 netloc=url_parsed.netloc.encode('idna').decode('ascii'),
1932 path=escape_rfc3986(url_parsed.path),
1933 params=escape_rfc3986(url_parsed.params),
1934 query=escape_rfc3986(url_parsed.query),
1935 fragment=escape_rfc3986(url_parsed.fragment)
1939 def read_batch_urls(batch_fd):
1941 if not isinstance(url, compat_str):
1942 url = url.decode('utf-8', 'replace')
1943 BOM_UTF8 = '\xef\xbb\xbf'
1944 if url.startswith(BOM_UTF8):
1945 url = url[len(BOM_UTF8):]
1947 if url.startswith(('#', ';', ']')):
1951 with contextlib.closing(batch_fd) as fd:
1952 return [url for url in map(fixup, fd) if url]
1955 def urlencode_postdata(*args, **kargs):
1956 return compat_urllib_parse_urlencode(*args, **kargs).encode('ascii')
1959 def update_url_query(url, query):
1962 parsed_url = compat_urlparse.urlparse(url)
1963 qs = compat_parse_qs(parsed_url.query)
1965 return compat_urlparse.urlunparse(parsed_url._replace(
1966 query=compat_urllib_parse_urlencode(qs, True)))
1969 def update_Request(req, url=None, data=None, headers={}, query={}):
1970 req_headers = req.headers.copy()
1971 req_headers.update(headers)
1972 req_data = data or req.data
1973 req_url = update_url_query(url or req.get_full_url(), query)
1974 req_get_method = req.get_method()
1975 if req_get_method == 'HEAD':
1976 req_type = HEADRequest
1977 elif req_get_method == 'PUT':
1978 req_type = PUTRequest
1980 req_type = compat_urllib_request.Request
1982 req_url, data=req_data, headers=req_headers,
1983 origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
1984 if hasattr(req, 'timeout'):
1985 new_req.timeout = req.timeout
1989 def dict_get(d, key_or_keys, default=None, skip_false_values=True):
1990 if isinstance(key_or_keys, (list, tuple)):
1991 for key in key_or_keys:
1992 if key not in d or d[key] is None or skip_false_values and not d[key]:
1996 return d.get(key_or_keys, default)
1999 def try_get(src, getter, expected_type=None):
2002 except (AttributeError, KeyError, TypeError, IndexError):
2005 if expected_type is None or isinstance(v, expected_type):
2009 def encode_compat_str(string, encoding=preferredencoding(), errors='strict'):
2010 return string if isinstance(string, compat_str) else compat_str(string, encoding, errors)
2022 TV_PARENTAL_GUIDELINES = {
2032 def parse_age_limit(s):
2034 return s if 0 <= s <= 21 else None
2035 if not isinstance(s, compat_basestring):
2037 m = re.match(r'^(?P<age>\d{1,2})\+?$', s)
2039 return int(m.group('age'))
2041 return US_RATINGS[s]
2042 return TV_PARENTAL_GUIDELINES.get(s)
2045 def strip_jsonp(code):
2047 r'(?s)^[a-zA-Z0-9_.$]+\s*\(\s*(.*)\);?\s*?(?://[^\n]*)*$', r'\1', code)
2050 def js_to_json(code):
2053 if v in ('true', 'false', 'null'):
2055 elif v.startswith('/*') or v == ',':
2058 if v[0] in ("'", '"'):
2059 v = re.sub(r'(?s)\\.|"', lambda m: {
2064 }.get(m.group(0), m.group(0)), v[1:-1])
2067 (r'^(0[xX][0-9a-fA-F]+)\s*:?$', 16),
2068 (r'^(0+[0-7]+)\s*:?$', 8),
2071 for regex, base in INTEGER_TABLE:
2072 im = re.match(regex, v)
2074 i = int(im.group(1), base)
2075 return '"%d":' % i if v.endswith(':') else '%d' % i
2079 return re.sub(r'''(?sx)
2080 "(?:[^"\\]*(?:\\\\|\\['"nurtbfx/\n]))*[^"\\]*"|
2081 '(?:[^'\\]*(?:\\\\|\\['"nurtbfx/\n]))*[^'\\]*'|
2082 /\*.*?\*/|,(?=\s*[\]}])|
2083 [a-zA-Z_][.a-zA-Z_0-9]*|
2084 \b(?:0[xX][0-9a-fA-F]+|0+[0-7]+)(?:\s*:)?|
2089 def qualities(quality_ids):
2090 """ Get a numeric quality value out of a list of possible values """
2093 return quality_ids.index(qid)
2099 DEFAULT_OUTTMPL = '%(title)s-%(id)s.%(ext)s'
2102 def limit_length(s, length):
2103 """ Add ellipses to overly long strings """
2108 return s[:length - len(ELLIPSES)] + ELLIPSES
2112 def version_tuple(v):
2113 return tuple(int(e) for e in re.split(r'[-.]', v))
2116 def is_outdated_version(version, limit, assume_new=True):
2118 return not assume_new
2120 return version_tuple(version) < version_tuple(limit)
2122 return not assume_new
2125 def ytdl_is_updateable():
2126 """ Returns if youtube-dl can be updated with -U """
2127 from zipimport import zipimporter
2129 return isinstance(globals().get('__loader__'), zipimporter) or hasattr(sys, 'frozen')
2132 def args_to_str(args):
2133 # Get a short string representation for a subprocess command
2134 return ' '.join(compat_shlex_quote(a) for a in args)
2137 def error_to_compat_str(err):
2139 # On python 2 error byte string must be decoded with proper
2140 # encoding rather than ascii
2141 if sys.version_info[0] < 3:
2142 err_str = err_str.decode(preferredencoding())
2146 def mimetype2ext(mt):
2152 # Per RFC 3003, audio/mpeg can be .mp1, .mp2 or .mp3. Here use .mp3 as
2153 # it's the most popular one
2154 'audio/mpeg': 'mp3',
2159 _, _, res = mt.rpartition('/')
2160 res = res.split(';')[0].strip().lower()
2164 'smptett+xml': 'tt',
2170 'x-mp4-fragmented': 'mp4',
2173 'x-mpegurl': 'm3u8',
2174 'vnd.apple.mpegurl': 'm3u8',
2179 'vnd.ms-sstr+xml': 'ism',
2184 def parse_codecs(codecs_str):
2185 # http://tools.ietf.org/html/rfc6381
2188 splited_codecs = list(filter(None, map(
2189 lambda str: str.strip(), codecs_str.strip().strip(',').split(','))))
2190 vcodec, acodec = None, None
2191 for full_codec in splited_codecs:
2192 codec = full_codec.split('.')[0]
2193 if codec in ('avc1', 'avc2', 'avc3', 'avc4', 'vp9', 'vp8', 'hev1', 'hev2', 'h263', 'h264', 'mp4v'):
2196 elif codec in ('mp4a', 'opus', 'vorbis', 'mp3', 'aac', 'ac-3'):
2200 write_string('WARNING: Unknown codec %s' % full_codec, sys.stderr)
2201 if not vcodec and not acodec:
2202 if len(splited_codecs) == 2:
2207 elif len(splited_codecs) == 1:
2214 'vcodec': vcodec or 'none',
2215 'acodec': acodec or 'none',
2220 def urlhandle_detect_ext(url_handle):
2221 getheader = url_handle.headers.get
2223 cd = getheader('Content-Disposition')
2225 m = re.match(r'attachment;\s*filename="(?P<filename>[^"]+)"', cd)
2227 e = determine_ext(m.group('filename'), default_ext=None)
2231 return mimetype2ext(getheader('Content-Type'))
2234 def encode_data_uri(data, mime_type):
2235 return 'data:%s;base64,%s' % (mime_type, base64.b64encode(data).decode('ascii'))
2238 def age_restricted(content_limit, age_limit):
2239 """ Returns True iff the content should be blocked """
2241 if age_limit is None: # No limit set
2243 if content_limit is None:
2244 return False # Content available for everyone
2245 return age_limit < content_limit
2248 def is_html(first_bytes):
2249 """ Detect whether a file contains HTML by examining its first bytes. """
2252 (b'\xef\xbb\xbf', 'utf-8'),
2253 (b'\x00\x00\xfe\xff', 'utf-32-be'),
2254 (b'\xff\xfe\x00\x00', 'utf-32-le'),
2255 (b'\xff\xfe', 'utf-16-le'),
2256 (b'\xfe\xff', 'utf-16-be'),
2258 for bom, enc in BOMS:
2259 if first_bytes.startswith(bom):
2260 s = first_bytes[len(bom):].decode(enc, 'replace')
2263 s = first_bytes.decode('utf-8', 'replace')
2265 return re.match(r'^\s*<', s)
2268 def determine_protocol(info_dict):
2269 protocol = info_dict.get('protocol')
2270 if protocol is not None:
2273 url = info_dict['url']
2274 if url.startswith('rtmp'):
2276 elif url.startswith('mms'):
2278 elif url.startswith('rtsp'):
2281 ext = determine_ext(url)
2287 return compat_urllib_parse_urlparse(url).scheme
2290 def render_table(header_row, data):
2291 """ Render a list of rows, each as a list of values """
2292 table = [header_row] + data
2293 max_lens = [max(len(compat_str(v)) for v in col) for col in zip(*table)]
2294 format_str = ' '.join('%-' + compat_str(ml + 1) + 's' for ml in max_lens[:-1]) + '%s'
2295 return '\n'.join(format_str % tuple(row) for row in table)
2298 def _match_one(filter_part, dct):
2299 COMPARISON_OPERATORS = {
2307 operator_rex = re.compile(r'''(?x)\s*
2309 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
2311 (?P<intval>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)|
2312 (?P<strval>(?![0-9.])[a-z0-9A-Z]*)
2315 ''' % '|'.join(map(re.escape, COMPARISON_OPERATORS.keys())))
2316 m = operator_rex.search(filter_part)
2318 op = COMPARISON_OPERATORS[m.group('op')]
2319 if m.group('strval') is not None:
2320 if m.group('op') not in ('=', '!='):
2322 'Operator %s does not support string values!' % m.group('op'))
2323 comparison_value = m.group('strval')
2326 comparison_value = int(m.group('intval'))
2328 comparison_value = parse_filesize(m.group('intval'))
2329 if comparison_value is None:
2330 comparison_value = parse_filesize(m.group('intval') + 'B')
2331 if comparison_value is None:
2333 'Invalid integer value %r in filter part %r' % (
2334 m.group('intval'), filter_part))
2335 actual_value = dct.get(m.group('key'))
2336 if actual_value is None:
2337 return m.group('none_inclusive')
2338 return op(actual_value, comparison_value)
2341 '': lambda v: v is not None,
2342 '!': lambda v: v is None,
2344 operator_rex = re.compile(r'''(?x)\s*
2345 (?P<op>%s)\s*(?P<key>[a-z_]+)
2347 ''' % '|'.join(map(re.escape, UNARY_OPERATORS.keys())))
2348 m = operator_rex.search(filter_part)
2350 op = UNARY_OPERATORS[m.group('op')]
2351 actual_value = dct.get(m.group('key'))
2352 return op(actual_value)
2354 raise ValueError('Invalid filter part %r' % filter_part)
2357 def match_str(filter_str, dct):
2358 """ Filter a dictionary with a simple string syntax. Returns True (=passes filter) or false """
2361 _match_one(filter_part, dct) for filter_part in filter_str.split('&'))
2364 def match_filter_func(filter_str):
2365 def _match_func(info_dict):
2366 if match_str(filter_str, info_dict):
2369 video_title = info_dict.get('title', info_dict.get('id', 'video'))
2370 return '%s does not pass filter %s, skipping ..' % (video_title, filter_str)
2374 def parse_dfxp_time_expr(time_expr):
2378 mobj = re.match(r'^(?P<time_offset>\d+(?:\.\d+)?)s?$', time_expr)
2380 return float(mobj.group('time_offset'))
2382 mobj = re.match(r'^(\d+):(\d\d):(\d\d(?:(?:\.|:)\d+)?)$', time_expr)
2384 return 3600 * int(mobj.group(1)) + 60 * int(mobj.group(2)) + float(mobj.group(3).replace(':', '.'))
2387 def srt_subtitles_timecode(seconds):
2388 return '%02d:%02d:%02d,%03d' % (seconds / 3600, (seconds % 3600) / 60, seconds % 60, (seconds % 1) * 1000)
2391 def dfxp2srt(dfxp_data):
2392 _x = functools.partial(xpath_with_ns, ns_map={
2393 'ttml': 'http://www.w3.org/ns/ttml',
2394 'ttaf1': 'http://www.w3.org/2006/10/ttaf1',
2395 'ttaf1_0604': 'http://www.w3.org/2006/04/ttaf1',
2398 class TTMLPElementParser(object):
2401 def start(self, tag, attrib):
2402 if tag in (_x('ttml:br'), _x('ttaf1:br'), 'br'):
2408 def data(self, data):
2412 return self.out.strip()
2414 def parse_node(node):
2415 target = TTMLPElementParser()
2416 parser = xml.etree.ElementTree.XMLParser(target=target)
2417 parser.feed(xml.etree.ElementTree.tostring(node))
2418 return parser.close()
2420 dfxp = compat_etree_fromstring(dfxp_data.encode('utf-8'))
2422 paras = dfxp.findall(_x('.//ttml:p')) or dfxp.findall(_x('.//ttaf1:p')) or dfxp.findall(_x('.//ttaf1_0604:p')) or dfxp.findall('.//p')
2425 raise ValueError('Invalid dfxp/TTML subtitle')
2427 for para, index in zip(paras, itertools.count(1)):
2428 begin_time = parse_dfxp_time_expr(para.attrib.get('begin'))
2429 end_time = parse_dfxp_time_expr(para.attrib.get('end'))
2430 dur = parse_dfxp_time_expr(para.attrib.get('dur'))
2431 if begin_time is None:
2436 end_time = begin_time + dur
2437 out.append('%d\n%s --> %s\n%s\n\n' % (
2439 srt_subtitles_timecode(begin_time),
2440 srt_subtitles_timecode(end_time),
2446 def cli_option(params, command_option, param):
2447 param = params.get(param)
2449 param = compat_str(param)
2450 return [command_option, param] if param is not None else []
2453 def cli_bool_option(params, command_option, param, true_value='true', false_value='false', separator=None):
2454 param = params.get(param)
2455 assert isinstance(param, bool)
2457 return [command_option + separator + (true_value if param else false_value)]
2458 return [command_option, true_value if param else false_value]
2461 def cli_valueless_option(params, command_option, param, expected_value=True):
2462 param = params.get(param)
2463 return [command_option] if param == expected_value else []
2466 def cli_configuration_args(params, param, default=[]):
2467 ex_args = params.get(param)
2470 assert isinstance(ex_args, list)
2474 class ISO639Utils(object):
2475 # See http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
2664 def short2long(cls, code):
2665 """Convert language code from ISO 639-1 to ISO 639-2/T"""
2666 return cls._lang_map.get(code[:2])
2669 def long2short(cls, code):
2670 """Convert language code from ISO 639-2/T to ISO 639-1"""
2671 for short_name, long_name in cls._lang_map.items():
2672 if long_name == code:
2676 class ISO3166Utils(object):
2677 # From http://data.okfn.org/data/core/country-list
2679 'AF': 'Afghanistan',
2680 'AX': 'Åland Islands',
2683 'AS': 'American Samoa',
2688 'AG': 'Antigua and Barbuda',
2705 'BO': 'Bolivia, Plurinational State of',
2706 'BQ': 'Bonaire, Sint Eustatius and Saba',
2707 'BA': 'Bosnia and Herzegovina',
2709 'BV': 'Bouvet Island',
2711 'IO': 'British Indian Ocean Territory',
2712 'BN': 'Brunei Darussalam',
2714 'BF': 'Burkina Faso',
2720 'KY': 'Cayman Islands',
2721 'CF': 'Central African Republic',
2725 'CX': 'Christmas Island',
2726 'CC': 'Cocos (Keeling) Islands',
2730 'CD': 'Congo, the Democratic Republic of the',
2731 'CK': 'Cook Islands',
2733 'CI': 'Côte d\'Ivoire',
2738 'CZ': 'Czech Republic',
2742 'DO': 'Dominican Republic',
2745 'SV': 'El Salvador',
2746 'GQ': 'Equatorial Guinea',
2750 'FK': 'Falkland Islands (Malvinas)',
2751 'FO': 'Faroe Islands',
2755 'GF': 'French Guiana',
2756 'PF': 'French Polynesia',
2757 'TF': 'French Southern Territories',
2772 'GW': 'Guinea-Bissau',
2775 'HM': 'Heard Island and McDonald Islands',
2776 'VA': 'Holy See (Vatican City State)',
2783 'IR': 'Iran, Islamic Republic of',
2786 'IM': 'Isle of Man',
2796 'KP': 'Korea, Democratic People\'s Republic of',
2797 'KR': 'Korea, Republic of',
2800 'LA': 'Lao People\'s Democratic Republic',
2806 'LI': 'Liechtenstein',
2810 'MK': 'Macedonia, the Former Yugoslav Republic of',
2817 'MH': 'Marshall Islands',
2823 'FM': 'Micronesia, Federated States of',
2824 'MD': 'Moldova, Republic of',
2835 'NL': 'Netherlands',
2836 'NC': 'New Caledonia',
2837 'NZ': 'New Zealand',
2842 'NF': 'Norfolk Island',
2843 'MP': 'Northern Mariana Islands',
2848 'PS': 'Palestine, State of',
2850 'PG': 'Papua New Guinea',
2853 'PH': 'Philippines',
2857 'PR': 'Puerto Rico',
2861 'RU': 'Russian Federation',
2863 'BL': 'Saint Barthélemy',
2864 'SH': 'Saint Helena, Ascension and Tristan da Cunha',
2865 'KN': 'Saint Kitts and Nevis',
2866 'LC': 'Saint Lucia',
2867 'MF': 'Saint Martin (French part)',
2868 'PM': 'Saint Pierre and Miquelon',
2869 'VC': 'Saint Vincent and the Grenadines',
2872 'ST': 'Sao Tome and Principe',
2873 'SA': 'Saudi Arabia',
2877 'SL': 'Sierra Leone',
2879 'SX': 'Sint Maarten (Dutch part)',
2882 'SB': 'Solomon Islands',
2884 'ZA': 'South Africa',
2885 'GS': 'South Georgia and the South Sandwich Islands',
2886 'SS': 'South Sudan',
2891 'SJ': 'Svalbard and Jan Mayen',
2894 'CH': 'Switzerland',
2895 'SY': 'Syrian Arab Republic',
2896 'TW': 'Taiwan, Province of China',
2898 'TZ': 'Tanzania, United Republic of',
2900 'TL': 'Timor-Leste',
2904 'TT': 'Trinidad and Tobago',
2907 'TM': 'Turkmenistan',
2908 'TC': 'Turks and Caicos Islands',
2912 'AE': 'United Arab Emirates',
2913 'GB': 'United Kingdom',
2914 'US': 'United States',
2915 'UM': 'United States Minor Outlying Islands',
2919 'VE': 'Venezuela, Bolivarian Republic of',
2921 'VG': 'Virgin Islands, British',
2922 'VI': 'Virgin Islands, U.S.',
2923 'WF': 'Wallis and Futuna',
2924 'EH': 'Western Sahara',
2931 def short2full(cls, code):
2932 """Convert an ISO 3166-2 country code to the corresponding full name"""
2933 return cls._country_map.get(code.upper())
2936 class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
2937 def __init__(self, proxies=None):
2938 # Set default handlers
2939 for type in ('http', 'https'):
2940 setattr(self, '%s_open' % type,
2941 lambda r, proxy='__noproxy__', type=type, meth=self.proxy_open:
2942 meth(r, proxy, type))
2943 return compat_urllib_request.ProxyHandler.__init__(self, proxies)
2945 def proxy_open(self, req, proxy, type):
2946 req_proxy = req.headers.get('Ytdl-request-proxy')
2947 if req_proxy is not None:
2949 del req.headers['Ytdl-request-proxy']
2951 if proxy == '__noproxy__':
2952 return None # No Proxy
2953 if compat_urlparse.urlparse(proxy).scheme.lower() in ('socks', 'socks4', 'socks4a', 'socks5'):
2954 req.add_header('Ytdl-socks-proxy', proxy)
2955 # youtube-dl's http/https handlers do wrapping the socket with socks
2957 return compat_urllib_request.ProxyHandler.proxy_open(
2958 self, req, proxy, type)
2961 def ohdave_rsa_encrypt(data, exponent, modulus):
2963 Implement OHDave's RSA algorithm. See http://www.ohdave.com/rsa/
2966 data: data to encrypt, bytes-like object
2967 exponent, modulus: parameter e and N of RSA algorithm, both integer
2968 Output: hex string of encrypted data
2970 Limitation: supports one block encryption only
2973 payload = int(binascii.hexlify(data[::-1]), 16)
2974 encrypted = pow(payload, exponent, modulus)
2975 return '%x' % encrypted
2978 def encode_base_n(num, n, table=None):
2979 FULL_TABLE = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
2981 table = FULL_TABLE[:n]
2984 raise ValueError('base %d exceeds table length %d' % (n, len(table)))
2991 ret = table[num % n] + ret
2996 def decode_packed_codes(code):
2998 r"}\('(.+)',(\d+),(\d+),'([^']+)'\.split\('\|'\)",
3000 obfucasted_code, base, count, symbols = mobj.groups()
3003 symbols = symbols.split('|')
3008 base_n_count = encode_base_n(count, base)
3009 symbol_table[base_n_count] = symbols[count] or base_n_count
3012 r'\b(\w+)\b', lambda mobj: symbol_table[mobj.group(0)],
3016 def parse_m3u8_attributes(attrib):
3018 for (key, val) in re.findall(r'(?P<key>[A-Z0-9-]+)=(?P<val>"[^"]+"|[^",]+)(?:,|$)', attrib):
3019 if val.startswith('"'):
3025 def urshift(val, n):
3026 return val >> n if val >= 0 else (val + 0x100000000) >> n
3029 # Based on png2str() written by @gdkchan and improved by @yokrysty
3030 # Originally posted at https://github.com/rg3/youtube-dl/issues/9706
3031 def decode_png(png_data):
3032 # Reference: https://www.w3.org/TR/PNG/
3033 header = png_data[8:]
3035 if png_data[:8] != b'\x89PNG\x0d\x0a\x1a\x0a' or header[4:8] != b'IHDR':
3036 raise IOError('Not a valid PNG file.')
3038 int_map = {1: '>B', 2: '>H', 4: '>I'}
3039 unpack_integer = lambda x: compat_struct_unpack(int_map[len(x)], x)[0]
3044 length = unpack_integer(header[:4])
3047 chunk_type = header[:4]
3050 chunk_data = header[:length]
3051 header = header[length:]
3053 header = header[4:] # Skip CRC
3061 ihdr = chunks[0]['data']
3063 width = unpack_integer(ihdr[:4])
3064 height = unpack_integer(ihdr[4:8])
3068 for chunk in chunks:
3069 if chunk['type'] == b'IDAT':
3070 idat += chunk['data']
3073 raise IOError('Unable to read PNG data.')
3075 decompressed_data = bytearray(zlib.decompress(idat))
3080 def _get_pixel(idx):
3085 for y in range(height):
3086 basePos = y * (1 + stride)
3087 filter_type = decompressed_data[basePos]
3091 pixels.append(current_row)
3093 for x in range(stride):
3094 color = decompressed_data[1 + basePos + x]
3095 basex = y * stride + x
3100 left = _get_pixel(basex - 3)
3102 up = _get_pixel(basex - stride)
3104 if filter_type == 1: # Sub
3105 color = (color + left) & 0xff
3106 elif filter_type == 2: # Up
3107 color = (color + up) & 0xff
3108 elif filter_type == 3: # Average
3109 color = (color + ((left + up) >> 1)) & 0xff
3110 elif filter_type == 4: # Paeth
3116 c = _get_pixel(basex - stride - 3)
3124 if pa <= pb and pa <= pc:
3125 color = (color + a) & 0xff
3127 color = (color + b) & 0xff
3129 color = (color + c) & 0xff
3131 current_row.append(color)
3133 return width, height, pixels