2 # -*- coding: utf-8 -*-
4 from __future__ import unicode_literals
33 import xml.etree.ElementTree
40 compat_etree_fromstring,
42 compat_html_entities_html5,
48 compat_socket_create_connection,
54 compat_urllib_parse_urlencode,
55 compat_urllib_parse_urlparse,
56 compat_urllib_parse_unquote_plus,
57 compat_urllib_request,
68 def register_socks_protocols():
69 # "Register" SOCKS protocols
70 # In Python < 2.6.5, urlsplit() suffers from bug https://bugs.python.org/issue7904
71 # URLs with protocols not in urlparse.uses_netloc are not handled correctly
72 for scheme in ('socks', 'socks4', 'socks4a', 'socks5'):
73 if scheme not in compat_urlparse.uses_netloc:
74 compat_urlparse.uses_netloc.append(scheme)
77 # This is not clearly defined otherwise
78 compiled_regex_type = type(re.compile(''))
81 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20150101 Firefox/47.0 (Chrome)',
82 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
83 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
84 'Accept-Encoding': 'gzip, deflate',
85 'Accept-Language': 'en-us,en;q=0.5',
91 ENGLISH_MONTH_NAMES = [
92 'January', 'February', 'March', 'April', 'May', 'June',
93 'July', 'August', 'September', 'October', 'November', 'December']
96 'en': ENGLISH_MONTH_NAMES,
98 'janvier', 'février', 'mars', 'avril', 'mai', 'juin',
99 'juillet', 'août', 'septembre', 'octobre', 'novembre', 'décembre'],
103 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'aac',
104 'flv', 'f4v', 'f4a', 'f4b',
105 'webm', 'ogg', 'ogv', 'oga', 'ogx', 'spx', 'opus',
106 'mkv', 'mka', 'mk3d',
115 'f4f', 'f4m', 'm3u8', 'smil')
117 # needed for sanitizing filenames in restricted mode
118 ACCENT_CHARS = dict(zip('ÂÃÄÀÁÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖŐØŒÙÚÛÜŰÝÞßàáâãäåæçèéêëìíîïðñòóôõöőøœùúûüűýþÿ',
119 itertools.chain('AAAAAA', ['AE'], 'CEEEEIIIIDNOOOOOOO', ['OE'], 'UUUUUYP', ['ss'],
120 'aaaaaa', ['ae'], 'ceeeeiiiionooooooo', ['oe'], 'uuuuuypy')))
136 '%Y-%m-%d %H:%M:%S.%f',
139 '%Y-%m-%dT%H:%M:%SZ',
140 '%Y-%m-%dT%H:%M:%S.%fZ',
141 '%Y-%m-%dT%H:%M:%S.%f0Z',
143 '%Y-%m-%dT%H:%M:%S.%f',
147 DATE_FORMATS_DAY_FIRST = list(DATE_FORMATS)
148 DATE_FORMATS_DAY_FIRST.extend([
157 DATE_FORMATS_MONTH_FIRST = list(DATE_FORMATS)
158 DATE_FORMATS_MONTH_FIRST.extend([
167 def preferredencoding():
168 """Get preferred encoding.
170 Returns the best encoding scheme for the system, based on
171 locale.getpreferredencoding() and some further tweaks.
174 pref = locale.getpreferredencoding()
182 def write_json_file(obj, fn):
183 """ Encode obj as JSON and write it to fn, atomically if possible """
185 fn = encodeFilename(fn)
186 if sys.version_info < (3, 0) and sys.platform != 'win32':
187 encoding = get_filesystem_encoding()
188 # os.path.basename returns a bytes object, but NamedTemporaryFile
189 # will fail if the filename contains non ascii characters unless we
190 # use a unicode object
191 path_basename = lambda f: os.path.basename(fn).decode(encoding)
192 # the same for os.path.dirname
193 path_dirname = lambda f: os.path.dirname(fn).decode(encoding)
195 path_basename = os.path.basename
196 path_dirname = os.path.dirname
200 'prefix': path_basename(fn) + '.',
201 'dir': path_dirname(fn),
205 # In Python 2.x, json.dump expects a bytestream.
206 # In Python 3.x, it writes to a character stream
207 if sys.version_info < (3, 0):
215 tf = tempfile.NamedTemporaryFile(**compat_kwargs(args))
220 if sys.platform == 'win32':
221 # Need to remove existing file on Windows, else os.rename raises
222 # WindowsError or FileExistsError.
227 os.rename(tf.name, fn)
236 if sys.version_info >= (2, 7):
237 def find_xpath_attr(node, xpath, key, val=None):
238 """ Find the xpath xpath[@key=val] """
239 assert re.match(r'^[a-zA-Z_-]+$', key)
240 expr = xpath + ('[@%s]' % key if val is None else "[@%s='%s']" % (key, val))
241 return node.find(expr)
243 def find_xpath_attr(node, xpath, key, val=None):
244 for f in node.findall(compat_xpath(xpath)):
245 if key not in f.attrib:
247 if val is None or f.attrib.get(key) == val:
251 # On python2.6 the xml.etree.ElementTree.Element methods don't support
252 # the namespace parameter
255 def xpath_with_ns(path, ns_map):
256 components = [c.split(':') for c in path.split('/')]
260 replaced.append(c[0])
263 replaced.append('{%s}%s' % (ns_map[ns], tag))
264 return '/'.join(replaced)
267 def xpath_element(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
268 def _find_xpath(xpath):
269 return node.find(compat_xpath(xpath))
271 if isinstance(xpath, (str, compat_str)):
272 n = _find_xpath(xpath)
280 if default is not NO_DEFAULT:
283 name = xpath if name is None else name
284 raise ExtractorError('Could not find XML element %s' % name)
290 def xpath_text(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
291 n = xpath_element(node, xpath, name, fatal=fatal, default=default)
292 if n is None or n == default:
295 if default is not NO_DEFAULT:
298 name = xpath if name is None else name
299 raise ExtractorError('Could not find XML element\'s text %s' % name)
305 def xpath_attr(node, xpath, key, name=None, fatal=False, default=NO_DEFAULT):
306 n = find_xpath_attr(node, xpath, key)
308 if default is not NO_DEFAULT:
311 name = '%s[@%s]' % (xpath, key) if name is None else name
312 raise ExtractorError('Could not find XML attribute %s' % name)
318 def get_element_by_id(id, html):
319 """Return the content of the tag with the specified ID in the passed HTML document"""
320 return get_element_by_attribute('id', id, html)
323 def get_element_by_class(class_name, html):
324 return get_element_by_attribute(
325 'class', r'[^\'"]*\b%s\b[^\'"]*' % re.escape(class_name),
326 html, escape_value=False)
329 def get_element_by_attribute(attribute, value, html, escape_value=True):
330 """Return the content of the tag with the specified attribute in the passed HTML document"""
332 value = re.escape(value) if escape_value else value
334 m = re.search(r'''(?xs)
336 (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^"]*"|='[^']*'))*?
338 (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^"]*"|='[^']*'))*?
342 ''' % (re.escape(attribute), value), html)
346 res = m.group('content')
348 if res.startswith('"') or res.startswith("'"):
351 return unescapeHTML(res)
354 class HTMLAttributeParser(compat_HTMLParser):
355 """Trivial HTML parser to gather the attributes for a single element"""
358 compat_HTMLParser.__init__(self)
360 def handle_starttag(self, tag, attrs):
361 self.attrs = dict(attrs)
364 def extract_attributes(html_element):
365 """Given a string for an HTML element such as
367 a="foo" B="bar" c="&98;az" d=boz
368 empty= noval entity="&"
371 Decode and return a dictionary of attributes.
373 'a': 'foo', 'b': 'bar', c: 'baz', d: 'boz',
374 'empty': '', 'noval': None, 'entity': '&',
375 'sq': '"', 'dq': '\''
377 NB HTMLParser is stricter in Python 2.6 & 3.2 than in later versions,
378 but the cases in the unit test will work for all of 2.6, 2.7, 3.2-3.5.
380 parser = HTMLAttributeParser()
381 parser.feed(html_element)
386 def clean_html(html):
387 """Clean an HTML snippet into a readable string"""
389 if html is None: # Convenience for sanitizing descriptions etc.
393 html = html.replace('\n', ' ')
394 html = re.sub(r'\s*<\s*br\s*/?\s*>\s*', '\n', html)
395 html = re.sub(r'<\s*/\s*p\s*>\s*<\s*p[^>]*>', '\n', html)
397 html = re.sub('<.*?>', '', html)
398 # Replace html entities
399 html = unescapeHTML(html)
403 def sanitize_open(filename, open_mode):
404 """Try to open the given filename, and slightly tweak it if this fails.
406 Attempts to open the given filename. If this fails, it tries to change
407 the filename slightly, step by step, until it's either able to open it
408 or it fails and raises a final exception, like the standard open()
411 It returns the tuple (stream, definitive_file_name).
415 if sys.platform == 'win32':
417 msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
418 return (sys.stdout.buffer if hasattr(sys.stdout, 'buffer') else sys.stdout, filename)
419 stream = open(encodeFilename(filename), open_mode)
420 return (stream, filename)
421 except (IOError, OSError) as err:
422 if err.errno in (errno.EACCES,):
425 # In case of error, try to remove win32 forbidden chars
426 alt_filename = sanitize_path(filename)
427 if alt_filename == filename:
430 # An exception here should be caught in the caller
431 stream = open(encodeFilename(alt_filename), open_mode)
432 return (stream, alt_filename)
435 def timeconvert(timestr):
436 """Convert RFC 2822 defined time string into system timestamp"""
438 timetuple = email.utils.parsedate_tz(timestr)
439 if timetuple is not None:
440 timestamp = email.utils.mktime_tz(timetuple)
444 def sanitize_filename(s, restricted=False, is_id=False):
445 """Sanitizes a string so it could be used as part of a filename.
446 If restricted is set, use a stricter subset of allowed characters.
447 Set is_id if this is not an arbitrary string, but an ID that should be kept if possible
449 def replace_insane(char):
450 if restricted and char in ACCENT_CHARS:
451 return ACCENT_CHARS[char]
452 if char == '?' or ord(char) < 32 or ord(char) == 127:
455 return '' if restricted else '\''
457 return '_-' if restricted else ' -'
458 elif char in '\\/|*<>':
460 if restricted and (char in '!&\'()[]{}$;`^,#' or char.isspace()):
462 if restricted and ord(char) > 127:
467 s = re.sub(r'[0-9]+(?::[0-9]+)+', lambda m: m.group(0).replace(':', '_'), s)
468 result = ''.join(map(replace_insane, s))
470 while '__' in result:
471 result = result.replace('__', '_')
472 result = result.strip('_')
473 # Common case of "Foreign band name - English song title"
474 if restricted and result.startswith('-_'):
476 if result.startswith('-'):
477 result = '_' + result[len('-'):]
478 result = result.lstrip('.')
484 def sanitize_path(s):
485 """Sanitizes and normalizes path on Windows"""
486 if sys.platform != 'win32':
488 drive_or_unc, _ = os.path.splitdrive(s)
489 if sys.version_info < (2, 7) and not drive_or_unc:
490 drive_or_unc, _ = os.path.splitunc(s)
491 norm_path = os.path.normpath(remove_start(s, drive_or_unc)).split(os.path.sep)
495 path_part if path_part in ['.', '..'] else re.sub('(?:[/<>:"\\|\\\\?\\*]|[\s.]$)', '#', path_part)
496 for path_part in norm_path]
498 sanitized_path.insert(0, drive_or_unc + os.path.sep)
499 return os.path.join(*sanitized_path)
502 # Prepend protocol-less URLs with `http:` scheme in order to mitigate the number of
503 # unwanted failures due to missing protocol
504 def sanitize_url(url):
505 return 'http:%s' % url if url.startswith('//') else url
508 def sanitized_Request(url, *args, **kwargs):
509 return compat_urllib_request.Request(sanitize_url(url), *args, **kwargs)
512 def orderedSet(iterable):
513 """ Remove all duplicates from the input iterable """
521 def _htmlentity_transform(entity_with_semicolon):
522 """Transforms an HTML entity to a character."""
523 entity = entity_with_semicolon[:-1]
525 # Known non-numeric HTML entity
526 if entity in compat_html_entities.name2codepoint:
527 return compat_chr(compat_html_entities.name2codepoint[entity])
529 # TODO: HTML5 allows entities without a semicolon. For example,
530 # 'Éric' should be decoded as 'Éric'.
531 if entity_with_semicolon in compat_html_entities_html5:
532 return compat_html_entities_html5[entity_with_semicolon]
534 mobj = re.match(r'#(x[0-9a-fA-F]+|[0-9]+)', entity)
536 numstr = mobj.group(1)
537 if numstr.startswith('x'):
539 numstr = '0%s' % numstr
542 # See https://github.com/rg3/youtube-dl/issues/7518
544 return compat_chr(int(numstr, base))
548 # Unknown entity in name, return its literal representation
549 return '&%s;' % entity
555 assert type(s) == compat_str
558 r'&([^;]+;)', lambda m: _htmlentity_transform(m.group(1)), s)
561 def get_subprocess_encoding():
562 if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
563 # For subprocess calls, encode with locale encoding
564 # Refer to http://stackoverflow.com/a/9951851/35070
565 encoding = preferredencoding()
567 encoding = sys.getfilesystemencoding()
573 def encodeFilename(s, for_subprocess=False):
575 @param s The name of the file
578 assert type(s) == compat_str
580 # Python 3 has a Unicode API
581 if sys.version_info >= (3, 0):
584 # Pass '' directly to use Unicode APIs on Windows 2000 and up
585 # (Detecting Windows NT 4 is tricky because 'major >= 4' would
586 # match Windows 9x series as well. Besides, NT 4 is obsolete.)
587 if not for_subprocess and sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
590 # Jython assumes filenames are Unicode strings though reported as Python 2.x compatible
591 if sys.platform.startswith('java'):
594 return s.encode(get_subprocess_encoding(), 'ignore')
597 def decodeFilename(b, for_subprocess=False):
599 if sys.version_info >= (3, 0):
602 if not isinstance(b, bytes):
605 return b.decode(get_subprocess_encoding(), 'ignore')
608 def encodeArgument(s):
609 if not isinstance(s, compat_str):
610 # Legacy code that uses byte strings
611 # Uncomment the following line after fixing all post processors
612 # assert False, 'Internal error: %r should be of type %r, is %r' % (s, compat_str, type(s))
613 s = s.decode('ascii')
614 return encodeFilename(s, True)
617 def decodeArgument(b):
618 return decodeFilename(b, True)
621 def decodeOption(optval):
624 if isinstance(optval, bytes):
625 optval = optval.decode(preferredencoding())
627 assert isinstance(optval, compat_str)
631 def formatSeconds(secs):
633 return '%d:%02d:%02d' % (secs // 3600, (secs % 3600) // 60, secs % 60)
635 return '%d:%02d' % (secs // 60, secs % 60)
640 def make_HTTPS_handler(params, **kwargs):
641 opts_no_check_certificate = params.get('nocheckcertificate', False)
642 if hasattr(ssl, 'create_default_context'): # Python >= 3.4 or 2.7.9
643 context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
644 if opts_no_check_certificate:
645 context.check_hostname = False
646 context.verify_mode = ssl.CERT_NONE
648 return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
651 # (create_default_context present but HTTPSHandler has no context=)
654 if sys.version_info < (3, 2):
655 return YoutubeDLHTTPSHandler(params, **kwargs)
657 context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
658 context.verify_mode = (ssl.CERT_NONE
659 if opts_no_check_certificate
660 else ssl.CERT_REQUIRED)
661 context.set_default_verify_paths()
662 return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
665 def bug_reports_message():
666 if ytdl_is_updateable():
667 update_cmd = 'type youtube-dl -U to update'
669 update_cmd = 'see https://yt-dl.org/update on how to update'
670 msg = '; please report this issue on https://yt-dl.org/bug .'
671 msg += ' Make sure you are using the latest version; %s.' % update_cmd
672 msg += ' Be sure to call youtube-dl with the --verbose flag and include its complete output.'
676 class ExtractorError(Exception):
677 """Error during info extraction."""
679 def __init__(self, msg, tb=None, expected=False, cause=None, video_id=None):
680 """ tb, if given, is the original traceback (so that it can be printed out).
681 If expected is set, this is a normal error message and most likely not a bug in youtube-dl.
684 if sys.exc_info()[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError):
686 if video_id is not None:
687 msg = video_id + ': ' + msg
689 msg += ' (caused by %r)' % cause
691 msg += bug_reports_message()
692 super(ExtractorError, self).__init__(msg)
695 self.exc_info = sys.exc_info() # preserve original exception
697 self.video_id = video_id
699 def format_traceback(self):
700 if self.traceback is None:
702 return ''.join(traceback.format_tb(self.traceback))
705 class UnsupportedError(ExtractorError):
706 def __init__(self, url):
707 super(UnsupportedError, self).__init__(
708 'Unsupported URL: %s' % url, expected=True)
712 class RegexNotFoundError(ExtractorError):
713 """Error when a regex didn't match"""
717 class DownloadError(Exception):
718 """Download Error exception.
720 This exception may be thrown by FileDownloader objects if they are not
721 configured to continue on errors. They will contain the appropriate
725 def __init__(self, msg, exc_info=None):
726 """ exc_info, if given, is the original exception that caused the trouble (as returned by sys.exc_info()). """
727 super(DownloadError, self).__init__(msg)
728 self.exc_info = exc_info
731 class SameFileError(Exception):
732 """Same File exception.
734 This exception will be thrown by FileDownloader objects if they detect
735 multiple files would have to be downloaded to the same file on disk.
740 class PostProcessingError(Exception):
741 """Post Processing exception.
743 This exception may be raised by PostProcessor's .run() method to
744 indicate an error in the postprocessing task.
747 def __init__(self, msg):
751 class MaxDownloadsReached(Exception):
752 """ --max-downloads limit has been reached. """
756 class UnavailableVideoError(Exception):
757 """Unavailable Format exception.
759 This exception will be thrown when a video is requested
760 in a format that is not available for that video.
765 class ContentTooShortError(Exception):
766 """Content Too Short exception.
768 This exception may be raised by FileDownloader objects when a file they
769 download is too small for what the server announced first, indicating
770 the connection was probably interrupted.
773 def __init__(self, downloaded, expected):
775 self.downloaded = downloaded
776 self.expected = expected
779 class XAttrMetadataError(Exception):
780 def __init__(self, code=None, msg='Unknown error'):
781 super(XAttrMetadataError, self).__init__(msg)
784 # Parsing code and msg
785 if (self.code in (errno.ENOSPC, errno.EDQUOT) or
786 'No space left' in self.msg or 'Disk quota excedded' in self.msg):
787 self.reason = 'NO_SPACE'
788 elif self.code == errno.E2BIG or 'Argument list too long' in self.msg:
789 self.reason = 'VALUE_TOO_LONG'
791 self.reason = 'NOT_SUPPORTED'
794 class XAttrUnavailableError(Exception):
798 def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs):
799 # Working around python 2 bug (see http://bugs.python.org/issue17849) by limiting
800 # expected HTTP responses to meet HTTP/1.0 or later (see also
801 # https://github.com/rg3/youtube-dl/issues/6727)
802 if sys.version_info < (3, 0):
803 kwargs[b'strict'] = True
804 hc = http_class(*args, **kwargs)
805 source_address = ydl_handler._params.get('source_address')
806 if source_address is not None:
807 sa = (source_address, 0)
808 if hasattr(hc, 'source_address'): # Python 2.7+
809 hc.source_address = sa
811 def _hc_connect(self, *args, **kwargs):
812 sock = compat_socket_create_connection(
813 (self.host, self.port), self.timeout, sa)
815 self.sock = ssl.wrap_socket(
816 sock, self.key_file, self.cert_file,
817 ssl_version=ssl.PROTOCOL_TLSv1)
820 hc.connect = functools.partial(_hc_connect, hc)
825 def handle_youtubedl_headers(headers):
826 filtered_headers = headers
828 if 'Youtubedl-no-compression' in filtered_headers:
829 filtered_headers = dict((k, v) for k, v in filtered_headers.items() if k.lower() != 'accept-encoding')
830 del filtered_headers['Youtubedl-no-compression']
832 return filtered_headers
835 class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
836 """Handler for HTTP requests and responses.
838 This class, when installed with an OpenerDirector, automatically adds
839 the standard headers to every HTTP request and handles gzipped and
840 deflated responses from web servers. If compression is to be avoided in
841 a particular request, the original request in the program code only has
842 to include the HTTP header "Youtubedl-no-compression", which will be
843 removed before making the real request.
845 Part of this code was copied from:
847 http://techknack.net/python-urllib2-handlers/
849 Andrew Rowls, the author of that code, agreed to release it to the
853 def __init__(self, params, *args, **kwargs):
854 compat_urllib_request.HTTPHandler.__init__(self, *args, **kwargs)
855 self._params = params
857 def http_open(self, req):
858 conn_class = compat_http_client.HTTPConnection
860 socks_proxy = req.headers.get('Ytdl-socks-proxy')
862 conn_class = make_socks_conn_class(conn_class, socks_proxy)
863 del req.headers['Ytdl-socks-proxy']
865 return self.do_open(functools.partial(
866 _create_http_connection, self, conn_class, False),
872 return zlib.decompress(data, -zlib.MAX_WBITS)
874 return zlib.decompress(data)
877 def addinfourl_wrapper(stream, headers, url, code):
878 if hasattr(compat_urllib_request.addinfourl, 'getcode'):
879 return compat_urllib_request.addinfourl(stream, headers, url, code)
880 ret = compat_urllib_request.addinfourl(stream, headers, url)
884 def http_request(self, req):
885 # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
886 # always respected by websites, some tend to give out URLs with non percent-encoded
887 # non-ASCII characters (see telemb.py, ard.py [#3412])
888 # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
889 # To work around aforementioned issue we will replace request's original URL with
890 # percent-encoded one
891 # Since redirects are also affected (e.g. http://www.southpark.de/alle-episoden/s18e09)
892 # the code of this workaround has been moved here from YoutubeDL.urlopen()
893 url = req.get_full_url()
894 url_escaped = escape_url(url)
896 # Substitute URL if any change after escaping
897 if url != url_escaped:
898 req = update_Request(req, url=url_escaped)
900 for h, v in std_headers.items():
901 # Capitalize is needed because of Python bug 2275: http://bugs.python.org/issue2275
902 # The dict keys are capitalized because of this bug by urllib
903 if h.capitalize() not in req.headers:
906 req.headers = handle_youtubedl_headers(req.headers)
908 if sys.version_info < (2, 7) and '#' in req.get_full_url():
909 # Python 2.6 is brain-dead when it comes to fragments
910 req._Request__original = req._Request__original.partition('#')[0]
911 req._Request__r_type = req._Request__r_type.partition('#')[0]
915 def http_response(self, req, resp):
918 if resp.headers.get('Content-encoding', '') == 'gzip':
919 content = resp.read()
920 gz = gzip.GzipFile(fileobj=io.BytesIO(content), mode='rb')
922 uncompressed = io.BytesIO(gz.read())
923 except IOError as original_ioerror:
924 # There may be junk add the end of the file
925 # See http://stackoverflow.com/q/4928560/35070 for details
926 for i in range(1, 1024):
928 gz = gzip.GzipFile(fileobj=io.BytesIO(content[:-i]), mode='rb')
929 uncompressed = io.BytesIO(gz.read())
934 raise original_ioerror
935 resp = self.addinfourl_wrapper(uncompressed, old_resp.headers, old_resp.url, old_resp.code)
936 resp.msg = old_resp.msg
937 del resp.headers['Content-encoding']
939 if resp.headers.get('Content-encoding', '') == 'deflate':
940 gz = io.BytesIO(self.deflate(resp.read()))
941 resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
942 resp.msg = old_resp.msg
943 del resp.headers['Content-encoding']
944 # Percent-encode redirect URL of Location HTTP header to satisfy RFC 3986 (see
945 # https://github.com/rg3/youtube-dl/issues/6457).
946 if 300 <= resp.code < 400:
947 location = resp.headers.get('Location')
949 # As of RFC 2616 default charset is iso-8859-1 that is respected by python 3
950 if sys.version_info >= (3, 0):
951 location = location.encode('iso-8859-1').decode('utf-8')
953 location = location.decode('utf-8')
954 location_escaped = escape_url(location)
955 if location != location_escaped:
956 del resp.headers['Location']
957 if sys.version_info < (3, 0):
958 location_escaped = location_escaped.encode('utf-8')
959 resp.headers['Location'] = location_escaped
962 https_request = http_request
963 https_response = http_response
966 def make_socks_conn_class(base_class, socks_proxy):
967 assert issubclass(base_class, (
968 compat_http_client.HTTPConnection, compat_http_client.HTTPSConnection))
970 url_components = compat_urlparse.urlparse(socks_proxy)
971 if url_components.scheme.lower() == 'socks5':
972 socks_type = ProxyType.SOCKS5
973 elif url_components.scheme.lower() in ('socks', 'socks4'):
974 socks_type = ProxyType.SOCKS4
975 elif url_components.scheme.lower() == 'socks4a':
976 socks_type = ProxyType.SOCKS4A
978 def unquote_if_non_empty(s):
981 return compat_urllib_parse_unquote_plus(s)
985 url_components.hostname, url_components.port or 1080,
987 unquote_if_non_empty(url_components.username),
988 unquote_if_non_empty(url_components.password),
991 class SocksConnection(base_class):
993 self.sock = sockssocket()
994 self.sock.setproxy(*proxy_args)
995 if type(self.timeout) in (int, float):
996 self.sock.settimeout(self.timeout)
997 self.sock.connect((self.host, self.port))
999 if isinstance(self, compat_http_client.HTTPSConnection):
1000 if hasattr(self, '_context'): # Python > 2.6
1001 self.sock = self._context.wrap_socket(
1002 self.sock, server_hostname=self.host)
1004 self.sock = ssl.wrap_socket(self.sock)
1006 return SocksConnection
1009 class YoutubeDLHTTPSHandler(compat_urllib_request.HTTPSHandler):
1010 def __init__(self, params, https_conn_class=None, *args, **kwargs):
1011 compat_urllib_request.HTTPSHandler.__init__(self, *args, **kwargs)
1012 self._https_conn_class = https_conn_class or compat_http_client.HTTPSConnection
1013 self._params = params
1015 def https_open(self, req):
1017 conn_class = self._https_conn_class
1019 if hasattr(self, '_context'): # python > 2.6
1020 kwargs['context'] = self._context
1021 if hasattr(self, '_check_hostname'): # python 3.x
1022 kwargs['check_hostname'] = self._check_hostname
1024 socks_proxy = req.headers.get('Ytdl-socks-proxy')
1026 conn_class = make_socks_conn_class(conn_class, socks_proxy)
1027 del req.headers['Ytdl-socks-proxy']
1029 return self.do_open(functools.partial(
1030 _create_http_connection, self, conn_class, True),
1034 class YoutubeDLCookieProcessor(compat_urllib_request.HTTPCookieProcessor):
1035 def __init__(self, cookiejar=None):
1036 compat_urllib_request.HTTPCookieProcessor.__init__(self, cookiejar)
1038 def http_response(self, request, response):
1039 # Python 2 will choke on next HTTP request in row if there are non-ASCII
1040 # characters in Set-Cookie HTTP header of last response (see
1041 # https://github.com/rg3/youtube-dl/issues/6769).
1042 # In order to at least prevent crashing we will percent encode Set-Cookie
1043 # header before HTTPCookieProcessor starts processing it.
1044 # if sys.version_info < (3, 0) and response.headers:
1045 # for set_cookie_header in ('Set-Cookie', 'Set-Cookie2'):
1046 # set_cookie = response.headers.get(set_cookie_header)
1048 # set_cookie_escaped = compat_urllib_parse.quote(set_cookie, b"%/;:@&=+$,!~*'()?#[] ")
1049 # if set_cookie != set_cookie_escaped:
1050 # del response.headers[set_cookie_header]
1051 # response.headers[set_cookie_header] = set_cookie_escaped
1052 return compat_urllib_request.HTTPCookieProcessor.http_response(self, request, response)
1054 https_request = compat_urllib_request.HTTPCookieProcessor.http_request
1055 https_response = http_response
1058 def extract_timezone(date_str):
1060 r'^.{8,}?(?P<tz>Z$| ?(?P<sign>\+|-)(?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})$)',
1063 timezone = datetime.timedelta()
1065 date_str = date_str[:-len(m.group('tz'))]
1066 if not m.group('sign'):
1067 timezone = datetime.timedelta()
1069 sign = 1 if m.group('sign') == '+' else -1
1070 timezone = datetime.timedelta(
1071 hours=sign * int(m.group('hours')),
1072 minutes=sign * int(m.group('minutes')))
1073 return timezone, date_str
1076 def parse_iso8601(date_str, delimiter='T', timezone=None):
1077 """ Return a UNIX timestamp from the given date """
1079 if date_str is None:
1082 date_str = re.sub(r'\.[0-9]+', '', date_str)
1084 if timezone is None:
1085 timezone, date_str = extract_timezone(date_str)
1088 date_format = '%Y-%m-%d{0}%H:%M:%S'.format(delimiter)
1089 dt = datetime.datetime.strptime(date_str, date_format) - timezone
1090 return calendar.timegm(dt.timetuple())
1095 def date_formats(day_first=True):
1096 return DATE_FORMATS_DAY_FIRST if day_first else DATE_FORMATS_MONTH_FIRST
1099 def unified_strdate(date_str, day_first=True):
1100 """Return a string with the date in the format YYYYMMDD"""
1102 if date_str is None:
1106 date_str = date_str.replace(',', ' ')
1107 # Remove AM/PM + timezone
1108 date_str = re.sub(r'(?i)\s*(?:AM|PM)(?:\s+[A-Z]+)?', '', date_str)
1109 _, date_str = extract_timezone(date_str)
1111 for expression in date_formats(day_first):
1113 upload_date = datetime.datetime.strptime(date_str, expression).strftime('%Y%m%d')
1116 if upload_date is None:
1117 timetuple = email.utils.parsedate_tz(date_str)
1120 upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d')
1123 if upload_date is not None:
1124 return compat_str(upload_date)
1127 def unified_timestamp(date_str, day_first=True):
1128 if date_str is None:
1131 date_str = date_str.replace(',', ' ')
1133 pm_delta = 12 if re.search(r'(?i)PM', date_str) else 0
1134 timezone, date_str = extract_timezone(date_str)
1136 # Remove AM/PM + timezone
1137 date_str = re.sub(r'(?i)\s*(?:AM|PM)(?:\s+[A-Z]+)?', '', date_str)
1139 for expression in date_formats(day_first):
1141 dt = datetime.datetime.strptime(date_str, expression) - timezone + datetime.timedelta(hours=pm_delta)
1142 return calendar.timegm(dt.timetuple())
1145 timetuple = email.utils.parsedate_tz(date_str)
1147 return calendar.timegm(timetuple) + pm_delta * 3600
1150 def determine_ext(url, default_ext='unknown_video'):
1153 guess = url.partition('?')[0].rpartition('.')[2]
1154 if re.match(r'^[A-Za-z0-9]+$', guess):
1156 # Try extract ext from URLs like http://example.com/foo/bar.mp4/?download
1157 elif guess.rstrip('/') in KNOWN_EXTENSIONS:
1158 return guess.rstrip('/')
1163 def subtitles_filename(filename, sub_lang, sub_format):
1164 return filename.rsplit('.', 1)[0] + '.' + sub_lang + '.' + sub_format
1167 def date_from_str(date_str):
1169 Return a datetime object from a string in the format YYYYMMDD or
1170 (now|today)[+-][0-9](day|week|month|year)(s)?"""
1171 today = datetime.date.today()
1172 if date_str in ('now', 'today'):
1174 if date_str == 'yesterday':
1175 return today - datetime.timedelta(days=1)
1176 match = re.match('(now|today)(?P<sign>[+-])(?P<time>\d+)(?P<unit>day|week|month|year)(s)?', date_str)
1177 if match is not None:
1178 sign = match.group('sign')
1179 time = int(match.group('time'))
1182 unit = match.group('unit')
1183 # A bad approximation?
1187 elif unit == 'year':
1191 delta = datetime.timedelta(**{unit: time})
1192 return today + delta
1193 return datetime.datetime.strptime(date_str, '%Y%m%d').date()
1196 def hyphenate_date(date_str):
1198 Convert a date in 'YYYYMMDD' format to 'YYYY-MM-DD' format"""
1199 match = re.match(r'^(\d\d\d\d)(\d\d)(\d\d)$', date_str)
1200 if match is not None:
1201 return '-'.join(match.groups())
1206 class DateRange(object):
1207 """Represents a time interval between two dates"""
1209 def __init__(self, start=None, end=None):
1210 """start and end must be strings in the format accepted by date"""
1211 if start is not None:
1212 self.start = date_from_str(start)
1214 self.start = datetime.datetime.min.date()
1216 self.end = date_from_str(end)
1218 self.end = datetime.datetime.max.date()
1219 if self.start > self.end:
1220 raise ValueError('Date range: "%s" , the start date must be before the end date' % self)
1224 """Returns a range that only contains the given day"""
1225 return cls(day, day)
1227 def __contains__(self, date):
1228 """Check if the date is in the range"""
1229 if not isinstance(date, datetime.date):
1230 date = date_from_str(date)
1231 return self.start <= date <= self.end
1234 return '%s - %s' % (self.start.isoformat(), self.end.isoformat())
1237 def platform_name():
1238 """ Returns the platform name as a compat_str """
1239 res = platform.platform()
1240 if isinstance(res, bytes):
1241 res = res.decode(preferredencoding())
1243 assert isinstance(res, compat_str)
1247 def _windows_write_string(s, out):
1248 """ Returns True if the string was written using special methods,
1249 False if it has yet to be written out."""
1250 # Adapted from http://stackoverflow.com/a/3259271/35070
1253 import ctypes.wintypes
1261 fileno = out.fileno()
1262 except AttributeError:
1263 # If the output stream doesn't have a fileno, it's virtual
1265 except io.UnsupportedOperation:
1266 # Some strange Windows pseudo files?
1268 if fileno not in WIN_OUTPUT_IDS:
1271 GetStdHandle = ctypes.WINFUNCTYPE(
1272 ctypes.wintypes.HANDLE, ctypes.wintypes.DWORD)(
1273 (b'GetStdHandle', ctypes.windll.kernel32))
1274 h = GetStdHandle(WIN_OUTPUT_IDS[fileno])
1276 WriteConsoleW = ctypes.WINFUNCTYPE(
1277 ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE, ctypes.wintypes.LPWSTR,
1278 ctypes.wintypes.DWORD, ctypes.POINTER(ctypes.wintypes.DWORD),
1279 ctypes.wintypes.LPVOID)((b'WriteConsoleW', ctypes.windll.kernel32))
1280 written = ctypes.wintypes.DWORD(0)
1282 GetFileType = ctypes.WINFUNCTYPE(ctypes.wintypes.DWORD, ctypes.wintypes.DWORD)((b'GetFileType', ctypes.windll.kernel32))
1283 FILE_TYPE_CHAR = 0x0002
1284 FILE_TYPE_REMOTE = 0x8000
1285 GetConsoleMode = ctypes.WINFUNCTYPE(
1286 ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE,
1287 ctypes.POINTER(ctypes.wintypes.DWORD))(
1288 (b'GetConsoleMode', ctypes.windll.kernel32))
1289 INVALID_HANDLE_VALUE = ctypes.wintypes.DWORD(-1).value
1291 def not_a_console(handle):
1292 if handle == INVALID_HANDLE_VALUE or handle is None:
1294 return ((GetFileType(handle) & ~FILE_TYPE_REMOTE) != FILE_TYPE_CHAR or
1295 GetConsoleMode(handle, ctypes.byref(ctypes.wintypes.DWORD())) == 0)
1297 if not_a_console(h):
1300 def next_nonbmp_pos(s):
1302 return next(i for i, c in enumerate(s) if ord(c) > 0xffff)
1303 except StopIteration:
1307 count = min(next_nonbmp_pos(s), 1024)
1309 ret = WriteConsoleW(
1310 h, s, count if count else 2, ctypes.byref(written), None)
1312 raise OSError('Failed to write string')
1313 if not count: # We just wrote a non-BMP character
1314 assert written.value == 2
1317 assert written.value > 0
1318 s = s[written.value:]
1322 def write_string(s, out=None, encoding=None):
1325 assert type(s) == compat_str
1327 if sys.platform == 'win32' and encoding is None and hasattr(out, 'fileno'):
1328 if _windows_write_string(s, out):
1331 if ('b' in getattr(out, 'mode', '') or
1332 sys.version_info[0] < 3): # Python 2 lies about mode of sys.stderr
1333 byt = s.encode(encoding or preferredencoding(), 'ignore')
1335 elif hasattr(out, 'buffer'):
1336 enc = encoding or getattr(out, 'encoding', None) or preferredencoding()
1337 byt = s.encode(enc, 'ignore')
1338 out.buffer.write(byt)
1344 def bytes_to_intlist(bs):
1347 if isinstance(bs[0], int): # Python 3
1350 return [ord(c) for c in bs]
1353 def intlist_to_bytes(xs):
1356 return compat_struct_pack('%dB' % len(xs), *xs)
1359 # Cross-platform file locking
1360 if sys.platform == 'win32':
1361 import ctypes.wintypes
1364 class OVERLAPPED(ctypes.Structure):
1366 ('Internal', ctypes.wintypes.LPVOID),
1367 ('InternalHigh', ctypes.wintypes.LPVOID),
1368 ('Offset', ctypes.wintypes.DWORD),
1369 ('OffsetHigh', ctypes.wintypes.DWORD),
1370 ('hEvent', ctypes.wintypes.HANDLE),
1373 kernel32 = ctypes.windll.kernel32
1374 LockFileEx = kernel32.LockFileEx
1375 LockFileEx.argtypes = [
1376 ctypes.wintypes.HANDLE, # hFile
1377 ctypes.wintypes.DWORD, # dwFlags
1378 ctypes.wintypes.DWORD, # dwReserved
1379 ctypes.wintypes.DWORD, # nNumberOfBytesToLockLow
1380 ctypes.wintypes.DWORD, # nNumberOfBytesToLockHigh
1381 ctypes.POINTER(OVERLAPPED) # Overlapped
1383 LockFileEx.restype = ctypes.wintypes.BOOL
1384 UnlockFileEx = kernel32.UnlockFileEx
1385 UnlockFileEx.argtypes = [
1386 ctypes.wintypes.HANDLE, # hFile
1387 ctypes.wintypes.DWORD, # dwReserved
1388 ctypes.wintypes.DWORD, # nNumberOfBytesToLockLow
1389 ctypes.wintypes.DWORD, # nNumberOfBytesToLockHigh
1390 ctypes.POINTER(OVERLAPPED) # Overlapped
1392 UnlockFileEx.restype = ctypes.wintypes.BOOL
1393 whole_low = 0xffffffff
1394 whole_high = 0x7fffffff
1396 def _lock_file(f, exclusive):
1397 overlapped = OVERLAPPED()
1398 overlapped.Offset = 0
1399 overlapped.OffsetHigh = 0
1400 overlapped.hEvent = 0
1401 f._lock_file_overlapped_p = ctypes.pointer(overlapped)
1402 handle = msvcrt.get_osfhandle(f.fileno())
1403 if not LockFileEx(handle, 0x2 if exclusive else 0x0, 0,
1404 whole_low, whole_high, f._lock_file_overlapped_p):
1405 raise OSError('Locking file failed: %r' % ctypes.FormatError())
1407 def _unlock_file(f):
1408 assert f._lock_file_overlapped_p
1409 handle = msvcrt.get_osfhandle(f.fileno())
1410 if not UnlockFileEx(handle, 0,
1411 whole_low, whole_high, f._lock_file_overlapped_p):
1412 raise OSError('Unlocking file failed: %r' % ctypes.FormatError())
1415 # Some platforms, such as Jython, is missing fcntl
1419 def _lock_file(f, exclusive):
1420 fcntl.flock(f, fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH)
1422 def _unlock_file(f):
1423 fcntl.flock(f, fcntl.LOCK_UN)
1425 UNSUPPORTED_MSG = 'file locking is not supported on this platform'
1427 def _lock_file(f, exclusive):
1428 raise IOError(UNSUPPORTED_MSG)
1430 def _unlock_file(f):
1431 raise IOError(UNSUPPORTED_MSG)
1434 class locked_file(object):
1435 def __init__(self, filename, mode, encoding=None):
1436 assert mode in ['r', 'a', 'w']
1437 self.f = io.open(filename, mode, encoding=encoding)
1440 def __enter__(self):
1441 exclusive = self.mode != 'r'
1443 _lock_file(self.f, exclusive)
1449 def __exit__(self, etype, value, traceback):
1451 _unlock_file(self.f)
1458 def write(self, *args):
1459 return self.f.write(*args)
1461 def read(self, *args):
1462 return self.f.read(*args)
1465 def get_filesystem_encoding():
1466 encoding = sys.getfilesystemencoding()
1467 return encoding if encoding is not None else 'utf-8'
1470 def shell_quote(args):
1472 encoding = get_filesystem_encoding()
1474 if isinstance(a, bytes):
1475 # We may get a filename encoded with 'encodeFilename'
1476 a = a.decode(encoding)
1477 quoted_args.append(pipes.quote(a))
1478 return ' '.join(quoted_args)
1481 def smuggle_url(url, data):
1482 """ Pass additional data in a URL for internal use. """
1484 url, idata = unsmuggle_url(url, {})
1486 sdata = compat_urllib_parse_urlencode(
1487 {'__youtubedl_smuggle': json.dumps(data)})
1488 return url + '#' + sdata
1491 def unsmuggle_url(smug_url, default=None):
1492 if '#__youtubedl_smuggle' not in smug_url:
1493 return smug_url, default
1494 url, _, sdata = smug_url.rpartition('#')
1495 jsond = compat_parse_qs(sdata)['__youtubedl_smuggle'][0]
1496 data = json.loads(jsond)
1500 def format_bytes(bytes):
1503 if type(bytes) is str:
1504 bytes = float(bytes)
1508 exponent = int(math.log(bytes, 1024.0))
1509 suffix = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'][exponent]
1510 converted = float(bytes) / float(1024 ** exponent)
1511 return '%.2f%s' % (converted, suffix)
1514 def lookup_unit_table(unit_table, s):
1515 units_re = '|'.join(re.escape(u) for u in unit_table)
1517 r'(?P<num>[0-9]+(?:[,.][0-9]*)?)\s*(?P<unit>%s)\b' % units_re, s)
1520 num_str = m.group('num').replace(',', '.')
1521 mult = unit_table[m.group('unit')]
1522 return int(float(num_str) * mult)
1525 def parse_filesize(s):
1529 # The lower-case forms are of course incorrect and unofficial,
1530 # but we support those too
1547 'megabytes': 1000 ** 2,
1548 'mebibytes': 1024 ** 2,
1554 'gigabytes': 1000 ** 3,
1555 'gibibytes': 1024 ** 3,
1561 'terabytes': 1000 ** 4,
1562 'tebibytes': 1024 ** 4,
1568 'petabytes': 1000 ** 5,
1569 'pebibytes': 1024 ** 5,
1575 'exabytes': 1000 ** 6,
1576 'exbibytes': 1024 ** 6,
1582 'zettabytes': 1000 ** 7,
1583 'zebibytes': 1024 ** 7,
1589 'yottabytes': 1000 ** 8,
1590 'yobibytes': 1024 ** 8,
1593 return lookup_unit_table(_UNIT_TABLE, s)
1602 if re.match(r'^[\d,.]+$', s):
1603 return str_to_int(s)
1614 return lookup_unit_table(_UNIT_TABLE, s)
1617 def month_by_name(name, lang='en'):
1618 """ Return the number of a month by (locale-independently) English name """
1620 month_names = MONTH_NAMES.get(lang, MONTH_NAMES['en'])
1623 return month_names.index(name) + 1
1628 def month_by_abbreviation(abbrev):
1629 """ Return the number of a month by (locale-independently) English
1633 return [s[:3] for s in ENGLISH_MONTH_NAMES].index(abbrev) + 1
1638 def fix_xml_ampersands(xml_str):
1639 """Replace all the '&' by '&' in XML"""
1641 r'&(?!amp;|lt;|gt;|apos;|quot;|#x[0-9a-fA-F]{,4};|#[0-9]{,4};)',
1646 def setproctitle(title):
1647 assert isinstance(title, compat_str)
1649 # ctypes in Jython is not complete
1650 # http://bugs.jython.org/issue2148
1651 if sys.platform.startswith('java'):
1655 libc = ctypes.cdll.LoadLibrary('libc.so.6')
1658 title_bytes = title.encode('utf-8')
1659 buf = ctypes.create_string_buffer(len(title_bytes))
1660 buf.value = title_bytes
1662 libc.prctl(15, buf, 0, 0, 0)
1663 except AttributeError:
1664 return # Strange libc, just skip this
1667 def remove_start(s, start):
1668 return s[len(start):] if s is not None and s.startswith(start) else s
1671 def remove_end(s, end):
1672 return s[:-len(end)] if s is not None and s.endswith(end) else s
1675 def remove_quotes(s):
1676 if s is None or len(s) < 2:
1678 for quote in ('"', "'", ):
1679 if s[0] == quote and s[-1] == quote:
1684 def url_basename(url):
1685 path = compat_urlparse.urlparse(url).path
1686 return path.strip('/').split('/')[-1]
1689 class HEADRequest(compat_urllib_request.Request):
1690 def get_method(self):
1694 class PUTRequest(compat_urllib_request.Request):
1695 def get_method(self):
1699 def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1):
1702 v = getattr(v, get_attr, None)
1708 return int(v) * invscale // scale
1713 def str_or_none(v, default=None):
1714 return default if v is None else compat_str(v)
1717 def str_to_int(int_str):
1718 """ A more relaxed version of int_or_none """
1721 int_str = re.sub(r'[,\.\+]', '', int_str)
1725 def float_or_none(v, scale=1, invscale=1, default=None):
1729 return float(v) * invscale / scale
1734 def strip_or_none(v):
1735 return None if v is None else v.strip()
1738 def parse_duration(s):
1739 if not isinstance(s, compat_basestring):
1744 days, hours, mins, secs, ms = [None] * 5
1745 m = re.match(r'(?:(?:(?:(?P<days>[0-9]+):)?(?P<hours>[0-9]+):)?(?P<mins>[0-9]+):)?(?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?$', s)
1747 days, hours, mins, secs, ms = m.groups()
1752 (?P<days>[0-9]+)\s*d(?:ays?)?\s*
1755 (?P<hours>[0-9]+)\s*h(?:ours?)?\s*
1758 (?P<mins>[0-9]+)\s*m(?:in(?:ute)?s?)?\s*
1761 (?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?\s*s(?:ec(?:ond)?s?)?\s*
1764 days, hours, mins, secs, ms = m.groups()
1766 m = re.match(r'(?i)(?:(?P<hours>[0-9.]+)\s*(?:hours?)|(?P<mins>[0-9.]+)\s*(?:mins?\.?|minutes?)\s*)$', s)
1768 hours, mins = m.groups()
1774 duration += float(secs)
1776 duration += float(mins) * 60
1778 duration += float(hours) * 60 * 60
1780 duration += float(days) * 24 * 60 * 60
1782 duration += float(ms)
1786 def prepend_extension(filename, ext, expected_real_ext=None):
1787 name, real_ext = os.path.splitext(filename)
1789 '{0}.{1}{2}'.format(name, ext, real_ext)
1790 if not expected_real_ext or real_ext[1:] == expected_real_ext
1791 else '{0}.{1}'.format(filename, ext))
1794 def replace_extension(filename, ext, expected_real_ext=None):
1795 name, real_ext = os.path.splitext(filename)
1796 return '{0}.{1}'.format(
1797 name if not expected_real_ext or real_ext[1:] == expected_real_ext else filename,
1801 def check_executable(exe, args=[]):
1802 """ Checks if the given binary is installed somewhere in PATH, and returns its name.
1803 args can be a list of arguments for a short output (like -version) """
1805 subprocess.Popen([exe] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
1811 def get_exe_version(exe, args=['--version'],
1812 version_re=None, unrecognized='present'):
1813 """ Returns the version of the specified executable,
1814 or False if the executable is not present """
1816 out, _ = subprocess.Popen(
1817 [encodeArgument(exe)] + args,
1818 stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()
1821 if isinstance(out, bytes): # Python 2.x
1822 out = out.decode('ascii', 'ignore')
1823 return detect_exe_version(out, version_re, unrecognized)
1826 def detect_exe_version(output, version_re=None, unrecognized='present'):
1827 assert isinstance(output, compat_str)
1828 if version_re is None:
1829 version_re = r'version\s+([-0-9._a-zA-Z]+)'
1830 m = re.search(version_re, output)
1837 class PagedList(object):
1839 # This is only useful for tests
1840 return len(self.getslice())
1843 class OnDemandPagedList(PagedList):
1844 def __init__(self, pagefunc, pagesize, use_cache=False):
1845 self._pagefunc = pagefunc
1846 self._pagesize = pagesize
1847 self._use_cache = use_cache
1851 def getslice(self, start=0, end=None):
1853 for pagenum in itertools.count(start // self._pagesize):
1854 firstid = pagenum * self._pagesize
1855 nextfirstid = pagenum * self._pagesize + self._pagesize
1856 if start >= nextfirstid:
1861 page_results = self._cache.get(pagenum)
1862 if page_results is None:
1863 page_results = list(self._pagefunc(pagenum))
1865 self._cache[pagenum] = page_results
1868 start % self._pagesize
1869 if firstid <= start < nextfirstid
1873 ((end - 1) % self._pagesize) + 1
1874 if (end is not None and firstid <= end <= nextfirstid)
1877 if startv != 0 or endv is not None:
1878 page_results = page_results[startv:endv]
1879 res.extend(page_results)
1881 # A little optimization - if current page is not "full", ie. does
1882 # not contain page_size videos then we can assume that this page
1883 # is the last one - there are no more ids on further pages -
1884 # i.e. no need to query again.
1885 if len(page_results) + startv < self._pagesize:
1888 # If we got the whole page, but the next page is not interesting,
1889 # break out early as well
1890 if end == nextfirstid:
1895 class InAdvancePagedList(PagedList):
1896 def __init__(self, pagefunc, pagecount, pagesize):
1897 self._pagefunc = pagefunc
1898 self._pagecount = pagecount
1899 self._pagesize = pagesize
1901 def getslice(self, start=0, end=None):
1903 start_page = start // self._pagesize
1905 self._pagecount if end is None else (end // self._pagesize + 1))
1906 skip_elems = start - start_page * self._pagesize
1907 only_more = None if end is None else end - start
1908 for pagenum in range(start_page, end_page):
1909 page = list(self._pagefunc(pagenum))
1911 page = page[skip_elems:]
1913 if only_more is not None:
1914 if len(page) < only_more:
1915 only_more -= len(page)
1917 page = page[:only_more]
1924 def uppercase_escape(s):
1925 unicode_escape = codecs.getdecoder('unicode_escape')
1927 r'\\U[0-9a-fA-F]{8}',
1928 lambda m: unicode_escape(m.group(0))[0],
1932 def lowercase_escape(s):
1933 unicode_escape = codecs.getdecoder('unicode_escape')
1935 r'\\u[0-9a-fA-F]{4}',
1936 lambda m: unicode_escape(m.group(0))[0],
1940 def escape_rfc3986(s):
1941 """Escape non-ASCII characters as suggested by RFC 3986"""
1942 if sys.version_info < (3, 0) and isinstance(s, compat_str):
1943 s = s.encode('utf-8')
1944 return compat_urllib_parse.quote(s, b"%/;:@&=+$,!~*'()?#[]")
1947 def escape_url(url):
1948 """Escape URL as suggested by RFC 3986"""
1949 url_parsed = compat_urllib_parse_urlparse(url)
1950 return url_parsed._replace(
1951 netloc=url_parsed.netloc.encode('idna').decode('ascii'),
1952 path=escape_rfc3986(url_parsed.path),
1953 params=escape_rfc3986(url_parsed.params),
1954 query=escape_rfc3986(url_parsed.query),
1955 fragment=escape_rfc3986(url_parsed.fragment)
1959 def read_batch_urls(batch_fd):
1961 if not isinstance(url, compat_str):
1962 url = url.decode('utf-8', 'replace')
1963 BOM_UTF8 = '\xef\xbb\xbf'
1964 if url.startswith(BOM_UTF8):
1965 url = url[len(BOM_UTF8):]
1967 if url.startswith(('#', ';', ']')):
1971 with contextlib.closing(batch_fd) as fd:
1972 return [url for url in map(fixup, fd) if url]
1975 def urlencode_postdata(*args, **kargs):
1976 return compat_urllib_parse_urlencode(*args, **kargs).encode('ascii')
1979 def update_url_query(url, query):
1982 parsed_url = compat_urlparse.urlparse(url)
1983 qs = compat_parse_qs(parsed_url.query)
1985 return compat_urlparse.urlunparse(parsed_url._replace(
1986 query=compat_urllib_parse_urlencode(qs, True)))
1989 def update_Request(req, url=None, data=None, headers={}, query={}):
1990 req_headers = req.headers.copy()
1991 req_headers.update(headers)
1992 req_data = data or req.data
1993 req_url = update_url_query(url or req.get_full_url(), query)
1994 req_get_method = req.get_method()
1995 if req_get_method == 'HEAD':
1996 req_type = HEADRequest
1997 elif req_get_method == 'PUT':
1998 req_type = PUTRequest
2000 req_type = compat_urllib_request.Request
2002 req_url, data=req_data, headers=req_headers,
2003 origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
2004 if hasattr(req, 'timeout'):
2005 new_req.timeout = req.timeout
2009 def dict_get(d, key_or_keys, default=None, skip_false_values=True):
2010 if isinstance(key_or_keys, (list, tuple)):
2011 for key in key_or_keys:
2012 if key not in d or d[key] is None or skip_false_values and not d[key]:
2016 return d.get(key_or_keys, default)
2019 def try_get(src, getter, expected_type=None):
2022 except (AttributeError, KeyError, TypeError, IndexError):
2025 if expected_type is None or isinstance(v, expected_type):
2029 def encode_compat_str(string, encoding=preferredencoding(), errors='strict'):
2030 return string if isinstance(string, compat_str) else compat_str(string, encoding, errors)
2042 TV_PARENTAL_GUIDELINES = {
2052 def parse_age_limit(s):
2054 return s if 0 <= s <= 21 else None
2055 if not isinstance(s, compat_basestring):
2057 m = re.match(r'^(?P<age>\d{1,2})\+?$', s)
2059 return int(m.group('age'))
2061 return US_RATINGS[s]
2062 return TV_PARENTAL_GUIDELINES.get(s)
2065 def strip_jsonp(code):
2067 r'(?s)^[a-zA-Z0-9_.$]+\s*\(\s*(.*)\);?\s*?(?://[^\n]*)*$', r'\1', code)
2070 def js_to_json(code):
2073 if v in ('true', 'false', 'null'):
2075 elif v.startswith('/*') or v == ',':
2078 if v[0] in ("'", '"'):
2079 v = re.sub(r'(?s)\\.|"', lambda m: {
2084 }.get(m.group(0), m.group(0)), v[1:-1])
2087 (r'^(0[xX][0-9a-fA-F]+)\s*:?$', 16),
2088 (r'^(0+[0-7]+)\s*:?$', 8),
2091 for regex, base in INTEGER_TABLE:
2092 im = re.match(regex, v)
2094 i = int(im.group(1), base)
2095 return '"%d":' % i if v.endswith(':') else '%d' % i
2099 return re.sub(r'''(?sx)
2100 "(?:[^"\\]*(?:\\\\|\\['"nurtbfx/\n]))*[^"\\]*"|
2101 '(?:[^'\\]*(?:\\\\|\\['"nurtbfx/\n]))*[^'\\]*'|
2102 /\*.*?\*/|,(?=\s*[\]}])|
2103 [a-zA-Z_][.a-zA-Z_0-9]*|
2104 \b(?:0[xX][0-9a-fA-F]+|0+[0-7]+)(?:\s*:)?|
2109 def qualities(quality_ids):
2110 """ Get a numeric quality value out of a list of possible values """
2113 return quality_ids.index(qid)
2119 DEFAULT_OUTTMPL = '%(title)s-%(id)s.%(ext)s'
2122 def limit_length(s, length):
2123 """ Add ellipses to overly long strings """
2128 return s[:length - len(ELLIPSES)] + ELLIPSES
2132 def version_tuple(v):
2133 return tuple(int(e) for e in re.split(r'[-.]', v))
2136 def is_outdated_version(version, limit, assume_new=True):
2138 return not assume_new
2140 return version_tuple(version) < version_tuple(limit)
2142 return not assume_new
2145 def ytdl_is_updateable():
2146 """ Returns if youtube-dl can be updated with -U """
2147 from zipimport import zipimporter
2149 return isinstance(globals().get('__loader__'), zipimporter) or hasattr(sys, 'frozen')
2152 def args_to_str(args):
2153 # Get a short string representation for a subprocess command
2154 return ' '.join(compat_shlex_quote(a) for a in args)
2157 def error_to_compat_str(err):
2159 # On python 2 error byte string must be decoded with proper
2160 # encoding rather than ascii
2161 if sys.version_info[0] < 3:
2162 err_str = err_str.decode(preferredencoding())
2166 def mimetype2ext(mt):
2172 # Per RFC 3003, audio/mpeg can be .mp1, .mp2 or .mp3. Here use .mp3 as
2173 # it's the most popular one
2174 'audio/mpeg': 'mp3',
2179 _, _, res = mt.rpartition('/')
2180 res = res.split(';')[0].strip().lower()
2184 'smptett+xml': 'tt',
2190 'x-mp4-fragmented': 'mp4',
2193 'x-mpegurl': 'm3u8',
2194 'vnd.apple.mpegurl': 'm3u8',
2199 'vnd.ms-sstr+xml': 'ism',
2204 def parse_codecs(codecs_str):
2205 # http://tools.ietf.org/html/rfc6381
2208 splited_codecs = list(filter(None, map(
2209 lambda str: str.strip(), codecs_str.strip().strip(',').split(','))))
2210 vcodec, acodec = None, None
2211 for full_codec in splited_codecs:
2212 codec = full_codec.split('.')[0]
2213 if codec in ('avc1', 'avc2', 'avc3', 'avc4', 'vp9', 'vp8', 'hev1', 'hev2', 'h263', 'h264', 'mp4v'):
2216 elif codec in ('mp4a', 'opus', 'vorbis', 'mp3', 'aac', 'ac-3'):
2220 write_string('WARNING: Unknown codec %s' % full_codec, sys.stderr)
2221 if not vcodec and not acodec:
2222 if len(splited_codecs) == 2:
2227 elif len(splited_codecs) == 1:
2234 'vcodec': vcodec or 'none',
2235 'acodec': acodec or 'none',
2240 def urlhandle_detect_ext(url_handle):
2241 getheader = url_handle.headers.get
2243 cd = getheader('Content-Disposition')
2245 m = re.match(r'attachment;\s*filename="(?P<filename>[^"]+)"', cd)
2247 e = determine_ext(m.group('filename'), default_ext=None)
2251 return mimetype2ext(getheader('Content-Type'))
2254 def encode_data_uri(data, mime_type):
2255 return 'data:%s;base64,%s' % (mime_type, base64.b64encode(data).decode('ascii'))
2258 def age_restricted(content_limit, age_limit):
2259 """ Returns True iff the content should be blocked """
2261 if age_limit is None: # No limit set
2263 if content_limit is None:
2264 return False # Content available for everyone
2265 return age_limit < content_limit
2268 def is_html(first_bytes):
2269 """ Detect whether a file contains HTML by examining its first bytes. """
2272 (b'\xef\xbb\xbf', 'utf-8'),
2273 (b'\x00\x00\xfe\xff', 'utf-32-be'),
2274 (b'\xff\xfe\x00\x00', 'utf-32-le'),
2275 (b'\xff\xfe', 'utf-16-le'),
2276 (b'\xfe\xff', 'utf-16-be'),
2278 for bom, enc in BOMS:
2279 if first_bytes.startswith(bom):
2280 s = first_bytes[len(bom):].decode(enc, 'replace')
2283 s = first_bytes.decode('utf-8', 'replace')
2285 return re.match(r'^\s*<', s)
2288 def determine_protocol(info_dict):
2289 protocol = info_dict.get('protocol')
2290 if protocol is not None:
2293 url = info_dict['url']
2294 if url.startswith('rtmp'):
2296 elif url.startswith('mms'):
2298 elif url.startswith('rtsp'):
2301 ext = determine_ext(url)
2307 return compat_urllib_parse_urlparse(url).scheme
2310 def render_table(header_row, data):
2311 """ Render a list of rows, each as a list of values """
2312 table = [header_row] + data
2313 max_lens = [max(len(compat_str(v)) for v in col) for col in zip(*table)]
2314 format_str = ' '.join('%-' + compat_str(ml + 1) + 's' for ml in max_lens[:-1]) + '%s'
2315 return '\n'.join(format_str % tuple(row) for row in table)
2318 def _match_one(filter_part, dct):
2319 COMPARISON_OPERATORS = {
2327 operator_rex = re.compile(r'''(?x)\s*
2329 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
2331 (?P<intval>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)|
2332 (?P<strval>(?![0-9.])[a-z0-9A-Z]*)
2335 ''' % '|'.join(map(re.escape, COMPARISON_OPERATORS.keys())))
2336 m = operator_rex.search(filter_part)
2338 op = COMPARISON_OPERATORS[m.group('op')]
2339 if m.group('strval') is not None:
2340 if m.group('op') not in ('=', '!='):
2342 'Operator %s does not support string values!' % m.group('op'))
2343 comparison_value = m.group('strval')
2346 comparison_value = int(m.group('intval'))
2348 comparison_value = parse_filesize(m.group('intval'))
2349 if comparison_value is None:
2350 comparison_value = parse_filesize(m.group('intval') + 'B')
2351 if comparison_value is None:
2353 'Invalid integer value %r in filter part %r' % (
2354 m.group('intval'), filter_part))
2355 actual_value = dct.get(m.group('key'))
2356 if actual_value is None:
2357 return m.group('none_inclusive')
2358 return op(actual_value, comparison_value)
2361 '': lambda v: v is not None,
2362 '!': lambda v: v is None,
2364 operator_rex = re.compile(r'''(?x)\s*
2365 (?P<op>%s)\s*(?P<key>[a-z_]+)
2367 ''' % '|'.join(map(re.escape, UNARY_OPERATORS.keys())))
2368 m = operator_rex.search(filter_part)
2370 op = UNARY_OPERATORS[m.group('op')]
2371 actual_value = dct.get(m.group('key'))
2372 return op(actual_value)
2374 raise ValueError('Invalid filter part %r' % filter_part)
2377 def match_str(filter_str, dct):
2378 """ Filter a dictionary with a simple string syntax. Returns True (=passes filter) or false """
2381 _match_one(filter_part, dct) for filter_part in filter_str.split('&'))
2384 def match_filter_func(filter_str):
2385 def _match_func(info_dict):
2386 if match_str(filter_str, info_dict):
2389 video_title = info_dict.get('title', info_dict.get('id', 'video'))
2390 return '%s does not pass filter %s, skipping ..' % (video_title, filter_str)
2394 def parse_dfxp_time_expr(time_expr):
2398 mobj = re.match(r'^(?P<time_offset>\d+(?:\.\d+)?)s?$', time_expr)
2400 return float(mobj.group('time_offset'))
2402 mobj = re.match(r'^(\d+):(\d\d):(\d\d(?:(?:\.|:)\d+)?)$', time_expr)
2404 return 3600 * int(mobj.group(1)) + 60 * int(mobj.group(2)) + float(mobj.group(3).replace(':', '.'))
2407 def srt_subtitles_timecode(seconds):
2408 return '%02d:%02d:%02d,%03d' % (seconds / 3600, (seconds % 3600) / 60, seconds % 60, (seconds % 1) * 1000)
2411 def dfxp2srt(dfxp_data):
2412 _x = functools.partial(xpath_with_ns, ns_map={
2413 'ttml': 'http://www.w3.org/ns/ttml',
2414 'ttaf1': 'http://www.w3.org/2006/10/ttaf1',
2415 'ttaf1_0604': 'http://www.w3.org/2006/04/ttaf1',
2418 class TTMLPElementParser(object):
2421 def start(self, tag, attrib):
2422 if tag in (_x('ttml:br'), _x('ttaf1:br'), 'br'):
2428 def data(self, data):
2432 return self.out.strip()
2434 def parse_node(node):
2435 target = TTMLPElementParser()
2436 parser = xml.etree.ElementTree.XMLParser(target=target)
2437 parser.feed(xml.etree.ElementTree.tostring(node))
2438 return parser.close()
2440 dfxp = compat_etree_fromstring(dfxp_data.encode('utf-8'))
2442 paras = dfxp.findall(_x('.//ttml:p')) or dfxp.findall(_x('.//ttaf1:p')) or dfxp.findall(_x('.//ttaf1_0604:p')) or dfxp.findall('.//p')
2445 raise ValueError('Invalid dfxp/TTML subtitle')
2447 for para, index in zip(paras, itertools.count(1)):
2448 begin_time = parse_dfxp_time_expr(para.attrib.get('begin'))
2449 end_time = parse_dfxp_time_expr(para.attrib.get('end'))
2450 dur = parse_dfxp_time_expr(para.attrib.get('dur'))
2451 if begin_time is None:
2456 end_time = begin_time + dur
2457 out.append('%d\n%s --> %s\n%s\n\n' % (
2459 srt_subtitles_timecode(begin_time),
2460 srt_subtitles_timecode(end_time),
2466 def cli_option(params, command_option, param):
2467 param = params.get(param)
2469 param = compat_str(param)
2470 return [command_option, param] if param is not None else []
2473 def cli_bool_option(params, command_option, param, true_value='true', false_value='false', separator=None):
2474 param = params.get(param)
2475 assert isinstance(param, bool)
2477 return [command_option + separator + (true_value if param else false_value)]
2478 return [command_option, true_value if param else false_value]
2481 def cli_valueless_option(params, command_option, param, expected_value=True):
2482 param = params.get(param)
2483 return [command_option] if param == expected_value else []
2486 def cli_configuration_args(params, param, default=[]):
2487 ex_args = params.get(param)
2490 assert isinstance(ex_args, list)
2494 class ISO639Utils(object):
2495 # See http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
2684 def short2long(cls, code):
2685 """Convert language code from ISO 639-1 to ISO 639-2/T"""
2686 return cls._lang_map.get(code[:2])
2689 def long2short(cls, code):
2690 """Convert language code from ISO 639-2/T to ISO 639-1"""
2691 for short_name, long_name in cls._lang_map.items():
2692 if long_name == code:
2696 class ISO3166Utils(object):
2697 # From http://data.okfn.org/data/core/country-list
2699 'AF': 'Afghanistan',
2700 'AX': 'Åland Islands',
2703 'AS': 'American Samoa',
2708 'AG': 'Antigua and Barbuda',
2725 'BO': 'Bolivia, Plurinational State of',
2726 'BQ': 'Bonaire, Sint Eustatius and Saba',
2727 'BA': 'Bosnia and Herzegovina',
2729 'BV': 'Bouvet Island',
2731 'IO': 'British Indian Ocean Territory',
2732 'BN': 'Brunei Darussalam',
2734 'BF': 'Burkina Faso',
2740 'KY': 'Cayman Islands',
2741 'CF': 'Central African Republic',
2745 'CX': 'Christmas Island',
2746 'CC': 'Cocos (Keeling) Islands',
2750 'CD': 'Congo, the Democratic Republic of the',
2751 'CK': 'Cook Islands',
2753 'CI': 'Côte d\'Ivoire',
2758 'CZ': 'Czech Republic',
2762 'DO': 'Dominican Republic',
2765 'SV': 'El Salvador',
2766 'GQ': 'Equatorial Guinea',
2770 'FK': 'Falkland Islands (Malvinas)',
2771 'FO': 'Faroe Islands',
2775 'GF': 'French Guiana',
2776 'PF': 'French Polynesia',
2777 'TF': 'French Southern Territories',
2792 'GW': 'Guinea-Bissau',
2795 'HM': 'Heard Island and McDonald Islands',
2796 'VA': 'Holy See (Vatican City State)',
2803 'IR': 'Iran, Islamic Republic of',
2806 'IM': 'Isle of Man',
2816 'KP': 'Korea, Democratic People\'s Republic of',
2817 'KR': 'Korea, Republic of',
2820 'LA': 'Lao People\'s Democratic Republic',
2826 'LI': 'Liechtenstein',
2830 'MK': 'Macedonia, the Former Yugoslav Republic of',
2837 'MH': 'Marshall Islands',
2843 'FM': 'Micronesia, Federated States of',
2844 'MD': 'Moldova, Republic of',
2855 'NL': 'Netherlands',
2856 'NC': 'New Caledonia',
2857 'NZ': 'New Zealand',
2862 'NF': 'Norfolk Island',
2863 'MP': 'Northern Mariana Islands',
2868 'PS': 'Palestine, State of',
2870 'PG': 'Papua New Guinea',
2873 'PH': 'Philippines',
2877 'PR': 'Puerto Rico',
2881 'RU': 'Russian Federation',
2883 'BL': 'Saint Barthélemy',
2884 'SH': 'Saint Helena, Ascension and Tristan da Cunha',
2885 'KN': 'Saint Kitts and Nevis',
2886 'LC': 'Saint Lucia',
2887 'MF': 'Saint Martin (French part)',
2888 'PM': 'Saint Pierre and Miquelon',
2889 'VC': 'Saint Vincent and the Grenadines',
2892 'ST': 'Sao Tome and Principe',
2893 'SA': 'Saudi Arabia',
2897 'SL': 'Sierra Leone',
2899 'SX': 'Sint Maarten (Dutch part)',
2902 'SB': 'Solomon Islands',
2904 'ZA': 'South Africa',
2905 'GS': 'South Georgia and the South Sandwich Islands',
2906 'SS': 'South Sudan',
2911 'SJ': 'Svalbard and Jan Mayen',
2914 'CH': 'Switzerland',
2915 'SY': 'Syrian Arab Republic',
2916 'TW': 'Taiwan, Province of China',
2918 'TZ': 'Tanzania, United Republic of',
2920 'TL': 'Timor-Leste',
2924 'TT': 'Trinidad and Tobago',
2927 'TM': 'Turkmenistan',
2928 'TC': 'Turks and Caicos Islands',
2932 'AE': 'United Arab Emirates',
2933 'GB': 'United Kingdom',
2934 'US': 'United States',
2935 'UM': 'United States Minor Outlying Islands',
2939 'VE': 'Venezuela, Bolivarian Republic of',
2941 'VG': 'Virgin Islands, British',
2942 'VI': 'Virgin Islands, U.S.',
2943 'WF': 'Wallis and Futuna',
2944 'EH': 'Western Sahara',
2951 def short2full(cls, code):
2952 """Convert an ISO 3166-2 country code to the corresponding full name"""
2953 return cls._country_map.get(code.upper())
2956 class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
2957 def __init__(self, proxies=None):
2958 # Set default handlers
2959 for type in ('http', 'https'):
2960 setattr(self, '%s_open' % type,
2961 lambda r, proxy='__noproxy__', type=type, meth=self.proxy_open:
2962 meth(r, proxy, type))
2963 return compat_urllib_request.ProxyHandler.__init__(self, proxies)
2965 def proxy_open(self, req, proxy, type):
2966 req_proxy = req.headers.get('Ytdl-request-proxy')
2967 if req_proxy is not None:
2969 del req.headers['Ytdl-request-proxy']
2971 if proxy == '__noproxy__':
2972 return None # No Proxy
2973 if compat_urlparse.urlparse(proxy).scheme.lower() in ('socks', 'socks4', 'socks4a', 'socks5'):
2974 req.add_header('Ytdl-socks-proxy', proxy)
2975 # youtube-dl's http/https handlers do wrapping the socket with socks
2977 return compat_urllib_request.ProxyHandler.proxy_open(
2978 self, req, proxy, type)
2981 def ohdave_rsa_encrypt(data, exponent, modulus):
2983 Implement OHDave's RSA algorithm. See http://www.ohdave.com/rsa/
2986 data: data to encrypt, bytes-like object
2987 exponent, modulus: parameter e and N of RSA algorithm, both integer
2988 Output: hex string of encrypted data
2990 Limitation: supports one block encryption only
2993 payload = int(binascii.hexlify(data[::-1]), 16)
2994 encrypted = pow(payload, exponent, modulus)
2995 return '%x' % encrypted
2998 def encode_base_n(num, n, table=None):
2999 FULL_TABLE = '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'
3001 table = FULL_TABLE[:n]
3004 raise ValueError('base %d exceeds table length %d' % (n, len(table)))
3011 ret = table[num % n] + ret
3016 def decode_packed_codes(code):
3018 r"}\('(.+)',(\d+),(\d+),'([^']+)'\.split\('\|'\)",
3020 obfucasted_code, base, count, symbols = mobj.groups()
3023 symbols = symbols.split('|')
3028 base_n_count = encode_base_n(count, base)
3029 symbol_table[base_n_count] = symbols[count] or base_n_count
3032 r'\b(\w+)\b', lambda mobj: symbol_table[mobj.group(0)],
3036 def parse_m3u8_attributes(attrib):
3038 for (key, val) in re.findall(r'(?P<key>[A-Z0-9-]+)=(?P<val>"[^"]+"|[^",]+)(?:,|$)', attrib):
3039 if val.startswith('"'):
3045 def urshift(val, n):
3046 return val >> n if val >= 0 else (val + 0x100000000) >> n
3049 # Based on png2str() written by @gdkchan and improved by @yokrysty
3050 # Originally posted at https://github.com/rg3/youtube-dl/issues/9706
3051 def decode_png(png_data):
3052 # Reference: https://www.w3.org/TR/PNG/
3053 header = png_data[8:]
3055 if png_data[:8] != b'\x89PNG\x0d\x0a\x1a\x0a' or header[4:8] != b'IHDR':
3056 raise IOError('Not a valid PNG file.')
3058 int_map = {1: '>B', 2: '>H', 4: '>I'}
3059 unpack_integer = lambda x: compat_struct_unpack(int_map[len(x)], x)[0]
3064 length = unpack_integer(header[:4])
3067 chunk_type = header[:4]
3070 chunk_data = header[:length]
3071 header = header[length:]
3073 header = header[4:] # Skip CRC
3081 ihdr = chunks[0]['data']
3083 width = unpack_integer(ihdr[:4])
3084 height = unpack_integer(ihdr[4:8])
3088 for chunk in chunks:
3089 if chunk['type'] == b'IDAT':
3090 idat += chunk['data']
3093 raise IOError('Unable to read PNG data.')
3095 decompressed_data = bytearray(zlib.decompress(idat))
3100 def _get_pixel(idx):
3105 for y in range(height):
3106 basePos = y * (1 + stride)
3107 filter_type = decompressed_data[basePos]
3111 pixels.append(current_row)
3113 for x in range(stride):
3114 color = decompressed_data[1 + basePos + x]
3115 basex = y * stride + x
3120 left = _get_pixel(basex - 3)
3122 up = _get_pixel(basex - stride)
3124 if filter_type == 1: # Sub
3125 color = (color + left) & 0xff
3126 elif filter_type == 2: # Up
3127 color = (color + up) & 0xff
3128 elif filter_type == 3: # Average
3129 color = (color + ((left + up) >> 1)) & 0xff
3130 elif filter_type == 4: # Paeth
3136 c = _get_pixel(basex - stride - 3)
3144 if pa <= pb and pa <= pc:
3145 color = (color + a) & 0xff
3147 color = (color + b) & 0xff
3149 color = (color + c) & 0xff
3151 current_row.append(color)
3153 return width, height, pixels
3156 def write_xattr(path, key, value):
3157 # This mess below finds the best xattr tool for the job
3159 # try the pyxattr module...
3162 # Unicode arguments are not supported in python-pyxattr until
3164 # See https://github.com/rg3/youtube-dl/issues/5498
3165 pyxattr_required_version = '0.5.0'
3166 if version_tuple(xattr.__version__) < version_tuple(pyxattr_required_version):
3167 # TODO: fallback to CLI tools
3168 raise XAttrUnavailableError(
3169 'python-pyxattr is detected but is too old. '
3170 'youtube-dl requires %s or above while your version is %s. '
3171 'Falling back to other xattr implementations' % (
3172 pyxattr_required_version, xattr.__version__))
3175 xattr.set(path, key, value)
3176 except EnvironmentError as e:
3177 raise XAttrMetadataError(e.errno, e.strerror)
3180 if compat_os_name == 'nt':
3181 # Write xattrs to NTFS Alternate Data Streams:
3182 # http://en.wikipedia.org/wiki/NTFS#Alternate_data_streams_.28ADS.29
3183 assert ':' not in key
3184 assert os.path.exists(path)
3186 ads_fn = path + ':' + key
3188 with open(ads_fn, 'wb') as f:
3190 except EnvironmentError as e:
3191 raise XAttrMetadataError(e.errno, e.strerror)
3193 user_has_setfattr = check_executable('setfattr', ['--version'])
3194 user_has_xattr = check_executable('xattr', ['-h'])
3196 if user_has_setfattr or user_has_xattr:
3198 value = value.decode('utf-8')
3199 if user_has_setfattr:
3200 executable = 'setfattr'
3201 opts = ['-n', key, '-v', value]
3202 elif user_has_xattr:
3203 executable = 'xattr'
3204 opts = ['-w', key, value]
3206 cmd = ([encodeFilename(executable, True)] +
3207 [encodeArgument(o) for o in opts] +
3208 [encodeFilename(path, True)])
3211 p = subprocess.Popen(
3212 cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
3213 except EnvironmentError as e:
3214 raise XAttrMetadataError(e.errno, e.strerror)
3215 stdout, stderr = p.communicate()
3216 stderr = stderr.decode('utf-8', 'replace')
3217 if p.returncode != 0:
3218 raise XAttrMetadataError(p.returncode, stderr)
3221 # On Unix, and can't find pyxattr, setfattr, or xattr.
3222 if sys.platform.startswith('linux'):
3223 raise XAttrUnavailableError(
3224 "Couldn't find a tool to set the xattrs. "
3225 "Install either the python 'pyxattr' or 'xattr' "
3226 "modules, or the GNU 'attr' package "
3227 "(which contains the 'setfattr' tool).")
3229 raise XAttrUnavailableError(
3230 "Couldn't find a tool to set the xattrs. "
3231 "Install either the python 'xattr' module, "
3232 "or the 'xattr' binary.")