2 # -*- coding: utf-8 -*-
4 from __future__ import unicode_literals
33 import xml.etree.ElementTree
39 compat_etree_fromstring,
44 compat_socket_create_connection,
48 compat_urllib_parse_urlparse,
49 compat_urllib_request,
55 # This is not clearly defined otherwise
56 compiled_regex_type = type(re.compile(''))
59 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20150101 Firefox/20.0 (Chrome)',
60 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
61 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
62 'Accept-Encoding': 'gzip, deflate',
63 'Accept-Language': 'en-us,en;q=0.5',
69 ENGLISH_MONTH_NAMES = [
70 'January', 'February', 'March', 'April', 'May', 'June',
71 'July', 'August', 'September', 'October', 'November', 'December']
74 def preferredencoding():
75 """Get preferred encoding.
77 Returns the best encoding scheme for the system, based on
78 locale.getpreferredencoding() and some further tweaks.
81 pref = locale.getpreferredencoding()
89 def write_json_file(obj, fn):
90 """ Encode obj as JSON and write it to fn, atomically if possible """
92 fn = encodeFilename(fn)
93 if sys.version_info < (3, 0) and sys.platform != 'win32':
94 encoding = get_filesystem_encoding()
95 # os.path.basename returns a bytes object, but NamedTemporaryFile
96 # will fail if the filename contains non ascii characters unless we
97 # use a unicode object
98 path_basename = lambda f: os.path.basename(fn).decode(encoding)
99 # the same for os.path.dirname
100 path_dirname = lambda f: os.path.dirname(fn).decode(encoding)
102 path_basename = os.path.basename
103 path_dirname = os.path.dirname
107 'prefix': path_basename(fn) + '.',
108 'dir': path_dirname(fn),
112 # In Python 2.x, json.dump expects a bytestream.
113 # In Python 3.x, it writes to a character stream
114 if sys.version_info < (3, 0):
122 tf = tempfile.NamedTemporaryFile(**compat_kwargs(args))
127 if sys.platform == 'win32':
128 # Need to remove existing file on Windows, else os.rename raises
129 # WindowsError or FileExistsError.
134 os.rename(tf.name, fn)
143 if sys.version_info >= (2, 7):
144 def find_xpath_attr(node, xpath, key, val=None):
145 """ Find the xpath xpath[@key=val] """
146 assert re.match(r'^[a-zA-Z_-]+$', key)
148 assert re.match(r'^[a-zA-Z0-9@\s:._-]*$', val)
149 expr = xpath + ('[@%s]' % key if val is None else "[@%s='%s']" % (key, val))
150 return node.find(expr)
152 def find_xpath_attr(node, xpath, key, val=None):
153 # Here comes the crazy part: In 2.6, if the xpath is a unicode,
154 # .//node does not match if a node is a direct child of . !
155 if isinstance(xpath, compat_str):
156 xpath = xpath.encode('ascii')
158 for f in node.findall(xpath):
159 if key not in f.attrib:
161 if val is None or f.attrib.get(key) == val:
165 # On python2.6 the xml.etree.ElementTree.Element methods don't support
166 # the namespace parameter
169 def xpath_with_ns(path, ns_map):
170 components = [c.split(':') for c in path.split('/')]
174 replaced.append(c[0])
177 replaced.append('{%s}%s' % (ns_map[ns], tag))
178 return '/'.join(replaced)
181 def xpath_element(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
182 if sys.version_info < (2, 7): # Crazy 2.6
183 xpath = xpath.encode('ascii')
187 if default is not NO_DEFAULT:
190 name = xpath if name is None else name
191 raise ExtractorError('Could not find XML element %s' % name)
197 def xpath_text(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
198 n = xpath_element(node, xpath, name, fatal=fatal, default=default)
199 if n is None or n == default:
202 if default is not NO_DEFAULT:
205 name = xpath if name is None else name
206 raise ExtractorError('Could not find XML element\'s text %s' % name)
212 def xpath_attr(node, xpath, key, name=None, fatal=False, default=NO_DEFAULT):
213 n = find_xpath_attr(node, xpath, key)
215 if default is not NO_DEFAULT:
218 name = '%s[@%s]' % (xpath, key) if name is None else name
219 raise ExtractorError('Could not find XML attribute %s' % name)
225 def get_element_by_id(id, html):
226 """Return the content of the tag with the specified ID in the passed HTML document"""
227 return get_element_by_attribute("id", id, html)
230 def get_element_by_attribute(attribute, value, html):
231 """Return the content of the tag with the specified attribute in the passed HTML document"""
233 m = re.search(r'''(?xs)
235 (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?
237 (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?
241 ''' % (re.escape(attribute), re.escape(value)), html)
245 res = m.group('content')
247 if res.startswith('"') or res.startswith("'"):
250 return unescapeHTML(res)
253 def clean_html(html):
254 """Clean an HTML snippet into a readable string"""
256 if html is None: # Convenience for sanitizing descriptions etc.
260 html = html.replace('\n', ' ')
261 html = re.sub(r'\s*<\s*br\s*/?\s*>\s*', '\n', html)
262 html = re.sub(r'<\s*/\s*p\s*>\s*<\s*p[^>]*>', '\n', html)
264 html = re.sub('<.*?>', '', html)
265 # Replace html entities
266 html = unescapeHTML(html)
270 def sanitize_open(filename, open_mode):
271 """Try to open the given filename, and slightly tweak it if this fails.
273 Attempts to open the given filename. If this fails, it tries to change
274 the filename slightly, step by step, until it's either able to open it
275 or it fails and raises a final exception, like the standard open()
278 It returns the tuple (stream, definitive_file_name).
282 if sys.platform == 'win32':
284 msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
285 return (sys.stdout.buffer if hasattr(sys.stdout, 'buffer') else sys.stdout, filename)
286 stream = open(encodeFilename(filename), open_mode)
287 return (stream, filename)
288 except (IOError, OSError) as err:
289 if err.errno in (errno.EACCES,):
292 # In case of error, try to remove win32 forbidden chars
293 alt_filename = sanitize_path(filename)
294 if alt_filename == filename:
297 # An exception here should be caught in the caller
298 stream = open(encodeFilename(alt_filename), open_mode)
299 return (stream, alt_filename)
302 def timeconvert(timestr):
303 """Convert RFC 2822 defined time string into system timestamp"""
305 timetuple = email.utils.parsedate_tz(timestr)
306 if timetuple is not None:
307 timestamp = email.utils.mktime_tz(timetuple)
311 def sanitize_filename(s, restricted=False, is_id=False):
312 """Sanitizes a string so it could be used as part of a filename.
313 If restricted is set, use a stricter subset of allowed characters.
314 Set is_id if this is not an arbitrary string, but an ID that should be kept if possible
316 def replace_insane(char):
317 if char == '?' or ord(char) < 32 or ord(char) == 127:
320 return '' if restricted else '\''
322 return '_-' if restricted else ' -'
323 elif char in '\\/|*<>':
325 if restricted and (char in '!&\'()[]{}$;`^,#' or char.isspace()):
327 if restricted and ord(char) > 127:
332 s = re.sub(r'[0-9]+(?::[0-9]+)+', lambda m: m.group(0).replace(':', '_'), s)
333 result = ''.join(map(replace_insane, s))
335 while '__' in result:
336 result = result.replace('__', '_')
337 result = result.strip('_')
338 # Common case of "Foreign band name - English song title"
339 if restricted and result.startswith('-_'):
341 if result.startswith('-'):
342 result = '_' + result[len('-'):]
343 result = result.lstrip('.')
349 def sanitize_path(s):
350 """Sanitizes and normalizes path on Windows"""
351 if sys.platform != 'win32':
353 drive_or_unc, _ = os.path.splitdrive(s)
354 if sys.version_info < (2, 7) and not drive_or_unc:
355 drive_or_unc, _ = os.path.splitunc(s)
356 norm_path = os.path.normpath(remove_start(s, drive_or_unc)).split(os.path.sep)
360 path_part if path_part in ['.', '..'] else re.sub('(?:[/<>:"\\|\\\\?\\*]|\.$)', '#', path_part)
361 for path_part in norm_path]
363 sanitized_path.insert(0, drive_or_unc + os.path.sep)
364 return os.path.join(*sanitized_path)
367 def orderedSet(iterable):
368 """ Remove all duplicates from the input iterable """
376 def _htmlentity_transform(entity):
377 """Transforms an HTML entity to a character."""
378 # Known non-numeric HTML entity
379 if entity in compat_html_entities.name2codepoint:
380 return compat_chr(compat_html_entities.name2codepoint[entity])
382 mobj = re.match(r'#(x[0-9a-fA-F]+|[0-9]+)', entity)
384 numstr = mobj.group(1)
385 if numstr.startswith('x'):
387 numstr = '0%s' % numstr
390 return compat_chr(int(numstr, base))
392 # Unknown entity in name, return its literal representation
393 return ('&%s;' % entity)
399 assert type(s) == compat_str
402 r'&([^;]+);', lambda m: _htmlentity_transform(m.group(1)), s)
405 def get_subprocess_encoding():
406 if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
407 # For subprocess calls, encode with locale encoding
408 # Refer to http://stackoverflow.com/a/9951851/35070
409 encoding = preferredencoding()
411 encoding = sys.getfilesystemencoding()
417 def encodeFilename(s, for_subprocess=False):
419 @param s The name of the file
422 assert type(s) == compat_str
424 # Python 3 has a Unicode API
425 if sys.version_info >= (3, 0):
428 # Pass '' directly to use Unicode APIs on Windows 2000 and up
429 # (Detecting Windows NT 4 is tricky because 'major >= 4' would
430 # match Windows 9x series as well. Besides, NT 4 is obsolete.)
431 if not for_subprocess and sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
434 return s.encode(get_subprocess_encoding(), 'ignore')
437 def decodeFilename(b, for_subprocess=False):
439 if sys.version_info >= (3, 0):
442 if not isinstance(b, bytes):
445 return b.decode(get_subprocess_encoding(), 'ignore')
448 def encodeArgument(s):
449 if not isinstance(s, compat_str):
450 # Legacy code that uses byte strings
451 # Uncomment the following line after fixing all post processors
452 # assert False, 'Internal error: %r should be of type %r, is %r' % (s, compat_str, type(s))
453 s = s.decode('ascii')
454 return encodeFilename(s, True)
457 def decodeArgument(b):
458 return decodeFilename(b, True)
461 def decodeOption(optval):
464 if isinstance(optval, bytes):
465 optval = optval.decode(preferredencoding())
467 assert isinstance(optval, compat_str)
471 def formatSeconds(secs):
473 return '%d:%02d:%02d' % (secs // 3600, (secs % 3600) // 60, secs % 60)
475 return '%d:%02d' % (secs // 60, secs % 60)
480 def make_HTTPS_handler(params, **kwargs):
481 opts_no_check_certificate = params.get('nocheckcertificate', False)
482 if hasattr(ssl, 'create_default_context'): # Python >= 3.4 or 2.7.9
483 context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
484 if opts_no_check_certificate:
485 context.check_hostname = False
486 context.verify_mode = ssl.CERT_NONE
488 return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
491 # (create_default_context present but HTTPSHandler has no context=)
494 if sys.version_info < (3, 2):
495 return YoutubeDLHTTPSHandler(params, **kwargs)
497 context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
498 context.verify_mode = (ssl.CERT_NONE
499 if opts_no_check_certificate
500 else ssl.CERT_REQUIRED)
501 context.set_default_verify_paths()
502 return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
505 def bug_reports_message():
506 if ytdl_is_updateable():
507 update_cmd = 'type youtube-dl -U to update'
509 update_cmd = 'see https://yt-dl.org/update on how to update'
510 msg = '; please report this issue on https://yt-dl.org/bug .'
511 msg += ' Make sure you are using the latest version; %s.' % update_cmd
512 msg += ' Be sure to call youtube-dl with the --verbose flag and include its complete output.'
516 class ExtractorError(Exception):
517 """Error during info extraction."""
519 def __init__(self, msg, tb=None, expected=False, cause=None, video_id=None):
520 """ tb, if given, is the original traceback (so that it can be printed out).
521 If expected is set, this is a normal error message and most likely not a bug in youtube-dl.
524 if sys.exc_info()[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError):
526 if video_id is not None:
527 msg = video_id + ': ' + msg
529 msg += ' (caused by %r)' % cause
531 msg += bug_reports_message()
532 super(ExtractorError, self).__init__(msg)
535 self.exc_info = sys.exc_info() # preserve original exception
537 self.video_id = video_id
539 def format_traceback(self):
540 if self.traceback is None:
542 return ''.join(traceback.format_tb(self.traceback))
545 class UnsupportedError(ExtractorError):
546 def __init__(self, url):
547 super(UnsupportedError, self).__init__(
548 'Unsupported URL: %s' % url, expected=True)
552 class RegexNotFoundError(ExtractorError):
553 """Error when a regex didn't match"""
557 class DownloadError(Exception):
558 """Download Error exception.
560 This exception may be thrown by FileDownloader objects if they are not
561 configured to continue on errors. They will contain the appropriate
565 def __init__(self, msg, exc_info=None):
566 """ exc_info, if given, is the original exception that caused the trouble (as returned by sys.exc_info()). """
567 super(DownloadError, self).__init__(msg)
568 self.exc_info = exc_info
571 class SameFileError(Exception):
572 """Same File exception.
574 This exception will be thrown by FileDownloader objects if they detect
575 multiple files would have to be downloaded to the same file on disk.
580 class PostProcessingError(Exception):
581 """Post Processing exception.
583 This exception may be raised by PostProcessor's .run() method to
584 indicate an error in the postprocessing task.
587 def __init__(self, msg):
591 class MaxDownloadsReached(Exception):
592 """ --max-downloads limit has been reached. """
596 class UnavailableVideoError(Exception):
597 """Unavailable Format exception.
599 This exception will be thrown when a video is requested
600 in a format that is not available for that video.
605 class ContentTooShortError(Exception):
606 """Content Too Short exception.
608 This exception may be raised by FileDownloader objects when a file they
609 download is too small for what the server announced first, indicating
610 the connection was probably interrupted.
613 def __init__(self, downloaded, expected):
615 self.downloaded = downloaded
616 self.expected = expected
619 def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs):
620 # Working around python 2 bug (see http://bugs.python.org/issue17849) by limiting
621 # expected HTTP responses to meet HTTP/1.0 or later (see also
622 # https://github.com/rg3/youtube-dl/issues/6727)
623 if sys.version_info < (3, 0):
624 kwargs[b'strict'] = True
625 hc = http_class(*args, **kwargs)
626 source_address = ydl_handler._params.get('source_address')
627 if source_address is not None:
628 sa = (source_address, 0)
629 if hasattr(hc, 'source_address'): # Python 2.7+
630 hc.source_address = sa
632 def _hc_connect(self, *args, **kwargs):
633 sock = compat_socket_create_connection(
634 (self.host, self.port), self.timeout, sa)
636 self.sock = ssl.wrap_socket(
637 sock, self.key_file, self.cert_file,
638 ssl_version=ssl.PROTOCOL_TLSv1)
641 hc.connect = functools.partial(_hc_connect, hc)
646 class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
647 """Handler for HTTP requests and responses.
649 This class, when installed with an OpenerDirector, automatically adds
650 the standard headers to every HTTP request and handles gzipped and
651 deflated responses from web servers. If compression is to be avoided in
652 a particular request, the original request in the program code only has
653 to include the HTTP header "Youtubedl-No-Compression", which will be
654 removed before making the real request.
656 Part of this code was copied from:
658 http://techknack.net/python-urllib2-handlers/
660 Andrew Rowls, the author of that code, agreed to release it to the
664 def __init__(self, params, *args, **kwargs):
665 compat_urllib_request.HTTPHandler.__init__(self, *args, **kwargs)
666 self._params = params
668 def http_open(self, req):
669 return self.do_open(functools.partial(
670 _create_http_connection, self, compat_http_client.HTTPConnection, False),
676 return zlib.decompress(data, -zlib.MAX_WBITS)
678 return zlib.decompress(data)
681 def addinfourl_wrapper(stream, headers, url, code):
682 if hasattr(compat_urllib_request.addinfourl, 'getcode'):
683 return compat_urllib_request.addinfourl(stream, headers, url, code)
684 ret = compat_urllib_request.addinfourl(stream, headers, url)
688 def http_request(self, req):
689 # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
690 # always respected by websites, some tend to give out URLs with non percent-encoded
691 # non-ASCII characters (see telemb.py, ard.py [#3412])
692 # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
693 # To work around aforementioned issue we will replace request's original URL with
694 # percent-encoded one
695 # Since redirects are also affected (e.g. http://www.southpark.de/alle-episoden/s18e09)
696 # the code of this workaround has been moved here from YoutubeDL.urlopen()
697 url = req.get_full_url()
698 url_escaped = escape_url(url)
700 # Substitute URL if any change after escaping
701 if url != url_escaped:
702 req_type = HEADRequest if req.get_method() == 'HEAD' else compat_urllib_request.Request
704 url_escaped, data=req.data, headers=req.headers,
705 origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
706 new_req.timeout = req.timeout
709 for h, v in std_headers.items():
710 # Capitalize is needed because of Python bug 2275: http://bugs.python.org/issue2275
711 # The dict keys are capitalized because of this bug by urllib
712 if h.capitalize() not in req.headers:
714 if 'Youtubedl-no-compression' in req.headers:
715 if 'Accept-encoding' in req.headers:
716 del req.headers['Accept-encoding']
717 del req.headers['Youtubedl-no-compression']
719 if sys.version_info < (2, 7) and '#' in req.get_full_url():
720 # Python 2.6 is brain-dead when it comes to fragments
721 req._Request__original = req._Request__original.partition('#')[0]
722 req._Request__r_type = req._Request__r_type.partition('#')[0]
726 def http_response(self, req, resp):
729 if resp.headers.get('Content-encoding', '') == 'gzip':
730 content = resp.read()
731 gz = gzip.GzipFile(fileobj=io.BytesIO(content), mode='rb')
733 uncompressed = io.BytesIO(gz.read())
734 except IOError as original_ioerror:
735 # There may be junk add the end of the file
736 # See http://stackoverflow.com/q/4928560/35070 for details
737 for i in range(1, 1024):
739 gz = gzip.GzipFile(fileobj=io.BytesIO(content[:-i]), mode='rb')
740 uncompressed = io.BytesIO(gz.read())
745 raise original_ioerror
746 resp = self.addinfourl_wrapper(uncompressed, old_resp.headers, old_resp.url, old_resp.code)
747 resp.msg = old_resp.msg
749 if resp.headers.get('Content-encoding', '') == 'deflate':
750 gz = io.BytesIO(self.deflate(resp.read()))
751 resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
752 resp.msg = old_resp.msg
753 # Percent-encode redirect URL of Location HTTP header to satisfy RFC 3986 (see
754 # https://github.com/rg3/youtube-dl/issues/6457).
755 if 300 <= resp.code < 400:
756 location = resp.headers.get('Location')
758 # As of RFC 2616 default charset is iso-8859-1 that is respected by python 3
759 if sys.version_info >= (3, 0):
760 location = location.encode('iso-8859-1').decode('utf-8')
761 location_escaped = escape_url(location)
762 if location != location_escaped:
763 del resp.headers['Location']
764 resp.headers['Location'] = location_escaped
767 https_request = http_request
768 https_response = http_response
771 class YoutubeDLHTTPSHandler(compat_urllib_request.HTTPSHandler):
772 def __init__(self, params, https_conn_class=None, *args, **kwargs):
773 compat_urllib_request.HTTPSHandler.__init__(self, *args, **kwargs)
774 self._https_conn_class = https_conn_class or compat_http_client.HTTPSConnection
775 self._params = params
777 def https_open(self, req):
779 if hasattr(self, '_context'): # python > 2.6
780 kwargs['context'] = self._context
781 if hasattr(self, '_check_hostname'): # python 3.x
782 kwargs['check_hostname'] = self._check_hostname
783 return self.do_open(functools.partial(
784 _create_http_connection, self, self._https_conn_class, True),
788 class YoutubeDLCookieProcessor(compat_urllib_request.HTTPCookieProcessor):
789 def __init__(self, cookiejar=None):
790 compat_urllib_request.HTTPCookieProcessor.__init__(self, cookiejar)
792 def http_response(self, request, response):
793 # Python 2 will choke on next HTTP request in row if there are non-ASCII
794 # characters in Set-Cookie HTTP header of last response (see
795 # https://github.com/rg3/youtube-dl/issues/6769).
796 # In order to at least prevent crashing we will percent encode Set-Cookie
797 # header before HTTPCookieProcessor starts processing it.
798 # if sys.version_info < (3, 0) and response.headers:
799 # for set_cookie_header in ('Set-Cookie', 'Set-Cookie2'):
800 # set_cookie = response.headers.get(set_cookie_header)
802 # set_cookie_escaped = compat_urllib_parse.quote(set_cookie, b"%/;:@&=+$,!~*'()?#[] ")
803 # if set_cookie != set_cookie_escaped:
804 # del response.headers[set_cookie_header]
805 # response.headers[set_cookie_header] = set_cookie_escaped
806 return compat_urllib_request.HTTPCookieProcessor.http_response(self, request, response)
808 https_request = compat_urllib_request.HTTPCookieProcessor.http_request
809 https_response = http_response
812 def parse_iso8601(date_str, delimiter='T', timezone=None):
813 """ Return a UNIX timestamp from the given date """
820 r'(\.[0-9]+)?(?:Z$| ?(?P<sign>\+|-)(?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})$)',
823 timezone = datetime.timedelta()
825 date_str = date_str[:-len(m.group(0))]
826 if not m.group('sign'):
827 timezone = datetime.timedelta()
829 sign = 1 if m.group('sign') == '+' else -1
830 timezone = datetime.timedelta(
831 hours=sign * int(m.group('hours')),
832 minutes=sign * int(m.group('minutes')))
833 date_format = '%Y-%m-%d{0}%H:%M:%S'.format(delimiter)
834 dt = datetime.datetime.strptime(date_str, date_format) - timezone
835 return calendar.timegm(dt.timetuple())
838 def unified_strdate(date_str, day_first=True):
839 """Return a string with the date in the format YYYYMMDD"""
845 date_str = date_str.replace(',', ' ')
846 # %z (UTC offset) is only supported in python>=3.2
847 if not re.match(r'^[0-9]{1,2}-[0-9]{1,2}-[0-9]{4}$', date_str):
848 date_str = re.sub(r' ?(\+|-)[0-9]{2}:?[0-9]{2}$', '', date_str)
849 # Remove AM/PM + timezone
850 date_str = re.sub(r'(?i)\s*(?:AM|PM)(?:\s+[A-Z]+)?', '', date_str)
852 format_expressions = [
857 '%b %dst %Y %I:%M%p',
858 '%b %dnd %Y %I:%M%p',
859 '%b %dth %Y %I:%M%p',
865 '%Y-%m-%d %H:%M:%S.%f',
868 '%Y-%m-%dT%H:%M:%SZ',
869 '%Y-%m-%dT%H:%M:%S.%fZ',
870 '%Y-%m-%dT%H:%M:%S.%f0Z',
872 '%Y-%m-%dT%H:%M:%S.%f',
876 format_expressions.extend([
884 format_expressions.extend([
891 for expression in format_expressions:
893 upload_date = datetime.datetime.strptime(date_str, expression).strftime('%Y%m%d')
896 if upload_date is None:
897 timetuple = email.utils.parsedate_tz(date_str)
899 upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d')
903 def determine_ext(url, default_ext='unknown_video'):
906 guess = url.partition('?')[0].rpartition('.')[2]
907 if re.match(r'^[A-Za-z0-9]+$', guess):
913 def subtitles_filename(filename, sub_lang, sub_format):
914 return filename.rsplit('.', 1)[0] + '.' + sub_lang + '.' + sub_format
917 def date_from_str(date_str):
919 Return a datetime object from a string in the format YYYYMMDD or
920 (now|today)[+-][0-9](day|week|month|year)(s)?"""
921 today = datetime.date.today()
922 if date_str in ('now', 'today'):
924 if date_str == 'yesterday':
925 return today - datetime.timedelta(days=1)
926 match = re.match('(now|today)(?P<sign>[+-])(?P<time>\d+)(?P<unit>day|week|month|year)(s)?', date_str)
927 if match is not None:
928 sign = match.group('sign')
929 time = int(match.group('time'))
932 unit = match.group('unit')
933 # A bad aproximation?
941 delta = datetime.timedelta(**{unit: time})
943 return datetime.datetime.strptime(date_str, "%Y%m%d").date()
946 def hyphenate_date(date_str):
948 Convert a date in 'YYYYMMDD' format to 'YYYY-MM-DD' format"""
949 match = re.match(r'^(\d\d\d\d)(\d\d)(\d\d)$', date_str)
950 if match is not None:
951 return '-'.join(match.groups())
956 class DateRange(object):
957 """Represents a time interval between two dates"""
959 def __init__(self, start=None, end=None):
960 """start and end must be strings in the format accepted by date"""
961 if start is not None:
962 self.start = date_from_str(start)
964 self.start = datetime.datetime.min.date()
966 self.end = date_from_str(end)
968 self.end = datetime.datetime.max.date()
969 if self.start > self.end:
970 raise ValueError('Date range: "%s" , the start date must be before the end date' % self)
974 """Returns a range that only contains the given day"""
977 def __contains__(self, date):
978 """Check if the date is in the range"""
979 if not isinstance(date, datetime.date):
980 date = date_from_str(date)
981 return self.start <= date <= self.end
984 return '%s - %s' % (self.start.isoformat(), self.end.isoformat())
988 """ Returns the platform name as a compat_str """
989 res = platform.platform()
990 if isinstance(res, bytes):
991 res = res.decode(preferredencoding())
993 assert isinstance(res, compat_str)
997 def _windows_write_string(s, out):
998 """ Returns True if the string was written using special methods,
999 False if it has yet to be written out."""
1000 # Adapted from http://stackoverflow.com/a/3259271/35070
1003 import ctypes.wintypes
1011 fileno = out.fileno()
1012 except AttributeError:
1013 # If the output stream doesn't have a fileno, it's virtual
1015 except io.UnsupportedOperation:
1016 # Some strange Windows pseudo files?
1018 if fileno not in WIN_OUTPUT_IDS:
1021 GetStdHandle = ctypes.WINFUNCTYPE(
1022 ctypes.wintypes.HANDLE, ctypes.wintypes.DWORD)(
1023 (b"GetStdHandle", ctypes.windll.kernel32))
1024 h = GetStdHandle(WIN_OUTPUT_IDS[fileno])
1026 WriteConsoleW = ctypes.WINFUNCTYPE(
1027 ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE, ctypes.wintypes.LPWSTR,
1028 ctypes.wintypes.DWORD, ctypes.POINTER(ctypes.wintypes.DWORD),
1029 ctypes.wintypes.LPVOID)((b"WriteConsoleW", ctypes.windll.kernel32))
1030 written = ctypes.wintypes.DWORD(0)
1032 GetFileType = ctypes.WINFUNCTYPE(ctypes.wintypes.DWORD, ctypes.wintypes.DWORD)((b"GetFileType", ctypes.windll.kernel32))
1033 FILE_TYPE_CHAR = 0x0002
1034 FILE_TYPE_REMOTE = 0x8000
1035 GetConsoleMode = ctypes.WINFUNCTYPE(
1036 ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE,
1037 ctypes.POINTER(ctypes.wintypes.DWORD))(
1038 (b"GetConsoleMode", ctypes.windll.kernel32))
1039 INVALID_HANDLE_VALUE = ctypes.wintypes.DWORD(-1).value
1041 def not_a_console(handle):
1042 if handle == INVALID_HANDLE_VALUE or handle is None:
1044 return ((GetFileType(handle) & ~FILE_TYPE_REMOTE) != FILE_TYPE_CHAR or
1045 GetConsoleMode(handle, ctypes.byref(ctypes.wintypes.DWORD())) == 0)
1047 if not_a_console(h):
1050 def next_nonbmp_pos(s):
1052 return next(i for i, c in enumerate(s) if ord(c) > 0xffff)
1053 except StopIteration:
1057 count = min(next_nonbmp_pos(s), 1024)
1059 ret = WriteConsoleW(
1060 h, s, count if count else 2, ctypes.byref(written), None)
1062 raise OSError('Failed to write string')
1063 if not count: # We just wrote a non-BMP character
1064 assert written.value == 2
1067 assert written.value > 0
1068 s = s[written.value:]
1072 def write_string(s, out=None, encoding=None):
1075 assert type(s) == compat_str
1077 if sys.platform == 'win32' and encoding is None and hasattr(out, 'fileno'):
1078 if _windows_write_string(s, out):
1081 if ('b' in getattr(out, 'mode', '') or
1082 sys.version_info[0] < 3): # Python 2 lies about mode of sys.stderr
1083 byt = s.encode(encoding or preferredencoding(), 'ignore')
1085 elif hasattr(out, 'buffer'):
1086 enc = encoding or getattr(out, 'encoding', None) or preferredencoding()
1087 byt = s.encode(enc, 'ignore')
1088 out.buffer.write(byt)
1094 def bytes_to_intlist(bs):
1097 if isinstance(bs[0], int): # Python 3
1100 return [ord(c) for c in bs]
1103 def intlist_to_bytes(xs):
1106 return struct_pack('%dB' % len(xs), *xs)
1109 # Cross-platform file locking
1110 if sys.platform == 'win32':
1111 import ctypes.wintypes
1114 class OVERLAPPED(ctypes.Structure):
1116 ('Internal', ctypes.wintypes.LPVOID),
1117 ('InternalHigh', ctypes.wintypes.LPVOID),
1118 ('Offset', ctypes.wintypes.DWORD),
1119 ('OffsetHigh', ctypes.wintypes.DWORD),
1120 ('hEvent', ctypes.wintypes.HANDLE),
1123 kernel32 = ctypes.windll.kernel32
1124 LockFileEx = kernel32.LockFileEx
1125 LockFileEx.argtypes = [
1126 ctypes.wintypes.HANDLE, # hFile
1127 ctypes.wintypes.DWORD, # dwFlags
1128 ctypes.wintypes.DWORD, # dwReserved
1129 ctypes.wintypes.DWORD, # nNumberOfBytesToLockLow
1130 ctypes.wintypes.DWORD, # nNumberOfBytesToLockHigh
1131 ctypes.POINTER(OVERLAPPED) # Overlapped
1133 LockFileEx.restype = ctypes.wintypes.BOOL
1134 UnlockFileEx = kernel32.UnlockFileEx
1135 UnlockFileEx.argtypes = [
1136 ctypes.wintypes.HANDLE, # hFile
1137 ctypes.wintypes.DWORD, # dwReserved
1138 ctypes.wintypes.DWORD, # nNumberOfBytesToLockLow
1139 ctypes.wintypes.DWORD, # nNumberOfBytesToLockHigh
1140 ctypes.POINTER(OVERLAPPED) # Overlapped
1142 UnlockFileEx.restype = ctypes.wintypes.BOOL
1143 whole_low = 0xffffffff
1144 whole_high = 0x7fffffff
1146 def _lock_file(f, exclusive):
1147 overlapped = OVERLAPPED()
1148 overlapped.Offset = 0
1149 overlapped.OffsetHigh = 0
1150 overlapped.hEvent = 0
1151 f._lock_file_overlapped_p = ctypes.pointer(overlapped)
1152 handle = msvcrt.get_osfhandle(f.fileno())
1153 if not LockFileEx(handle, 0x2 if exclusive else 0x0, 0,
1154 whole_low, whole_high, f._lock_file_overlapped_p):
1155 raise OSError('Locking file failed: %r' % ctypes.FormatError())
1157 def _unlock_file(f):
1158 assert f._lock_file_overlapped_p
1159 handle = msvcrt.get_osfhandle(f.fileno())
1160 if not UnlockFileEx(handle, 0,
1161 whole_low, whole_high, f._lock_file_overlapped_p):
1162 raise OSError('Unlocking file failed: %r' % ctypes.FormatError())
1167 def _lock_file(f, exclusive):
1168 fcntl.flock(f, fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH)
1170 def _unlock_file(f):
1171 fcntl.flock(f, fcntl.LOCK_UN)
1174 class locked_file(object):
1175 def __init__(self, filename, mode, encoding=None):
1176 assert mode in ['r', 'a', 'w']
1177 self.f = io.open(filename, mode, encoding=encoding)
1180 def __enter__(self):
1181 exclusive = self.mode != 'r'
1183 _lock_file(self.f, exclusive)
1189 def __exit__(self, etype, value, traceback):
1191 _unlock_file(self.f)
1198 def write(self, *args):
1199 return self.f.write(*args)
1201 def read(self, *args):
1202 return self.f.read(*args)
1205 def get_filesystem_encoding():
1206 encoding = sys.getfilesystemencoding()
1207 return encoding if encoding is not None else 'utf-8'
1210 def shell_quote(args):
1212 encoding = get_filesystem_encoding()
1214 if isinstance(a, bytes):
1215 # We may get a filename encoded with 'encodeFilename'
1216 a = a.decode(encoding)
1217 quoted_args.append(pipes.quote(a))
1218 return ' '.join(quoted_args)
1221 def smuggle_url(url, data):
1222 """ Pass additional data in a URL for internal use. """
1224 sdata = compat_urllib_parse.urlencode(
1225 {'__youtubedl_smuggle': json.dumps(data)})
1226 return url + '#' + sdata
1229 def unsmuggle_url(smug_url, default=None):
1230 if '#__youtubedl_smuggle' not in smug_url:
1231 return smug_url, default
1232 url, _, sdata = smug_url.rpartition('#')
1233 jsond = compat_parse_qs(sdata)['__youtubedl_smuggle'][0]
1234 data = json.loads(jsond)
1238 def format_bytes(bytes):
1241 if type(bytes) is str:
1242 bytes = float(bytes)
1246 exponent = int(math.log(bytes, 1024.0))
1247 suffix = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'][exponent]
1248 converted = float(bytes) / float(1024 ** exponent)
1249 return '%.2f%s' % (converted, suffix)
1252 def parse_filesize(s):
1256 # The lower-case forms are of course incorrect and inofficial,
1257 # but we support those too
1295 units_re = '|'.join(re.escape(u) for u in _UNIT_TABLE)
1297 r'(?P<num>[0-9]+(?:[,.][0-9]*)?)\s*(?P<unit>%s)' % units_re, s)
1301 num_str = m.group('num').replace(',', '.')
1302 mult = _UNIT_TABLE[m.group('unit')]
1303 return int(float(num_str) * mult)
1306 def month_by_name(name):
1307 """ Return the number of a month by (locale-independently) English name """
1310 return ENGLISH_MONTH_NAMES.index(name) + 1
1315 def month_by_abbreviation(abbrev):
1316 """ Return the number of a month by (locale-independently) English
1320 return [s[:3] for s in ENGLISH_MONTH_NAMES].index(abbrev) + 1
1325 def fix_xml_ampersands(xml_str):
1326 """Replace all the '&' by '&' in XML"""
1328 r'&(?!amp;|lt;|gt;|apos;|quot;|#x[0-9a-fA-F]{,4};|#[0-9]{,4};)',
1333 def setproctitle(title):
1334 assert isinstance(title, compat_str)
1336 libc = ctypes.cdll.LoadLibrary("libc.so.6")
1339 title_bytes = title.encode('utf-8')
1340 buf = ctypes.create_string_buffer(len(title_bytes))
1341 buf.value = title_bytes
1343 libc.prctl(15, buf, 0, 0, 0)
1344 except AttributeError:
1345 return # Strange libc, just skip this
1348 def remove_start(s, start):
1349 if s.startswith(start):
1350 return s[len(start):]
1354 def remove_end(s, end):
1356 return s[:-len(end)]
1360 def url_basename(url):
1361 path = compat_urlparse.urlparse(url).path
1362 return path.strip('/').split('/')[-1]
1365 class HEADRequest(compat_urllib_request.Request):
1366 def get_method(self):
1370 def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1):
1373 v = getattr(v, get_attr, None)
1379 return int(v) * invscale // scale
1384 def str_or_none(v, default=None):
1385 return default if v is None else compat_str(v)
1388 def str_to_int(int_str):
1389 """ A more relaxed version of int_or_none """
1392 int_str = re.sub(r'[,\.\+]', '', int_str)
1396 def float_or_none(v, scale=1, invscale=1, default=None):
1400 return float(v) * invscale / scale
1405 def parse_duration(s):
1406 if not isinstance(s, compat_basestring):
1414 (?P<only_mins>[0-9.]+)\s*(?:mins?\.?|minutes?)\s*|
1415 (?P<only_hours>[0-9.]+)\s*(?:hours?)|
1417 \s*(?P<hours_reversed>[0-9]+)\s*(?:[:h]|hours?)\s*(?P<mins_reversed>[0-9]+)\s*(?:[:m]|mins?\.?|minutes?)\s*|
1420 (?:(?P<days>[0-9]+)\s*(?:[:d]|days?)\s*)?
1421 (?P<hours>[0-9]+)\s*(?:[:h]|hours?)\s*
1423 (?P<mins>[0-9]+)\s*(?:[:m]|mins?|minutes?)\s*
1425 (?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?\s*(?:s|secs?|seconds?)?
1430 if m.group('only_mins'):
1431 return float_or_none(m.group('only_mins'), invscale=60)
1432 if m.group('only_hours'):
1433 return float_or_none(m.group('only_hours'), invscale=60 * 60)
1435 res += int(m.group('secs'))
1436 if m.group('mins_reversed'):
1437 res += int(m.group('mins_reversed')) * 60
1439 res += int(m.group('mins')) * 60
1440 if m.group('hours'):
1441 res += int(m.group('hours')) * 60 * 60
1442 if m.group('hours_reversed'):
1443 res += int(m.group('hours_reversed')) * 60 * 60
1445 res += int(m.group('days')) * 24 * 60 * 60
1447 res += float(m.group('ms'))
1451 def prepend_extension(filename, ext, expected_real_ext=None):
1452 name, real_ext = os.path.splitext(filename)
1454 '{0}.{1}{2}'.format(name, ext, real_ext)
1455 if not expected_real_ext or real_ext[1:] == expected_real_ext
1456 else '{0}.{1}'.format(filename, ext))
1459 def replace_extension(filename, ext, expected_real_ext=None):
1460 name, real_ext = os.path.splitext(filename)
1461 return '{0}.{1}'.format(
1462 name if not expected_real_ext or real_ext[1:] == expected_real_ext else filename,
1466 def check_executable(exe, args=[]):
1467 """ Checks if the given binary is installed somewhere in PATH, and returns its name.
1468 args can be a list of arguments for a short output (like -version) """
1470 subprocess.Popen([exe] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
1476 def get_exe_version(exe, args=['--version'],
1477 version_re=None, unrecognized='present'):
1478 """ Returns the version of the specified executable,
1479 or False if the executable is not present """
1481 out, _ = subprocess.Popen(
1482 [encodeArgument(exe)] + args,
1483 stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()
1486 if isinstance(out, bytes): # Python 2.x
1487 out = out.decode('ascii', 'ignore')
1488 return detect_exe_version(out, version_re, unrecognized)
1491 def detect_exe_version(output, version_re=None, unrecognized='present'):
1492 assert isinstance(output, compat_str)
1493 if version_re is None:
1494 version_re = r'version\s+([-0-9._a-zA-Z]+)'
1495 m = re.search(version_re, output)
1502 class PagedList(object):
1504 # This is only useful for tests
1505 return len(self.getslice())
1508 class OnDemandPagedList(PagedList):
1509 def __init__(self, pagefunc, pagesize):
1510 self._pagefunc = pagefunc
1511 self._pagesize = pagesize
1513 def getslice(self, start=0, end=None):
1515 for pagenum in itertools.count(start // self._pagesize):
1516 firstid = pagenum * self._pagesize
1517 nextfirstid = pagenum * self._pagesize + self._pagesize
1518 if start >= nextfirstid:
1521 page_results = list(self._pagefunc(pagenum))
1524 start % self._pagesize
1525 if firstid <= start < nextfirstid
1529 ((end - 1) % self._pagesize) + 1
1530 if (end is not None and firstid <= end <= nextfirstid)
1533 if startv != 0 or endv is not None:
1534 page_results = page_results[startv:endv]
1535 res.extend(page_results)
1537 # A little optimization - if current page is not "full", ie. does
1538 # not contain page_size videos then we can assume that this page
1539 # is the last one - there are no more ids on further pages -
1540 # i.e. no need to query again.
1541 if len(page_results) + startv < self._pagesize:
1544 # If we got the whole page, but the next page is not interesting,
1545 # break out early as well
1546 if end == nextfirstid:
1551 class InAdvancePagedList(PagedList):
1552 def __init__(self, pagefunc, pagecount, pagesize):
1553 self._pagefunc = pagefunc
1554 self._pagecount = pagecount
1555 self._pagesize = pagesize
1557 def getslice(self, start=0, end=None):
1559 start_page = start // self._pagesize
1561 self._pagecount if end is None else (end // self._pagesize + 1))
1562 skip_elems = start - start_page * self._pagesize
1563 only_more = None if end is None else end - start
1564 for pagenum in range(start_page, end_page):
1565 page = list(self._pagefunc(pagenum))
1567 page = page[skip_elems:]
1569 if only_more is not None:
1570 if len(page) < only_more:
1571 only_more -= len(page)
1573 page = page[:only_more]
1580 def uppercase_escape(s):
1581 unicode_escape = codecs.getdecoder('unicode_escape')
1583 r'\\U[0-9a-fA-F]{8}',
1584 lambda m: unicode_escape(m.group(0))[0],
1588 def lowercase_escape(s):
1589 unicode_escape = codecs.getdecoder('unicode_escape')
1591 r'\\u[0-9a-fA-F]{4}',
1592 lambda m: unicode_escape(m.group(0))[0],
1596 def escape_rfc3986(s):
1597 """Escape non-ASCII characters as suggested by RFC 3986"""
1598 if sys.version_info < (3, 0) and isinstance(s, compat_str):
1599 s = s.encode('utf-8')
1600 return compat_urllib_parse.quote(s, b"%/;:@&=+$,!~*'()?#[]")
1603 def escape_url(url):
1604 """Escape URL as suggested by RFC 3986"""
1605 url_parsed = compat_urllib_parse_urlparse(url)
1606 return url_parsed._replace(
1607 path=escape_rfc3986(url_parsed.path),
1608 params=escape_rfc3986(url_parsed.params),
1609 query=escape_rfc3986(url_parsed.query),
1610 fragment=escape_rfc3986(url_parsed.fragment)
1614 struct.pack('!I', 0)
1616 # In Python 2.6 (and some 2.7 versions), struct requires a bytes argument
1617 def struct_pack(spec, *args):
1618 if isinstance(spec, compat_str):
1619 spec = spec.encode('ascii')
1620 return struct.pack(spec, *args)
1622 def struct_unpack(spec, *args):
1623 if isinstance(spec, compat_str):
1624 spec = spec.encode('ascii')
1625 return struct.unpack(spec, *args)
1627 struct_pack = struct.pack
1628 struct_unpack = struct.unpack
1631 def read_batch_urls(batch_fd):
1633 if not isinstance(url, compat_str):
1634 url = url.decode('utf-8', 'replace')
1635 BOM_UTF8 = '\xef\xbb\xbf'
1636 if url.startswith(BOM_UTF8):
1637 url = url[len(BOM_UTF8):]
1639 if url.startswith(('#', ';', ']')):
1643 with contextlib.closing(batch_fd) as fd:
1644 return [url for url in map(fixup, fd) if url]
1647 def urlencode_postdata(*args, **kargs):
1648 return compat_urllib_parse.urlencode(*args, **kargs).encode('ascii')
1651 def encode_dict(d, encoding='utf-8'):
1652 return dict((k.encode(encoding), v.encode(encoding)) for k, v in d.items())
1664 def parse_age_limit(s):
1667 m = re.match(r'^(?P<age>\d{1,2})\+?$', s)
1668 return int(m.group('age')) if m else US_RATINGS.get(s, None)
1671 def strip_jsonp(code):
1673 r'(?s)^[a-zA-Z0-9_]+\s*\(\s*(.*)\);?\s*?(?://[^\n]*)*$', r'\1', code)
1676 def js_to_json(code):
1679 if v in ('true', 'false', 'null'):
1681 if v.startswith('"'):
1682 v = re.sub(r"\\'", "'", v[1:-1])
1683 elif v.startswith("'"):
1685 v = re.sub(r"\\\\|\\'|\"", lambda m: {
1692 res = re.sub(r'''(?x)
1693 "(?:[^"\\]*(?:\\\\|\\['"nu]))*[^"\\]*"|
1694 '(?:[^'\\]*(?:\\\\|\\['"nu]))*[^'\\]*'|
1695 [a-zA-Z_][.a-zA-Z_0-9]*
1697 res = re.sub(r',(\s*[\]}])', lambda m: m.group(1), res)
1701 def qualities(quality_ids):
1702 """ Get a numeric quality value out of a list of possible values """
1705 return quality_ids.index(qid)
1711 DEFAULT_OUTTMPL = '%(title)s-%(id)s.%(ext)s'
1714 def limit_length(s, length):
1715 """ Add ellipses to overly long strings """
1720 return s[:length - len(ELLIPSES)] + ELLIPSES
1724 def version_tuple(v):
1725 return tuple(int(e) for e in re.split(r'[-.]', v))
1728 def is_outdated_version(version, limit, assume_new=True):
1730 return not assume_new
1732 return version_tuple(version) < version_tuple(limit)
1734 return not assume_new
1737 def ytdl_is_updateable():
1738 """ Returns if youtube-dl can be updated with -U """
1739 from zipimport import zipimporter
1741 return isinstance(globals().get('__loader__'), zipimporter) or hasattr(sys, 'frozen')
1744 def args_to_str(args):
1745 # Get a short string representation for a subprocess command
1746 return ' '.join(shlex_quote(a) for a in args)
1749 def mimetype2ext(mt):
1750 _, _, res = mt.rpartition('/')
1754 'x-mp4-fragmented': 'mp4',
1759 def urlhandle_detect_ext(url_handle):
1762 getheader = lambda h: url_handle.headers[h]
1763 except AttributeError: # Python < 3
1764 getheader = url_handle.info().getheader
1766 cd = getheader('Content-Disposition')
1768 m = re.match(r'attachment;\s*filename="(?P<filename>[^"]+)"', cd)
1770 e = determine_ext(m.group('filename'), default_ext=None)
1774 return mimetype2ext(getheader('Content-Type'))
1777 def encode_data_uri(data, mime_type):
1778 return 'data:%s;base64,%s' % (mime_type, base64.b64encode(data).decode('ascii'))
1781 def age_restricted(content_limit, age_limit):
1782 """ Returns True iff the content should be blocked """
1784 if age_limit is None: # No limit set
1786 if content_limit is None:
1787 return False # Content available for everyone
1788 return age_limit < content_limit
1791 def is_html(first_bytes):
1792 """ Detect whether a file contains HTML by examining its first bytes. """
1795 (b'\xef\xbb\xbf', 'utf-8'),
1796 (b'\x00\x00\xfe\xff', 'utf-32-be'),
1797 (b'\xff\xfe\x00\x00', 'utf-32-le'),
1798 (b'\xff\xfe', 'utf-16-le'),
1799 (b'\xfe\xff', 'utf-16-be'),
1801 for bom, enc in BOMS:
1802 if first_bytes.startswith(bom):
1803 s = first_bytes[len(bom):].decode(enc, 'replace')
1806 s = first_bytes.decode('utf-8', 'replace')
1808 return re.match(r'^\s*<', s)
1811 def determine_protocol(info_dict):
1812 protocol = info_dict.get('protocol')
1813 if protocol is not None:
1816 url = info_dict['url']
1817 if url.startswith('rtmp'):
1819 elif url.startswith('mms'):
1821 elif url.startswith('rtsp'):
1824 ext = determine_ext(url)
1830 return compat_urllib_parse_urlparse(url).scheme
1833 def render_table(header_row, data):
1834 """ Render a list of rows, each as a list of values """
1835 table = [header_row] + data
1836 max_lens = [max(len(compat_str(v)) for v in col) for col in zip(*table)]
1837 format_str = ' '.join('%-' + compat_str(ml + 1) + 's' for ml in max_lens[:-1]) + '%s'
1838 return '\n'.join(format_str % tuple(row) for row in table)
1841 def _match_one(filter_part, dct):
1842 COMPARISON_OPERATORS = {
1850 operator_rex = re.compile(r'''(?x)\s*
1852 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
1854 (?P<intval>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)|
1855 (?P<strval>(?![0-9.])[a-z0-9A-Z]*)
1858 ''' % '|'.join(map(re.escape, COMPARISON_OPERATORS.keys())))
1859 m = operator_rex.search(filter_part)
1861 op = COMPARISON_OPERATORS[m.group('op')]
1862 if m.group('strval') is not None:
1863 if m.group('op') not in ('=', '!='):
1865 'Operator %s does not support string values!' % m.group('op'))
1866 comparison_value = m.group('strval')
1869 comparison_value = int(m.group('intval'))
1871 comparison_value = parse_filesize(m.group('intval'))
1872 if comparison_value is None:
1873 comparison_value = parse_filesize(m.group('intval') + 'B')
1874 if comparison_value is None:
1876 'Invalid integer value %r in filter part %r' % (
1877 m.group('intval'), filter_part))
1878 actual_value = dct.get(m.group('key'))
1879 if actual_value is None:
1880 return m.group('none_inclusive')
1881 return op(actual_value, comparison_value)
1884 '': lambda v: v is not None,
1885 '!': lambda v: v is None,
1887 operator_rex = re.compile(r'''(?x)\s*
1888 (?P<op>%s)\s*(?P<key>[a-z_]+)
1890 ''' % '|'.join(map(re.escape, UNARY_OPERATORS.keys())))
1891 m = operator_rex.search(filter_part)
1893 op = UNARY_OPERATORS[m.group('op')]
1894 actual_value = dct.get(m.group('key'))
1895 return op(actual_value)
1897 raise ValueError('Invalid filter part %r' % filter_part)
1900 def match_str(filter_str, dct):
1901 """ Filter a dictionary with a simple string syntax. Returns True (=passes filter) or false """
1904 _match_one(filter_part, dct) for filter_part in filter_str.split('&'))
1907 def match_filter_func(filter_str):
1908 def _match_func(info_dict):
1909 if match_str(filter_str, info_dict):
1912 video_title = info_dict.get('title', info_dict.get('id', 'video'))
1913 return '%s does not pass filter %s, skipping ..' % (video_title, filter_str)
1917 def parse_dfxp_time_expr(time_expr):
1921 mobj = re.match(r'^(?P<time_offset>\d+(?:\.\d+)?)s?$', time_expr)
1923 return float(mobj.group('time_offset'))
1925 mobj = re.match(r'^(\d+):(\d\d):(\d\d(?:\.\d+)?)$', time_expr)
1927 return 3600 * int(mobj.group(1)) + 60 * int(mobj.group(2)) + float(mobj.group(3))
1930 def srt_subtitles_timecode(seconds):
1931 return '%02d:%02d:%02d,%03d' % (seconds / 3600, (seconds % 3600) / 60, seconds % 60, (seconds % 1) * 1000)
1934 def dfxp2srt(dfxp_data):
1935 _x = functools.partial(xpath_with_ns, ns_map={
1936 'ttml': 'http://www.w3.org/ns/ttml',
1937 'ttaf1': 'http://www.w3.org/2006/10/ttaf1',
1940 def parse_node(node):
1941 str_or_empty = functools.partial(str_or_none, default='')
1943 out = str_or_empty(node.text)
1946 if child.tag in (_x('ttml:br'), _x('ttaf1:br'), 'br'):
1947 out += '\n' + str_or_empty(child.tail)
1948 elif child.tag in (_x('ttml:span'), _x('ttaf1:span'), 'span'):
1949 out += str_or_empty(parse_node(child))
1951 out += str_or_empty(xml.etree.ElementTree.tostring(child))
1955 dfxp = compat_etree_fromstring(dfxp_data.encode('utf-8'))
1957 paras = dfxp.findall(_x('.//ttml:p')) or dfxp.findall(_x('.//ttaf1:p')) or dfxp.findall('.//p')
1960 raise ValueError('Invalid dfxp/TTML subtitle')
1962 for para, index in zip(paras, itertools.count(1)):
1963 begin_time = parse_dfxp_time_expr(para.attrib['begin'])
1964 end_time = parse_dfxp_time_expr(para.attrib.get('end'))
1966 end_time = begin_time + parse_dfxp_time_expr(para.attrib['dur'])
1967 out.append('%d\n%s --> %s\n%s\n\n' % (
1969 srt_subtitles_timecode(begin_time),
1970 srt_subtitles_timecode(end_time),
1976 def cli_option(params, command_option, param):
1977 param = params.get(param)
1978 return [command_option, param] if param is not None else []
1981 def cli_bool_option(params, command_option, param, true_value='true', false_value='false', separator=None):
1982 param = params.get(param)
1983 assert isinstance(param, bool)
1985 return [command_option + separator + (true_value if param else false_value)]
1986 return [command_option, true_value if param else false_value]
1989 def cli_valueless_option(params, command_option, param, expected_value=True):
1990 param = params.get(param)
1991 return [command_option] if param == expected_value else []
1994 def cli_configuration_args(params, param, default=[]):
1995 ex_args = params.get(param)
1998 assert isinstance(ex_args, list)
2002 class ISO639Utils(object):
2003 # See http://www.loc.gov/standards/iso639-2/ISO-639-2_utf-8.txt
2192 def short2long(cls, code):
2193 """Convert language code from ISO 639-1 to ISO 639-2/T"""
2194 return cls._lang_map.get(code[:2])
2197 def long2short(cls, code):
2198 """Convert language code from ISO 639-2/T to ISO 639-1"""
2199 for short_name, long_name in cls._lang_map.items():
2200 if long_name == code:
2204 class ISO3166Utils(object):
2205 # From http://data.okfn.org/data/core/country-list
2207 'AF': 'Afghanistan',
2208 'AX': 'Ã…land Islands',
2211 'AS': 'American Samoa',
2216 'AG': 'Antigua and Barbuda',
2233 'BO': 'Bolivia, Plurinational State of',
2234 'BQ': 'Bonaire, Sint Eustatius and Saba',
2235 'BA': 'Bosnia and Herzegovina',
2237 'BV': 'Bouvet Island',
2239 'IO': 'British Indian Ocean Territory',
2240 'BN': 'Brunei Darussalam',
2242 'BF': 'Burkina Faso',
2248 'KY': 'Cayman Islands',
2249 'CF': 'Central African Republic',
2253 'CX': 'Christmas Island',
2254 'CC': 'Cocos (Keeling) Islands',
2258 'CD': 'Congo, the Democratic Republic of the',
2259 'CK': 'Cook Islands',
2261 'CI': 'Côte d\'Ivoire',
2266 'CZ': 'Czech Republic',
2270 'DO': 'Dominican Republic',
2273 'SV': 'El Salvador',
2274 'GQ': 'Equatorial Guinea',
2278 'FK': 'Falkland Islands (Malvinas)',
2279 'FO': 'Faroe Islands',
2283 'GF': 'French Guiana',
2284 'PF': 'French Polynesia',
2285 'TF': 'French Southern Territories',
2300 'GW': 'Guinea-Bissau',
2303 'HM': 'Heard Island and McDonald Islands',
2304 'VA': 'Holy See (Vatican City State)',
2311 'IR': 'Iran, Islamic Republic of',
2314 'IM': 'Isle of Man',
2324 'KP': 'Korea, Democratic People\'s Republic of',
2325 'KR': 'Korea, Republic of',
2328 'LA': 'Lao People\'s Democratic Republic',
2334 'LI': 'Liechtenstein',
2338 'MK': 'Macedonia, the Former Yugoslav Republic of',
2345 'MH': 'Marshall Islands',
2351 'FM': 'Micronesia, Federated States of',
2352 'MD': 'Moldova, Republic of',
2363 'NL': 'Netherlands',
2364 'NC': 'New Caledonia',
2365 'NZ': 'New Zealand',
2370 'NF': 'Norfolk Island',
2371 'MP': 'Northern Mariana Islands',
2376 'PS': 'Palestine, State of',
2378 'PG': 'Papua New Guinea',
2381 'PH': 'Philippines',
2385 'PR': 'Puerto Rico',
2389 'RU': 'Russian Federation',
2391 'BL': 'Saint Barthélemy',
2392 'SH': 'Saint Helena, Ascension and Tristan da Cunha',
2393 'KN': 'Saint Kitts and Nevis',
2394 'LC': 'Saint Lucia',
2395 'MF': 'Saint Martin (French part)',
2396 'PM': 'Saint Pierre and Miquelon',
2397 'VC': 'Saint Vincent and the Grenadines',
2400 'ST': 'Sao Tome and Principe',
2401 'SA': 'Saudi Arabia',
2405 'SL': 'Sierra Leone',
2407 'SX': 'Sint Maarten (Dutch part)',
2410 'SB': 'Solomon Islands',
2412 'ZA': 'South Africa',
2413 'GS': 'South Georgia and the South Sandwich Islands',
2414 'SS': 'South Sudan',
2419 'SJ': 'Svalbard and Jan Mayen',
2422 'CH': 'Switzerland',
2423 'SY': 'Syrian Arab Republic',
2424 'TW': 'Taiwan, Province of China',
2426 'TZ': 'Tanzania, United Republic of',
2428 'TL': 'Timor-Leste',
2432 'TT': 'Trinidad and Tobago',
2435 'TM': 'Turkmenistan',
2436 'TC': 'Turks and Caicos Islands',
2440 'AE': 'United Arab Emirates',
2441 'GB': 'United Kingdom',
2442 'US': 'United States',
2443 'UM': 'United States Minor Outlying Islands',
2447 'VE': 'Venezuela, Bolivarian Republic of',
2449 'VG': 'Virgin Islands, British',
2450 'VI': 'Virgin Islands, U.S.',
2451 'WF': 'Wallis and Futuna',
2452 'EH': 'Western Sahara',
2459 def short2full(cls, code):
2460 """Convert an ISO 3166-2 country code to the corresponding full name"""
2461 return cls._country_map.get(code.upper())
2464 class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
2465 def __init__(self, proxies=None):
2466 # Set default handlers
2467 for type in ('http', 'https'):
2468 setattr(self, '%s_open' % type,
2469 lambda r, proxy='__noproxy__', type=type, meth=self.proxy_open:
2470 meth(r, proxy, type))
2471 return compat_urllib_request.ProxyHandler.__init__(self, proxies)
2473 def proxy_open(self, req, proxy, type):
2474 req_proxy = req.headers.get('Ytdl-request-proxy')
2475 if req_proxy is not None:
2477 del req.headers['Ytdl-request-proxy']
2479 if proxy == '__noproxy__':
2480 return None # No Proxy
2481 return compat_urllib_request.ProxyHandler.proxy_open(
2482 self, req, proxy, type)