2 # -*- coding: utf-8 -*-
4 from __future__ import unicode_literals
32 import xml.etree.ElementTree
41 compat_socket_create_connection,
45 compat_urllib_parse_urlparse,
46 compat_urllib_request,
52 # This is not clearly defined otherwise
53 compiled_regex_type = type(re.compile(''))
56 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20150101 Firefox/20.0 (Chrome)',
57 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
58 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
59 'Accept-Encoding': 'gzip, deflate',
60 'Accept-Language': 'en-us,en;q=0.5',
64 ENGLISH_MONTH_NAMES = [
65 'January', 'February', 'March', 'April', 'May', 'June',
66 'July', 'August', 'September', 'October', 'November', 'December']
69 def preferredencoding():
70 """Get preferred encoding.
72 Returns the best encoding scheme for the system, based on
73 locale.getpreferredencoding() and some further tweaks.
76 pref = locale.getpreferredencoding()
84 def write_json_file(obj, fn):
85 """ Encode obj as JSON and write it to fn, atomically if possible """
87 fn = encodeFilename(fn)
88 if sys.version_info < (3, 0) and sys.platform != 'win32':
89 encoding = get_filesystem_encoding()
90 # os.path.basename returns a bytes object, but NamedTemporaryFile
91 # will fail if the filename contains non ascii characters unless we
92 # use a unicode object
93 path_basename = lambda f: os.path.basename(fn).decode(encoding)
94 # the same for os.path.dirname
95 path_dirname = lambda f: os.path.dirname(fn).decode(encoding)
97 path_basename = os.path.basename
98 path_dirname = os.path.dirname
102 'prefix': path_basename(fn) + '.',
103 'dir': path_dirname(fn),
107 # In Python 2.x, json.dump expects a bytestream.
108 # In Python 3.x, it writes to a character stream
109 if sys.version_info < (3, 0):
117 tf = tempfile.NamedTemporaryFile(**args)
122 if sys.platform == 'win32':
123 # Need to remove existing file on Windows, else os.rename raises
124 # WindowsError or FileExistsError.
129 os.rename(tf.name, fn)
138 if sys.version_info >= (2, 7):
139 def find_xpath_attr(node, xpath, key, val):
140 """ Find the xpath xpath[@key=val] """
141 assert re.match(r'^[a-zA-Z-]+$', key)
142 assert re.match(r'^[a-zA-Z0-9@\s:._-]*$', val)
143 expr = xpath + "[@%s='%s']" % (key, val)
144 return node.find(expr)
146 def find_xpath_attr(node, xpath, key, val):
147 # Here comes the crazy part: In 2.6, if the xpath is a unicode,
148 # .//node does not match if a node is a direct child of . !
149 if isinstance(xpath, compat_str):
150 xpath = xpath.encode('ascii')
152 for f in node.findall(xpath):
153 if f.attrib.get(key) == val:
157 # On python2.6 the xml.etree.ElementTree.Element methods don't support
158 # the namespace parameter
161 def xpath_with_ns(path, ns_map):
162 components = [c.split(':') for c in path.split('/')]
166 replaced.append(c[0])
169 replaced.append('{%s}%s' % (ns_map[ns], tag))
170 return '/'.join(replaced)
173 def xpath_text(node, xpath, name=None, fatal=False):
174 if sys.version_info < (2, 7): # Crazy 2.6
175 xpath = xpath.encode('ascii')
178 if n is None or n.text is None:
180 name = xpath if name is None else name
181 raise ExtractorError('Could not find XML element %s' % name)
187 def get_element_by_id(id, html):
188 """Return the content of the tag with the specified ID in the passed HTML document"""
189 return get_element_by_attribute("id", id, html)
192 def get_element_by_attribute(attribute, value, html):
193 """Return the content of the tag with the specified attribute in the passed HTML document"""
195 m = re.search(r'''(?xs)
197 (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?
199 (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]+|="[^"]+"|='[^']+'))*?
203 ''' % (re.escape(attribute), re.escape(value)), html)
207 res = m.group('content')
209 if res.startswith('"') or res.startswith("'"):
212 return unescapeHTML(res)
215 def clean_html(html):
216 """Clean an HTML snippet into a readable string"""
218 if html is None: # Convenience for sanitizing descriptions etc.
222 html = html.replace('\n', ' ')
223 html = re.sub(r'\s*<\s*br\s*/?\s*>\s*', '\n', html)
224 html = re.sub(r'<\s*/\s*p\s*>\s*<\s*p[^>]*>', '\n', html)
226 html = re.sub('<.*?>', '', html)
227 # Replace html entities
228 html = unescapeHTML(html)
232 def sanitize_open(filename, open_mode):
233 """Try to open the given filename, and slightly tweak it if this fails.
235 Attempts to open the given filename. If this fails, it tries to change
236 the filename slightly, step by step, until it's either able to open it
237 or it fails and raises a final exception, like the standard open()
240 It returns the tuple (stream, definitive_file_name).
244 if sys.platform == 'win32':
246 msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
247 return (sys.stdout.buffer if hasattr(sys.stdout, 'buffer') else sys.stdout, filename)
248 stream = open(encodeFilename(filename), open_mode)
249 return (stream, filename)
250 except (IOError, OSError) as err:
251 if err.errno in (errno.EACCES,):
254 # In case of error, try to remove win32 forbidden chars
255 alt_filename = sanitize_path(filename)
256 if alt_filename == filename:
259 # An exception here should be caught in the caller
260 stream = open(encodeFilename(alt_filename), open_mode)
261 return (stream, alt_filename)
264 def timeconvert(timestr):
265 """Convert RFC 2822 defined time string into system timestamp"""
267 timetuple = email.utils.parsedate_tz(timestr)
268 if timetuple is not None:
269 timestamp = email.utils.mktime_tz(timetuple)
273 def sanitize_filename(s, restricted=False, is_id=False):
274 """Sanitizes a string so it could be used as part of a filename.
275 If restricted is set, use a stricter subset of allowed characters.
276 Set is_id if this is not an arbitrary string, but an ID that should be kept if possible
278 def replace_insane(char):
279 if char == '?' or ord(char) < 32 or ord(char) == 127:
282 return '' if restricted else '\''
284 return '_-' if restricted else ' -'
285 elif char in '\\/|*<>':
287 if restricted and (char in '!&\'()[]{}$;`^,#' or char.isspace()):
289 if restricted and ord(char) > 127:
294 s = re.sub(r'[0-9]+(?::[0-9]+)+', lambda m: m.group(0).replace(':', '_'), s)
295 result = ''.join(map(replace_insane, s))
297 while '__' in result:
298 result = result.replace('__', '_')
299 result = result.strip('_')
300 # Common case of "Foreign band name - English song title"
301 if restricted and result.startswith('-_'):
303 if result.startswith('-'):
304 result = '_' + result[len('-'):]
305 result = result.lstrip('.')
311 def sanitize_path(s):
312 """Sanitizes and normalizes path on Windows"""
313 if sys.platform != 'win32':
315 drive_or_unc, _ = os.path.splitdrive(s)
316 if sys.version_info < (2, 7) and not drive_or_unc:
317 drive_or_unc, _ = os.path.splitunc(s)
318 norm_path = os.path.normpath(remove_start(s, drive_or_unc)).split(os.path.sep)
322 path_part if path_part in ['.', '..'] else re.sub('(?:[/<>:"\\|\\\\?\\*]|\.$)', '#', path_part)
323 for path_part in norm_path]
325 sanitized_path.insert(0, drive_or_unc + os.path.sep)
326 return os.path.join(*sanitized_path)
329 def sanitize_url_path_consecutive_slashes(url):
330 """Collapses consecutive slashes in URLs' path"""
331 parsed_url = list(compat_urlparse.urlparse(url))
332 parsed_url[2] = re.sub(r'/{2,}', '/', parsed_url[2])
333 return compat_urlparse.urlunparse(parsed_url)
336 def orderedSet(iterable):
337 """ Remove all duplicates from the input iterable """
345 def _htmlentity_transform(entity):
346 """Transforms an HTML entity to a character."""
347 # Known non-numeric HTML entity
348 if entity in compat_html_entities.name2codepoint:
349 return compat_chr(compat_html_entities.name2codepoint[entity])
351 mobj = re.match(r'#(x[0-9a-fA-F]+|[0-9]+)', entity)
353 numstr = mobj.group(1)
354 if numstr.startswith('x'):
356 numstr = '0%s' % numstr
359 return compat_chr(int(numstr, base))
361 # Unknown entity in name, return its literal representation
362 return ('&%s;' % entity)
368 assert type(s) == compat_str
371 r'&([^;]+);', lambda m: _htmlentity_transform(m.group(1)), s)
374 def encodeFilename(s, for_subprocess=False):
376 @param s The name of the file
379 assert type(s) == compat_str
381 # Python 3 has a Unicode API
382 if sys.version_info >= (3, 0):
385 if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
386 # Pass '' directly to use Unicode APIs on Windows 2000 and up
387 # (Detecting Windows NT 4 is tricky because 'major >= 4' would
388 # match Windows 9x series as well. Besides, NT 4 is obsolete.)
389 if not for_subprocess:
392 # For subprocess calls, encode with locale encoding
393 # Refer to http://stackoverflow.com/a/9951851/35070
394 encoding = preferredencoding()
396 encoding = sys.getfilesystemencoding()
399 return s.encode(encoding, 'ignore')
402 def encodeArgument(s):
403 if not isinstance(s, compat_str):
404 # Legacy code that uses byte strings
405 # Uncomment the following line after fixing all post processors
406 # assert False, 'Internal error: %r should be of type %r, is %r' % (s, compat_str, type(s))
407 s = s.decode('ascii')
408 return encodeFilename(s, True)
411 def decodeOption(optval):
414 if isinstance(optval, bytes):
415 optval = optval.decode(preferredencoding())
417 assert isinstance(optval, compat_str)
421 def formatSeconds(secs):
423 return '%d:%02d:%02d' % (secs // 3600, (secs % 3600) // 60, secs % 60)
425 return '%d:%02d' % (secs // 60, secs % 60)
430 def make_HTTPS_handler(params, **kwargs):
431 opts_no_check_certificate = params.get('nocheckcertificate', False)
432 if hasattr(ssl, 'create_default_context'): # Python >= 3.4 or 2.7.9
433 context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
434 if opts_no_check_certificate:
435 context.check_hostname = False
436 context.verify_mode = ssl.CERT_NONE
438 return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
441 # (create_default_context present but HTTPSHandler has no context=)
444 if sys.version_info < (3, 2):
445 return YoutubeDLHTTPSHandler(params, **kwargs)
447 context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
448 context.verify_mode = (ssl.CERT_NONE
449 if opts_no_check_certificate
450 else ssl.CERT_REQUIRED)
451 context.set_default_verify_paths()
452 return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
455 def bug_reports_message():
456 if ytdl_is_updateable():
457 update_cmd = 'type youtube-dl -U to update'
459 update_cmd = 'see https://yt-dl.org/update on how to update'
460 msg = '; please report this issue on https://yt-dl.org/bug .'
461 msg += ' Make sure you are using the latest version; %s.' % update_cmd
462 msg += ' Be sure to call youtube-dl with the --verbose flag and include its complete output.'
466 class ExtractorError(Exception):
467 """Error during info extraction."""
469 def __init__(self, msg, tb=None, expected=False, cause=None, video_id=None):
470 """ tb, if given, is the original traceback (so that it can be printed out).
471 If expected is set, this is a normal error message and most likely not a bug in youtube-dl.
474 if sys.exc_info()[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError):
476 if video_id is not None:
477 msg = video_id + ': ' + msg
479 msg += ' (caused by %r)' % cause
481 msg += bug_reports_message()
482 super(ExtractorError, self).__init__(msg)
485 self.exc_info = sys.exc_info() # preserve original exception
487 self.video_id = video_id
489 def format_traceback(self):
490 if self.traceback is None:
492 return ''.join(traceback.format_tb(self.traceback))
495 class UnsupportedError(ExtractorError):
496 def __init__(self, url):
497 super(UnsupportedError, self).__init__(
498 'Unsupported URL: %s' % url, expected=True)
502 class RegexNotFoundError(ExtractorError):
503 """Error when a regex didn't match"""
507 class DownloadError(Exception):
508 """Download Error exception.
510 This exception may be thrown by FileDownloader objects if they are not
511 configured to continue on errors. They will contain the appropriate
515 def __init__(self, msg, exc_info=None):
516 """ exc_info, if given, is the original exception that caused the trouble (as returned by sys.exc_info()). """
517 super(DownloadError, self).__init__(msg)
518 self.exc_info = exc_info
521 class SameFileError(Exception):
522 """Same File exception.
524 This exception will be thrown by FileDownloader objects if they detect
525 multiple files would have to be downloaded to the same file on disk.
530 class PostProcessingError(Exception):
531 """Post Processing exception.
533 This exception may be raised by PostProcessor's .run() method to
534 indicate an error in the postprocessing task.
537 def __init__(self, msg):
541 class MaxDownloadsReached(Exception):
542 """ --max-downloads limit has been reached. """
546 class UnavailableVideoError(Exception):
547 """Unavailable Format exception.
549 This exception will be thrown when a video is requested
550 in a format that is not available for that video.
555 class ContentTooShortError(Exception):
556 """Content Too Short exception.
558 This exception may be raised by FileDownloader objects when a file they
559 download is too small for what the server announced first, indicating
560 the connection was probably interrupted.
566 def __init__(self, downloaded, expected):
567 self.downloaded = downloaded
568 self.expected = expected
571 def _create_http_connection(ydl_handler, http_class, is_https, *args, **kwargs):
572 hc = http_class(*args, **kwargs)
573 source_address = ydl_handler._params.get('source_address')
574 if source_address is not None:
575 sa = (source_address, 0)
576 if hasattr(hc, 'source_address'): # Python 2.7+
577 hc.source_address = sa
579 def _hc_connect(self, *args, **kwargs):
580 sock = compat_socket_create_connection(
581 (self.host, self.port), self.timeout, sa)
583 self.sock = ssl.wrap_socket(
584 sock, self.key_file, self.cert_file,
585 ssl_version=ssl.PROTOCOL_TLSv1)
588 hc.connect = functools.partial(_hc_connect, hc)
593 class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
594 """Handler for HTTP requests and responses.
596 This class, when installed with an OpenerDirector, automatically adds
597 the standard headers to every HTTP request and handles gzipped and
598 deflated responses from web servers. If compression is to be avoided in
599 a particular request, the original request in the program code only has
600 to include the HTTP header "Youtubedl-No-Compression", which will be
601 removed before making the real request.
603 Part of this code was copied from:
605 http://techknack.net/python-urllib2-handlers/
607 Andrew Rowls, the author of that code, agreed to release it to the
611 def __init__(self, params, *args, **kwargs):
612 compat_urllib_request.HTTPHandler.__init__(self, *args, **kwargs)
613 self._params = params
615 def http_open(self, req):
616 return self.do_open(functools.partial(
617 _create_http_connection, self, compat_http_client.HTTPConnection, False),
623 return zlib.decompress(data, -zlib.MAX_WBITS)
625 return zlib.decompress(data)
628 def addinfourl_wrapper(stream, headers, url, code):
629 if hasattr(compat_urllib_request.addinfourl, 'getcode'):
630 return compat_urllib_request.addinfourl(stream, headers, url, code)
631 ret = compat_urllib_request.addinfourl(stream, headers, url)
635 def http_request(self, req):
636 for h, v in std_headers.items():
637 # Capitalize is needed because of Python bug 2275: http://bugs.python.org/issue2275
638 # The dict keys are capitalized because of this bug by urllib
639 if h.capitalize() not in req.headers:
641 if 'Youtubedl-no-compression' in req.headers:
642 if 'Accept-encoding' in req.headers:
643 del req.headers['Accept-encoding']
644 del req.headers['Youtubedl-no-compression']
646 if sys.version_info < (2, 7) and '#' in req.get_full_url():
647 # Python 2.6 is brain-dead when it comes to fragments
648 req._Request__original = req._Request__original.partition('#')[0]
649 req._Request__r_type = req._Request__r_type.partition('#')[0]
653 def http_response(self, req, resp):
656 if resp.headers.get('Content-encoding', '') == 'gzip':
657 content = resp.read()
658 gz = gzip.GzipFile(fileobj=io.BytesIO(content), mode='rb')
660 uncompressed = io.BytesIO(gz.read())
661 except IOError as original_ioerror:
662 # There may be junk add the end of the file
663 # See http://stackoverflow.com/q/4928560/35070 for details
664 for i in range(1, 1024):
666 gz = gzip.GzipFile(fileobj=io.BytesIO(content[:-i]), mode='rb')
667 uncompressed = io.BytesIO(gz.read())
672 raise original_ioerror
673 resp = self.addinfourl_wrapper(uncompressed, old_resp.headers, old_resp.url, old_resp.code)
674 resp.msg = old_resp.msg
676 if resp.headers.get('Content-encoding', '') == 'deflate':
677 gz = io.BytesIO(self.deflate(resp.read()))
678 resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
679 resp.msg = old_resp.msg
682 https_request = http_request
683 https_response = http_response
686 class YoutubeDLHTTPSHandler(compat_urllib_request.HTTPSHandler):
687 def __init__(self, params, https_conn_class=None, *args, **kwargs):
688 compat_urllib_request.HTTPSHandler.__init__(self, *args, **kwargs)
689 self._https_conn_class = https_conn_class or compat_http_client.HTTPSConnection
690 self._params = params
692 def https_open(self, req):
694 if hasattr(self, '_context'): # python > 2.6
695 kwargs['context'] = self._context
696 if hasattr(self, '_check_hostname'): # python 3.x
697 kwargs['check_hostname'] = self._check_hostname
698 return self.do_open(functools.partial(
699 _create_http_connection, self, self._https_conn_class, True),
703 def parse_iso8601(date_str, delimiter='T', timezone=None):
704 """ Return a UNIX timestamp from the given date """
711 r'(\.[0-9]+)?(?:Z$| ?(?P<sign>\+|-)(?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})$)',
714 timezone = datetime.timedelta()
716 date_str = date_str[:-len(m.group(0))]
717 if not m.group('sign'):
718 timezone = datetime.timedelta()
720 sign = 1 if m.group('sign') == '+' else -1
721 timezone = datetime.timedelta(
722 hours=sign * int(m.group('hours')),
723 minutes=sign * int(m.group('minutes')))
724 date_format = '%Y-%m-%d{0}%H:%M:%S'.format(delimiter)
725 dt = datetime.datetime.strptime(date_str, date_format) - timezone
726 return calendar.timegm(dt.timetuple())
729 def unified_strdate(date_str, day_first=True):
730 """Return a string with the date in the format YYYYMMDD"""
736 date_str = date_str.replace(',', ' ')
737 # %z (UTC offset) is only supported in python>=3.2
738 if not re.match(r'^[0-9]{1,2}-[0-9]{1,2}-[0-9]{4}$', date_str):
739 date_str = re.sub(r' ?(\+|-)[0-9]{2}:?[0-9]{2}$', '', date_str)
740 # Remove AM/PM + timezone
741 date_str = re.sub(r'(?i)\s*(?:AM|PM)(?:\s+[A-Z]+)?', '', date_str)
743 format_expressions = [
748 '%b %dst %Y %I:%M%p',
749 '%b %dnd %Y %I:%M%p',
750 '%b %dth %Y %I:%M%p',
756 '%Y-%m-%d %H:%M:%S.%f',
759 '%Y-%m-%dT%H:%M:%SZ',
760 '%Y-%m-%dT%H:%M:%S.%fZ',
761 '%Y-%m-%dT%H:%M:%S.%f0Z',
763 '%Y-%m-%dT%H:%M:%S.%f',
767 format_expressions.extend([
775 format_expressions.extend([
782 for expression in format_expressions:
784 upload_date = datetime.datetime.strptime(date_str, expression).strftime('%Y%m%d')
787 if upload_date is None:
788 timetuple = email.utils.parsedate_tz(date_str)
790 upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d')
794 def determine_ext(url, default_ext='unknown_video'):
797 guess = url.partition('?')[0].rpartition('.')[2]
798 if re.match(r'^[A-Za-z0-9]+$', guess):
804 def subtitles_filename(filename, sub_lang, sub_format):
805 return filename.rsplit('.', 1)[0] + '.' + sub_lang + '.' + sub_format
808 def date_from_str(date_str):
810 Return a datetime object from a string in the format YYYYMMDD or
811 (now|today)[+-][0-9](day|week|month|year)(s)?"""
812 today = datetime.date.today()
813 if date_str in ('now', 'today'):
815 if date_str == 'yesterday':
816 return today - datetime.timedelta(days=1)
817 match = re.match('(now|today)(?P<sign>[+-])(?P<time>\d+)(?P<unit>day|week|month|year)(s)?', date_str)
818 if match is not None:
819 sign = match.group('sign')
820 time = int(match.group('time'))
823 unit = match.group('unit')
824 # A bad aproximation?
832 delta = datetime.timedelta(**{unit: time})
834 return datetime.datetime.strptime(date_str, "%Y%m%d").date()
837 def hyphenate_date(date_str):
839 Convert a date in 'YYYYMMDD' format to 'YYYY-MM-DD' format"""
840 match = re.match(r'^(\d\d\d\d)(\d\d)(\d\d)$', date_str)
841 if match is not None:
842 return '-'.join(match.groups())
847 class DateRange(object):
848 """Represents a time interval between two dates"""
850 def __init__(self, start=None, end=None):
851 """start and end must be strings in the format accepted by date"""
852 if start is not None:
853 self.start = date_from_str(start)
855 self.start = datetime.datetime.min.date()
857 self.end = date_from_str(end)
859 self.end = datetime.datetime.max.date()
860 if self.start > self.end:
861 raise ValueError('Date range: "%s" , the start date must be before the end date' % self)
865 """Returns a range that only contains the given day"""
868 def __contains__(self, date):
869 """Check if the date is in the range"""
870 if not isinstance(date, datetime.date):
871 date = date_from_str(date)
872 return self.start <= date <= self.end
875 return '%s - %s' % (self.start.isoformat(), self.end.isoformat())
879 """ Returns the platform name as a compat_str """
880 res = platform.platform()
881 if isinstance(res, bytes):
882 res = res.decode(preferredencoding())
884 assert isinstance(res, compat_str)
888 def _windows_write_string(s, out):
889 """ Returns True if the string was written using special methods,
890 False if it has yet to be written out."""
891 # Adapted from http://stackoverflow.com/a/3259271/35070
894 import ctypes.wintypes
902 fileno = out.fileno()
903 except AttributeError:
904 # If the output stream doesn't have a fileno, it's virtual
906 except io.UnsupportedOperation:
907 # Some strange Windows pseudo files?
909 if fileno not in WIN_OUTPUT_IDS:
912 GetStdHandle = ctypes.WINFUNCTYPE(
913 ctypes.wintypes.HANDLE, ctypes.wintypes.DWORD)(
914 (b"GetStdHandle", ctypes.windll.kernel32))
915 h = GetStdHandle(WIN_OUTPUT_IDS[fileno])
917 WriteConsoleW = ctypes.WINFUNCTYPE(
918 ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE, ctypes.wintypes.LPWSTR,
919 ctypes.wintypes.DWORD, ctypes.POINTER(ctypes.wintypes.DWORD),
920 ctypes.wintypes.LPVOID)((b"WriteConsoleW", ctypes.windll.kernel32))
921 written = ctypes.wintypes.DWORD(0)
923 GetFileType = ctypes.WINFUNCTYPE(ctypes.wintypes.DWORD, ctypes.wintypes.DWORD)((b"GetFileType", ctypes.windll.kernel32))
924 FILE_TYPE_CHAR = 0x0002
925 FILE_TYPE_REMOTE = 0x8000
926 GetConsoleMode = ctypes.WINFUNCTYPE(
927 ctypes.wintypes.BOOL, ctypes.wintypes.HANDLE,
928 ctypes.POINTER(ctypes.wintypes.DWORD))(
929 (b"GetConsoleMode", ctypes.windll.kernel32))
930 INVALID_HANDLE_VALUE = ctypes.wintypes.DWORD(-1).value
932 def not_a_console(handle):
933 if handle == INVALID_HANDLE_VALUE or handle is None:
935 return ((GetFileType(handle) & ~FILE_TYPE_REMOTE) != FILE_TYPE_CHAR or
936 GetConsoleMode(handle, ctypes.byref(ctypes.wintypes.DWORD())) == 0)
941 def next_nonbmp_pos(s):
943 return next(i for i, c in enumerate(s) if ord(c) > 0xffff)
944 except StopIteration:
948 count = min(next_nonbmp_pos(s), 1024)
951 h, s, count if count else 2, ctypes.byref(written), None)
953 raise OSError('Failed to write string')
954 if not count: # We just wrote a non-BMP character
955 assert written.value == 2
958 assert written.value > 0
959 s = s[written.value:]
963 def write_string(s, out=None, encoding=None):
966 assert type(s) == compat_str
968 if sys.platform == 'win32' and encoding is None and hasattr(out, 'fileno'):
969 if _windows_write_string(s, out):
972 if ('b' in getattr(out, 'mode', '') or
973 sys.version_info[0] < 3): # Python 2 lies about mode of sys.stderr
974 byt = s.encode(encoding or preferredencoding(), 'ignore')
976 elif hasattr(out, 'buffer'):
977 enc = encoding or getattr(out, 'encoding', None) or preferredencoding()
978 byt = s.encode(enc, 'ignore')
979 out.buffer.write(byt)
985 def bytes_to_intlist(bs):
988 if isinstance(bs[0], int): # Python 3
991 return [ord(c) for c in bs]
994 def intlist_to_bytes(xs):
997 return struct_pack('%dB' % len(xs), *xs)
1000 # Cross-platform file locking
1001 if sys.platform == 'win32':
1002 import ctypes.wintypes
1005 class OVERLAPPED(ctypes.Structure):
1007 ('Internal', ctypes.wintypes.LPVOID),
1008 ('InternalHigh', ctypes.wintypes.LPVOID),
1009 ('Offset', ctypes.wintypes.DWORD),
1010 ('OffsetHigh', ctypes.wintypes.DWORD),
1011 ('hEvent', ctypes.wintypes.HANDLE),
1014 kernel32 = ctypes.windll.kernel32
1015 LockFileEx = kernel32.LockFileEx
1016 LockFileEx.argtypes = [
1017 ctypes.wintypes.HANDLE, # hFile
1018 ctypes.wintypes.DWORD, # dwFlags
1019 ctypes.wintypes.DWORD, # dwReserved
1020 ctypes.wintypes.DWORD, # nNumberOfBytesToLockLow
1021 ctypes.wintypes.DWORD, # nNumberOfBytesToLockHigh
1022 ctypes.POINTER(OVERLAPPED) # Overlapped
1024 LockFileEx.restype = ctypes.wintypes.BOOL
1025 UnlockFileEx = kernel32.UnlockFileEx
1026 UnlockFileEx.argtypes = [
1027 ctypes.wintypes.HANDLE, # hFile
1028 ctypes.wintypes.DWORD, # dwReserved
1029 ctypes.wintypes.DWORD, # nNumberOfBytesToLockLow
1030 ctypes.wintypes.DWORD, # nNumberOfBytesToLockHigh
1031 ctypes.POINTER(OVERLAPPED) # Overlapped
1033 UnlockFileEx.restype = ctypes.wintypes.BOOL
1034 whole_low = 0xffffffff
1035 whole_high = 0x7fffffff
1037 def _lock_file(f, exclusive):
1038 overlapped = OVERLAPPED()
1039 overlapped.Offset = 0
1040 overlapped.OffsetHigh = 0
1041 overlapped.hEvent = 0
1042 f._lock_file_overlapped_p = ctypes.pointer(overlapped)
1043 handle = msvcrt.get_osfhandle(f.fileno())
1044 if not LockFileEx(handle, 0x2 if exclusive else 0x0, 0,
1045 whole_low, whole_high, f._lock_file_overlapped_p):
1046 raise OSError('Locking file failed: %r' % ctypes.FormatError())
1048 def _unlock_file(f):
1049 assert f._lock_file_overlapped_p
1050 handle = msvcrt.get_osfhandle(f.fileno())
1051 if not UnlockFileEx(handle, 0,
1052 whole_low, whole_high, f._lock_file_overlapped_p):
1053 raise OSError('Unlocking file failed: %r' % ctypes.FormatError())
1058 def _lock_file(f, exclusive):
1059 fcntl.flock(f, fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH)
1061 def _unlock_file(f):
1062 fcntl.flock(f, fcntl.LOCK_UN)
1065 class locked_file(object):
1066 def __init__(self, filename, mode, encoding=None):
1067 assert mode in ['r', 'a', 'w']
1068 self.f = io.open(filename, mode, encoding=encoding)
1071 def __enter__(self):
1072 exclusive = self.mode != 'r'
1074 _lock_file(self.f, exclusive)
1080 def __exit__(self, etype, value, traceback):
1082 _unlock_file(self.f)
1089 def write(self, *args):
1090 return self.f.write(*args)
1092 def read(self, *args):
1093 return self.f.read(*args)
1096 def get_filesystem_encoding():
1097 encoding = sys.getfilesystemencoding()
1098 return encoding if encoding is not None else 'utf-8'
1101 def shell_quote(args):
1103 encoding = get_filesystem_encoding()
1105 if isinstance(a, bytes):
1106 # We may get a filename encoded with 'encodeFilename'
1107 a = a.decode(encoding)
1108 quoted_args.append(pipes.quote(a))
1109 return ' '.join(quoted_args)
1112 def takewhile_inclusive(pred, seq):
1113 """ Like itertools.takewhile, but include the latest evaluated element
1114 (the first element so that Not pred(e)) """
1121 def smuggle_url(url, data):
1122 """ Pass additional data in a URL for internal use. """
1124 sdata = compat_urllib_parse.urlencode(
1125 {'__youtubedl_smuggle': json.dumps(data)})
1126 return url + '#' + sdata
1129 def unsmuggle_url(smug_url, default=None):
1130 if '#__youtubedl_smuggle' not in smug_url:
1131 return smug_url, default
1132 url, _, sdata = smug_url.rpartition('#')
1133 jsond = compat_parse_qs(sdata)['__youtubedl_smuggle'][0]
1134 data = json.loads(jsond)
1138 def format_bytes(bytes):
1141 if type(bytes) is str:
1142 bytes = float(bytes)
1146 exponent = int(math.log(bytes, 1024.0))
1147 suffix = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB', 'EiB', 'ZiB', 'YiB'][exponent]
1148 converted = float(bytes) / float(1024 ** exponent)
1149 return '%.2f%s' % (converted, suffix)
1152 def parse_filesize(s):
1156 # The lower-case forms are of course incorrect and inofficial,
1157 # but we support those too
1195 units_re = '|'.join(re.escape(u) for u in _UNIT_TABLE)
1197 r'(?P<num>[0-9]+(?:[,.][0-9]*)?)\s*(?P<unit>%s)' % units_re, s)
1201 num_str = m.group('num').replace(',', '.')
1202 mult = _UNIT_TABLE[m.group('unit')]
1203 return int(float(num_str) * mult)
1206 def month_by_name(name):
1207 """ Return the number of a month by (locale-independently) English name """
1210 return ENGLISH_MONTH_NAMES.index(name) + 1
1215 def month_by_abbreviation(abbrev):
1216 """ Return the number of a month by (locale-independently) English
1220 return [s[:3] for s in ENGLISH_MONTH_NAMES].index(abbrev) + 1
1225 def fix_xml_ampersands(xml_str):
1226 """Replace all the '&' by '&' in XML"""
1228 r'&(?!amp;|lt;|gt;|apos;|quot;|#x[0-9a-fA-F]{,4};|#[0-9]{,4};)',
1233 def setproctitle(title):
1234 assert isinstance(title, compat_str)
1236 libc = ctypes.cdll.LoadLibrary("libc.so.6")
1239 title_bytes = title.encode('utf-8')
1240 buf = ctypes.create_string_buffer(len(title_bytes))
1241 buf.value = title_bytes
1243 libc.prctl(15, buf, 0, 0, 0)
1244 except AttributeError:
1245 return # Strange libc, just skip this
1248 def remove_start(s, start):
1249 if s.startswith(start):
1250 return s[len(start):]
1254 def remove_end(s, end):
1256 return s[:-len(end)]
1260 def url_basename(url):
1261 path = compat_urlparse.urlparse(url).path
1262 return path.strip('/').split('/')[-1]
1265 class HEADRequest(compat_urllib_request.Request):
1266 def get_method(self):
1270 def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1):
1273 v = getattr(v, get_attr, None)
1276 return default if v is None else (int(v) * invscale // scale)
1279 def str_or_none(v, default=None):
1280 return default if v is None else compat_str(v)
1283 def str_to_int(int_str):
1284 """ A more relaxed version of int_or_none """
1287 int_str = re.sub(r'[,\.\+]', '', int_str)
1291 def float_or_none(v, scale=1, invscale=1, default=None):
1292 return default if v is None else (float(v) * invscale / scale)
1295 def parse_duration(s):
1296 if not isinstance(s, compat_basestring):
1304 (?P<only_mins>[0-9.]+)\s*(?:mins?|minutes?)\s*|
1305 (?P<only_hours>[0-9.]+)\s*(?:hours?)|
1307 \s*(?P<hours_reversed>[0-9]+)\s*(?:[:h]|hours?)\s*(?P<mins_reversed>[0-9]+)\s*(?:[:m]|mins?|minutes?)\s*|
1310 (?:(?P<days>[0-9]+)\s*(?:[:d]|days?)\s*)?
1311 (?P<hours>[0-9]+)\s*(?:[:h]|hours?)\s*
1313 (?P<mins>[0-9]+)\s*(?:[:m]|mins?|minutes?)\s*
1315 (?P<secs>[0-9]+)(?P<ms>\.[0-9]+)?\s*(?:s|secs?|seconds?)?
1320 if m.group('only_mins'):
1321 return float_or_none(m.group('only_mins'), invscale=60)
1322 if m.group('only_hours'):
1323 return float_or_none(m.group('only_hours'), invscale=60 * 60)
1325 res += int(m.group('secs'))
1326 if m.group('mins_reversed'):
1327 res += int(m.group('mins_reversed')) * 60
1329 res += int(m.group('mins')) * 60
1330 if m.group('hours'):
1331 res += int(m.group('hours')) * 60 * 60
1332 if m.group('hours_reversed'):
1333 res += int(m.group('hours_reversed')) * 60 * 60
1335 res += int(m.group('days')) * 24 * 60 * 60
1337 res += float(m.group('ms'))
1341 def prepend_extension(filename, ext):
1342 name, real_ext = os.path.splitext(filename)
1343 return '{0}.{1}{2}'.format(name, ext, real_ext)
1346 def check_executable(exe, args=[]):
1347 """ Checks if the given binary is installed somewhere in PATH, and returns its name.
1348 args can be a list of arguments for a short output (like -version) """
1350 subprocess.Popen([exe] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
1356 def get_exe_version(exe, args=['--version'],
1357 version_re=None, unrecognized='present'):
1358 """ Returns the version of the specified executable,
1359 or False if the executable is not present """
1361 out, _ = subprocess.Popen(
1363 stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()
1366 if isinstance(out, bytes): # Python 2.x
1367 out = out.decode('ascii', 'ignore')
1368 return detect_exe_version(out, version_re, unrecognized)
1371 def detect_exe_version(output, version_re=None, unrecognized='present'):
1372 assert isinstance(output, compat_str)
1373 if version_re is None:
1374 version_re = r'version\s+([-0-9._a-zA-Z]+)'
1375 m = re.search(version_re, output)
1382 class PagedList(object):
1384 # This is only useful for tests
1385 return len(self.getslice())
1388 class OnDemandPagedList(PagedList):
1389 def __init__(self, pagefunc, pagesize):
1390 self._pagefunc = pagefunc
1391 self._pagesize = pagesize
1393 def getslice(self, start=0, end=None):
1395 for pagenum in itertools.count(start // self._pagesize):
1396 firstid = pagenum * self._pagesize
1397 nextfirstid = pagenum * self._pagesize + self._pagesize
1398 if start >= nextfirstid:
1401 page_results = list(self._pagefunc(pagenum))
1404 start % self._pagesize
1405 if firstid <= start < nextfirstid
1409 ((end - 1) % self._pagesize) + 1
1410 if (end is not None and firstid <= end <= nextfirstid)
1413 if startv != 0 or endv is not None:
1414 page_results = page_results[startv:endv]
1415 res.extend(page_results)
1417 # A little optimization - if current page is not "full", ie. does
1418 # not contain page_size videos then we can assume that this page
1419 # is the last one - there are no more ids on further pages -
1420 # i.e. no need to query again.
1421 if len(page_results) + startv < self._pagesize:
1424 # If we got the whole page, but the next page is not interesting,
1425 # break out early as well
1426 if end == nextfirstid:
1431 class InAdvancePagedList(PagedList):
1432 def __init__(self, pagefunc, pagecount, pagesize):
1433 self._pagefunc = pagefunc
1434 self._pagecount = pagecount
1435 self._pagesize = pagesize
1437 def getslice(self, start=0, end=None):
1439 start_page = start // self._pagesize
1441 self._pagecount if end is None else (end // self._pagesize + 1))
1442 skip_elems = start - start_page * self._pagesize
1443 only_more = None if end is None else end - start
1444 for pagenum in range(start_page, end_page):
1445 page = list(self._pagefunc(pagenum))
1447 page = page[skip_elems:]
1449 if only_more is not None:
1450 if len(page) < only_more:
1451 only_more -= len(page)
1453 page = page[:only_more]
1460 def uppercase_escape(s):
1461 unicode_escape = codecs.getdecoder('unicode_escape')
1463 r'\\U[0-9a-fA-F]{8}',
1464 lambda m: unicode_escape(m.group(0))[0],
1468 def escape_rfc3986(s):
1469 """Escape non-ASCII characters as suggested by RFC 3986"""
1470 if sys.version_info < (3, 0) and isinstance(s, compat_str):
1471 s = s.encode('utf-8')
1472 return compat_urllib_parse.quote(s, b"%/;:@&=+$,!~*'()?#[]")
1475 def escape_url(url):
1476 """Escape URL as suggested by RFC 3986"""
1477 url_parsed = compat_urllib_parse_urlparse(url)
1478 return url_parsed._replace(
1479 path=escape_rfc3986(url_parsed.path),
1480 params=escape_rfc3986(url_parsed.params),
1481 query=escape_rfc3986(url_parsed.query),
1482 fragment=escape_rfc3986(url_parsed.fragment)
1486 struct.pack('!I', 0)
1488 # In Python 2.6 (and some 2.7 versions), struct requires a bytes argument
1489 def struct_pack(spec, *args):
1490 if isinstance(spec, compat_str):
1491 spec = spec.encode('ascii')
1492 return struct.pack(spec, *args)
1494 def struct_unpack(spec, *args):
1495 if isinstance(spec, compat_str):
1496 spec = spec.encode('ascii')
1497 return struct.unpack(spec, *args)
1499 struct_pack = struct.pack
1500 struct_unpack = struct.unpack
1503 def read_batch_urls(batch_fd):
1505 if not isinstance(url, compat_str):
1506 url = url.decode('utf-8', 'replace')
1507 BOM_UTF8 = '\xef\xbb\xbf'
1508 if url.startswith(BOM_UTF8):
1509 url = url[len(BOM_UTF8):]
1511 if url.startswith(('#', ';', ']')):
1515 with contextlib.closing(batch_fd) as fd:
1516 return [url for url in map(fixup, fd) if url]
1519 def urlencode_postdata(*args, **kargs):
1520 return compat_urllib_parse.urlencode(*args, **kargs).encode('ascii')
1524 etree_iter = xml.etree.ElementTree.Element.iter
1525 except AttributeError: # Python <=2.6
1526 etree_iter = lambda n: n.findall('.//*')
1530 class TreeBuilder(xml.etree.ElementTree.TreeBuilder):
1531 def doctype(self, name, pubid, system):
1532 pass # Ignore doctypes
1534 parser = xml.etree.ElementTree.XMLParser(target=TreeBuilder())
1535 kwargs = {'parser': parser} if sys.version_info >= (2, 7) else {}
1536 tree = xml.etree.ElementTree.XML(s.encode('utf-8'), **kwargs)
1537 # Fix up XML parser in Python 2.x
1538 if sys.version_info < (3, 0):
1539 for n in etree_iter(tree):
1540 if n.text is not None:
1541 if not isinstance(n.text, compat_str):
1542 n.text = n.text.decode('utf-8')
1555 def parse_age_limit(s):
1558 m = re.match(r'^(?P<age>\d{1,2})\+?$', s)
1559 return int(m.group('age')) if m else US_RATINGS.get(s, None)
1562 def strip_jsonp(code):
1564 r'(?s)^[a-zA-Z0-9_]+\s*\(\s*(.*)\);?\s*?(?://[^\n]*)*$', r'\1', code)
1567 def js_to_json(code):
1570 if v in ('true', 'false', 'null'):
1572 if v.startswith('"'):
1574 if v.startswith("'"):
1576 v = re.sub(r"\\\\|\\'|\"", lambda m: {
1583 res = re.sub(r'''(?x)
1584 "(?:[^"\\]*(?:\\\\|\\['"nu]))*[^"\\]*"|
1585 '(?:[^'\\]*(?:\\\\|\\['"nu]))*[^'\\]*'|
1586 [a-zA-Z_][.a-zA-Z_0-9]*
1588 res = re.sub(r',(\s*[\]}])', lambda m: m.group(1), res)
1592 def qualities(quality_ids):
1593 """ Get a numeric quality value out of a list of possible values """
1596 return quality_ids.index(qid)
1602 DEFAULT_OUTTMPL = '%(title)s-%(id)s.%(ext)s'
1605 def limit_length(s, length):
1606 """ Add ellipses to overly long strings """
1611 return s[:length - len(ELLIPSES)] + ELLIPSES
1615 def version_tuple(v):
1616 return tuple(int(e) for e in re.split(r'[-.]', v))
1619 def is_outdated_version(version, limit, assume_new=True):
1621 return not assume_new
1623 return version_tuple(version) < version_tuple(limit)
1625 return not assume_new
1628 def ytdl_is_updateable():
1629 """ Returns if youtube-dl can be updated with -U """
1630 from zipimport import zipimporter
1632 return isinstance(globals().get('__loader__'), zipimporter) or hasattr(sys, 'frozen')
1635 def args_to_str(args):
1636 # Get a short string representation for a subprocess command
1637 return ' '.join(shlex_quote(a) for a in args)
1640 def mimetype2ext(mt):
1641 _, _, res = mt.rpartition('/')
1645 'x-mp4-fragmented': 'mp4',
1649 def urlhandle_detect_ext(url_handle):
1652 getheader = lambda h: url_handle.headers[h]
1653 except AttributeError: # Python < 3
1654 getheader = url_handle.info().getheader
1656 cd = getheader('Content-Disposition')
1658 m = re.match(r'attachment;\s*filename="(?P<filename>[^"]+)"', cd)
1660 e = determine_ext(m.group('filename'), default_ext=None)
1664 return mimetype2ext(getheader('Content-Type'))
1667 def age_restricted(content_limit, age_limit):
1668 """ Returns True iff the content should be blocked """
1670 if age_limit is None: # No limit set
1672 if content_limit is None:
1673 return False # Content available for everyone
1674 return age_limit < content_limit
1677 def is_html(first_bytes):
1678 """ Detect whether a file contains HTML by examining its first bytes. """
1681 (b'\xef\xbb\xbf', 'utf-8'),
1682 (b'\x00\x00\xfe\xff', 'utf-32-be'),
1683 (b'\xff\xfe\x00\x00', 'utf-32-le'),
1684 (b'\xff\xfe', 'utf-16-le'),
1685 (b'\xfe\xff', 'utf-16-be'),
1687 for bom, enc in BOMS:
1688 if first_bytes.startswith(bom):
1689 s = first_bytes[len(bom):].decode(enc, 'replace')
1692 s = first_bytes.decode('utf-8', 'replace')
1694 return re.match(r'^\s*<', s)
1697 def determine_protocol(info_dict):
1698 protocol = info_dict.get('protocol')
1699 if protocol is not None:
1702 url = info_dict['url']
1703 if url.startswith('rtmp'):
1705 elif url.startswith('mms'):
1707 elif url.startswith('rtsp'):
1710 ext = determine_ext(url)
1716 return compat_urllib_parse_urlparse(url).scheme
1719 def render_table(header_row, data):
1720 """ Render a list of rows, each as a list of values """
1721 table = [header_row] + data
1722 max_lens = [max(len(compat_str(v)) for v in col) for col in zip(*table)]
1723 format_str = ' '.join('%-' + compat_str(ml + 1) + 's' for ml in max_lens[:-1]) + '%s'
1724 return '\n'.join(format_str % tuple(row) for row in table)
1727 def _match_one(filter_part, dct):
1728 COMPARISON_OPERATORS = {
1736 operator_rex = re.compile(r'''(?x)\s*
1738 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
1740 (?P<intval>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)|
1741 (?P<strval>(?![0-9.])[a-z0-9A-Z]*)
1744 ''' % '|'.join(map(re.escape, COMPARISON_OPERATORS.keys())))
1745 m = operator_rex.search(filter_part)
1747 op = COMPARISON_OPERATORS[m.group('op')]
1748 if m.group('strval') is not None:
1749 if m.group('op') not in ('=', '!='):
1751 'Operator %s does not support string values!' % m.group('op'))
1752 comparison_value = m.group('strval')
1755 comparison_value = int(m.group('intval'))
1757 comparison_value = parse_filesize(m.group('intval'))
1758 if comparison_value is None:
1759 comparison_value = parse_filesize(m.group('intval') + 'B')
1760 if comparison_value is None:
1762 'Invalid integer value %r in filter part %r' % (
1763 m.group('intval'), filter_part))
1764 actual_value = dct.get(m.group('key'))
1765 if actual_value is None:
1766 return m.group('none_inclusive')
1767 return op(actual_value, comparison_value)
1770 '': lambda v: v is not None,
1771 '!': lambda v: v is None,
1773 operator_rex = re.compile(r'''(?x)\s*
1774 (?P<op>%s)\s*(?P<key>[a-z_]+)
1776 ''' % '|'.join(map(re.escape, UNARY_OPERATORS.keys())))
1777 m = operator_rex.search(filter_part)
1779 op = UNARY_OPERATORS[m.group('op')]
1780 actual_value = dct.get(m.group('key'))
1781 return op(actual_value)
1783 raise ValueError('Invalid filter part %r' % filter_part)
1786 def match_str(filter_str, dct):
1787 """ Filter a dictionary with a simple string syntax. Returns True (=passes filter) or false """
1790 _match_one(filter_part, dct) for filter_part in filter_str.split('&'))
1793 def match_filter_func(filter_str):
1794 def _match_func(info_dict):
1795 if match_str(filter_str, info_dict):
1798 video_title = info_dict.get('title', info_dict.get('id', 'video'))
1799 return '%s does not pass filter %s, skipping ..' % (video_title, filter_str)
1803 class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
1804 def __init__(self, proxies=None):
1805 # Set default handlers
1806 for type in ('http', 'https'):
1807 setattr(self, '%s_open' % type,
1808 lambda r, proxy='__noproxy__', type=type, meth=self.proxy_open:
1809 meth(r, proxy, type))
1810 return compat_urllib_request.ProxyHandler.__init__(self, proxies)
1812 def proxy_open(self, req, proxy, type):
1813 req_proxy = req.headers.get('Ytdl-request-proxy')
1814 if req_proxy is not None:
1816 del req.headers['Ytdl-request-proxy']
1818 if proxy == '__noproxy__':
1819 return None # No Proxy
1820 return compat_urllib_request.ProxyHandler.proxy_open(
1821 self, req, proxy, type)