1 from __future__ import unicode_literals
15 import urllib.request as compat_urllib_request
16 except ImportError: # Python 2
17 import urllib2 as compat_urllib_request
20 import urllib.error as compat_urllib_error
21 except ImportError: # Python 2
22 import urllib2 as compat_urllib_error
25 import urllib.parse as compat_urllib_parse
26 except ImportError: # Python 2
27 import urllib as compat_urllib_parse
30 from urllib.parse import urlparse as compat_urllib_parse_urlparse
31 except ImportError: # Python 2
32 from urlparse import urlparse as compat_urllib_parse_urlparse
35 import urllib.parse as compat_urlparse
36 except ImportError: # Python 2
37 import urlparse as compat_urlparse
40 import http.cookiejar as compat_cookiejar
41 except ImportError: # Python 2
42 import cookielib as compat_cookiejar
45 import html.entities as compat_html_entities
46 except ImportError: # Python 2
47 import htmlentitydefs as compat_html_entities
50 import html.parser as compat_html_parser
51 except ImportError: # Python 2
52 import HTMLParser as compat_html_parser
55 import http.client as compat_http_client
56 except ImportError: # Python 2
57 import httplib as compat_http_client
60 from urllib.error import HTTPError as compat_HTTPError
61 except ImportError: # Python 2
62 from urllib2 import HTTPError as compat_HTTPError
65 from urllib.request import urlretrieve as compat_urlretrieve
66 except ImportError: # Python 2
67 from urllib import urlretrieve as compat_urlretrieve
71 from subprocess import DEVNULL
72 compat_subprocess_get_DEVNULL = lambda: DEVNULL
74 compat_subprocess_get_DEVNULL = lambda: open(os.path.devnull, 'w')
77 import http.server as compat_http_server
79 import BaseHTTPServer as compat_http_server
82 from urllib.parse import unquote as compat_urllib_parse_unquote
84 def compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'):
87 res = string.split('%')
94 # pct_sequence: contiguous sequence of percent-encoded bytes, decoded
101 pct_sequence += item[:2].decode('hex')
104 # This segment was just a single percent-encoded character.
105 # May be part of a sequence of code units, so delay decoding.
106 # (Stored in pct_sequence).
110 # Encountered non-percent-encoded characters. Flush the current
112 string += pct_sequence.decode(encoding, errors) + rest
115 # Flush the final pct_sequence
116 string += pct_sequence.decode(encoding, errors)
120 compat_str = unicode # Python 2
125 compat_basestring = basestring # Python 2
127 compat_basestring = str
130 compat_chr = unichr # Python 2
135 from xml.etree.ElementTree import ParseError as compat_xml_parse_error
136 except ImportError: # Python 2.6
137 from xml.parsers.expat import ExpatError as compat_xml_parse_error
141 from urllib.parse import parse_qs as compat_parse_qs
142 except ImportError: # Python 2
143 # HACK: The following is the correct parse_qs implementation from cpython 3's stdlib.
144 # Python 2's version is apparently totally broken
146 def _parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
147 encoding='utf-8', errors='replace'):
148 qs, _coerce_result = qs, compat_str
149 pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
151 for name_value in pairs:
152 if not name_value and not strict_parsing:
154 nv = name_value.split('=', 1)
157 raise ValueError("bad query field: %r" % (name_value,))
158 # Handle case of a control-name with no equal sign
159 if keep_blank_values:
163 if len(nv[1]) or keep_blank_values:
164 name = nv[0].replace('+', ' ')
165 name = compat_urllib_parse_unquote(
166 name, encoding=encoding, errors=errors)
167 name = _coerce_result(name)
168 value = nv[1].replace('+', ' ')
169 value = compat_urllib_parse_unquote(
170 value, encoding=encoding, errors=errors)
171 value = _coerce_result(value)
172 r.append((name, value))
175 def compat_parse_qs(qs, keep_blank_values=False, strict_parsing=False,
176 encoding='utf-8', errors='replace'):
178 pairs = _parse_qsl(qs, keep_blank_values, strict_parsing,
179 encoding=encoding, errors=errors)
180 for name, value in pairs:
181 if name in parsed_result:
182 parsed_result[name].append(value)
184 parsed_result[name] = [value]
188 from shlex import quote as shlex_quote
189 except ImportError: # Python < 3.3
191 if re.match(r'^[-_\w./]+$', s):
194 return "'" + s.replace("'", "'\"'\"'") + "'"
204 if sys.version_info >= (3, 0):
205 compat_getenv = os.getenv
206 compat_expanduser = os.path.expanduser
208 # Environment variables should be decoded with filesystem encoding.
209 # Otherwise it will fail if any non-ASCII characters present (see #3854 #3217 #2918)
211 def compat_getenv(key, default=None):
212 from .utils import get_filesystem_encoding
213 env = os.getenv(key, default)
215 env = env.decode(get_filesystem_encoding())
218 # HACK: The default implementations of os.path.expanduser from cpython do not decode
219 # environment variables with filesystem encoding. We will work around this by
220 # providing adjusted implementations.
221 # The following are os.path.expanduser implementations from cpython 2.7.8 stdlib
222 # for different platforms with correct environment variables decoding.
224 if os.name == 'posix':
225 def compat_expanduser(path):
226 """Expand ~ and ~user constructions. If user or $HOME is unknown,
228 if not path.startswith('~'):
230 i = path.find('/', 1)
234 if 'HOME' not in os.environ:
236 userhome = pwd.getpwuid(os.getuid()).pw_dir
238 userhome = compat_getenv('HOME')
242 pwent = pwd.getpwnam(path[1:i])
245 userhome = pwent.pw_dir
246 userhome = userhome.rstrip('/')
247 return (userhome + path[i:]) or '/'
248 elif os.name == 'nt' or os.name == 'ce':
249 def compat_expanduser(path):
250 """Expand ~ and ~user constructs.
252 If user or $HOME is unknown, do nothing."""
256 while i < n and path[i] not in '/\\':
259 if 'HOME' in os.environ:
260 userhome = compat_getenv('HOME')
261 elif 'USERPROFILE' in os.environ:
262 userhome = compat_getenv('USERPROFILE')
263 elif 'HOMEPATH' not in os.environ:
267 drive = compat_getenv('HOMEDRIVE')
270 userhome = os.path.join(drive, compat_getenv('HOMEPATH'))
273 userhome = os.path.join(os.path.dirname(userhome), path[1:i])
275 return userhome + path[i:]
277 compat_expanduser = os.path.expanduser
280 if sys.version_info < (3, 0):
282 from .utils import preferredencoding
283 print(s.encode(preferredencoding(), 'xmlcharrefreplace'))
286 assert isinstance(s, compat_str)
291 subprocess_check_output = subprocess.check_output
292 except AttributeError:
293 def subprocess_check_output(*args, **kwargs):
294 assert 'input' not in kwargs
295 p = subprocess.Popen(*args, stdout=subprocess.PIPE, **kwargs)
296 output, _ = p.communicate()
299 raise subprocess.CalledProcessError(ret, p.args, output=output)
302 if sys.version_info < (3, 0) and sys.platform == 'win32':
303 def compat_getpass(prompt, *args, **kwargs):
304 if isinstance(prompt, compat_str):
305 from .utils import preferredencoding
306 prompt = prompt.encode(preferredencoding())
307 return getpass.getpass(prompt, *args, **kwargs)
309 compat_getpass = getpass.getpass
311 # Old 2.6 and 2.7 releases require kwargs to be bytes
315 _testfunc(**{'x': 0})
317 def compat_kwargs(kwargs):
318 return dict((bytes(k), v) for k, v in kwargs.items())
320 compat_kwargs = lambda kwargs: kwargs
323 if sys.version_info < (2, 7):
324 def compat_socket_create_connection(address, timeout, source_address=None):
327 for res in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM):
328 af, socktype, proto, canonname, sa = res
331 sock = socket.socket(af, socktype, proto)
332 sock.settimeout(timeout)
334 sock.bind(source_address)
337 except socket.error as _:
344 raise socket.error("getaddrinfo returns an empty list")
346 compat_socket_create_connection = socket.create_connection
349 # Fix https://github.com/rg3/youtube-dl/issues/4223
350 # See http://bugs.python.org/issue9161 for what is broken
351 def workaround_optparse_bug9161():
352 op = optparse.OptionParser()
353 og = optparse.OptionGroup(op, 'foo')
357 real_add_option = optparse.OptionGroup.add_option
359 def _compat_add_option(self, *args, **kwargs):
361 v.encode('ascii', 'replace') if isinstance(v, compat_str)
363 bargs = [enc(a) for a in args]
365 (k, enc(v)) for k, v in kwargs.items())
366 return real_add_option(self, *bargs, **bkwargs)
367 optparse.OptionGroup.add_option = _compat_add_option
369 if hasattr(shutil, 'get_terminal_size'): # Python >= 3.3
370 compat_get_terminal_size = shutil.get_terminal_size
372 _terminal_size = collections.namedtuple('terminal_size', ['columns', 'lines'])
374 def compat_get_terminal_size():
375 columns = compat_getenv('COLUMNS', None)
377 columns = int(columns)
380 lines = compat_getenv('LINES', None)
387 sp = subprocess.Popen(
389 stdout=subprocess.PIPE, stderr=subprocess.PIPE)
390 out, err = sp.communicate()
391 lines, columns = map(int, out.split())
394 return _terminal_size(columns, lines)
403 'compat_get_terminal_size',
406 'compat_html_entities',
407 'compat_html_parser',
408 'compat_http_client',
409 'compat_http_server',
414 'compat_socket_create_connection',
416 'compat_subprocess_get_DEVNULL',
417 'compat_urllib_error',
418 'compat_urllib_parse',
419 'compat_urllib_parse_unquote',
420 'compat_urllib_parse_urlparse',
421 'compat_urllib_request',
423 'compat_urlretrieve',
424 'compat_xml_parse_error',
426 'subprocess_check_output',
427 'workaround_optparse_bug9161',