8 import urllib.request as compat_urllib_request
9 except ImportError: # Python 2
10 import urllib2 as compat_urllib_request
13 import urllib.error as compat_urllib_error
14 except ImportError: # Python 2
15 import urllib2 as compat_urllib_error
18 import urllib.parse as compat_urllib_parse
19 except ImportError: # Python 2
20 import urllib as compat_urllib_parse
23 from urllib.parse import urlparse as compat_urllib_parse_urlparse
24 except ImportError: # Python 2
25 from urlparse import urlparse as compat_urllib_parse_urlparse
28 import urllib.parse as compat_urlparse
29 except ImportError: # Python 2
30 import urlparse as compat_urlparse
33 import http.cookiejar as compat_cookiejar
34 except ImportError: # Python 2
35 import cookielib as compat_cookiejar
38 import html.entities as compat_html_entities
39 except ImportError: # Python 2
40 import htmlentitydefs as compat_html_entities
43 import html.parser as compat_html_parser
44 except ImportError: # Python 2
45 import HTMLParser as compat_html_parser
48 import http.client as compat_http_client
49 except ImportError: # Python 2
50 import httplib as compat_http_client
53 from urllib.error import HTTPError as compat_HTTPError
54 except ImportError: # Python 2
55 from urllib2 import HTTPError as compat_HTTPError
58 from urllib.request import urlretrieve as compat_urlretrieve
59 except ImportError: # Python 2
60 from urllib import urlretrieve as compat_urlretrieve
64 from subprocess import DEVNULL
65 compat_subprocess_get_DEVNULL = lambda: DEVNULL
67 compat_subprocess_get_DEVNULL = lambda: open(os.path.devnull, 'w')
70 from urllib.parse import unquote as compat_urllib_parse_unquote
72 def compat_urllib_parse_unquote(string, encoding='utf-8', errors='replace'):
75 res = string.split('%')
82 # pct_sequence: contiguous sequence of percent-encoded bytes, decoded
89 pct_sequence += item[:2].decode('hex')
92 # This segment was just a single percent-encoded character.
93 # May be part of a sequence of code units, so delay decoding.
94 # (Stored in pct_sequence).
98 # Encountered non-percent-encoded characters. Flush the current
100 string += pct_sequence.decode(encoding, errors) + rest
103 # Flush the final pct_sequence
104 string += pct_sequence.decode(encoding, errors)
109 from urllib.parse import parse_qs as compat_parse_qs
110 except ImportError: # Python 2
111 # HACK: The following is the correct parse_qs implementation from cpython 3's stdlib.
112 # Python 2's version is apparently totally broken
114 def _parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
115 encoding='utf-8', errors='replace'):
116 qs, _coerce_result = qs, unicode
117 pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
119 for name_value in pairs:
120 if not name_value and not strict_parsing:
122 nv = name_value.split('=', 1)
125 raise ValueError("bad query field: %r" % (name_value,))
126 # Handle case of a control-name with no equal sign
127 if keep_blank_values:
131 if len(nv[1]) or keep_blank_values:
132 name = nv[0].replace('+', ' ')
133 name = compat_urllib_parse_unquote(
134 name, encoding=encoding, errors=errors)
135 name = _coerce_result(name)
136 value = nv[1].replace('+', ' ')
137 value = compat_urllib_parse_unquote(
138 value, encoding=encoding, errors=errors)
139 value = _coerce_result(value)
140 r.append((name, value))
143 def compat_parse_qs(qs, keep_blank_values=False, strict_parsing=False,
144 encoding='utf-8', errors='replace'):
146 pairs = _parse_qsl(qs, keep_blank_values, strict_parsing,
147 encoding=encoding, errors=errors)
148 for name, value in pairs:
149 if name in parsed_result:
150 parsed_result[name].append(value)
152 parsed_result[name] = [value]
156 compat_str = unicode # Python 2
161 compat_chr = unichr # Python 2
166 from xml.etree.ElementTree import ParseError as compat_xml_parse_error
167 except ImportError: # Python 2.6
168 from xml.parsers.expat import ExpatError as compat_xml_parse_error
171 from shlex import quote as shlex_quote
172 except ImportError: # Python < 3.3
174 return "'" + s.replace("'", "'\"'\"'") + "'"
178 if type(c) is int: return c
182 if sys.version_info >= (3, 0):
183 compat_getenv = os.getenv
184 compat_expanduser = os.path.expanduser
186 # Environment variables should be decoded with filesystem encoding.
187 # Otherwise it will fail if any non-ASCII characters present (see #3854 #3217 #2918)
189 def compat_getenv(key, default=None):
190 from .utils import get_filesystem_encoding
191 env = os.getenv(key, default)
193 env = env.decode(get_filesystem_encoding())
196 # HACK: The default implementations of os.path.expanduser from cpython do not decode
197 # environment variables with filesystem encoding. We will work around this by
198 # providing adjusted implementations.
199 # The following are os.path.expanduser implementations from cpython 2.7.8 stdlib
200 # for different platforms with correct environment variables decoding.
202 if os.name == 'posix':
203 def compat_expanduser(path):
204 """Expand ~ and ~user constructions. If user or $HOME is unknown,
206 if not path.startswith('~'):
208 i = path.find('/', 1)
212 if 'HOME' not in os.environ:
214 userhome = pwd.getpwuid(os.getuid()).pw_dir
216 userhome = compat_getenv('HOME')
220 pwent = pwd.getpwnam(path[1:i])
223 userhome = pwent.pw_dir
224 userhome = userhome.rstrip('/')
225 return (userhome + path[i:]) or '/'
226 elif os.name == 'nt' or os.name == 'ce':
227 def compat_expanduser(path):
228 """Expand ~ and ~user constructs.
230 If user or $HOME is unknown, do nothing."""
234 while i < n and path[i] not in '/\\':
237 if 'HOME' in os.environ:
238 userhome = compat_getenv('HOME')
239 elif 'USERPROFILE' in os.environ:
240 userhome = compat_getenv('USERPROFILE')
241 elif not 'HOMEPATH' in os.environ:
245 drive = compat_getenv('HOMEDRIVE')
248 userhome = os.path.join(drive, compat_getenv('HOMEPATH'))
251 userhome = os.path.join(os.path.dirname(userhome), path[1:i])
253 return userhome + path[i:]
255 compat_expanduser = os.path.expanduser
258 if sys.version_info < (3, 0):
260 from .utils import preferredencoding
261 print(s.encode(preferredencoding(), 'xmlcharrefreplace'))
264 assert type(s) == type(u'')
269 subprocess_check_output = subprocess.check_output
270 except AttributeError:
271 def subprocess_check_output(*args, **kwargs):
272 assert 'input' not in kwargs
273 p = subprocess.Popen(*args, stdout=subprocess.PIPE, **kwargs)
274 output, _ = p.communicate()
277 raise subprocess.CalledProcessError(ret, p.args, output=output)
280 if sys.version_info < (3, 0) and sys.platform == 'win32':
281 def compat_getpass(prompt, *args, **kwargs):
282 if isinstance(prompt, compat_str):
283 from .utils import preferredencoding
284 prompt = prompt.encode(preferredencoding())
285 return getpass.getpass(prompt, *args, **kwargs)
287 compat_getpass = getpass.getpass
297 'compat_html_entities',
298 'compat_html_parser',
299 'compat_http_client',
304 'compat_subprocess_get_DEVNULL',
305 'compat_urllib_error',
306 'compat_urllib_parse',
307 'compat_urllib_parse_unquote',
308 'compat_urllib_parse_urlparse',
309 'compat_urllib_request',
311 'compat_urlretrieve',
312 'compat_xml_parse_error',
314 'subprocess_check_output',