2 # -*- coding: utf-8 -*-
17 import urllib.request as compat_urllib_request
18 except ImportError: # Python 2
19 import urllib2 as compat_urllib_request
22 import urllib.error as compat_urllib_error
23 except ImportError: # Python 2
24 import urllib2 as compat_urllib_error
27 import urllib.parse as compat_urllib_parse
28 except ImportError: # Python 2
29 import urllib as compat_urllib_parse
32 from urllib.parse import urlparse as compat_urllib_parse_urlparse
33 except ImportError: # Python 2
34 from urlparse import urlparse as compat_urllib_parse_urlparse
37 import http.cookiejar as compat_cookiejar
38 except ImportError: # Python 2
39 import cookielib as compat_cookiejar
42 import html.entities as compat_html_entities
43 except ImportError: # Python 2
44 import htmlentitydefs as compat_html_entities
47 import html.parser as compat_html_parser
48 except ImportError: # Python 2
49 import HTMLParser as compat_html_parser
52 import http.client as compat_http_client
53 except ImportError: # Python 2
54 import httplib as compat_http_client
57 from subprocess import DEVNULL
58 compat_subprocess_get_DEVNULL = lambda: DEVNULL
60 compat_subprocess_get_DEVNULL = lambda: open(os.path.devnull, 'w')
63 from urllib.parse import parse_qs as compat_parse_qs
64 except ImportError: # Python 2
65 # HACK: The following is the correct parse_qs implementation from cpython 3's stdlib.
66 # Python 2's version is apparently totally broken
67 def _unquote(string, encoding='utf-8', errors='replace'):
70 res = string.split('%')
77 # pct_sequence: contiguous sequence of percent-encoded bytes, decoded
84 pct_sequence += item[:2].decode('hex')
87 # This segment was just a single percent-encoded character.
88 # May be part of a sequence of code units, so delay decoding.
89 # (Stored in pct_sequence).
93 # Encountered non-percent-encoded characters. Flush the current
95 string += pct_sequence.decode(encoding, errors) + rest
98 # Flush the final pct_sequence
99 string += pct_sequence.decode(encoding, errors)
102 def _parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
103 encoding='utf-8', errors='replace'):
104 qs, _coerce_result = qs, unicode
105 pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
107 for name_value in pairs:
108 if not name_value and not strict_parsing:
110 nv = name_value.split('=', 1)
113 raise ValueError("bad query field: %r" % (name_value,))
114 # Handle case of a control-name with no equal sign
115 if keep_blank_values:
119 if len(nv[1]) or keep_blank_values:
120 name = nv[0].replace('+', ' ')
121 name = _unquote(name, encoding=encoding, errors=errors)
122 name = _coerce_result(name)
123 value = nv[1].replace('+', ' ')
124 value = _unquote(value, encoding=encoding, errors=errors)
125 value = _coerce_result(value)
126 r.append((name, value))
129 def compat_parse_qs(qs, keep_blank_values=False, strict_parsing=False,
130 encoding='utf-8', errors='replace'):
132 pairs = _parse_qsl(qs, keep_blank_values, strict_parsing,
133 encoding=encoding, errors=errors)
134 for name, value in pairs:
135 if name in parsed_result:
136 parsed_result[name].append(value)
138 parsed_result[name] = [value]
142 compat_str = unicode # Python 2
147 compat_chr = unichr # Python 2
152 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 Firefox/10.0',
153 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
154 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
155 'Accept-Encoding': 'gzip, deflate',
156 'Accept-Language': 'en-us,en;q=0.5',
159 def preferredencoding():
160 """Get preferred encoding.
162 Returns the best encoding scheme for the system, based on
163 locale.getpreferredencoding() and some further tweaks.
166 pref = locale.getpreferredencoding()
173 if sys.version_info < (3,0):
175 print(s.encode(preferredencoding(), 'xmlcharrefreplace'))
178 assert type(s) == type(u'')
181 # In Python 2.x, json.dump expects a bytestream.
182 # In Python 3.x, it writes to a character stream
183 if sys.version_info < (3,0):
184 def write_json_file(obj, fn):
185 with open(fn, 'wb') as f:
188 def write_json_file(obj, fn):
189 with open(fn, 'w', encoding='utf-8') as f:
192 def htmlentity_transform(matchobj):
193 """Transforms an HTML entity to a character.
195 This function receives a match object and is intended to be used with
196 the re.sub() function.
198 entity = matchobj.group(1)
200 # Known non-numeric HTML entity
201 if entity in compat_html_entities.name2codepoint:
202 return compat_chr(compat_html_entities.name2codepoint[entity])
204 mobj = re.match(u'(?u)#(x?\\d+)', entity)
206 numstr = mobj.group(1)
207 if numstr.startswith(u'x'):
209 numstr = u'0%s' % numstr
212 return compat_chr(int(numstr, base))
214 # Unknown entity in name, return its literal representation
215 return (u'&%s;' % entity)
217 compat_html_parser.locatestarttagend = re.compile(r"""<[a-zA-Z][-.a-zA-Z0-9:_]*(?:\s+(?:(?<=['"\s])[^\s/>][^\s/=>]*(?:\s*=+\s*(?:'[^']*'|"[^"]*"|(?!['"])[^>\s]*))?\s*)*)?\s*""", re.VERBOSE) # backport bugfix
218 class AttrParser(compat_html_parser.HTMLParser):
219 """Modified HTMLParser that isolates a tag with the specified attribute"""
220 def __init__(self, attribute, value):
221 self.attribute = attribute
227 self.watch_startpos = False
229 compat_html_parser.HTMLParser.__init__(self)
231 def error(self, message):
232 if self.error_count > 10 or self.started:
233 raise compat_html_parser.HTMLParseError(message, self.getpos())
234 self.rawdata = '\n'.join(self.html.split('\n')[self.getpos()[0]:]) # skip one line
235 self.error_count += 1
238 def loads(self, html):
243 def handle_starttag(self, tag, attrs):
246 self.find_startpos(None)
247 if self.attribute in attrs and attrs[self.attribute] == self.value:
250 self.watch_startpos = True
252 if not tag in self.depth: self.depth[tag] = 0
255 def handle_endtag(self, tag):
257 if tag in self.depth: self.depth[tag] -= 1
258 if self.depth[self.result[0]] == 0:
260 self.result.append(self.getpos())
262 def find_startpos(self, x):
263 """Needed to put the start position of the result (self.result[1])
264 after the opening tag with the requested id"""
265 if self.watch_startpos:
266 self.watch_startpos = False
267 self.result.append(self.getpos())
268 handle_entityref = handle_charref = handle_data = handle_comment = \
269 handle_decl = handle_pi = unknown_decl = find_startpos
271 def get_result(self):
272 if self.result is None:
274 if len(self.result) != 3:
276 lines = self.html.split('\n')
277 lines = lines[self.result[1][0]-1:self.result[2][0]]
278 lines[0] = lines[0][self.result[1][1]:]
280 lines[-1] = lines[-1][:self.result[2][1]-self.result[1][1]]
281 lines[-1] = lines[-1][:self.result[2][1]]
282 return '\n'.join(lines).strip()
284 def get_element_by_id(id, html):
285 """Return the content of the tag with the specified ID in the passed HTML document"""
286 return get_element_by_attribute("id", id, html)
288 def get_element_by_attribute(attribute, value, html):
289 """Return the content of the tag with the specified attribute in the passed HTML document"""
290 parser = AttrParser(attribute, value)
293 except compat_html_parser.HTMLParseError:
295 return parser.get_result()
298 def clean_html(html):
299 """Clean an HTML snippet into a readable string"""
301 html = html.replace('\n', ' ')
302 html = re.sub(r'\s*<\s*br\s*/?\s*>\s*', '\n', html)
303 html = re.sub(r'<\s*/\s*p\s*>\s*<\s*p[^>]*>', '\n', html)
305 html = re.sub('<.*?>', '', html)
306 # Replace html entities
307 html = unescapeHTML(html)
311 def sanitize_open(filename, open_mode):
312 """Try to open the given filename, and slightly tweak it if this fails.
314 Attempts to open the given filename. If this fails, it tries to change
315 the filename slightly, step by step, until it's either able to open it
316 or it fails and raises a final exception, like the standard open()
319 It returns the tuple (stream, definitive_file_name).
323 if sys.platform == 'win32':
325 msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
326 return (sys.stdout, filename)
327 stream = open(encodeFilename(filename), open_mode)
328 return (stream, filename)
329 except (IOError, OSError) as err:
330 # In case of error, try to remove win32 forbidden chars
331 filename = re.sub(u'[/<>:"\\|\\\\?\\*]', u'#', filename)
333 # An exception here should be caught in the caller
334 stream = open(encodeFilename(filename), open_mode)
335 return (stream, filename)
338 def timeconvert(timestr):
339 """Convert RFC 2822 defined time string into system timestamp"""
341 timetuple = email.utils.parsedate_tz(timestr)
342 if timetuple is not None:
343 timestamp = email.utils.mktime_tz(timetuple)
346 def sanitize_filename(s, restricted=False, is_id=False):
347 """Sanitizes a string so it could be used as part of a filename.
348 If restricted is set, use a stricter subset of allowed characters.
349 Set is_id if this is not an arbitrary string, but an ID that should be kept if possible
351 def replace_insane(char):
352 if char == '?' or ord(char) < 32 or ord(char) == 127:
355 return '' if restricted else '\''
357 return '_-' if restricted else ' -'
358 elif char in '\\/|*<>':
360 if restricted and (char in '!&\'()[]{}$;`^,#' or char.isspace()):
362 if restricted and ord(char) > 127:
366 result = u''.join(map(replace_insane, s))
368 while '__' in result:
369 result = result.replace('__', '_')
370 result = result.strip('_')
371 # Common case of "Foreign band name - English song title"
372 if restricted and result.startswith('-_'):
378 def orderedSet(iterable):
379 """ Remove all duplicates from the input iterable """
390 assert type(s) == type(u'')
392 result = re.sub(u'(?u)&(.+?);', htmlentity_transform, s)
395 def encodeFilename(s):
397 @param s The name of the file
400 assert type(s) == type(u'')
402 # Python 3 has a Unicode API
403 if sys.version_info >= (3, 0):
406 if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
407 # Pass u'' directly to use Unicode APIs on Windows 2000 and up
408 # (Detecting Windows NT 4 is tricky because 'major >= 4' would
409 # match Windows 9x series as well. Besides, NT 4 is obsolete.)
412 return s.encode(sys.getfilesystemencoding(), 'ignore')
415 class ExtractorError(Exception):
416 """Error during info extraction."""
417 def __init__(self, msg, tb=None):
418 """ tb, if given, is the original traceback (so that it can be printed out). """
419 super(ExtractorError, self).__init__(msg)
422 def format_traceback(self):
423 if self.traceback is None:
425 return u''.join(traceback.format_tb(self.traceback))
428 class DownloadError(Exception):
429 """Download Error exception.
431 This exception may be thrown by FileDownloader objects if they are not
432 configured to continue on errors. They will contain the appropriate
438 class SameFileError(Exception):
439 """Same File exception.
441 This exception will be thrown by FileDownloader objects if they detect
442 multiple files would have to be downloaded to the same file on disk.
447 class PostProcessingError(Exception):
448 """Post Processing exception.
450 This exception may be raised by PostProcessor's .run() method to
451 indicate an error in the postprocessing task.
453 def __init__(self, msg):
456 class MaxDownloadsReached(Exception):
457 """ --max-downloads limit has been reached. """
461 class UnavailableVideoError(Exception):
462 """Unavailable Format exception.
464 This exception will be thrown when a video is requested
465 in a format that is not available for that video.
470 class ContentTooShortError(Exception):
471 """Content Too Short exception.
473 This exception may be raised by FileDownloader objects when a file they
474 download is too small for what the server announced first, indicating
475 the connection was probably interrupted.
481 def __init__(self, downloaded, expected):
482 self.downloaded = downloaded
483 self.expected = expected
485 class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
486 """Handler for HTTP requests and responses.
488 This class, when installed with an OpenerDirector, automatically adds
489 the standard headers to every HTTP request and handles gzipped and
490 deflated responses from web servers. If compression is to be avoided in
491 a particular request, the original request in the program code only has
492 to include the HTTP header "Youtubedl-No-Compression", which will be
493 removed before making the real request.
495 Part of this code was copied from:
497 http://techknack.net/python-urllib2-handlers/
499 Andrew Rowls, the author of that code, agreed to release it to the
506 return zlib.decompress(data, -zlib.MAX_WBITS)
508 return zlib.decompress(data)
511 def addinfourl_wrapper(stream, headers, url, code):
512 if hasattr(compat_urllib_request.addinfourl, 'getcode'):
513 return compat_urllib_request.addinfourl(stream, headers, url, code)
514 ret = compat_urllib_request.addinfourl(stream, headers, url)
518 def http_request(self, req):
519 for h in std_headers:
522 req.add_header(h, std_headers[h])
523 if 'Youtubedl-no-compression' in req.headers:
524 if 'Accept-encoding' in req.headers:
525 del req.headers['Accept-encoding']
526 del req.headers['Youtubedl-no-compression']
527 if 'Youtubedl-user-agent' in req.headers:
528 if 'User-Agent' in req.headers:
529 del req.headers['User-Agent']
530 req.headers['User-Agent'] = req.headers['Youtubedl-user-agent']
531 del req.headers['Youtubedl-user-agent']
534 def http_response(self, req, resp):
537 if resp.headers.get('Content-encoding', '') == 'gzip':
538 gz = gzip.GzipFile(fileobj=io.BytesIO(resp.read()), mode='r')
539 resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
540 resp.msg = old_resp.msg
542 if resp.headers.get('Content-encoding', '') == 'deflate':
543 gz = io.BytesIO(self.deflate(resp.read()))
544 resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
545 resp.msg = old_resp.msg
548 https_request = http_request
549 https_response = http_response