2 # -*- coding: utf-8 -*-
16 import cStringIO as StringIO
22 except ImportError: # Python <2.6, use trivialjson (https://github.com/phihag/trivialjson):
23 import trivialjson as json
26 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:5.0.1) Gecko/20100101 Firefox/5.0.1',
27 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
28 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
29 'Accept-Encoding': 'gzip, deflate',
30 'Accept-Language': 'en-us,en;q=0.5',
33 def preferredencoding():
34 """Get preferred encoding.
36 Returns the best encoding scheme for the system, based on
37 locale.getpreferredencoding() and some further tweaks.
39 def yield_preferredencoding():
41 pref = locale.getpreferredencoding()
47 return yield_preferredencoding().next()
50 def htmlentity_transform(matchobj):
51 """Transforms an HTML entity to a Unicode character.
53 This function receives a match object and is intended to be used with
54 the re.sub() function.
56 entity = matchobj.group(1)
58 # Known non-numeric HTML entity
59 if entity in htmlentitydefs.name2codepoint:
60 return unichr(htmlentitydefs.name2codepoint[entity])
63 mobj = re.match(ur'(?u)#(x?\d+)', entity)
65 numstr = mobj.group(1)
66 if numstr.startswith(u'x'):
68 numstr = u'0%s' % numstr
71 return unichr(long(numstr, base))
73 # Unknown entity in name, return its literal representation
74 return (u'&%s;' % entity)
77 def sanitize_title(utitle):
78 """Sanitizes a video title so it could be used as part of a filename."""
79 utitle = re.sub(ur'(?u)&(.+?);', htmlentity_transform, utitle)
80 return utitle.replace(unicode(os.sep), u'%')
83 def sanitize_open(filename, open_mode):
84 """Try to open the given filename, and slightly tweak it if this fails.
86 Attempts to open the given filename. If this fails, it tries to change
87 the filename slightly, step by step, until it's either able to open it
88 or it fails and raises a final exception, like the standard open()
91 It returns the tuple (stream, definitive_file_name).
95 if sys.platform == 'win32':
97 msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
98 return (sys.stdout, filename)
99 stream = open(encodeFilename(filename), open_mode)
100 return (stream, filename)
101 except (IOError, OSError), err:
102 # In case of error, try to remove win32 forbidden chars
103 filename = re.sub(ur'[/<>:"\|\?\*]', u'#', filename)
105 # An exception here should be caught in the caller
106 stream = open(encodeFilename(filename), open_mode)
107 return (stream, filename)
110 def timeconvert(timestr):
111 """Convert RFC 2822 defined time string into system timestamp"""
113 timetuple = email.utils.parsedate_tz(timestr)
114 if timetuple is not None:
115 timestamp = email.utils.mktime_tz(timetuple)
118 def simplify_title(title):
119 expr = re.compile(ur'[^\w\d_\-]+', flags=re.UNICODE)
120 return expr.sub(u'_', title).strip(u'_')
122 def orderedSet(iterable):
123 """ Remove all duplicates from the input iterable """
132 @param s a string (of type unicode)
134 assert type(s) == type(u'')
136 htmlParser = HTMLParser.HTMLParser()
137 return htmlParser.unescape(s)
139 def encodeFilename(s):
141 @param s The name of the file (of type unicode)
144 assert type(s) == type(u'')
146 if sys.platform == 'win32' and sys.getwindowsversion().major >= 5:
147 # Pass u'' directly to use Unicode APIs on Windows 2000 and up
148 # (Detecting Windows NT 4 is tricky because 'major >= 4' would
149 # match Windows 9x series as well. Besides, NT 4 is obsolete.)
152 return s.encode(sys.getfilesystemencoding(), 'ignore')
154 class DownloadError(Exception):
155 """Download Error exception.
157 This exception may be thrown by FileDownloader objects if they are not
158 configured to continue on errors. They will contain the appropriate
164 class SameFileError(Exception):
165 """Same File exception.
167 This exception will be thrown by FileDownloader objects if they detect
168 multiple files would have to be downloaded to the same file on disk.
173 class PostProcessingError(Exception):
174 """Post Processing exception.
176 This exception may be raised by PostProcessor's .run() method to
177 indicate an error in the postprocessing task.
181 class MaxDownloadsReached(Exception):
182 """ --max-downloads limit has been reached. """
186 class UnavailableVideoError(Exception):
187 """Unavailable Format exception.
189 This exception will be thrown when a video is requested
190 in a format that is not available for that video.
195 class ContentTooShortError(Exception):
196 """Content Too Short exception.
198 This exception may be raised by FileDownloader objects when a file they
199 download is too small for what the server announced first, indicating
200 the connection was probably interrupted.
206 def __init__(self, downloaded, expected):
207 self.downloaded = downloaded
208 self.expected = expected
211 class YoutubeDLHandler(urllib2.HTTPHandler):
212 """Handler for HTTP requests and responses.
214 This class, when installed with an OpenerDirector, automatically adds
215 the standard headers to every HTTP request and handles gzipped and
216 deflated responses from web servers. If compression is to be avoided in
217 a particular request, the original request in the program code only has
218 to include the HTTP header "Youtubedl-No-Compression", which will be
219 removed before making the real request.
221 Part of this code was copied from:
223 http://techknack.net/python-urllib2-handlers/
225 Andrew Rowls, the author of that code, agreed to release it to the
232 return zlib.decompress(data, -zlib.MAX_WBITS)
234 return zlib.decompress(data)
237 def addinfourl_wrapper(stream, headers, url, code):
238 if hasattr(urllib2.addinfourl, 'getcode'):
239 return urllib2.addinfourl(stream, headers, url, code)
240 ret = urllib2.addinfourl(stream, headers, url)
244 def http_request(self, req):
245 for h in std_headers:
248 req.add_header(h, std_headers[h])
249 if 'Youtubedl-no-compression' in req.headers:
250 if 'Accept-encoding' in req.headers:
251 del req.headers['Accept-encoding']
252 del req.headers['Youtubedl-no-compression']
255 def http_response(self, req, resp):
258 if resp.headers.get('Content-encoding', '') == 'gzip':
259 gz = gzip.GzipFile(fileobj=StringIO.StringIO(resp.read()), mode='r')
260 resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
261 resp.msg = old_resp.msg
263 if resp.headers.get('Content-encoding', '') == 'deflate':
264 gz = StringIO.StringIO(self.deflate(resp.read()))
265 resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
266 resp.msg = old_resp.msg