2 # -*- coding: utf-8 -*-
3 # Author: Ricardo Garcia Gonzalez
4 # Author: Danny Colligan
5 # License: Public domain code
22 'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.0.8) Gecko/2009032609 Firefox/3.0.8',
23 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
24 'Accept': 'text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5',
25 'Accept-Language': 'en-us,en;q=0.5',
28 simple_title_chars = string.ascii_letters.decode('ascii') + string.digits.decode('ascii')
30 class DownloadError(Exception):
31 """Download Error exception.
33 This exception may be thrown by FileDownloader objects if they are not
34 configured to continue on errors. They will contain the appropriate
39 class SameFileError(Exception):
40 """Same File exception.
42 This exception will be thrown by FileDownloader objects if they detect
43 multiple files would have to be downloaded to the same file on disk.
47 class PostProcessingError(Exception):
48 """Post Processing exception.
50 This exception may be raised by PostProcessor's .run() method to
51 indicate an error in the postprocessing task.
55 class UnavailableFormatError(Exception):
56 """Unavailable Format exception.
58 This exception will be thrown when a video is requested
59 in a format that is not available for that video.
63 class ContentTooShortError(Exception):
64 """Content Too Short exception.
66 This exception may be raised by FileDownloader objects when a file they
67 download is too small for what the server announced first, indicating
68 the connection was probably interrupted.
74 def __init__(self, downloaded, expected):
75 self.downloaded = downloaded
76 self.expected = expected
78 class FileDownloader(object):
79 """File Downloader class.
81 File downloader objects are the ones responsible of downloading the
82 actual video file and writing it to disk if the user has requested
83 it, among some other tasks. In most cases there should be one per
84 program. As, given a video URL, the downloader doesn't know how to
85 extract all the needed information, task that InfoExtractors do, it
86 has to pass the URL to one of them.
88 For this, file downloader objects have a method that allows
89 InfoExtractors to be registered in a given order. When it is passed
90 a URL, the file downloader handles it to the first InfoExtractor it
91 finds that reports being able to handle it. The InfoExtractor extracts
92 all the information about the video or videos the URL refers to, and
93 asks the FileDownloader to process the video information, possibly
94 downloading the video.
96 File downloaders accept a lot of parameters. In order not to saturate
97 the object constructor with arguments, it receives a dictionary of
98 options instead. These options are available through the params
99 attribute for the InfoExtractors to use. The FileDownloader also
100 registers itself as the downloader in charge for the InfoExtractors
101 that are added to it, so this is a "mutual registration".
105 username: Username for authentication purposes.
106 password: Password for authentication purposes.
107 usenetrc: Use netrc for authentication instead.
108 quiet: Do not print messages to stdout.
109 forceurl: Force printing final URL.
110 forcetitle: Force printing title.
111 simulate: Do not download the video files.
112 format: Video format code.
113 outtmpl: Template for output names.
114 ignoreerrors: Do not stop on download errors.
115 ratelimit: Download speed limit, in bytes/sec.
116 nooverwrites: Prevent overwriting files.
122 _download_retcode = None
124 def __init__(self, params):
125 """Create a FileDownloader object with the given options."""
128 self._download_retcode = 0
132 def pmkdir(filename):
133 """Create directory components in filename. Similar to Unix "mkdir -p"."""
134 components = filename.split(os.sep)
135 aggregate = [os.sep.join(components[0:x]) for x in xrange(1, len(components))]
136 aggregate = ['%s%s' % (x, os.sep) for x in aggregate] # Finish names with separator
137 for dir in aggregate:
138 if not os.path.exists(dir):
142 def format_bytes(bytes):
148 exponent = long(math.log(float(bytes), 1024.0))
149 suffix = 'bkMGTPEZY'[exponent]
150 converted = float(bytes) / float(1024**exponent)
151 return '%.2f%s' % (converted, suffix)
154 def calc_percent(byte_counter, data_len):
157 return '%6s' % ('%3.1f%%' % (float(byte_counter) / float(data_len) * 100.0))
160 def calc_eta(start, now, total, current):
164 if current == 0 or dif < 0.001: # One millisecond
166 rate = float(current) / dif
167 eta = long((float(total) - float(current)) / rate)
168 (eta_mins, eta_secs) = divmod(eta, 60)
171 return '%02d:%02d' % (eta_mins, eta_secs)
174 def calc_speed(start, now, bytes):
176 if bytes == 0 or dif < 0.001: # One millisecond
177 return '%10s' % '---b/s'
178 return '%10s' % ('%s/s' % FileDownloader.format_bytes(float(bytes) / dif))
181 def best_block_size(elapsed_time, bytes):
182 new_min = max(bytes / 2.0, 1.0)
183 new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
184 if elapsed_time < 0.001:
186 rate = bytes / elapsed_time
194 def parse_bytes(bytestr):
195 """Parse a string indicating a byte quantity into a long integer."""
196 matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
199 number = float(matchobj.group(1))
200 multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
201 return long(round(number * multiplier))
205 """Verify a URL is valid and data could be downloaded."""
206 request = urllib2.Request(url, None, std_headers)
207 data = urllib2.urlopen(request)
211 def add_info_extractor(self, ie):
212 """Add an InfoExtractor object to the end of the list."""
214 ie.set_downloader(self)
216 def add_post_processor(self, pp):
217 """Add a PostProcessor object to the end of the chain."""
219 pp.set_downloader(self)
221 def to_stdout(self, message, skip_eol=False):
222 """Print message to stdout if not in quiet mode."""
223 if not self.params.get('quiet', False):
224 print (u'%s%s' % (message, [u'\n', u''][skip_eol])).encode(locale.getpreferredencoding()),
227 def to_stderr(self, message):
228 """Print message to stderr."""
229 print >>sys.stderr, message
231 def fixed_template(self):
232 """Checks if the output template is fixed."""
233 return (re.search(ur'(?u)%\(.+?\)s', self.params['outtmpl']) is None)
235 def trouble(self, message=None):
236 """Determine action to take when a download problem appears.
238 Depending on if the downloader has been configured to ignore
239 download errors or not, this method may throw an exception or
240 not when errors are found, after printing the message.
242 if message is not None:
243 self.to_stderr(message)
244 if not self.params.get('ignoreerrors', False):
245 raise DownloadError(message)
246 self._download_retcode = 1
248 def slow_down(self, start_time, byte_counter):
249 """Sleep if the download speed is over the rate limit."""
250 rate_limit = self.params.get('ratelimit', None)
251 if rate_limit is None or byte_counter == 0:
254 elapsed = now - start_time
257 speed = float(byte_counter) / elapsed
258 if speed > rate_limit:
259 time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
261 def report_destination(self, filename):
262 """Report destination filename."""
263 self.to_stdout(u'[download] Destination: %s' % filename)
265 def report_progress(self, percent_str, data_len_str, speed_str, eta_str):
266 """Report download progress."""
267 self.to_stdout(u'\r[download] %s of %s at %s ETA %s' %
268 (percent_str, data_len_str, speed_str, eta_str), skip_eol=True)
270 def report_resuming_byte(self, resume_len):
271 """Report attemtp to resume at given byte."""
272 self.to_stdout(u'[download] Resuming download at byte %s' % resume_len)
274 def report_file_already_downloaded(self, file_name):
275 """Report file has already been fully downloaded."""
276 self.to_stdout(u'[download] %s has already been downloaded' % file_name)
278 def report_unable_to_resume(self):
279 """Report it was impossible to resume download."""
280 self.to_stdout(u'[download] Unable to resume')
282 def report_finish(self):
283 """Report download finished."""
286 def process_info(self, info_dict):
287 """Process a single dictionary returned by an InfoExtractor."""
288 # Do nothing else if in simulate mode
289 if self.params.get('simulate', False):
291 self.verify_url(info_dict['url'])
292 except (OSError, IOError, urllib2.URLError, httplib.HTTPException, socket.error), err:
293 raise UnavailableFormatError
296 if self.params.get('forcetitle', False):
297 print info_dict['title'].encode(locale.getpreferredencoding())
298 if self.params.get('forceurl', False):
299 print info_dict['url'].encode(locale.getpreferredencoding())
304 template_dict = dict(info_dict)
305 template_dict['epoch'] = unicode(long(time.time()))
306 filename = self.params['outtmpl'] % template_dict
307 self.report_destination(filename)
308 except (ValueError, KeyError), err:
309 self.trouble('ERROR: invalid output template or system charset: %s' % str(err))
310 if self.params['nooverwrites'] and os.path.exists(filename):
311 self.to_stderr('WARNING: file exists: %s; skipping' % filename)
315 self.pmkdir(filename)
316 except (OSError, IOError), err:
317 self.trouble('ERROR: unable to create directories: %s' % str(err))
321 outstream = open(filename, 'ab')
322 except (OSError, IOError), err:
323 self.trouble('ERROR: unable to open for writing: %s' % str(err))
327 self._do_download(outstream, info_dict['url'])
329 except (OSError, IOError), err:
332 raise UnavailableFormatError
333 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
334 self.trouble('ERROR: unable to download video data: %s' % str(err))
336 except (ContentTooShortError, ), err:
337 self.trouble('ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
341 self.post_process(filename, info_dict)
342 except (PostProcessingError), err:
343 self.trouble('ERROR: postprocessing: %s' % str(err))
346 def download(self, url_list):
347 """Download a given list of URLs."""
348 if len(url_list) > 1 and self.fixed_template():
349 raise SameFileError(self.params['outtmpl'])
352 suitable_found = False
354 # Go to next InfoExtractor if not suitable
355 if not ie.suitable(url):
358 # Suitable InfoExtractor found
359 suitable_found = True
361 # Extract information from URL and process it
364 # Suitable InfoExtractor had been found; go to next URL
367 if not suitable_found:
368 self.trouble('ERROR: no suitable InfoExtractor: %s' % url)
370 return self._download_retcode
372 def post_process(self, filename, ie_info):
373 """Run the postprocessing chain on the given file."""
375 info['filepath'] = filename
381 def _do_download(self, stream, url):
382 basic_request = urllib2.Request(url, None, std_headers)
383 request = urllib2.Request(url, None, std_headers)
385 # Resume transfer if filesize is non-zero
386 resume_len = stream.tell()
387 if self.params['continuedl'] and resume_len != 0:
388 self.report_resuming_byte(resume_len)
389 request.add_header('Range','bytes=%d-' % resume_len)
392 stream = open(stream.name,'wb')
394 data = urllib2.urlopen(request)
395 except urllib2.HTTPError, e:
396 if not e.code == 416: # 416 is 'Requested range not satisfiable'
398 data = urllib2.urlopen(basic_request)
399 content_length = data.info()['Content-Length']
400 if content_length is not None and long(content_length) == resume_len:
401 self.report_file_already_downloaded(stream.name)
404 self.report_unable_to_resume()
406 stream = open(stream.name,'wb')
408 data_len = data.info().get('Content-length', None)
409 data_len_str = self.format_bytes(data_len)
415 percent_str = self.calc_percent(byte_counter, data_len)
416 eta_str = self.calc_eta(start, time.time(), data_len, byte_counter)
417 speed_str = self.calc_speed(start, time.time(), byte_counter)
418 self.report_progress(percent_str, data_len_str, speed_str, eta_str)
422 data_block = data.read(block_size)
424 data_block_len = len(data_block)
425 if data_block_len == 0:
427 byte_counter += data_block_len
428 stream.write(data_block)
429 block_size = self.best_block_size(after - before, data_block_len)
432 self.slow_down(start, byte_counter)
435 if data_len is not None and str(byte_counter) != data_len:
436 raise ContentTooShortError(byte_counter, long(data_len))
438 class InfoExtractor(object):
439 """Information Extractor class.
441 Information extractors are the classes that, given a URL, extract
442 information from the video (or videos) the URL refers to. This
443 information includes the real video URL, the video title and simplified
444 title, author and others. The information is stored in a dictionary
445 which is then passed to the FileDownloader. The FileDownloader
446 processes this information possibly downloading the video to the file
447 system, among other possible outcomes. The dictionaries must include
448 the following fields:
450 id: Video identifier.
451 url: Final video URL.
452 uploader: Nickname of the video uploader.
453 title: Literal title.
454 stitle: Simplified title.
455 ext: Video filename extension.
457 Subclasses of this one should re-define the _real_initialize() and
458 _real_extract() methods, as well as the suitable() static method.
459 Probably, they should also be instantiated and added to the main
466 def __init__(self, downloader=None):
467 """Constructor. Receives an optional downloader."""
469 self.set_downloader(downloader)
473 """Receives a URL and returns True if suitable for this IE."""
476 def initialize(self):
477 """Initializes an instance (authentication, etc)."""
479 self._real_initialize()
482 def extract(self, url):
483 """Extracts URL information and returns it in list of dicts."""
485 return self._real_extract(url)
487 def set_downloader(self, downloader):
488 """Sets the downloader for this IE."""
489 self._downloader = downloader
491 def _real_initialize(self):
492 """Real initialization process. Redefine in subclasses."""
495 def _real_extract(self, url):
496 """Real extraction process. Redefine in subclasses."""
499 class YoutubeIE(InfoExtractor):
500 """Information extractor for youtube.com."""
502 _VALID_URL = r'^((?:http://)?(?:\w+\.)?youtube\.com/(?:(?:v/)|(?:(?:watch(?:\.php)?)?\?(?:.+&)?v=)))?([0-9A-Za-z_-]+)(?(1).+)?$'
503 _LANG_URL = r'http://uk.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
504 _LOGIN_URL = 'http://www.youtube.com/signup?next=/&gl=US&hl=en'
505 _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
506 _NETRC_MACHINE = 'youtube'
507 _available_formats = ['22', '35', '18', '17', '13'] # listed in order of priority for -b flag
508 _video_extensions = {
517 return (re.match(YoutubeIE._VALID_URL, url) is not None)
520 def htmlentity_transform(matchobj):
521 """Transforms an HTML entity to a Unicode character."""
522 entity = matchobj.group(1)
524 # Known non-numeric HTML entity
525 if entity in htmlentitydefs.name2codepoint:
526 return unichr(htmlentitydefs.name2codepoint[entity])
529 mobj = re.match(ur'(?u)#(x?\d+)', entity)
531 numstr = mobj.group(1)
532 if numstr.startswith(u'x'):
534 numstr = u'0%s' % numstr
537 return unichr(long(numstr, base))
539 # Unknown entity in name, return its literal representation
540 return (u'&%s;' % entity)
542 def report_lang(self):
543 """Report attempt to set language."""
544 self._downloader.to_stdout(u'[youtube] Setting language')
546 def report_login(self):
547 """Report attempt to log in."""
548 self._downloader.to_stdout(u'[youtube] Logging in')
550 def report_age_confirmation(self):
551 """Report attempt to confirm age."""
552 self._downloader.to_stdout(u'[youtube] Confirming age')
554 def report_webpage_download(self, video_id):
555 """Report attempt to download webpage."""
556 self._downloader.to_stdout(u'[youtube] %s: Downloading video webpage' % video_id)
558 def report_information_extraction(self, video_id):
559 """Report attempt to extract video information."""
560 self._downloader.to_stdout(u'[youtube] %s: Extracting video information' % video_id)
562 def report_video_url(self, video_id, video_real_url):
563 """Report extracted video URL."""
564 self._downloader.to_stdout(u'[youtube] %s: URL: %s' % (video_id, video_real_url))
566 def report_unavailable_format(self, video_id, format):
567 """Report extracted video URL."""
568 self._downloader.to_stdout(u'[youtube] %s: Format %s not available' % (video_id, format))
570 def _real_initialize(self):
571 if self._downloader is None:
576 downloader_params = self._downloader.params
578 # Attempt to use provided username and password or .netrc data
579 if downloader_params.get('username', None) is not None:
580 username = downloader_params['username']
581 password = downloader_params['password']
582 elif downloader_params.get('usenetrc', False):
584 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
589 raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
590 except (IOError, netrc.NetrcParseError), err:
591 self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % str(err))
595 request = urllib2.Request(self._LANG_URL, None, std_headers)
598 urllib2.urlopen(request).read()
599 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
600 self._downloader.to_stderr(u'WARNING: unable to set language: %s' % str(err))
603 # No authentication to be performed
609 'current_form': 'loginForm',
611 'action_login': 'Log In',
612 'username': username,
613 'password': password,
615 request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form), std_headers)
618 login_results = urllib2.urlopen(request).read()
619 if re.search(r'(?i)<form[^>]* name="loginForm"', login_results) is not None:
620 self._downloader.to_stderr(u'WARNING: unable to log in: bad username or password')
622 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
623 self._downloader.to_stderr(u'WARNING: unable to log in: %s' % str(err))
629 'action_confirm': 'Confirm',
631 request = urllib2.Request(self._AGE_URL, urllib.urlencode(age_form), std_headers)
633 self.report_age_confirmation()
634 age_results = urllib2.urlopen(request).read()
635 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
636 self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
639 def _real_extract(self, url):
640 # Extract video id from URL
641 mobj = re.match(self._VALID_URL, url)
643 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
645 video_id = mobj.group(2)
647 # Downloader parameters
651 if self._downloader is not None:
652 params = self._downloader.params
653 format_param = params.get('format', None)
654 if format_param == '0':
655 format_param = self._available_formats[quality_index]
660 video_extension = self._video_extensions.get(format_param, 'flv')
662 # Normalize URL, including format
663 normalized_url = 'http://www.youtube.com/watch?v=%s&gl=US&hl=en' % video_id
664 if format_param is not None:
665 normalized_url = '%s&fmt=%s' % (normalized_url, format_param)
666 request = urllib2.Request(normalized_url, None, std_headers)
668 self.report_webpage_download(video_id)
669 video_webpage = urllib2.urlopen(request).read()
670 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
671 self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
673 self.report_information_extraction(video_id)
676 mobj = re.search(r', "t": "([^"]+)"', video_webpage)
678 self._downloader.trouble(u'ERROR: unable to extract "t" parameter')
680 video_real_url = 'http://www.youtube.com/get_video?video_id=%s&t=%s&el=detailpage&ps=' % (video_id, mobj.group(1))
681 if format_param is not None:
682 video_real_url = '%s&fmt=%s' % (video_real_url, format_param)
683 self.report_video_url(video_id, video_real_url)
686 mobj = re.search(r"var watchUsername = '([^']+)';", video_webpage)
688 self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
690 video_uploader = mobj.group(1)
693 mobj = re.search(r'(?im)<title>YouTube - ([^<]*)</title>', video_webpage)
695 self._downloader.trouble(u'ERROR: unable to extract video title')
697 video_title = mobj.group(1).decode('utf-8')
698 video_title = re.sub(ur'(?u)&(.+?);', self.htmlentity_transform, video_title)
699 video_title = video_title.replace(os.sep, u'%')
702 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
703 simple_title = simple_title.strip(ur'_')
706 # Process video information
707 self._downloader.process_info({
708 'id': video_id.decode('utf-8'),
709 'url': video_real_url.decode('utf-8'),
710 'uploader': video_uploader.decode('utf-8'),
711 'title': video_title,
712 'stitle': simple_title,
713 'ext': video_extension.decode('utf-8'),
718 except UnavailableFormatError, err:
720 if quality_index == len(self._available_formats) - 1:
721 # I don't ever expect this to happen
722 self._downloader.trouble(u'ERROR: no known formats available for video')
725 self.report_unavailable_format(video_id, format_param)
727 format_param = self._available_formats[quality_index]
730 self._downloader.trouble('ERROR: format not available for video')
734 class MetacafeIE(InfoExtractor):
735 """Information Extractor for metacafe.com."""
737 _VALID_URL = r'(?:http://)?(?:www\.)?metacafe\.com/watch/([^/]+)/([^/]+)/.*'
738 _DISCLAIMER = 'http://www.metacafe.com/family_filter/'
739 _FILTER_POST = 'http://www.metacafe.com/f/index.php?inputType=filter&controllerGroup=user'
742 def __init__(self, youtube_ie, downloader=None):
743 InfoExtractor.__init__(self, downloader)
744 self._youtube_ie = youtube_ie
748 return (re.match(MetacafeIE._VALID_URL, url) is not None)
750 def report_disclaimer(self):
751 """Report disclaimer retrieval."""
752 self._downloader.to_stdout(u'[metacafe] Retrieving disclaimer')
754 def report_age_confirmation(self):
755 """Report attempt to confirm age."""
756 self._downloader.to_stdout(u'[metacafe] Confirming age')
758 def report_download_webpage(self, video_id):
759 """Report webpage download."""
760 self._downloader.to_stdout(u'[metacafe] %s: Downloading webpage' % video_id)
762 def report_extraction(self, video_id):
763 """Report information extraction."""
764 self._downloader.to_stdout(u'[metacafe] %s: Extracting information' % video_id)
766 def _real_initialize(self):
767 # Retrieve disclaimer
768 request = urllib2.Request(self._DISCLAIMER, None, std_headers)
770 self.report_disclaimer()
771 disclaimer = urllib2.urlopen(request).read()
772 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
773 self._downloader.trouble(u'ERROR: unable to retrieve disclaimer: %s' % str(err))
779 'submit': "Continue - I'm over 18",
781 request = urllib2.Request(self._FILTER_POST, urllib.urlencode(disclaimer_form), std_headers)
783 self.report_age_confirmation()
784 disclaimer = urllib2.urlopen(request).read()
785 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
786 self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
789 def _real_extract(self, url):
790 # Extract id and simplified title from URL
791 mobj = re.match(self._VALID_URL, url)
793 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
796 video_id = mobj.group(1)
798 # Check if video comes from YouTube
799 mobj2 = re.match(r'^yt-(.*)$', video_id)
800 if mobj2 is not None:
801 self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % mobj2.group(1))
804 simple_title = mobj.group(2).decode('utf-8')
805 video_extension = 'flv'
807 # Retrieve video webpage to extract further information
808 request = urllib2.Request('http://www.metacafe.com/watch/%s/' % video_id)
810 self.report_download_webpage(video_id)
811 webpage = urllib2.urlopen(request).read()
812 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
813 self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % str(err))
816 # Extract URL, uploader and title from webpage
817 self.report_extraction(video_id)
818 mobj = re.search(r'(?m)&mediaURL=(http.*?\.flv)', webpage)
820 self._downloader.trouble(u'ERROR: unable to extract media URL')
822 mediaURL = urllib.unquote(mobj.group(1))
824 mobj = re.search(r'(?m)&gdaKey=(.*?)&', webpage)
826 self._downloader.trouble(u'ERROR: unable to extract gdaKey')
828 gdaKey = mobj.group(1)
830 video_url = '%s?__gda__=%s' % (mediaURL, gdaKey)
832 mobj = re.search(r'(?im)<title>(.*) - Video</title>', webpage)
834 self._downloader.trouble(u'ERROR: unable to extract title')
836 video_title = mobj.group(1).decode('utf-8')
838 mobj = re.search(r'(?ms)<li id="ChnlUsr">.*?Submitter:.*?<a .*?>(.*?)<', webpage)
840 self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
842 video_uploader = mobj.group(1)
845 # Process video information
846 self._downloader.process_info({
847 'id': video_id.decode('utf-8'),
848 'url': video_url.decode('utf-8'),
849 'uploader': video_uploader.decode('utf-8'),
850 'title': video_title,
851 'stitle': simple_title,
852 'ext': video_extension.decode('utf-8'),
854 except UnavailableFormatError:
855 self._downloader.trouble(u'ERROR: format not available for video')
858 class YoutubeSearchIE(InfoExtractor):
859 """Information Extractor for YouTube search queries."""
860 _VALID_QUERY = r'ytsearch(\d+|all)?:[\s\S]+'
861 _TEMPLATE_URL = 'http://www.youtube.com/results?search_query=%s&page=%s&gl=US&hl=en'
862 _VIDEO_INDICATOR = r'href="/watch\?v=.+?"'
863 _MORE_PAGES_INDICATOR = r'>Next</a>'
865 _max_youtube_results = 1000
867 def __init__(self, youtube_ie, downloader=None):
868 InfoExtractor.__init__(self, downloader)
869 self._youtube_ie = youtube_ie
873 return (re.match(YoutubeSearchIE._VALID_QUERY, url) is not None)
875 def report_download_page(self, query, pagenum):
876 """Report attempt to download playlist page with given number."""
877 self._downloader.to_stdout(u'[youtube] query "%s": Downloading page %s' % (query, pagenum))
879 def _real_initialize(self):
880 self._youtube_ie.initialize()
882 def _real_extract(self, query):
883 mobj = re.match(self._VALID_QUERY, query)
885 self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
888 prefix, query = query.split(':')
891 self._download_n_results(query, 1)
893 elif prefix == 'all':
894 self._download_n_results(query, self._max_youtube_results)
900 self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
902 elif n > self._max_youtube_results:
903 self._downloader.to_stderr(u'WARNING: ytsearch returns max %i results (you requested %i)' % (self._max_youtube_results, n))
904 n = self._max_youtube_results
905 self._download_n_results(query, n)
907 except ValueError: # parsing prefix as integer fails
908 self._download_n_results(query, 1)
911 def _download_n_results(self, query, n):
912 """Downloads a specified number of results for a query"""
919 self.report_download_page(query, pagenum)
920 result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
921 request = urllib2.Request(result_url, None, std_headers)
923 page = urllib2.urlopen(request).read()
924 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
925 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
928 # Extract video identifiers
929 for mobj in re.finditer(self._VIDEO_INDICATOR, page):
930 video_id = page[mobj.span()[0]:mobj.span()[1]].split('=')[2][:-1]
931 if video_id not in already_seen:
932 video_ids.append(video_id)
933 already_seen.add(video_id)
934 if len(video_ids) == n:
935 # Specified n videos reached
937 self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
940 if self._MORE_PAGES_INDICATOR not in page:
942 self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
945 pagenum = pagenum + 1
947 class YoutubePlaylistIE(InfoExtractor):
948 """Information Extractor for YouTube playlists."""
950 _VALID_URL = r'(?:http://)?(?:\w+\.)?youtube.com/view_play_list\?p=(.+)'
951 _TEMPLATE_URL = 'http://www.youtube.com/view_play_list?p=%s&page=%s&gl=US&hl=en'
952 _VIDEO_INDICATOR = r'/watch\?v=(.+?)&'
953 _MORE_PAGES_INDICATOR = r'/view_play_list?p=%s&page=%s'
956 def __init__(self, youtube_ie, downloader=None):
957 InfoExtractor.__init__(self, downloader)
958 self._youtube_ie = youtube_ie
962 return (re.match(YoutubePlaylistIE._VALID_URL, url) is not None)
964 def report_download_page(self, playlist_id, pagenum):
965 """Report attempt to download playlist page with given number."""
966 self._downloader.to_stdout(u'[youtube] PL %s: Downloading page #%s' % (playlist_id, pagenum))
968 def _real_initialize(self):
969 self._youtube_ie.initialize()
971 def _real_extract(self, url):
972 # Extract playlist id
973 mobj = re.match(self._VALID_URL, url)
975 self._downloader.trouble(u'ERROR: invalid url: %s' % url)
978 # Download playlist pages
979 playlist_id = mobj.group(1)
984 self.report_download_page(playlist_id, pagenum)
985 request = urllib2.Request(self._TEMPLATE_URL % (playlist_id, pagenum), None, std_headers)
987 page = urllib2.urlopen(request).read()
988 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
989 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
992 # Extract video identifiers
994 for mobj in re.finditer(self._VIDEO_INDICATOR, page):
995 if mobj.group(1) not in ids_in_page:
996 ids_in_page.append(mobj.group(1))
997 video_ids.extend(ids_in_page)
999 if (self._MORE_PAGES_INDICATOR % (playlist_id, pagenum + 1)) not in page:
1001 pagenum = pagenum + 1
1003 for id in video_ids:
1004 self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
1007 class PostProcessor(object):
1008 """Post Processor class.
1010 PostProcessor objects can be added to downloaders with their
1011 add_post_processor() method. When the downloader has finished a
1012 successful download, it will take its internal chain of PostProcessors
1013 and start calling the run() method on each one of them, first with
1014 an initial argument and then with the returned value of the previous
1017 The chain will be stopped if one of them ever returns None or the end
1018 of the chain is reached.
1020 PostProcessor objects follow a "mutual registration" process similar
1021 to InfoExtractor objects.
1026 def __init__(self, downloader=None):
1027 self._downloader = downloader
1029 def set_downloader(self, downloader):
1030 """Sets the downloader for this PP."""
1031 self._downloader = downloader
1033 def run(self, information):
1034 """Run the PostProcessor.
1036 The "information" argument is a dictionary like the ones
1037 composed by InfoExtractors. The only difference is that this
1038 one has an extra field called "filepath" that points to the
1041 When this method returns None, the postprocessing chain is
1042 stopped. However, this method may return an information
1043 dictionary that will be passed to the next postprocessing
1044 object in the chain. It can be the one it received after
1045 changing some fields.
1047 In addition, this method may raise a PostProcessingError
1048 exception that will be taken into account by the downloader
1051 return information # by default, do nothing
1053 ### MAIN PROGRAM ###
1054 if __name__ == '__main__':
1056 # Modules needed only when running the main program
1060 # General configuration
1061 urllib2.install_opener(urllib2.build_opener(urllib2.ProxyHandler()))
1062 urllib2.install_opener(urllib2.build_opener(urllib2.HTTPCookieProcessor()))
1063 socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words)
1065 # Parse command line
1066 parser = optparse.OptionParser(
1067 usage='Usage: %prog [options] url...',
1069 conflict_handler='resolve',
1072 parser.add_option('-h', '--help',
1073 action='help', help='print this help text and exit')
1074 parser.add_option('-v', '--version',
1075 action='version', help='print program version and exit')
1076 parser.add_option('-i', '--ignore-errors',
1077 action='store_true', dest='ignoreerrors', help='continue on download errors', default=False)
1078 parser.add_option('-r', '--rate-limit',
1079 dest='ratelimit', metavar='L', help='download rate limit (e.g. 50k or 44.6m)')
1081 authentication = optparse.OptionGroup(parser, 'Authentication Options')
1082 authentication.add_option('-u', '--username',
1083 dest='username', metavar='UN', help='account username')
1084 authentication.add_option('-p', '--password',
1085 dest='password', metavar='PW', help='account password')
1086 authentication.add_option('-n', '--netrc',
1087 action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
1088 parser.add_option_group(authentication)
1090 video_format = optparse.OptionGroup(parser, 'Video Format Options')
1091 video_format.add_option('-f', '--format',
1092 action='store', dest='format', metavar='FMT', help='video format code')
1093 video_format.add_option('-b', '--best-quality',
1094 action='store_const', dest='format', help='download the best quality video possible', const='0')
1095 video_format.add_option('-m', '--mobile-version',
1096 action='store_const', dest='format', help='alias for -f 17', const='17')
1097 video_format.add_option('-d', '--high-def',
1098 action='store_const', dest='format', help='alias for -f 22', const='22')
1099 parser.add_option_group(video_format)
1101 verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
1102 verbosity.add_option('-q', '--quiet',
1103 action='store_true', dest='quiet', help='activates quiet mode', default=False)
1104 verbosity.add_option('-s', '--simulate',
1105 action='store_true', dest='simulate', help='do not download video', default=False)
1106 verbosity.add_option('-g', '--get-url',
1107 action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
1108 verbosity.add_option('-e', '--get-title',
1109 action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
1110 parser.add_option_group(verbosity)
1112 filesystem = optparse.OptionGroup(parser, 'Filesystem Options')
1113 filesystem.add_option('-t', '--title',
1114 action='store_true', dest='usetitle', help='use title in file name', default=False)
1115 filesystem.add_option('-l', '--literal',
1116 action='store_true', dest='useliteral', help='use literal title in file name', default=False)
1117 filesystem.add_option('-o', '--output',
1118 dest='outtmpl', metavar='TPL', help='output filename template')
1119 filesystem.add_option('-a', '--batch-file',
1120 dest='batchfile', metavar='F', help='file containing URLs to download')
1121 filesystem.add_option('-w', '--no-overwrites',
1122 action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
1123 filesystem.add_option('-c', '--continue',
1124 action='store_true', dest='continue_dl', help='resume partially downloaded files', default=False)
1125 parser.add_option_group(filesystem)
1127 (opts, args) = parser.parse_args()
1129 # Batch file verification
1131 if opts.batchfile is not None:
1133 batchurls = open(opts.batchfile, 'r').readlines()
1134 batchurls = [x.strip() for x in batchurls]
1135 batchurls = [x for x in batchurls if len(x) > 0]
1137 sys.exit(u'ERROR: batch file could not be read')
1138 all_urls = batchurls + args
1140 # Conflicting, missing and erroneous options
1141 if len(all_urls) < 1:
1142 parser.error(u'you must provide at least one URL')
1143 if opts.usenetrc and (opts.username is not None or opts.password is not None):
1144 parser.error(u'using .netrc conflicts with giving username/password')
1145 if opts.password is not None and opts.username is None:
1146 parser.error(u'account username missing')
1147 if opts.outtmpl is not None and (opts.useliteral or opts.usetitle):
1148 parser.error(u'using output template conflicts with using title or literal title')
1149 if opts.usetitle and opts.useliteral:
1150 parser.error(u'using title conflicts with using literal title')
1151 if opts.username is not None and opts.password is None:
1152 opts.password = getpass.getpass(u'Type account password and press return:')
1153 if opts.ratelimit is not None:
1154 numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
1155 if numeric_limit is None:
1156 parser.error(u'invalid rate limit specified')
1157 opts.ratelimit = numeric_limit
1159 # Information extractors
1160 youtube_ie = YoutubeIE()
1161 metacafe_ie = MetacafeIE(youtube_ie)
1162 youtube_pl_ie = YoutubePlaylistIE(youtube_ie)
1163 youtube_search_ie = YoutubeSearchIE(youtube_ie)
1166 fd = FileDownloader({
1167 'usenetrc': opts.usenetrc,
1168 'username': opts.username,
1169 'password': opts.password,
1170 'quiet': (opts.quiet or opts.geturl or opts.gettitle),
1171 'forceurl': opts.geturl,
1172 'forcetitle': opts.gettitle,
1173 'simulate': (opts.simulate or opts.geturl or opts.gettitle),
1174 'format': opts.format,
1175 'outtmpl': ((opts.outtmpl is not None and opts.outtmpl.decode(locale.getpreferredencoding()))
1176 or (opts.usetitle and u'%(stitle)s-%(id)s.%(ext)s')
1177 or (opts.useliteral and u'%(title)s-%(id)s.%(ext)s')
1178 or u'%(id)s.%(ext)s'),
1179 'ignoreerrors': opts.ignoreerrors,
1180 'ratelimit': opts.ratelimit,
1181 'nooverwrites': opts.nooverwrites,
1182 'continuedl': opts.continue_dl,
1184 fd.add_info_extractor(youtube_search_ie)
1185 fd.add_info_extractor(youtube_pl_ie)
1186 fd.add_info_extractor(metacafe_ie)
1187 fd.add_info_extractor(youtube_ie)
1188 retcode = fd.download(all_urls)
1191 except DownloadError:
1193 except SameFileError:
1194 sys.exit(u'ERROR: fixed output name but more than one file to download')
1195 except KeyboardInterrupt:
1196 sys.exit(u'\nERROR: Interrupted by user')