2 # -*- coding: utf-8 -*-
3 # Author: Ricardo Garcia Gonzalez
4 # Author: Danny Colligan
5 # License: Public domain code
22 'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.0.8) Gecko/2009032609 Firefox/3.0.8',
23 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
24 'Accept': 'text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5',
25 'Accept-Language': 'en-us,en;q=0.5',
28 simple_title_chars = string.ascii_letters.decode('ascii') + string.digits.decode('ascii')
30 class DownloadError(Exception):
31 """Download Error exception.
33 This exception may be thrown by FileDownloader objects if they are not
34 configured to continue on errors. They will contain the appropriate
39 class SameFileError(Exception):
40 """Same File exception.
42 This exception will be thrown by FileDownloader objects if they detect
43 multiple files would have to be downloaded to the same file on disk.
47 class PostProcessingError(Exception):
48 """Post Processing exception.
50 This exception may be raised by PostProcessor's .run() method to
51 indicate an error in the postprocessing task.
55 class UnavailableFormatError(Exception):
56 """Unavailable Format exception.
58 This exception will be thrown when a video is requested
59 in a format that is not available for that video.
63 class ContentTooShortError(Exception):
64 """Content Too Short exception.
66 This exception may be raised by FileDownloader objects when a file they
67 download is too small for what the server announced first, indicating
68 the connection was probably interrupted.
74 def __init__(self, downloaded, expected):
75 self.downloaded = downloaded
76 self.expected = expected
78 class FileDownloader(object):
79 """File Downloader class.
81 File downloader objects are the ones responsible of downloading the
82 actual video file and writing it to disk if the user has requested
83 it, among some other tasks. In most cases there should be one per
84 program. As, given a video URL, the downloader doesn't know how to
85 extract all the needed information, task that InfoExtractors do, it
86 has to pass the URL to one of them.
88 For this, file downloader objects have a method that allows
89 InfoExtractors to be registered in a given order. When it is passed
90 a URL, the file downloader handles it to the first InfoExtractor it
91 finds that reports being able to handle it. The InfoExtractor extracts
92 all the information about the video or videos the URL refers to, and
93 asks the FileDownloader to process the video information, possibly
94 downloading the video.
96 File downloaders accept a lot of parameters. In order not to saturate
97 the object constructor with arguments, it receives a dictionary of
98 options instead. These options are available through the params
99 attribute for the InfoExtractors to use. The FileDownloader also
100 registers itself as the downloader in charge for the InfoExtractors
101 that are added to it, so this is a "mutual registration".
105 username: Username for authentication purposes.
106 password: Password for authentication purposes.
107 usenetrc: Use netrc for authentication instead.
108 quiet: Do not print messages to stdout.
109 forceurl: Force printing final URL.
110 forcetitle: Force printing title.
111 simulate: Do not download the video files.
112 format: Video format code.
113 outtmpl: Template for output names.
114 ignoreerrors: Do not stop on download errors.
115 ratelimit: Download speed limit, in bytes/sec.
116 nooverwrites: Prevent overwriting files.
117 continuedl: Try to continue downloads if possible.
123 _download_retcode = None
125 def __init__(self, params):
126 """Create a FileDownloader object with the given options."""
129 self._download_retcode = 0
133 def pmkdir(filename):
134 """Create directory components in filename. Similar to Unix "mkdir -p"."""
135 components = filename.split(os.sep)
136 aggregate = [os.sep.join(components[0:x]) for x in xrange(1, len(components))]
137 aggregate = ['%s%s' % (x, os.sep) for x in aggregate] # Finish names with separator
138 for dir in aggregate:
139 if not os.path.exists(dir):
143 def format_bytes(bytes):
149 exponent = long(math.log(float(bytes), 1024.0))
150 suffix = 'bkMGTPEZY'[exponent]
151 converted = float(bytes) / float(1024**exponent)
152 return '%.2f%s' % (converted, suffix)
155 def calc_percent(byte_counter, data_len):
158 return '%6s' % ('%3.1f%%' % (float(byte_counter) / float(data_len) * 100.0))
161 def calc_eta(start, now, total, current):
165 if current == 0 or dif < 0.001: # One millisecond
167 rate = float(current) / dif
168 eta = long((float(total) - float(current)) / rate)
169 (eta_mins, eta_secs) = divmod(eta, 60)
172 return '%02d:%02d' % (eta_mins, eta_secs)
175 def calc_speed(start, now, bytes):
177 if bytes == 0 or dif < 0.001: # One millisecond
178 return '%10s' % '---b/s'
179 return '%10s' % ('%s/s' % FileDownloader.format_bytes(float(bytes) / dif))
182 def best_block_size(elapsed_time, bytes):
183 new_min = max(bytes / 2.0, 1.0)
184 new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
185 if elapsed_time < 0.001:
187 rate = bytes / elapsed_time
195 def parse_bytes(bytestr):
196 """Parse a string indicating a byte quantity into a long integer."""
197 matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
200 number = float(matchobj.group(1))
201 multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
202 return long(round(number * multiplier))
206 """Verify a URL is valid and data could be downloaded."""
207 request = urllib2.Request(url, None, std_headers)
208 data = urllib2.urlopen(request)
212 def add_info_extractor(self, ie):
213 """Add an InfoExtractor object to the end of the list."""
215 ie.set_downloader(self)
217 def add_post_processor(self, pp):
218 """Add a PostProcessor object to the end of the chain."""
220 pp.set_downloader(self)
222 def to_stdout(self, message, skip_eol=False):
223 """Print message to stdout if not in quiet mode."""
224 if not self.params.get('quiet', False):
225 print (u'%s%s' % (message, [u'\n', u''][skip_eol])).encode(locale.getpreferredencoding()),
228 def to_stderr(self, message):
229 """Print message to stderr."""
230 print >>sys.stderr, message
232 def fixed_template(self):
233 """Checks if the output template is fixed."""
234 return (re.search(ur'(?u)%\(.+?\)s', self.params['outtmpl']) is None)
236 def trouble(self, message=None):
237 """Determine action to take when a download problem appears.
239 Depending on if the downloader has been configured to ignore
240 download errors or not, this method may throw an exception or
241 not when errors are found, after printing the message.
243 if message is not None:
244 self.to_stderr(message)
245 if not self.params.get('ignoreerrors', False):
246 raise DownloadError(message)
247 self._download_retcode = 1
249 def slow_down(self, start_time, byte_counter):
250 """Sleep if the download speed is over the rate limit."""
251 rate_limit = self.params.get('ratelimit', None)
252 if rate_limit is None or byte_counter == 0:
255 elapsed = now - start_time
258 speed = float(byte_counter) / elapsed
259 if speed > rate_limit:
260 time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
262 def report_destination(self, filename):
263 """Report destination filename."""
264 self.to_stdout(u'[download] Destination: %s' % filename)
266 def report_progress(self, percent_str, data_len_str, speed_str, eta_str):
267 """Report download progress."""
268 self.to_stdout(u'\r[download] %s of %s at %s ETA %s' %
269 (percent_str, data_len_str, speed_str, eta_str), skip_eol=True)
271 def report_resuming_byte(self, resume_len):
272 """Report attemtp to resume at given byte."""
273 self.to_stdout(u'[download] Resuming download at byte %s' % resume_len)
275 def report_file_already_downloaded(self, file_name):
276 """Report file has already been fully downloaded."""
277 self.to_stdout(u'[download] %s has already been downloaded' % file_name)
279 def report_unable_to_resume(self):
280 """Report it was impossible to resume download."""
281 self.to_stdout(u'[download] Unable to resume')
283 def report_finish(self):
284 """Report download finished."""
287 def process_info(self, info_dict):
288 """Process a single dictionary returned by an InfoExtractor."""
289 # Do nothing else if in simulate mode
290 if self.params.get('simulate', False):
292 self.verify_url(info_dict['url'])
293 except (OSError, IOError, urllib2.URLError, httplib.HTTPException, socket.error), err:
294 raise UnavailableFormatError
297 if self.params.get('forcetitle', False):
298 print info_dict['title'].encode(locale.getpreferredencoding())
299 if self.params.get('forceurl', False):
300 print info_dict['url'].encode(locale.getpreferredencoding())
305 template_dict = dict(info_dict)
306 template_dict['epoch'] = unicode(long(time.time()))
307 filename = self.params['outtmpl'] % template_dict
308 except (ValueError, KeyError), err:
309 self.trouble('ERROR: invalid output template or system charset: %s' % str(err))
310 if self.params['nooverwrites'] and os.path.exists(filename):
311 self.to_stderr('WARNING: file exists: %s; skipping' % filename)
315 self.pmkdir(filename)
316 except (OSError, IOError), err:
317 self.trouble('ERROR: unable to create directories: %s' % str(err))
321 success = self._do_download(filename, info_dict['url'])
322 except (OSError, IOError), err:
323 raise UnavailableFormatError
324 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
325 self.trouble('ERROR: unable to download video data: %s' % str(err))
327 except (ContentTooShortError, ), err:
328 self.trouble('ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
333 self.post_process(filename, info_dict)
334 except (PostProcessingError), err:
335 self.trouble('ERROR: postprocessing: %s' % str(err))
338 def download(self, url_list):
339 """Download a given list of URLs."""
340 if len(url_list) > 1 and self.fixed_template():
341 raise SameFileError(self.params['outtmpl'])
344 suitable_found = False
346 # Go to next InfoExtractor if not suitable
347 if not ie.suitable(url):
350 # Suitable InfoExtractor found
351 suitable_found = True
353 # Extract information from URL and process it
356 # Suitable InfoExtractor had been found; go to next URL
359 if not suitable_found:
360 self.trouble('ERROR: no suitable InfoExtractor: %s' % url)
362 return self._download_retcode
364 def post_process(self, filename, ie_info):
365 """Run the postprocessing chain on the given file."""
367 info['filepath'] = filename
373 def _do_download(self, filename, url):
377 basic_request = urllib2.Request(url, None, std_headers)
378 request = urllib2.Request(url, None, std_headers)
380 # Attempt to resume download with "continuedl" option
381 if os.path.isfile(filename):
382 resume_len = os.path.getsize(filename)
385 if self.params['continuedl'] and resume_len != 0:
386 self.report_resuming_byte(resume_len)
387 request.add_header('Range','bytes=%d-' % resume_len)
389 # Establish connection
391 data = urllib2.urlopen(request)
392 except (urllib2.HTTPError, ), err:
393 if err.code != 416: # 416 is 'Requested range not satisfiable'
395 data = urllib2.urlopen(basic_request)
396 content_length = data.info()['Content-Length']
397 if content_length is not None and long(content_length) == resume_len:
398 self.report_file_already_downloaded(filename)
401 self.report_unable_to_resume()
404 data_len = data.info().get('Content-length', None)
405 data_len_str = self.format_bytes(data_len)
412 data_block = data.read(block_size)
414 data_block_len = len(data_block)
415 if data_block_len == 0:
417 byte_counter += data_block_len
419 # Open file just in time
422 stream = open(filename, open_mode)
423 self.report_destination(filename)
424 except (OSError, IOError), err:
425 self.trouble('ERROR: unable to open for writing: %s' % str(err))
427 stream.write(data_block)
428 block_size = self.best_block_size(after - before, data_block_len)
431 percent_str = self.calc_percent(byte_counter, data_len)
432 eta_str = self.calc_eta(start, time.time(), data_len, byte_counter)
433 speed_str = self.calc_speed(start, time.time(), byte_counter)
434 self.report_progress(percent_str, data_len_str, speed_str, eta_str)
437 self.slow_down(start, byte_counter)
440 if data_len is not None and str(byte_counter) != data_len:
441 raise ContentTooShortError(byte_counter, long(data_len))
444 class InfoExtractor(object):
445 """Information Extractor class.
447 Information extractors are the classes that, given a URL, extract
448 information from the video (or videos) the URL refers to. This
449 information includes the real video URL, the video title and simplified
450 title, author and others. The information is stored in a dictionary
451 which is then passed to the FileDownloader. The FileDownloader
452 processes this information possibly downloading the video to the file
453 system, among other possible outcomes. The dictionaries must include
454 the following fields:
456 id: Video identifier.
457 url: Final video URL.
458 uploader: Nickname of the video uploader.
459 title: Literal title.
460 stitle: Simplified title.
461 ext: Video filename extension.
463 Subclasses of this one should re-define the _real_initialize() and
464 _real_extract() methods, as well as the suitable() static method.
465 Probably, they should also be instantiated and added to the main
472 def __init__(self, downloader=None):
473 """Constructor. Receives an optional downloader."""
475 self.set_downloader(downloader)
479 """Receives a URL and returns True if suitable for this IE."""
482 def initialize(self):
483 """Initializes an instance (authentication, etc)."""
485 self._real_initialize()
488 def extract(self, url):
489 """Extracts URL information and returns it in list of dicts."""
491 return self._real_extract(url)
493 def set_downloader(self, downloader):
494 """Sets the downloader for this IE."""
495 self._downloader = downloader
497 def _real_initialize(self):
498 """Real initialization process. Redefine in subclasses."""
501 def _real_extract(self, url):
502 """Real extraction process. Redefine in subclasses."""
505 class YoutubeIE(InfoExtractor):
506 """Information extractor for youtube.com."""
508 _VALID_URL = r'^((?:http://)?(?:\w+\.)?youtube\.com/(?:(?:v/)|(?:(?:watch(?:\.php)?)?\?(?:.+&)?v=)))?([0-9A-Za-z_-]+)(?(1).+)?$'
509 _LANG_URL = r'http://uk.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
510 _LOGIN_URL = 'http://www.youtube.com/signup?next=/&gl=US&hl=en'
511 _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
512 _NETRC_MACHINE = 'youtube'
513 _available_formats = ['22', '35', '18', '17', '13'] # listed in order of priority for -b flag
514 _video_extensions = {
523 return (re.match(YoutubeIE._VALID_URL, url) is not None)
526 def htmlentity_transform(matchobj):
527 """Transforms an HTML entity to a Unicode character."""
528 entity = matchobj.group(1)
530 # Known non-numeric HTML entity
531 if entity in htmlentitydefs.name2codepoint:
532 return unichr(htmlentitydefs.name2codepoint[entity])
535 mobj = re.match(ur'(?u)#(x?\d+)', entity)
537 numstr = mobj.group(1)
538 if numstr.startswith(u'x'):
540 numstr = u'0%s' % numstr
543 return unichr(long(numstr, base))
545 # Unknown entity in name, return its literal representation
546 return (u'&%s;' % entity)
548 def report_lang(self):
549 """Report attempt to set language."""
550 self._downloader.to_stdout(u'[youtube] Setting language')
552 def report_login(self):
553 """Report attempt to log in."""
554 self._downloader.to_stdout(u'[youtube] Logging in')
556 def report_age_confirmation(self):
557 """Report attempt to confirm age."""
558 self._downloader.to_stdout(u'[youtube] Confirming age')
560 def report_webpage_download(self, video_id):
561 """Report attempt to download webpage."""
562 self._downloader.to_stdout(u'[youtube] %s: Downloading video webpage' % video_id)
564 def report_information_extraction(self, video_id):
565 """Report attempt to extract video information."""
566 self._downloader.to_stdout(u'[youtube] %s: Extracting video information' % video_id)
568 def report_video_url(self, video_id, video_real_url):
569 """Report extracted video URL."""
570 self._downloader.to_stdout(u'[youtube] %s: URL: %s' % (video_id, video_real_url))
572 def report_unavailable_format(self, video_id, format):
573 """Report extracted video URL."""
574 self._downloader.to_stdout(u'[youtube] %s: Format %s not available' % (video_id, format))
576 def _real_initialize(self):
577 if self._downloader is None:
582 downloader_params = self._downloader.params
584 # Attempt to use provided username and password or .netrc data
585 if downloader_params.get('username', None) is not None:
586 username = downloader_params['username']
587 password = downloader_params['password']
588 elif downloader_params.get('usenetrc', False):
590 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
595 raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
596 except (IOError, netrc.NetrcParseError), err:
597 self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % str(err))
601 request = urllib2.Request(self._LANG_URL, None, std_headers)
604 urllib2.urlopen(request).read()
605 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
606 self._downloader.to_stderr(u'WARNING: unable to set language: %s' % str(err))
609 # No authentication to be performed
615 'current_form': 'loginForm',
617 'action_login': 'Log In',
618 'username': username,
619 'password': password,
621 request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form), std_headers)
624 login_results = urllib2.urlopen(request).read()
625 if re.search(r'(?i)<form[^>]* name="loginForm"', login_results) is not None:
626 self._downloader.to_stderr(u'WARNING: unable to log in: bad username or password')
628 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
629 self._downloader.to_stderr(u'WARNING: unable to log in: %s' % str(err))
635 'action_confirm': 'Confirm',
637 request = urllib2.Request(self._AGE_URL, urllib.urlencode(age_form), std_headers)
639 self.report_age_confirmation()
640 age_results = urllib2.urlopen(request).read()
641 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
642 self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
645 def _real_extract(self, url):
646 # Extract video id from URL
647 mobj = re.match(self._VALID_URL, url)
649 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
651 video_id = mobj.group(2)
653 # Downloader parameters
657 if self._downloader is not None:
658 params = self._downloader.params
659 format_param = params.get('format', None)
660 if format_param == '0':
661 format_param = self._available_formats[quality_index]
666 video_extension = self._video_extensions.get(format_param, 'flv')
668 # Normalize URL, including format
669 normalized_url = 'http://www.youtube.com/watch?v=%s&gl=US&hl=en' % video_id
670 if format_param is not None:
671 normalized_url = '%s&fmt=%s' % (normalized_url, format_param)
672 request = urllib2.Request(normalized_url, None, std_headers)
674 self.report_webpage_download(video_id)
675 video_webpage = urllib2.urlopen(request).read()
676 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
677 self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
679 self.report_information_extraction(video_id)
682 mobj = re.search(r', "t": "([^"]+)"', video_webpage)
684 self._downloader.trouble(u'ERROR: unable to extract "t" parameter')
686 video_real_url = 'http://www.youtube.com/get_video?video_id=%s&t=%s&el=detailpage&ps=' % (video_id, mobj.group(1))
687 if format_param is not None:
688 video_real_url = '%s&fmt=%s' % (video_real_url, format_param)
689 self.report_video_url(video_id, video_real_url)
692 mobj = re.search(r"var watchUsername = '([^']+)';", video_webpage)
694 self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
696 video_uploader = mobj.group(1)
699 mobj = re.search(r'(?im)<title>YouTube - ([^<]*)</title>', video_webpage)
701 self._downloader.trouble(u'ERROR: unable to extract video title')
703 video_title = mobj.group(1).decode('utf-8')
704 video_title = re.sub(ur'(?u)&(.+?);', self.htmlentity_transform, video_title)
705 video_title = video_title.replace(os.sep, u'%')
708 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
709 simple_title = simple_title.strip(ur'_')
712 # Process video information
713 self._downloader.process_info({
714 'id': video_id.decode('utf-8'),
715 'url': video_real_url.decode('utf-8'),
716 'uploader': video_uploader.decode('utf-8'),
717 'title': video_title,
718 'stitle': simple_title,
719 'ext': video_extension.decode('utf-8'),
724 except UnavailableFormatError, err:
726 if quality_index == len(self._available_formats) - 1:
727 # I don't ever expect this to happen
728 self._downloader.trouble(u'ERROR: no known formats available for video')
731 self.report_unavailable_format(video_id, format_param)
733 format_param = self._available_formats[quality_index]
736 self._downloader.trouble('ERROR: format not available for video')
740 class MetacafeIE(InfoExtractor):
741 """Information Extractor for metacafe.com."""
743 _VALID_URL = r'(?:http://)?(?:www\.)?metacafe\.com/watch/([^/]+)/([^/]+)/.*'
744 _DISCLAIMER = 'http://www.metacafe.com/family_filter/'
745 _FILTER_POST = 'http://www.metacafe.com/f/index.php?inputType=filter&controllerGroup=user'
748 def __init__(self, youtube_ie, downloader=None):
749 InfoExtractor.__init__(self, downloader)
750 self._youtube_ie = youtube_ie
754 return (re.match(MetacafeIE._VALID_URL, url) is not None)
756 def report_disclaimer(self):
757 """Report disclaimer retrieval."""
758 self._downloader.to_stdout(u'[metacafe] Retrieving disclaimer')
760 def report_age_confirmation(self):
761 """Report attempt to confirm age."""
762 self._downloader.to_stdout(u'[metacafe] Confirming age')
764 def report_download_webpage(self, video_id):
765 """Report webpage download."""
766 self._downloader.to_stdout(u'[metacafe] %s: Downloading webpage' % video_id)
768 def report_extraction(self, video_id):
769 """Report information extraction."""
770 self._downloader.to_stdout(u'[metacafe] %s: Extracting information' % video_id)
772 def _real_initialize(self):
773 # Retrieve disclaimer
774 request = urllib2.Request(self._DISCLAIMER, None, std_headers)
776 self.report_disclaimer()
777 disclaimer = urllib2.urlopen(request).read()
778 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
779 self._downloader.trouble(u'ERROR: unable to retrieve disclaimer: %s' % str(err))
785 'submit': "Continue - I'm over 18",
787 request = urllib2.Request(self._FILTER_POST, urllib.urlencode(disclaimer_form), std_headers)
789 self.report_age_confirmation()
790 disclaimer = urllib2.urlopen(request).read()
791 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
792 self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
795 def _real_extract(self, url):
796 # Extract id and simplified title from URL
797 mobj = re.match(self._VALID_URL, url)
799 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
802 video_id = mobj.group(1)
804 # Check if video comes from YouTube
805 mobj2 = re.match(r'^yt-(.*)$', video_id)
806 if mobj2 is not None:
807 self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % mobj2.group(1))
810 simple_title = mobj.group(2).decode('utf-8')
811 video_extension = 'flv'
813 # Retrieve video webpage to extract further information
814 request = urllib2.Request('http://www.metacafe.com/watch/%s/' % video_id)
816 self.report_download_webpage(video_id)
817 webpage = urllib2.urlopen(request).read()
818 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
819 self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % str(err))
822 # Extract URL, uploader and title from webpage
823 self.report_extraction(video_id)
824 mobj = re.search(r'(?m)&mediaURL=(http.*?\.flv)', webpage)
826 self._downloader.trouble(u'ERROR: unable to extract media URL')
828 mediaURL = urllib.unquote(mobj.group(1))
830 mobj = re.search(r'(?m)&gdaKey=(.*?)&', webpage)
832 self._downloader.trouble(u'ERROR: unable to extract gdaKey')
834 gdaKey = mobj.group(1)
836 video_url = '%s?__gda__=%s' % (mediaURL, gdaKey)
838 mobj = re.search(r'(?im)<title>(.*) - Video</title>', webpage)
840 self._downloader.trouble(u'ERROR: unable to extract title')
842 video_title = mobj.group(1).decode('utf-8')
844 mobj = re.search(r'(?ms)<li id="ChnlUsr">.*?Submitter:.*?<a .*?>(.*?)<', webpage)
846 self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
848 video_uploader = mobj.group(1)
851 # Process video information
852 self._downloader.process_info({
853 'id': video_id.decode('utf-8'),
854 'url': video_url.decode('utf-8'),
855 'uploader': video_uploader.decode('utf-8'),
856 'title': video_title,
857 'stitle': simple_title,
858 'ext': video_extension.decode('utf-8'),
860 except UnavailableFormatError:
861 self._downloader.trouble(u'ERROR: format not available for video')
864 class YoutubeSearchIE(InfoExtractor):
865 """Information Extractor for YouTube search queries."""
866 _VALID_QUERY = r'ytsearch(\d+|all)?:[\s\S]+'
867 _TEMPLATE_URL = 'http://www.youtube.com/results?search_query=%s&page=%s&gl=US&hl=en'
868 _VIDEO_INDICATOR = r'href="/watch\?v=.+?"'
869 _MORE_PAGES_INDICATOR = r'>Next</a>'
871 _max_youtube_results = 1000
873 def __init__(self, youtube_ie, downloader=None):
874 InfoExtractor.__init__(self, downloader)
875 self._youtube_ie = youtube_ie
879 return (re.match(YoutubeSearchIE._VALID_QUERY, url) is not None)
881 def report_download_page(self, query, pagenum):
882 """Report attempt to download playlist page with given number."""
883 self._downloader.to_stdout(u'[youtube] query "%s": Downloading page %s' % (query, pagenum))
885 def _real_initialize(self):
886 self._youtube_ie.initialize()
888 def _real_extract(self, query):
889 mobj = re.match(self._VALID_QUERY, query)
891 self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
894 prefix, query = query.split(':')
897 self._download_n_results(query, 1)
899 elif prefix == 'all':
900 self._download_n_results(query, self._max_youtube_results)
906 self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
908 elif n > self._max_youtube_results:
909 self._downloader.to_stderr(u'WARNING: ytsearch returns max %i results (you requested %i)' % (self._max_youtube_results, n))
910 n = self._max_youtube_results
911 self._download_n_results(query, n)
913 except ValueError: # parsing prefix as integer fails
914 self._download_n_results(query, 1)
917 def _download_n_results(self, query, n):
918 """Downloads a specified number of results for a query"""
925 self.report_download_page(query, pagenum)
926 result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
927 request = urllib2.Request(result_url, None, std_headers)
929 page = urllib2.urlopen(request).read()
930 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
931 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
934 # Extract video identifiers
935 for mobj in re.finditer(self._VIDEO_INDICATOR, page):
936 video_id = page[mobj.span()[0]:mobj.span()[1]].split('=')[2][:-1]
937 if video_id not in already_seen:
938 video_ids.append(video_id)
939 already_seen.add(video_id)
940 if len(video_ids) == n:
941 # Specified n videos reached
943 self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
946 if self._MORE_PAGES_INDICATOR not in page:
948 self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
951 pagenum = pagenum + 1
953 class YoutubePlaylistIE(InfoExtractor):
954 """Information Extractor for YouTube playlists."""
956 _VALID_URL = r'(?:http://)?(?:\w+\.)?youtube.com/view_play_list\?p=(.+)'
957 _TEMPLATE_URL = 'http://www.youtube.com/view_play_list?p=%s&page=%s&gl=US&hl=en'
958 _VIDEO_INDICATOR = r'/watch\?v=(.+?)&'
959 _MORE_PAGES_INDICATOR = r'/view_play_list?p=%s&page=%s'
962 def __init__(self, youtube_ie, downloader=None):
963 InfoExtractor.__init__(self, downloader)
964 self._youtube_ie = youtube_ie
968 return (re.match(YoutubePlaylistIE._VALID_URL, url) is not None)
970 def report_download_page(self, playlist_id, pagenum):
971 """Report attempt to download playlist page with given number."""
972 self._downloader.to_stdout(u'[youtube] PL %s: Downloading page #%s' % (playlist_id, pagenum))
974 def _real_initialize(self):
975 self._youtube_ie.initialize()
977 def _real_extract(self, url):
978 # Extract playlist id
979 mobj = re.match(self._VALID_URL, url)
981 self._downloader.trouble(u'ERROR: invalid url: %s' % url)
984 # Download playlist pages
985 playlist_id = mobj.group(1)
990 self.report_download_page(playlist_id, pagenum)
991 request = urllib2.Request(self._TEMPLATE_URL % (playlist_id, pagenum), None, std_headers)
993 page = urllib2.urlopen(request).read()
994 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
995 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
998 # Extract video identifiers
1000 for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1001 if mobj.group(1) not in ids_in_page:
1002 ids_in_page.append(mobj.group(1))
1003 video_ids.extend(ids_in_page)
1005 if (self._MORE_PAGES_INDICATOR % (playlist_id, pagenum + 1)) not in page:
1007 pagenum = pagenum + 1
1009 for id in video_ids:
1010 self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
1013 class PostProcessor(object):
1014 """Post Processor class.
1016 PostProcessor objects can be added to downloaders with their
1017 add_post_processor() method. When the downloader has finished a
1018 successful download, it will take its internal chain of PostProcessors
1019 and start calling the run() method on each one of them, first with
1020 an initial argument and then with the returned value of the previous
1023 The chain will be stopped if one of them ever returns None or the end
1024 of the chain is reached.
1026 PostProcessor objects follow a "mutual registration" process similar
1027 to InfoExtractor objects.
1032 def __init__(self, downloader=None):
1033 self._downloader = downloader
1035 def set_downloader(self, downloader):
1036 """Sets the downloader for this PP."""
1037 self._downloader = downloader
1039 def run(self, information):
1040 """Run the PostProcessor.
1042 The "information" argument is a dictionary like the ones
1043 composed by InfoExtractors. The only difference is that this
1044 one has an extra field called "filepath" that points to the
1047 When this method returns None, the postprocessing chain is
1048 stopped. However, this method may return an information
1049 dictionary that will be passed to the next postprocessing
1050 object in the chain. It can be the one it received after
1051 changing some fields.
1053 In addition, this method may raise a PostProcessingError
1054 exception that will be taken into account by the downloader
1057 return information # by default, do nothing
1059 ### MAIN PROGRAM ###
1060 if __name__ == '__main__':
1062 # Modules needed only when running the main program
1066 # General configuration
1067 urllib2.install_opener(urllib2.build_opener(urllib2.ProxyHandler()))
1068 urllib2.install_opener(urllib2.build_opener(urllib2.HTTPCookieProcessor()))
1069 socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words)
1071 # Parse command line
1072 parser = optparse.OptionParser(
1073 usage='Usage: %prog [options] url...',
1075 conflict_handler='resolve',
1078 parser.add_option('-h', '--help',
1079 action='help', help='print this help text and exit')
1080 parser.add_option('-v', '--version',
1081 action='version', help='print program version and exit')
1082 parser.add_option('-i', '--ignore-errors',
1083 action='store_true', dest='ignoreerrors', help='continue on download errors', default=False)
1084 parser.add_option('-r', '--rate-limit',
1085 dest='ratelimit', metavar='L', help='download rate limit (e.g. 50k or 44.6m)')
1087 authentication = optparse.OptionGroup(parser, 'Authentication Options')
1088 authentication.add_option('-u', '--username',
1089 dest='username', metavar='UN', help='account username')
1090 authentication.add_option('-p', '--password',
1091 dest='password', metavar='PW', help='account password')
1092 authentication.add_option('-n', '--netrc',
1093 action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
1094 parser.add_option_group(authentication)
1096 video_format = optparse.OptionGroup(parser, 'Video Format Options')
1097 video_format.add_option('-f', '--format',
1098 action='store', dest='format', metavar='FMT', help='video format code')
1099 video_format.add_option('-b', '--best-quality',
1100 action='store_const', dest='format', help='download the best quality video possible', const='0')
1101 video_format.add_option('-m', '--mobile-version',
1102 action='store_const', dest='format', help='alias for -f 17', const='17')
1103 video_format.add_option('-d', '--high-def',
1104 action='store_const', dest='format', help='alias for -f 22', const='22')
1105 parser.add_option_group(video_format)
1107 verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
1108 verbosity.add_option('-q', '--quiet',
1109 action='store_true', dest='quiet', help='activates quiet mode', default=False)
1110 verbosity.add_option('-s', '--simulate',
1111 action='store_true', dest='simulate', help='do not download video', default=False)
1112 verbosity.add_option('-g', '--get-url',
1113 action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
1114 verbosity.add_option('-e', '--get-title',
1115 action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
1116 parser.add_option_group(verbosity)
1118 filesystem = optparse.OptionGroup(parser, 'Filesystem Options')
1119 filesystem.add_option('-t', '--title',
1120 action='store_true', dest='usetitle', help='use title in file name', default=False)
1121 filesystem.add_option('-l', '--literal',
1122 action='store_true', dest='useliteral', help='use literal title in file name', default=False)
1123 filesystem.add_option('-o', '--output',
1124 dest='outtmpl', metavar='TPL', help='output filename template')
1125 filesystem.add_option('-a', '--batch-file',
1126 dest='batchfile', metavar='F', help='file containing URLs to download')
1127 filesystem.add_option('-w', '--no-overwrites',
1128 action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
1129 filesystem.add_option('-c', '--continue',
1130 action='store_true', dest='continue_dl', help='resume partially downloaded files', default=False)
1131 parser.add_option_group(filesystem)
1133 (opts, args) = parser.parse_args()
1135 # Batch file verification
1137 if opts.batchfile is not None:
1139 batchurls = open(opts.batchfile, 'r').readlines()
1140 batchurls = [x.strip() for x in batchurls]
1141 batchurls = [x for x in batchurls if len(x) > 0]
1143 sys.exit(u'ERROR: batch file could not be read')
1144 all_urls = batchurls + args
1146 # Conflicting, missing and erroneous options
1147 if len(all_urls) < 1:
1148 parser.error(u'you must provide at least one URL')
1149 if opts.usenetrc and (opts.username is not None or opts.password is not None):
1150 parser.error(u'using .netrc conflicts with giving username/password')
1151 if opts.password is not None and opts.username is None:
1152 parser.error(u'account username missing')
1153 if opts.outtmpl is not None and (opts.useliteral or opts.usetitle):
1154 parser.error(u'using output template conflicts with using title or literal title')
1155 if opts.usetitle and opts.useliteral:
1156 parser.error(u'using title conflicts with using literal title')
1157 if opts.username is not None and opts.password is None:
1158 opts.password = getpass.getpass(u'Type account password and press return:')
1159 if opts.ratelimit is not None:
1160 numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
1161 if numeric_limit is None:
1162 parser.error(u'invalid rate limit specified')
1163 opts.ratelimit = numeric_limit
1165 # Information extractors
1166 youtube_ie = YoutubeIE()
1167 metacafe_ie = MetacafeIE(youtube_ie)
1168 youtube_pl_ie = YoutubePlaylistIE(youtube_ie)
1169 youtube_search_ie = YoutubeSearchIE(youtube_ie)
1172 fd = FileDownloader({
1173 'usenetrc': opts.usenetrc,
1174 'username': opts.username,
1175 'password': opts.password,
1176 'quiet': (opts.quiet or opts.geturl or opts.gettitle),
1177 'forceurl': opts.geturl,
1178 'forcetitle': opts.gettitle,
1179 'simulate': (opts.simulate or opts.geturl or opts.gettitle),
1180 'format': opts.format,
1181 'outtmpl': ((opts.outtmpl is not None and opts.outtmpl.decode(locale.getpreferredencoding()))
1182 or (opts.usetitle and u'%(stitle)s-%(id)s.%(ext)s')
1183 or (opts.useliteral and u'%(title)s-%(id)s.%(ext)s')
1184 or u'%(id)s.%(ext)s'),
1185 'ignoreerrors': opts.ignoreerrors,
1186 'ratelimit': opts.ratelimit,
1187 'nooverwrites': opts.nooverwrites,
1188 'continuedl': opts.continue_dl,
1190 fd.add_info_extractor(youtube_search_ie)
1191 fd.add_info_extractor(youtube_pl_ie)
1192 fd.add_info_extractor(metacafe_ie)
1193 fd.add_info_extractor(youtube_ie)
1194 retcode = fd.download(all_urls)
1197 except DownloadError:
1199 except SameFileError:
1200 sys.exit(u'ERROR: fixed output name but more than one file to download')
1201 except KeyboardInterrupt:
1202 sys.exit(u'\nERROR: Interrupted by user')