2 # -*- coding: utf-8 -*-
3 # Author: Ricardo Garcia Gonzalez
4 # Author: Danny Colligan
5 # License: Public domain code
22 'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.1.2) Gecko/20090729 Firefox/3.5.2',
23 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
24 'Accept': 'text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5',
25 'Accept-Language': 'en-us,en;q=0.5',
28 simple_title_chars = string.ascii_letters.decode('ascii') + string.digits.decode('ascii')
30 class DownloadError(Exception):
31 """Download Error exception.
33 This exception may be thrown by FileDownloader objects if they are not
34 configured to continue on errors. They will contain the appropriate
39 class SameFileError(Exception):
40 """Same File exception.
42 This exception will be thrown by FileDownloader objects if they detect
43 multiple files would have to be downloaded to the same file on disk.
47 class PostProcessingError(Exception):
48 """Post Processing exception.
50 This exception may be raised by PostProcessor's .run() method to
51 indicate an error in the postprocessing task.
55 class UnavailableFormatError(Exception):
56 """Unavailable Format exception.
58 This exception will be thrown when a video is requested
59 in a format that is not available for that video.
63 class ContentTooShortError(Exception):
64 """Content Too Short exception.
66 This exception may be raised by FileDownloader objects when a file they
67 download is too small for what the server announced first, indicating
68 the connection was probably interrupted.
74 def __init__(self, downloaded, expected):
75 self.downloaded = downloaded
76 self.expected = expected
78 class FileDownloader(object):
79 """File Downloader class.
81 File downloader objects are the ones responsible of downloading the
82 actual video file and writing it to disk if the user has requested
83 it, among some other tasks. In most cases there should be one per
84 program. As, given a video URL, the downloader doesn't know how to
85 extract all the needed information, task that InfoExtractors do, it
86 has to pass the URL to one of them.
88 For this, file downloader objects have a method that allows
89 InfoExtractors to be registered in a given order. When it is passed
90 a URL, the file downloader handles it to the first InfoExtractor it
91 finds that reports being able to handle it. The InfoExtractor extracts
92 all the information about the video or videos the URL refers to, and
93 asks the FileDownloader to process the video information, possibly
94 downloading the video.
96 File downloaders accept a lot of parameters. In order not to saturate
97 the object constructor with arguments, it receives a dictionary of
98 options instead. These options are available through the params
99 attribute for the InfoExtractors to use. The FileDownloader also
100 registers itself as the downloader in charge for the InfoExtractors
101 that are added to it, so this is a "mutual registration".
105 username: Username for authentication purposes.
106 password: Password for authentication purposes.
107 usenetrc: Use netrc for authentication instead.
108 quiet: Do not print messages to stdout.
109 forceurl: Force printing final URL.
110 forcetitle: Force printing title.
111 simulate: Do not download the video files.
112 format: Video format code.
113 outtmpl: Template for output names.
114 ignoreerrors: Do not stop on download errors.
115 ratelimit: Download speed limit, in bytes/sec.
116 nooverwrites: Prevent overwriting files.
117 continuedl: Try to continue downloads if possible.
123 _download_retcode = None
125 def __init__(self, params):
126 """Create a FileDownloader object with the given options."""
129 self._download_retcode = 0
133 def pmkdir(filename):
134 """Create directory components in filename. Similar to Unix "mkdir -p"."""
135 components = filename.split(os.sep)
136 aggregate = [os.sep.join(components[0:x]) for x in xrange(1, len(components))]
137 aggregate = ['%s%s' % (x, os.sep) for x in aggregate] # Finish names with separator
138 for dir in aggregate:
139 if not os.path.exists(dir):
143 def format_bytes(bytes):
146 if type(bytes) is str:
151 exponent = long(math.log(bytes, 1024.0))
152 suffix = 'bkMGTPEZY'[exponent]
153 converted = float(bytes) / float(1024**exponent)
154 return '%.2f%s' % (converted, suffix)
157 def calc_percent(byte_counter, data_len):
160 return '%6s' % ('%3.1f%%' % (float(byte_counter) / float(data_len) * 100.0))
163 def calc_eta(start, now, total, current):
167 if current == 0 or dif < 0.001: # One millisecond
169 rate = float(current) / dif
170 eta = long((float(total) - float(current)) / rate)
171 (eta_mins, eta_secs) = divmod(eta, 60)
174 return '%02d:%02d' % (eta_mins, eta_secs)
177 def calc_speed(start, now, bytes):
179 if bytes == 0 or dif < 0.001: # One millisecond
180 return '%10s' % '---b/s'
181 return '%10s' % ('%s/s' % FileDownloader.format_bytes(float(bytes) / dif))
184 def best_block_size(elapsed_time, bytes):
185 new_min = max(bytes / 2.0, 1.0)
186 new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
187 if elapsed_time < 0.001:
189 rate = bytes / elapsed_time
197 def parse_bytes(bytestr):
198 """Parse a string indicating a byte quantity into a long integer."""
199 matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
202 number = float(matchobj.group(1))
203 multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
204 return long(round(number * multiplier))
208 """Verify a URL is valid and data could be downloaded."""
209 request = urllib2.Request(url, None, std_headers)
210 data = urllib2.urlopen(request)
214 def add_info_extractor(self, ie):
215 """Add an InfoExtractor object to the end of the list."""
217 ie.set_downloader(self)
219 def add_post_processor(self, pp):
220 """Add a PostProcessor object to the end of the chain."""
222 pp.set_downloader(self)
224 def to_stdout(self, message, skip_eol=False):
225 """Print message to stdout if not in quiet mode."""
226 if not self.params.get('quiet', False):
227 print (u'%s%s' % (message, [u'\n', u''][skip_eol])).encode(locale.getpreferredencoding()),
230 def to_stderr(self, message):
231 """Print message to stderr."""
232 print >>sys.stderr, message.encode(locale.getpreferredencoding())
234 def fixed_template(self):
235 """Checks if the output template is fixed."""
236 return (re.search(ur'(?u)%\(.+?\)s', self.params['outtmpl']) is None)
238 def trouble(self, message=None):
239 """Determine action to take when a download problem appears.
241 Depending on if the downloader has been configured to ignore
242 download errors or not, this method may throw an exception or
243 not when errors are found, after printing the message.
245 if message is not None:
246 self.to_stderr(message)
247 if not self.params.get('ignoreerrors', False):
248 raise DownloadError(message)
249 self._download_retcode = 1
251 def slow_down(self, start_time, byte_counter):
252 """Sleep if the download speed is over the rate limit."""
253 rate_limit = self.params.get('ratelimit', None)
254 if rate_limit is None or byte_counter == 0:
257 elapsed = now - start_time
260 speed = float(byte_counter) / elapsed
261 if speed > rate_limit:
262 time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
264 def report_destination(self, filename):
265 """Report destination filename."""
266 self.to_stdout(u'[download] Destination: %s' % filename)
268 def report_progress(self, percent_str, data_len_str, speed_str, eta_str):
269 """Report download progress."""
270 self.to_stdout(u'\r[download] %s of %s at %s ETA %s' %
271 (percent_str, data_len_str, speed_str, eta_str), skip_eol=True)
273 def report_resuming_byte(self, resume_len):
274 """Report attemtp to resume at given byte."""
275 self.to_stdout(u'[download] Resuming download at byte %s' % resume_len)
277 def report_file_already_downloaded(self, file_name):
278 """Report file has already been fully downloaded."""
279 self.to_stdout(u'[download] %s has already been downloaded' % file_name)
281 def report_unable_to_resume(self):
282 """Report it was impossible to resume download."""
283 self.to_stdout(u'[download] Unable to resume')
285 def report_finish(self):
286 """Report download finished."""
289 def process_info(self, info_dict):
290 """Process a single dictionary returned by an InfoExtractor."""
291 # Do nothing else if in simulate mode
292 if self.params.get('simulate', False):
294 self.verify_url(info_dict['url'])
295 except (OSError, IOError, urllib2.URLError, httplib.HTTPException, socket.error), err:
296 raise UnavailableFormatError
299 if self.params.get('forcetitle', False):
300 print info_dict['title'].encode(locale.getpreferredencoding())
301 if self.params.get('forceurl', False):
302 print info_dict['url'].encode(locale.getpreferredencoding())
307 template_dict = dict(info_dict)
308 template_dict['epoch'] = unicode(long(time.time()))
309 filename = self.params['outtmpl'] % template_dict
310 except (ValueError, KeyError), err:
311 self.trouble('ERROR: invalid output template or system charset: %s' % str(err))
312 if self.params['nooverwrites'] and os.path.exists(filename):
313 self.to_stderr(u'WARNING: file exists: %s; skipping' % filename)
317 self.pmkdir(filename)
318 except (OSError, IOError), err:
319 self.trouble('ERROR: unable to create directories: %s' % str(err))
323 success = self._do_download(filename, info_dict['url'])
324 except (OSError, IOError), err:
325 raise UnavailableFormatError
326 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
327 self.trouble('ERROR: unable to download video data: %s' % str(err))
329 except (ContentTooShortError, ), err:
330 self.trouble('ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
335 self.post_process(filename, info_dict)
336 except (PostProcessingError), err:
337 self.trouble('ERROR: postprocessing: %s' % str(err))
340 def download(self, url_list):
341 """Download a given list of URLs."""
342 if len(url_list) > 1 and self.fixed_template():
343 raise SameFileError(self.params['outtmpl'])
346 suitable_found = False
348 # Go to next InfoExtractor if not suitable
349 if not ie.suitable(url):
352 # Suitable InfoExtractor found
353 suitable_found = True
355 # Extract information from URL and process it
358 # Suitable InfoExtractor had been found; go to next URL
361 if not suitable_found:
362 self.trouble('ERROR: no suitable InfoExtractor: %s' % url)
364 return self._download_retcode
366 def post_process(self, filename, ie_info):
367 """Run the postprocessing chain on the given file."""
369 info['filepath'] = filename
375 def _do_download(self, filename, url):
379 basic_request = urllib2.Request(url, None, std_headers)
380 request = urllib2.Request(url, None, std_headers)
382 # Attempt to resume download with "continuedl" option
383 if os.path.isfile(filename):
384 resume_len = os.path.getsize(filename)
387 if self.params['continuedl'] and resume_len != 0:
388 self.report_resuming_byte(resume_len)
389 request.add_header('Range','bytes=%d-' % resume_len)
391 # Establish connection
393 data = urllib2.urlopen(request)
394 except (urllib2.HTTPError, ), err:
395 if err.code != 416: # 416 is 'Requested range not satisfiable'
397 data = urllib2.urlopen(basic_request)
398 content_length = data.info()['Content-Length']
399 if content_length is not None and long(content_length) == resume_len:
400 self.report_file_already_downloaded(filename)
403 self.report_unable_to_resume()
406 data_len = data.info().get('Content-length', None)
407 data_len_str = self.format_bytes(data_len)
414 data_block = data.read(block_size)
416 data_block_len = len(data_block)
417 if data_block_len == 0:
419 byte_counter += data_block_len
421 # Open file just in time
424 stream = open(filename, open_mode)
425 self.report_destination(filename)
426 except (OSError, IOError), err:
427 self.trouble('ERROR: unable to open for writing: %s' % str(err))
429 stream.write(data_block)
430 block_size = self.best_block_size(after - before, data_block_len)
433 percent_str = self.calc_percent(byte_counter, data_len)
434 eta_str = self.calc_eta(start, time.time(), data_len, byte_counter)
435 speed_str = self.calc_speed(start, time.time(), byte_counter)
436 self.report_progress(percent_str, data_len_str, speed_str, eta_str)
439 self.slow_down(start, byte_counter)
442 if data_len is not None and str(byte_counter) != data_len:
443 raise ContentTooShortError(byte_counter, long(data_len))
446 class InfoExtractor(object):
447 """Information Extractor class.
449 Information extractors are the classes that, given a URL, extract
450 information from the video (or videos) the URL refers to. This
451 information includes the real video URL, the video title and simplified
452 title, author and others. The information is stored in a dictionary
453 which is then passed to the FileDownloader. The FileDownloader
454 processes this information possibly downloading the video to the file
455 system, among other possible outcomes. The dictionaries must include
456 the following fields:
458 id: Video identifier.
459 url: Final video URL.
460 uploader: Nickname of the video uploader.
461 title: Literal title.
462 stitle: Simplified title.
463 ext: Video filename extension.
465 Subclasses of this one should re-define the _real_initialize() and
466 _real_extract() methods, as well as the suitable() static method.
467 Probably, they should also be instantiated and added to the main
474 def __init__(self, downloader=None):
475 """Constructor. Receives an optional downloader."""
477 self.set_downloader(downloader)
481 """Receives a URL and returns True if suitable for this IE."""
484 def initialize(self):
485 """Initializes an instance (authentication, etc)."""
487 self._real_initialize()
490 def extract(self, url):
491 """Extracts URL information and returns it in list of dicts."""
493 return self._real_extract(url)
495 def set_downloader(self, downloader):
496 """Sets the downloader for this IE."""
497 self._downloader = downloader
499 def _real_initialize(self):
500 """Real initialization process. Redefine in subclasses."""
503 def _real_extract(self, url):
504 """Real extraction process. Redefine in subclasses."""
507 class YoutubeIE(InfoExtractor):
508 """Information extractor for youtube.com."""
510 _VALID_URL = r'^((?:http://)?(?:\w+\.)?youtube\.com/(?:(?:v/)|(?:(?:watch(?:\.php)?)?\?(?:.+&)?v=)))?([0-9A-Za-z_-]+)(?(1).+)?$'
511 _LANG_URL = r'http://uk.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
512 _LOGIN_URL = 'http://www.youtube.com/signup?next=/&gl=US&hl=en'
513 _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
514 _NETRC_MACHINE = 'youtube'
515 _available_formats = ['22', '35', '18', '5', '17', '13'] # listed in order of priority for -b flag
516 _video_extensions = {
525 return (re.match(YoutubeIE._VALID_URL, url) is not None)
528 def htmlentity_transform(matchobj):
529 """Transforms an HTML entity to a Unicode character."""
530 entity = matchobj.group(1)
532 # Known non-numeric HTML entity
533 if entity in htmlentitydefs.name2codepoint:
534 return unichr(htmlentitydefs.name2codepoint[entity])
537 mobj = re.match(ur'(?u)#(x?\d+)', entity)
539 numstr = mobj.group(1)
540 if numstr.startswith(u'x'):
542 numstr = u'0%s' % numstr
545 return unichr(long(numstr, base))
547 # Unknown entity in name, return its literal representation
548 return (u'&%s;' % entity)
550 def report_lang(self):
551 """Report attempt to set language."""
552 self._downloader.to_stdout(u'[youtube] Setting language')
554 def report_login(self):
555 """Report attempt to log in."""
556 self._downloader.to_stdout(u'[youtube] Logging in')
558 def report_age_confirmation(self):
559 """Report attempt to confirm age."""
560 self._downloader.to_stdout(u'[youtube] Confirming age')
562 def report_video_info_webpage_download(self, video_id):
563 """Report attempt to download video info webpage."""
564 self._downloader.to_stdout(u'[youtube] %s: Downloading video info webpage' % video_id)
566 def report_information_extraction(self, video_id):
567 """Report attempt to extract video information."""
568 self._downloader.to_stdout(u'[youtube] %s: Extracting video information' % video_id)
570 def report_video_url(self, video_id, video_real_url):
571 """Report extracted video URL."""
572 self._downloader.to_stdout(u'[youtube] %s: URL: %s' % (video_id, video_real_url))
574 def report_unavailable_format(self, video_id, format):
575 """Report extracted video URL."""
576 self._downloader.to_stdout(u'[youtube] %s: Format %s not available' % (video_id, format))
578 def _real_initialize(self):
579 if self._downloader is None:
584 downloader_params = self._downloader.params
586 # Attempt to use provided username and password or .netrc data
587 if downloader_params.get('username', None) is not None:
588 username = downloader_params['username']
589 password = downloader_params['password']
590 elif downloader_params.get('usenetrc', False):
592 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
597 raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
598 except (IOError, netrc.NetrcParseError), err:
599 self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % str(err))
603 request = urllib2.Request(self._LANG_URL, None, std_headers)
606 urllib2.urlopen(request).read()
607 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
608 self._downloader.to_stderr(u'WARNING: unable to set language: %s' % str(err))
611 # No authentication to be performed
617 'current_form': 'loginForm',
619 'action_login': 'Log In',
620 'username': username,
621 'password': password,
623 request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form), std_headers)
626 login_results = urllib2.urlopen(request).read()
627 if re.search(r'(?i)<form[^>]* name="loginForm"', login_results) is not None:
628 self._downloader.to_stderr(u'WARNING: unable to log in: bad username or password')
630 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
631 self._downloader.to_stderr(u'WARNING: unable to log in: %s' % str(err))
637 'action_confirm': 'Confirm',
639 request = urllib2.Request(self._AGE_URL, urllib.urlencode(age_form), std_headers)
641 self.report_age_confirmation()
642 age_results = urllib2.urlopen(request).read()
643 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
644 self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
647 def _real_extract(self, url):
648 # Extract video id from URL
649 mobj = re.match(self._VALID_URL, url)
651 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
653 video_id = mobj.group(2)
655 # Downloader parameters
659 if self._downloader is not None:
660 params = self._downloader.params
661 format_param = params.get('format', None)
662 if format_param == '0':
663 format_param = self._available_formats[quality_index]
668 video_extension = self._video_extensions.get(format_param, 'flv')
671 video_info_url = 'http://www.youtube.com/get_video_info?&video_id=%s&el=detailpage&ps=default&eurl=&gl=US&hl=en' % video_id
672 request = urllib2.Request(video_info_url, None, std_headers)
674 self.report_video_info_webpage_download(video_id)
675 video_info_webpage = urllib2.urlopen(request).read()
676 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
677 self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % str(err))
679 self.report_information_extraction(video_id)
682 mobj = re.search(r'(?m)&token=([^&]+)(?:&|$)', video_info_webpage)
684 # Attempt to see if YouTube has issued an error message
685 mobj = re.search(r'(?m)&reason=([^&]+)(?:&|$)', video_info_webpage)
687 self._downloader.trouble(u'ERROR: unable to extract "t" parameter for unknown reason')
688 stream = open('reportme-ydl-%s.dat' % time.time(), 'wb')
689 stream.write(video_info_webpage)
692 reason = urllib.unquote_plus(mobj.group(1))
693 self._downloader.trouble(u'ERROR: YouTube said: %s' % reason.decode('utf-8'))
695 token = urllib.unquote(mobj.group(1))
696 video_real_url = 'http://www.youtube.com/get_video?video_id=%s&t=%s&eurl=&el=detailpage&ps=default&gl=US&hl=en' % (video_id, token)
697 if format_param is not None:
698 video_real_url = '%s&fmt=%s' % (video_real_url, format_param)
699 self.report_video_url(video_id, video_real_url)
702 mobj = re.search(r'(?m)&author=([^&]+)(?:&|$)', video_info_webpage)
704 self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
706 video_uploader = urllib.unquote(mobj.group(1))
709 mobj = re.search(r'(?m)&title=([^&]+)(?:&|$)', video_info_webpage)
711 self._downloader.trouble(u'ERROR: unable to extract video title')
713 video_title = urllib.unquote(mobj.group(1))
714 video_title = video_title.decode('utf-8')
715 video_title = re.sub(ur'(?u)&(.+?);', self.htmlentity_transform, video_title)
716 video_title = video_title.replace(os.sep, u'%')
719 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
720 simple_title = simple_title.strip(ur'_')
723 # Process video information
724 self._downloader.process_info({
725 'id': video_id.decode('utf-8'),
726 'url': video_real_url.decode('utf-8'),
727 'uploader': video_uploader.decode('utf-8'),
728 'title': video_title,
729 'stitle': simple_title,
730 'ext': video_extension.decode('utf-8'),
735 except UnavailableFormatError, err:
737 if quality_index == len(self._available_formats) - 1:
738 # I don't ever expect this to happen
739 self._downloader.trouble(u'ERROR: no known formats available for video')
742 self.report_unavailable_format(video_id, format_param)
744 format_param = self._available_formats[quality_index]
747 self._downloader.trouble('ERROR: format not available for video')
751 class MetacafeIE(InfoExtractor):
752 """Information Extractor for metacafe.com."""
754 _VALID_URL = r'(?:http://)?(?:www\.)?metacafe\.com/watch/([^/]+)/([^/]+)/.*'
755 _DISCLAIMER = 'http://www.metacafe.com/family_filter/'
756 _FILTER_POST = 'http://www.metacafe.com/f/index.php?inputType=filter&controllerGroup=user'
759 def __init__(self, youtube_ie, downloader=None):
760 InfoExtractor.__init__(self, downloader)
761 self._youtube_ie = youtube_ie
765 return (re.match(MetacafeIE._VALID_URL, url) is not None)
767 def report_disclaimer(self):
768 """Report disclaimer retrieval."""
769 self._downloader.to_stdout(u'[metacafe] Retrieving disclaimer')
771 def report_age_confirmation(self):
772 """Report attempt to confirm age."""
773 self._downloader.to_stdout(u'[metacafe] Confirming age')
775 def report_download_webpage(self, video_id):
776 """Report webpage download."""
777 self._downloader.to_stdout(u'[metacafe] %s: Downloading webpage' % video_id)
779 def report_extraction(self, video_id):
780 """Report information extraction."""
781 self._downloader.to_stdout(u'[metacafe] %s: Extracting information' % video_id)
783 def _real_initialize(self):
784 # Retrieve disclaimer
785 request = urllib2.Request(self._DISCLAIMER, None, std_headers)
787 self.report_disclaimer()
788 disclaimer = urllib2.urlopen(request).read()
789 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
790 self._downloader.trouble(u'ERROR: unable to retrieve disclaimer: %s' % str(err))
796 'submit': "Continue - I'm over 18",
798 request = urllib2.Request(self._FILTER_POST, urllib.urlencode(disclaimer_form), std_headers)
800 self.report_age_confirmation()
801 disclaimer = urllib2.urlopen(request).read()
802 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
803 self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
806 def _real_extract(self, url):
807 # Extract id and simplified title from URL
808 mobj = re.match(self._VALID_URL, url)
810 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
813 video_id = mobj.group(1)
815 # Check if video comes from YouTube
816 mobj2 = re.match(r'^yt-(.*)$', video_id)
817 if mobj2 is not None:
818 self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % mobj2.group(1))
821 simple_title = mobj.group(2).decode('utf-8')
822 video_extension = 'flv'
824 # Retrieve video webpage to extract further information
825 request = urllib2.Request('http://www.metacafe.com/watch/%s/' % video_id)
827 self.report_download_webpage(video_id)
828 webpage = urllib2.urlopen(request).read()
829 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
830 self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % str(err))
833 # Extract URL, uploader and title from webpage
834 self.report_extraction(video_id)
835 mobj = re.search(r'(?m)&mediaURL=([^&]+)', webpage)
837 self._downloader.trouble(u'ERROR: unable to extract media URL')
839 mediaURL = urllib.unquote(mobj.group(1))
841 #mobj = re.search(r'(?m)&gdaKey=(.*?)&', webpage)
843 # self._downloader.trouble(u'ERROR: unable to extract gdaKey')
845 #gdaKey = mobj.group(1)
847 #video_url = '%s?__gda__=%s' % (mediaURL, gdaKey)
851 mobj = re.search(r'(?im)<title>(.*) - Video</title>', webpage)
853 self._downloader.trouble(u'ERROR: unable to extract title')
855 video_title = mobj.group(1).decode('utf-8')
857 mobj = re.search(r'(?ms)<li id="ChnlUsr">.*?Submitter:.*?<a .*?>(.*?)<', webpage)
859 self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
861 video_uploader = mobj.group(1)
864 # Process video information
865 self._downloader.process_info({
866 'id': video_id.decode('utf-8'),
867 'url': video_url.decode('utf-8'),
868 'uploader': video_uploader.decode('utf-8'),
869 'title': video_title,
870 'stitle': simple_title,
871 'ext': video_extension.decode('utf-8'),
873 except UnavailableFormatError:
874 self._downloader.trouble(u'ERROR: format not available for video')
877 class YoutubeSearchIE(InfoExtractor):
878 """Information Extractor for YouTube search queries."""
879 _VALID_QUERY = r'ytsearch(\d+|all)?:[\s\S]+'
880 _TEMPLATE_URL = 'http://www.youtube.com/results?search_query=%s&page=%s&gl=US&hl=en'
881 _VIDEO_INDICATOR = r'href="/watch\?v=.+?"'
882 _MORE_PAGES_INDICATOR = r'>Next</a>'
884 _max_youtube_results = 1000
886 def __init__(self, youtube_ie, downloader=None):
887 InfoExtractor.__init__(self, downloader)
888 self._youtube_ie = youtube_ie
892 return (re.match(YoutubeSearchIE._VALID_QUERY, url) is not None)
894 def report_download_page(self, query, pagenum):
895 """Report attempt to download playlist page with given number."""
896 self._downloader.to_stdout(u'[youtube] query "%s": Downloading page %s' % (query, pagenum))
898 def _real_initialize(self):
899 self._youtube_ie.initialize()
901 def _real_extract(self, query):
902 mobj = re.match(self._VALID_QUERY, query)
904 self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
907 prefix, query = query.split(':')
910 self._download_n_results(query, 1)
912 elif prefix == 'all':
913 self._download_n_results(query, self._max_youtube_results)
919 self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
921 elif n > self._max_youtube_results:
922 self._downloader.to_stderr(u'WARNING: ytsearch returns max %i results (you requested %i)' % (self._max_youtube_results, n))
923 n = self._max_youtube_results
924 self._download_n_results(query, n)
926 except ValueError: # parsing prefix as integer fails
927 self._download_n_results(query, 1)
930 def _download_n_results(self, query, n):
931 """Downloads a specified number of results for a query"""
938 self.report_download_page(query, pagenum)
939 result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
940 request = urllib2.Request(result_url, None, std_headers)
942 page = urllib2.urlopen(request).read()
943 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
944 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
947 # Extract video identifiers
948 for mobj in re.finditer(self._VIDEO_INDICATOR, page):
949 video_id = page[mobj.span()[0]:mobj.span()[1]].split('=')[2][:-1]
950 if video_id not in already_seen:
951 video_ids.append(video_id)
952 already_seen.add(video_id)
953 if len(video_ids) == n:
954 # Specified n videos reached
956 self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
959 if self._MORE_PAGES_INDICATOR not in page:
961 self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
964 pagenum = pagenum + 1
966 class YoutubePlaylistIE(InfoExtractor):
967 """Information Extractor for YouTube playlists."""
969 _VALID_URL = r'(?:http://)?(?:\w+\.)?youtube.com/view_play_list\?p=(.+)'
970 _TEMPLATE_URL = 'http://www.youtube.com/view_play_list?p=%s&page=%s&gl=US&hl=en'
971 _VIDEO_INDICATOR = r'/watch\?v=(.+?)&'
972 _MORE_PAGES_INDICATOR = r'/view_play_list?p=%s&page=%s'
975 def __init__(self, youtube_ie, downloader=None):
976 InfoExtractor.__init__(self, downloader)
977 self._youtube_ie = youtube_ie
981 return (re.match(YoutubePlaylistIE._VALID_URL, url) is not None)
983 def report_download_page(self, playlist_id, pagenum):
984 """Report attempt to download playlist page with given number."""
985 self._downloader.to_stdout(u'[youtube] PL %s: Downloading page #%s' % (playlist_id, pagenum))
987 def _real_initialize(self):
988 self._youtube_ie.initialize()
990 def _real_extract(self, url):
991 # Extract playlist id
992 mobj = re.match(self._VALID_URL, url)
994 self._downloader.trouble(u'ERROR: invalid url: %s' % url)
997 # Download playlist pages
998 playlist_id = mobj.group(1)
1003 self.report_download_page(playlist_id, pagenum)
1004 request = urllib2.Request(self._TEMPLATE_URL % (playlist_id, pagenum), None, std_headers)
1006 page = urllib2.urlopen(request).read()
1007 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1008 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1011 # Extract video identifiers
1013 for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1014 if mobj.group(1) not in ids_in_page:
1015 ids_in_page.append(mobj.group(1))
1016 video_ids.extend(ids_in_page)
1018 if (self._MORE_PAGES_INDICATOR % (playlist_id.upper(), pagenum + 1)) not in page:
1020 pagenum = pagenum + 1
1022 for id in video_ids:
1023 self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
1026 class PostProcessor(object):
1027 """Post Processor class.
1029 PostProcessor objects can be added to downloaders with their
1030 add_post_processor() method. When the downloader has finished a
1031 successful download, it will take its internal chain of PostProcessors
1032 and start calling the run() method on each one of them, first with
1033 an initial argument and then with the returned value of the previous
1036 The chain will be stopped if one of them ever returns None or the end
1037 of the chain is reached.
1039 PostProcessor objects follow a "mutual registration" process similar
1040 to InfoExtractor objects.
1045 def __init__(self, downloader=None):
1046 self._downloader = downloader
1048 def set_downloader(self, downloader):
1049 """Sets the downloader for this PP."""
1050 self._downloader = downloader
1052 def run(self, information):
1053 """Run the PostProcessor.
1055 The "information" argument is a dictionary like the ones
1056 composed by InfoExtractors. The only difference is that this
1057 one has an extra field called "filepath" that points to the
1060 When this method returns None, the postprocessing chain is
1061 stopped. However, this method may return an information
1062 dictionary that will be passed to the next postprocessing
1063 object in the chain. It can be the one it received after
1064 changing some fields.
1066 In addition, this method may raise a PostProcessingError
1067 exception that will be taken into account by the downloader
1070 return information # by default, do nothing
1072 ### MAIN PROGRAM ###
1073 if __name__ == '__main__':
1075 # Modules needed only when running the main program
1079 # General configuration
1080 urllib2.install_opener(urllib2.build_opener(urllib2.ProxyHandler()))
1081 urllib2.install_opener(urllib2.build_opener(urllib2.HTTPCookieProcessor()))
1082 socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words)
1084 # Parse command line
1085 parser = optparse.OptionParser(
1086 usage='Usage: %prog [options] url...',
1087 version='2009.08.08',
1088 conflict_handler='resolve',
1091 parser.add_option('-h', '--help',
1092 action='help', help='print this help text and exit')
1093 parser.add_option('-v', '--version',
1094 action='version', help='print program version and exit')
1095 parser.add_option('-i', '--ignore-errors',
1096 action='store_true', dest='ignoreerrors', help='continue on download errors', default=False)
1097 parser.add_option('-r', '--rate-limit',
1098 dest='ratelimit', metavar='L', help='download rate limit (e.g. 50k or 44.6m)')
1100 authentication = optparse.OptionGroup(parser, 'Authentication Options')
1101 authentication.add_option('-u', '--username',
1102 dest='username', metavar='UN', help='account username')
1103 authentication.add_option('-p', '--password',
1104 dest='password', metavar='PW', help='account password')
1105 authentication.add_option('-n', '--netrc',
1106 action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
1107 parser.add_option_group(authentication)
1109 video_format = optparse.OptionGroup(parser, 'Video Format Options')
1110 video_format.add_option('-f', '--format',
1111 action='store', dest='format', metavar='FMT', help='video format code')
1112 video_format.add_option('-b', '--best-quality',
1113 action='store_const', dest='format', help='download the best quality video possible', const='0')
1114 video_format.add_option('-m', '--mobile-version',
1115 action='store_const', dest='format', help='alias for -f 17', const='17')
1116 video_format.add_option('-d', '--high-def',
1117 action='store_const', dest='format', help='alias for -f 22', const='22')
1118 parser.add_option_group(video_format)
1120 verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
1121 verbosity.add_option('-q', '--quiet',
1122 action='store_true', dest='quiet', help='activates quiet mode', default=False)
1123 verbosity.add_option('-s', '--simulate',
1124 action='store_true', dest='simulate', help='do not download video', default=False)
1125 verbosity.add_option('-g', '--get-url',
1126 action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
1127 verbosity.add_option('-e', '--get-title',
1128 action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
1129 parser.add_option_group(verbosity)
1131 filesystem = optparse.OptionGroup(parser, 'Filesystem Options')
1132 filesystem.add_option('-t', '--title',
1133 action='store_true', dest='usetitle', help='use title in file name', default=False)
1134 filesystem.add_option('-l', '--literal',
1135 action='store_true', dest='useliteral', help='use literal title in file name', default=False)
1136 filesystem.add_option('-o', '--output',
1137 dest='outtmpl', metavar='TPL', help='output filename template')
1138 filesystem.add_option('-a', '--batch-file',
1139 dest='batchfile', metavar='F', help='file containing URLs to download')
1140 filesystem.add_option('-w', '--no-overwrites',
1141 action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
1142 filesystem.add_option('-c', '--continue',
1143 action='store_true', dest='continue_dl', help='resume partially downloaded files', default=False)
1144 parser.add_option_group(filesystem)
1146 (opts, args) = parser.parse_args()
1148 # Batch file verification
1150 if opts.batchfile is not None:
1152 batchurls = open(opts.batchfile, 'r').readlines()
1153 batchurls = [x.strip() for x in batchurls]
1154 batchurls = [x for x in batchurls if len(x) > 0]
1156 sys.exit(u'ERROR: batch file could not be read')
1157 all_urls = batchurls + args
1159 # Conflicting, missing and erroneous options
1160 if len(all_urls) < 1:
1161 parser.error(u'you must provide at least one URL')
1162 if opts.usenetrc and (opts.username is not None or opts.password is not None):
1163 parser.error(u'using .netrc conflicts with giving username/password')
1164 if opts.password is not None and opts.username is None:
1165 parser.error(u'account username missing')
1166 if opts.outtmpl is not None and (opts.useliteral or opts.usetitle):
1167 parser.error(u'using output template conflicts with using title or literal title')
1168 if opts.usetitle and opts.useliteral:
1169 parser.error(u'using title conflicts with using literal title')
1170 if opts.username is not None and opts.password is None:
1171 opts.password = getpass.getpass(u'Type account password and press return:')
1172 if opts.ratelimit is not None:
1173 numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
1174 if numeric_limit is None:
1175 parser.error(u'invalid rate limit specified')
1176 opts.ratelimit = numeric_limit
1178 # Information extractors
1179 youtube_ie = YoutubeIE()
1180 metacafe_ie = MetacafeIE(youtube_ie)
1181 youtube_pl_ie = YoutubePlaylistIE(youtube_ie)
1182 youtube_search_ie = YoutubeSearchIE(youtube_ie)
1185 fd = FileDownloader({
1186 'usenetrc': opts.usenetrc,
1187 'username': opts.username,
1188 'password': opts.password,
1189 'quiet': (opts.quiet or opts.geturl or opts.gettitle),
1190 'forceurl': opts.geturl,
1191 'forcetitle': opts.gettitle,
1192 'simulate': (opts.simulate or opts.geturl or opts.gettitle),
1193 'format': opts.format,
1194 'outtmpl': ((opts.outtmpl is not None and opts.outtmpl.decode(locale.getpreferredencoding()))
1195 or (opts.usetitle and u'%(stitle)s-%(id)s.%(ext)s')
1196 or (opts.useliteral and u'%(title)s-%(id)s.%(ext)s')
1197 or u'%(id)s.%(ext)s'),
1198 'ignoreerrors': opts.ignoreerrors,
1199 'ratelimit': opts.ratelimit,
1200 'nooverwrites': opts.nooverwrites,
1201 'continuedl': opts.continue_dl,
1203 fd.add_info_extractor(youtube_search_ie)
1204 fd.add_info_extractor(youtube_pl_ie)
1205 fd.add_info_extractor(metacafe_ie)
1206 fd.add_info_extractor(youtube_ie)
1207 retcode = fd.download(all_urls)
1210 except DownloadError:
1212 except SameFileError:
1213 sys.exit(u'ERROR: fixed output name but more than one file to download')
1214 except KeyboardInterrupt:
1215 sys.exit(u'\nERROR: Interrupted by user')