2 # -*- coding: utf-8 -*-
3 # Author: Ricardo Garcia Gonzalez
4 # Author: Danny Colligan
5 # License: Public domain code
22 'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.1.2) Gecko/20090729 Firefox/3.5.2',
23 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
24 'Accept': 'text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5',
25 'Accept-Language': 'en-us,en;q=0.5',
28 simple_title_chars = string.ascii_letters.decode('ascii') + string.digits.decode('ascii')
30 def preferredencoding():
31 """Get preferred encoding.
33 Returns the best encoding scheme for the system, based on
34 locale.getpreferredencoding() and some further tweaks.
37 pref = locale.getpreferredencoding()
38 # Mac OSX systems have this problem sometimes
43 sys.stderr.write('WARNING: problem obtaining preferred encoding. Falling back to UTF-8.\n')
46 class DownloadError(Exception):
47 """Download Error exception.
49 This exception may be thrown by FileDownloader objects if they are not
50 configured to continue on errors. They will contain the appropriate
55 class SameFileError(Exception):
56 """Same File exception.
58 This exception will be thrown by FileDownloader objects if they detect
59 multiple files would have to be downloaded to the same file on disk.
63 class PostProcessingError(Exception):
64 """Post Processing exception.
66 This exception may be raised by PostProcessor's .run() method to
67 indicate an error in the postprocessing task.
71 class UnavailableFormatError(Exception):
72 """Unavailable Format exception.
74 This exception will be thrown when a video is requested
75 in a format that is not available for that video.
79 class ContentTooShortError(Exception):
80 """Content Too Short exception.
82 This exception may be raised by FileDownloader objects when a file they
83 download is too small for what the server announced first, indicating
84 the connection was probably interrupted.
90 def __init__(self, downloaded, expected):
91 self.downloaded = downloaded
92 self.expected = expected
94 class FileDownloader(object):
95 """File Downloader class.
97 File downloader objects are the ones responsible of downloading the
98 actual video file and writing it to disk if the user has requested
99 it, among some other tasks. In most cases there should be one per
100 program. As, given a video URL, the downloader doesn't know how to
101 extract all the needed information, task that InfoExtractors do, it
102 has to pass the URL to one of them.
104 For this, file downloader objects have a method that allows
105 InfoExtractors to be registered in a given order. When it is passed
106 a URL, the file downloader handles it to the first InfoExtractor it
107 finds that reports being able to handle it. The InfoExtractor extracts
108 all the information about the video or videos the URL refers to, and
109 asks the FileDownloader to process the video information, possibly
110 downloading the video.
112 File downloaders accept a lot of parameters. In order not to saturate
113 the object constructor with arguments, it receives a dictionary of
114 options instead. These options are available through the params
115 attribute for the InfoExtractors to use. The FileDownloader also
116 registers itself as the downloader in charge for the InfoExtractors
117 that are added to it, so this is a "mutual registration".
121 username: Username for authentication purposes.
122 password: Password for authentication purposes.
123 usenetrc: Use netrc for authentication instead.
124 quiet: Do not print messages to stdout.
125 forceurl: Force printing final URL.
126 forcetitle: Force printing title.
127 simulate: Do not download the video files.
128 format: Video format code.
129 outtmpl: Template for output names.
130 ignoreerrors: Do not stop on download errors.
131 ratelimit: Download speed limit, in bytes/sec.
132 nooverwrites: Prevent overwriting files.
133 continuedl: Try to continue downloads if possible.
139 _download_retcode = None
141 def __init__(self, params):
142 """Create a FileDownloader object with the given options."""
145 self._download_retcode = 0
149 def pmkdir(filename):
150 """Create directory components in filename. Similar to Unix "mkdir -p"."""
151 components = filename.split(os.sep)
152 aggregate = [os.sep.join(components[0:x]) for x in xrange(1, len(components))]
153 aggregate = ['%s%s' % (x, os.sep) for x in aggregate] # Finish names with separator
154 for dir in aggregate:
155 if not os.path.exists(dir):
159 def format_bytes(bytes):
162 if type(bytes) is str:
167 exponent = long(math.log(bytes, 1024.0))
168 suffix = 'bkMGTPEZY'[exponent]
169 converted = float(bytes) / float(1024**exponent)
170 return '%.2f%s' % (converted, suffix)
173 def calc_percent(byte_counter, data_len):
176 return '%6s' % ('%3.1f%%' % (float(byte_counter) / float(data_len) * 100.0))
179 def calc_eta(start, now, total, current):
183 if current == 0 or dif < 0.001: # One millisecond
185 rate = float(current) / dif
186 eta = long((float(total) - float(current)) / rate)
187 (eta_mins, eta_secs) = divmod(eta, 60)
190 return '%02d:%02d' % (eta_mins, eta_secs)
193 def calc_speed(start, now, bytes):
195 if bytes == 0 or dif < 0.001: # One millisecond
196 return '%10s' % '---b/s'
197 return '%10s' % ('%s/s' % FileDownloader.format_bytes(float(bytes) / dif))
200 def best_block_size(elapsed_time, bytes):
201 new_min = max(bytes / 2.0, 1.0)
202 new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
203 if elapsed_time < 0.001:
205 rate = bytes / elapsed_time
213 def parse_bytes(bytestr):
214 """Parse a string indicating a byte quantity into a long integer."""
215 matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
218 number = float(matchobj.group(1))
219 multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
220 return long(round(number * multiplier))
224 """Verify a URL is valid and data could be downloaded."""
225 request = urllib2.Request(url, None, std_headers)
226 data = urllib2.urlopen(request)
230 def add_info_extractor(self, ie):
231 """Add an InfoExtractor object to the end of the list."""
233 ie.set_downloader(self)
235 def add_post_processor(self, pp):
236 """Add a PostProcessor object to the end of the chain."""
238 pp.set_downloader(self)
240 def to_stdout(self, message, skip_eol=False):
241 """Print message to stdout if not in quiet mode."""
242 if not self.params.get('quiet', False):
243 print (u'%s%s' % (message, [u'\n', u''][skip_eol])).encode(preferredencoding()),
246 def to_stderr(self, message):
247 """Print message to stderr."""
248 print >>sys.stderr, message.encode(preferredencoding())
250 def fixed_template(self):
251 """Checks if the output template is fixed."""
252 return (re.search(ur'(?u)%\(.+?\)s', self.params['outtmpl']) is None)
254 def trouble(self, message=None):
255 """Determine action to take when a download problem appears.
257 Depending on if the downloader has been configured to ignore
258 download errors or not, this method may throw an exception or
259 not when errors are found, after printing the message.
261 if message is not None:
262 self.to_stderr(message)
263 if not self.params.get('ignoreerrors', False):
264 raise DownloadError(message)
265 self._download_retcode = 1
267 def slow_down(self, start_time, byte_counter):
268 """Sleep if the download speed is over the rate limit."""
269 rate_limit = self.params.get('ratelimit', None)
270 if rate_limit is None or byte_counter == 0:
273 elapsed = now - start_time
276 speed = float(byte_counter) / elapsed
277 if speed > rate_limit:
278 time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
280 def report_destination(self, filename):
281 """Report destination filename."""
282 self.to_stdout(u'[download] Destination: %s' % filename)
284 def report_progress(self, percent_str, data_len_str, speed_str, eta_str):
285 """Report download progress."""
286 self.to_stdout(u'\r[download] %s of %s at %s ETA %s' %
287 (percent_str, data_len_str, speed_str, eta_str), skip_eol=True)
289 def report_resuming_byte(self, resume_len):
290 """Report attemtp to resume at given byte."""
291 self.to_stdout(u'[download] Resuming download at byte %s' % resume_len)
293 def report_file_already_downloaded(self, file_name):
294 """Report file has already been fully downloaded."""
295 self.to_stdout(u'[download] %s has already been downloaded' % file_name)
297 def report_unable_to_resume(self):
298 """Report it was impossible to resume download."""
299 self.to_stdout(u'[download] Unable to resume')
301 def report_finish(self):
302 """Report download finished."""
305 def process_info(self, info_dict):
306 """Process a single dictionary returned by an InfoExtractor."""
307 # Do nothing else if in simulate mode
308 if self.params.get('simulate', False):
310 self.verify_url(info_dict['url'])
311 except (OSError, IOError, urllib2.URLError, httplib.HTTPException, socket.error), err:
312 raise UnavailableFormatError
315 if self.params.get('forcetitle', False):
316 print info_dict['title'].encode(preferredencoding())
317 if self.params.get('forceurl', False):
318 print info_dict['url'].encode(preferredencoding())
323 template_dict = dict(info_dict)
324 template_dict['epoch'] = unicode(long(time.time()))
325 filename = self.params['outtmpl'] % template_dict
326 except (ValueError, KeyError), err:
327 self.trouble('ERROR: invalid output template or system charset: %s' % str(err))
328 if self.params['nooverwrites'] and os.path.exists(filename):
329 self.to_stderr(u'WARNING: file exists: %s; skipping' % filename)
333 self.pmkdir(filename)
334 except (OSError, IOError), err:
335 self.trouble('ERROR: unable to create directories: %s' % str(err))
339 success = self._do_download(filename, info_dict['url'])
340 except (OSError, IOError), err:
341 raise UnavailableFormatError
342 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
343 self.trouble('ERROR: unable to download video data: %s' % str(err))
345 except (ContentTooShortError, ), err:
346 self.trouble('ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
351 self.post_process(filename, info_dict)
352 except (PostProcessingError), err:
353 self.trouble('ERROR: postprocessing: %s' % str(err))
356 def download(self, url_list):
357 """Download a given list of URLs."""
358 if len(url_list) > 1 and self.fixed_template():
359 raise SameFileError(self.params['outtmpl'])
362 suitable_found = False
364 # Go to next InfoExtractor if not suitable
365 if not ie.suitable(url):
368 # Suitable InfoExtractor found
369 suitable_found = True
371 # Extract information from URL and process it
374 # Suitable InfoExtractor had been found; go to next URL
377 if not suitable_found:
378 self.trouble('ERROR: no suitable InfoExtractor: %s' % url)
380 return self._download_retcode
382 def post_process(self, filename, ie_info):
383 """Run the postprocessing chain on the given file."""
385 info['filepath'] = filename
391 def _do_download(self, filename, url):
395 basic_request = urllib2.Request(url, None, std_headers)
396 request = urllib2.Request(url, None, std_headers)
398 # Attempt to resume download with "continuedl" option
399 if os.path.isfile(filename):
400 resume_len = os.path.getsize(filename)
403 if self.params['continuedl'] and resume_len != 0:
404 self.report_resuming_byte(resume_len)
405 request.add_header('Range','bytes=%d-' % resume_len)
407 # Establish connection
409 data = urllib2.urlopen(request)
410 except (urllib2.HTTPError, ), err:
411 if err.code != 416: # 416 is 'Requested range not satisfiable'
413 data = urllib2.urlopen(basic_request)
414 content_length = data.info()['Content-Length']
415 if content_length is not None and long(content_length) == resume_len:
416 self.report_file_already_downloaded(filename)
419 self.report_unable_to_resume()
422 data_len = data.info().get('Content-length', None)
423 data_len_str = self.format_bytes(data_len)
430 data_block = data.read(block_size)
432 data_block_len = len(data_block)
433 if data_block_len == 0:
435 byte_counter += data_block_len
437 # Open file just in time
440 stream = open(filename, open_mode)
441 self.report_destination(filename)
442 except (OSError, IOError), err:
443 self.trouble('ERROR: unable to open for writing: %s' % str(err))
445 stream.write(data_block)
446 block_size = self.best_block_size(after - before, data_block_len)
449 percent_str = self.calc_percent(byte_counter, data_len)
450 eta_str = self.calc_eta(start, time.time(), data_len, byte_counter)
451 speed_str = self.calc_speed(start, time.time(), byte_counter)
452 self.report_progress(percent_str, data_len_str, speed_str, eta_str)
455 self.slow_down(start, byte_counter)
458 if data_len is not None and str(byte_counter) != data_len:
459 raise ContentTooShortError(byte_counter, long(data_len))
462 class InfoExtractor(object):
463 """Information Extractor class.
465 Information extractors are the classes that, given a URL, extract
466 information from the video (or videos) the URL refers to. This
467 information includes the real video URL, the video title and simplified
468 title, author and others. The information is stored in a dictionary
469 which is then passed to the FileDownloader. The FileDownloader
470 processes this information possibly downloading the video to the file
471 system, among other possible outcomes. The dictionaries must include
472 the following fields:
474 id: Video identifier.
475 url: Final video URL.
476 uploader: Nickname of the video uploader.
477 title: Literal title.
478 stitle: Simplified title.
479 ext: Video filename extension.
481 Subclasses of this one should re-define the _real_initialize() and
482 _real_extract() methods, as well as the suitable() static method.
483 Probably, they should also be instantiated and added to the main
490 def __init__(self, downloader=None):
491 """Constructor. Receives an optional downloader."""
493 self.set_downloader(downloader)
497 """Receives a URL and returns True if suitable for this IE."""
500 def initialize(self):
501 """Initializes an instance (authentication, etc)."""
503 self._real_initialize()
506 def extract(self, url):
507 """Extracts URL information and returns it in list of dicts."""
509 return self._real_extract(url)
511 def set_downloader(self, downloader):
512 """Sets the downloader for this IE."""
513 self._downloader = downloader
515 def _real_initialize(self):
516 """Real initialization process. Redefine in subclasses."""
519 def _real_extract(self, url):
520 """Real extraction process. Redefine in subclasses."""
523 class YoutubeIE(InfoExtractor):
524 """Information extractor for youtube.com."""
526 _VALID_URL = r'^((?:http://)?(?:\w+\.)?youtube\.com/(?:(?:v/)|(?:(?:watch(?:\.php)?)?\?(?:.+&)?v=)))?([0-9A-Za-z_-]+)(?(1).+)?$'
527 _LANG_URL = r'http://uk.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
528 _LOGIN_URL = 'http://www.youtube.com/signup?next=/&gl=US&hl=en'
529 _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
530 _NETRC_MACHINE = 'youtube'
531 _available_formats = ['22', '35', '18', '5', '17', '13', None] # listed in order of priority for -b flag
532 _video_extensions = {
541 return (re.match(YoutubeIE._VALID_URL, url) is not None)
544 def htmlentity_transform(matchobj):
545 """Transforms an HTML entity to a Unicode character."""
546 entity = matchobj.group(1)
548 # Known non-numeric HTML entity
549 if entity in htmlentitydefs.name2codepoint:
550 return unichr(htmlentitydefs.name2codepoint[entity])
553 mobj = re.match(ur'(?u)#(x?\d+)', entity)
555 numstr = mobj.group(1)
556 if numstr.startswith(u'x'):
558 numstr = u'0%s' % numstr
561 return unichr(long(numstr, base))
563 # Unknown entity in name, return its literal representation
564 return (u'&%s;' % entity)
566 def report_lang(self):
567 """Report attempt to set language."""
568 self._downloader.to_stdout(u'[youtube] Setting language')
570 def report_login(self):
571 """Report attempt to log in."""
572 self._downloader.to_stdout(u'[youtube] Logging in')
574 def report_age_confirmation(self):
575 """Report attempt to confirm age."""
576 self._downloader.to_stdout(u'[youtube] Confirming age')
578 def report_video_info_webpage_download(self, video_id):
579 """Report attempt to download video info webpage."""
580 self._downloader.to_stdout(u'[youtube] %s: Downloading video info webpage' % video_id)
582 def report_information_extraction(self, video_id):
583 """Report attempt to extract video information."""
584 self._downloader.to_stdout(u'[youtube] %s: Extracting video information' % video_id)
586 def report_video_url(self, video_id, video_real_url):
587 """Report extracted video URL."""
588 self._downloader.to_stdout(u'[youtube] %s: URL: %s' % (video_id, video_real_url))
590 def report_unavailable_format(self, video_id, format):
591 """Report extracted video URL."""
592 self._downloader.to_stdout(u'[youtube] %s: Format %s not available' % (video_id, format))
594 def _real_initialize(self):
595 if self._downloader is None:
600 downloader_params = self._downloader.params
602 # Attempt to use provided username and password or .netrc data
603 if downloader_params.get('username', None) is not None:
604 username = downloader_params['username']
605 password = downloader_params['password']
606 elif downloader_params.get('usenetrc', False):
608 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
613 raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
614 except (IOError, netrc.NetrcParseError), err:
615 self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % str(err))
619 request = urllib2.Request(self._LANG_URL, None, std_headers)
622 urllib2.urlopen(request).read()
623 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
624 self._downloader.to_stderr(u'WARNING: unable to set language: %s' % str(err))
627 # No authentication to be performed
633 'current_form': 'loginForm',
635 'action_login': 'Log In',
636 'username': username,
637 'password': password,
639 request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form), std_headers)
642 login_results = urllib2.urlopen(request).read()
643 if re.search(r'(?i)<form[^>]* name="loginForm"', login_results) is not None:
644 self._downloader.to_stderr(u'WARNING: unable to log in: bad username or password')
646 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
647 self._downloader.to_stderr(u'WARNING: unable to log in: %s' % str(err))
653 'action_confirm': 'Confirm',
655 request = urllib2.Request(self._AGE_URL, urllib.urlencode(age_form), std_headers)
657 self.report_age_confirmation()
658 age_results = urllib2.urlopen(request).read()
659 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
660 self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
663 def _real_extract(self, url):
664 # Extract video id from URL
665 mobj = re.match(self._VALID_URL, url)
667 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
669 video_id = mobj.group(2)
671 # Downloader parameters
675 if self._downloader is not None:
676 params = self._downloader.params
677 format_param = params.get('format', None)
678 if format_param == '0':
679 format_param = self._available_formats[quality_index]
684 video_extension = self._video_extensions.get(format_param, 'flv')
687 video_info_url = 'http://www.youtube.com/get_video_info?&video_id=%s&el=detailpage&ps=default&eurl=&gl=US&hl=en' % video_id
688 request = urllib2.Request(video_info_url, None, std_headers)
690 self.report_video_info_webpage_download(video_id)
691 video_info_webpage = urllib2.urlopen(request).read()
692 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
693 self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % str(err))
695 self.report_information_extraction(video_id)
698 mobj = re.search(r'(?m)&token=([^&]+)(?:&|$)', video_info_webpage)
700 # Attempt to see if YouTube has issued an error message
701 mobj = re.search(r'(?m)&reason=([^&]+)(?:&|$)', video_info_webpage)
703 self._downloader.trouble(u'ERROR: unable to extract "t" parameter for unknown reason')
704 stream = open('reportme-ydl-%s.dat' % time.time(), 'wb')
705 stream.write(video_info_webpage)
708 reason = urllib.unquote_plus(mobj.group(1))
709 self._downloader.trouble(u'ERROR: YouTube said: %s' % reason.decode('utf-8'))
711 token = urllib.unquote(mobj.group(1))
712 video_real_url = 'http://www.youtube.com/get_video?video_id=%s&t=%s&eurl=&el=detailpage&ps=default&gl=US&hl=en' % (video_id, token)
713 if format_param is not None:
714 video_real_url = '%s&fmt=%s' % (video_real_url, format_param)
715 self.report_video_url(video_id, video_real_url)
718 mobj = re.search(r'(?m)&author=([^&]+)(?:&|$)', video_info_webpage)
720 self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
722 video_uploader = urllib.unquote(mobj.group(1))
725 mobj = re.search(r'(?m)&title=([^&]+)(?:&|$)', video_info_webpage)
727 self._downloader.trouble(u'ERROR: unable to extract video title')
729 video_title = urllib.unquote(mobj.group(1))
730 video_title = video_title.decode('utf-8')
731 video_title = re.sub(ur'(?u)&(.+?);', self.htmlentity_transform, video_title)
732 video_title = video_title.replace(os.sep, u'%')
735 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
736 simple_title = simple_title.strip(ur'_')
739 # Process video information
740 self._downloader.process_info({
741 'id': video_id.decode('utf-8'),
742 'url': video_real_url.decode('utf-8'),
743 'uploader': video_uploader.decode('utf-8'),
744 'title': video_title,
745 'stitle': simple_title,
746 'ext': video_extension.decode('utf-8'),
751 except UnavailableFormatError, err:
753 if quality_index == len(self._available_formats) - 1:
754 # I don't ever expect this to happen
755 self._downloader.trouble(u'ERROR: no known formats available for video')
758 self.report_unavailable_format(video_id, format_param)
760 format_param = self._available_formats[quality_index]
763 self._downloader.trouble('ERROR: format not available for video')
767 class MetacafeIE(InfoExtractor):
768 """Information Extractor for metacafe.com."""
770 _VALID_URL = r'(?:http://)?(?:www\.)?metacafe\.com/watch/([^/]+)/([^/]+)/.*'
771 _DISCLAIMER = 'http://www.metacafe.com/family_filter/'
772 _FILTER_POST = 'http://www.metacafe.com/f/index.php?inputType=filter&controllerGroup=user'
775 def __init__(self, youtube_ie, downloader=None):
776 InfoExtractor.__init__(self, downloader)
777 self._youtube_ie = youtube_ie
781 return (re.match(MetacafeIE._VALID_URL, url) is not None)
783 def report_disclaimer(self):
784 """Report disclaimer retrieval."""
785 self._downloader.to_stdout(u'[metacafe] Retrieving disclaimer')
787 def report_age_confirmation(self):
788 """Report attempt to confirm age."""
789 self._downloader.to_stdout(u'[metacafe] Confirming age')
791 def report_download_webpage(self, video_id):
792 """Report webpage download."""
793 self._downloader.to_stdout(u'[metacafe] %s: Downloading webpage' % video_id)
795 def report_extraction(self, video_id):
796 """Report information extraction."""
797 self._downloader.to_stdout(u'[metacafe] %s: Extracting information' % video_id)
799 def _real_initialize(self):
800 # Retrieve disclaimer
801 request = urllib2.Request(self._DISCLAIMER, None, std_headers)
803 self.report_disclaimer()
804 disclaimer = urllib2.urlopen(request).read()
805 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
806 self._downloader.trouble(u'ERROR: unable to retrieve disclaimer: %s' % str(err))
812 'submit': "Continue - I'm over 18",
814 request = urllib2.Request(self._FILTER_POST, urllib.urlencode(disclaimer_form), std_headers)
816 self.report_age_confirmation()
817 disclaimer = urllib2.urlopen(request).read()
818 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
819 self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
822 def _real_extract(self, url):
823 # Extract id and simplified title from URL
824 mobj = re.match(self._VALID_URL, url)
826 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
829 video_id = mobj.group(1)
831 # Check if video comes from YouTube
832 mobj2 = re.match(r'^yt-(.*)$', video_id)
833 if mobj2 is not None:
834 self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % mobj2.group(1))
837 simple_title = mobj.group(2).decode('utf-8')
838 video_extension = 'flv'
840 # Retrieve video webpage to extract further information
841 request = urllib2.Request('http://www.metacafe.com/watch/%s/' % video_id)
843 self.report_download_webpage(video_id)
844 webpage = urllib2.urlopen(request).read()
845 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
846 self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % str(err))
849 # Extract URL, uploader and title from webpage
850 self.report_extraction(video_id)
851 mobj = re.search(r'(?m)&mediaURL=([^&]+)', webpage)
853 self._downloader.trouble(u'ERROR: unable to extract media URL')
855 mediaURL = urllib.unquote(mobj.group(1))
857 #mobj = re.search(r'(?m)&gdaKey=(.*?)&', webpage)
859 # self._downloader.trouble(u'ERROR: unable to extract gdaKey')
861 #gdaKey = mobj.group(1)
863 #video_url = '%s?__gda__=%s' % (mediaURL, gdaKey)
867 mobj = re.search(r'(?im)<title>(.*) - Video</title>', webpage)
869 self._downloader.trouble(u'ERROR: unable to extract title')
871 video_title = mobj.group(1).decode('utf-8')
873 mobj = re.search(r'(?ms)<li id="ChnlUsr">.*?Submitter:.*?<a .*?>(.*?)<', webpage)
875 self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
877 video_uploader = mobj.group(1)
880 # Process video information
881 self._downloader.process_info({
882 'id': video_id.decode('utf-8'),
883 'url': video_url.decode('utf-8'),
884 'uploader': video_uploader.decode('utf-8'),
885 'title': video_title,
886 'stitle': simple_title,
887 'ext': video_extension.decode('utf-8'),
889 except UnavailableFormatError:
890 self._downloader.trouble(u'ERROR: format not available for video')
893 class YoutubeSearchIE(InfoExtractor):
894 """Information Extractor for YouTube search queries."""
895 _VALID_QUERY = r'ytsearch(\d+|all)?:[\s\S]+'
896 _TEMPLATE_URL = 'http://www.youtube.com/results?search_query=%s&page=%s&gl=US&hl=en'
897 _VIDEO_INDICATOR = r'href="/watch\?v=.+?"'
898 _MORE_PAGES_INDICATOR = r'(?m)>\s*Next\s*</a>'
900 _max_youtube_results = 1000
902 def __init__(self, youtube_ie, downloader=None):
903 InfoExtractor.__init__(self, downloader)
904 self._youtube_ie = youtube_ie
908 return (re.match(YoutubeSearchIE._VALID_QUERY, url) is not None)
910 def report_download_page(self, query, pagenum):
911 """Report attempt to download playlist page with given number."""
912 self._downloader.to_stdout(u'[youtube] query "%s": Downloading page %s' % (query, pagenum))
914 def _real_initialize(self):
915 self._youtube_ie.initialize()
917 def _real_extract(self, query):
918 mobj = re.match(self._VALID_QUERY, query)
920 self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
923 prefix, query = query.split(':')
926 self._download_n_results(query, 1)
928 elif prefix == 'all':
929 self._download_n_results(query, self._max_youtube_results)
935 self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
937 elif n > self._max_youtube_results:
938 self._downloader.to_stderr(u'WARNING: ytsearch returns max %i results (you requested %i)' % (self._max_youtube_results, n))
939 n = self._max_youtube_results
940 self._download_n_results(query, n)
942 except ValueError: # parsing prefix as integer fails
943 self._download_n_results(query, 1)
946 def _download_n_results(self, query, n):
947 """Downloads a specified number of results for a query"""
954 self.report_download_page(query, pagenum)
955 result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
956 request = urllib2.Request(result_url, None, std_headers)
958 page = urllib2.urlopen(request).read()
959 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
960 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
963 # Extract video identifiers
964 for mobj in re.finditer(self._VIDEO_INDICATOR, page):
965 video_id = page[mobj.span()[0]:mobj.span()[1]].split('=')[2][:-1]
966 if video_id not in already_seen:
967 video_ids.append(video_id)
968 already_seen.add(video_id)
969 if len(video_ids) == n:
970 # Specified n videos reached
972 self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
975 if re.search(self._MORE_PAGES_INDICATOR, page) is None:
977 self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
980 pagenum = pagenum + 1
982 class YoutubePlaylistIE(InfoExtractor):
983 """Information Extractor for YouTube playlists."""
985 _VALID_URL = r'(?:http://)?(?:\w+\.)?youtube.com/(?:view_play_list|my_playlists)\?.*?p=([^&]+).*'
986 _TEMPLATE_URL = 'http://www.youtube.com/view_play_list?p=%s&page=%s&gl=US&hl=en'
987 _VIDEO_INDICATOR = r'/watch\?v=(.+?)&'
988 _MORE_PAGES_INDICATOR = r'/view_play_list?p=%s&page=%s'
991 def __init__(self, youtube_ie, downloader=None):
992 InfoExtractor.__init__(self, downloader)
993 self._youtube_ie = youtube_ie
997 return (re.match(YoutubePlaylistIE._VALID_URL, url) is not None)
999 def report_download_page(self, playlist_id, pagenum):
1000 """Report attempt to download playlist page with given number."""
1001 self._downloader.to_stdout(u'[youtube] PL %s: Downloading page #%s' % (playlist_id, pagenum))
1003 def _real_initialize(self):
1004 self._youtube_ie.initialize()
1006 def _real_extract(self, url):
1007 # Extract playlist id
1008 mobj = re.match(self._VALID_URL, url)
1010 self._downloader.trouble(u'ERROR: invalid url: %s' % url)
1013 # Download playlist pages
1014 playlist_id = mobj.group(1)
1019 self.report_download_page(playlist_id, pagenum)
1020 request = urllib2.Request(self._TEMPLATE_URL % (playlist_id, pagenum), None, std_headers)
1022 page = urllib2.urlopen(request).read()
1023 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1024 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1027 # Extract video identifiers
1029 for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1030 if mobj.group(1) not in ids_in_page:
1031 ids_in_page.append(mobj.group(1))
1032 video_ids.extend(ids_in_page)
1034 if (self._MORE_PAGES_INDICATOR % (playlist_id.upper(), pagenum + 1)) not in page:
1036 pagenum = pagenum + 1
1038 for id in video_ids:
1039 self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
1042 class PostProcessor(object):
1043 """Post Processor class.
1045 PostProcessor objects can be added to downloaders with their
1046 add_post_processor() method. When the downloader has finished a
1047 successful download, it will take its internal chain of PostProcessors
1048 and start calling the run() method on each one of them, first with
1049 an initial argument and then with the returned value of the previous
1052 The chain will be stopped if one of them ever returns None or the end
1053 of the chain is reached.
1055 PostProcessor objects follow a "mutual registration" process similar
1056 to InfoExtractor objects.
1061 def __init__(self, downloader=None):
1062 self._downloader = downloader
1064 def set_downloader(self, downloader):
1065 """Sets the downloader for this PP."""
1066 self._downloader = downloader
1068 def run(self, information):
1069 """Run the PostProcessor.
1071 The "information" argument is a dictionary like the ones
1072 composed by InfoExtractors. The only difference is that this
1073 one has an extra field called "filepath" that points to the
1076 When this method returns None, the postprocessing chain is
1077 stopped. However, this method may return an information
1078 dictionary that will be passed to the next postprocessing
1079 object in the chain. It can be the one it received after
1080 changing some fields.
1082 In addition, this method may raise a PostProcessingError
1083 exception that will be taken into account by the downloader
1086 return information # by default, do nothing
1088 ### MAIN PROGRAM ###
1089 if __name__ == '__main__':
1091 # Modules needed only when running the main program
1095 # General configuration
1096 urllib2.install_opener(urllib2.build_opener(urllib2.ProxyHandler()))
1097 urllib2.install_opener(urllib2.build_opener(urllib2.HTTPCookieProcessor()))
1098 socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words)
1100 # Parse command line
1101 parser = optparse.OptionParser(
1102 usage='Usage: %prog [options] url...',
1104 conflict_handler='resolve',
1107 parser.add_option('-h', '--help',
1108 action='help', help='print this help text and exit')
1109 parser.add_option('-v', '--version',
1110 action='version', help='print program version and exit')
1111 parser.add_option('-i', '--ignore-errors',
1112 action='store_true', dest='ignoreerrors', help='continue on download errors', default=False)
1113 parser.add_option('-r', '--rate-limit',
1114 dest='ratelimit', metavar='L', help='download rate limit (e.g. 50k or 44.6m)')
1116 authentication = optparse.OptionGroup(parser, 'Authentication Options')
1117 authentication.add_option('-u', '--username',
1118 dest='username', metavar='UN', help='account username')
1119 authentication.add_option('-p', '--password',
1120 dest='password', metavar='PW', help='account password')
1121 authentication.add_option('-n', '--netrc',
1122 action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
1123 parser.add_option_group(authentication)
1125 video_format = optparse.OptionGroup(parser, 'Video Format Options')
1126 video_format.add_option('-f', '--format',
1127 action='store', dest='format', metavar='FMT', help='video format code')
1128 video_format.add_option('-b', '--best-quality',
1129 action='store_const', dest='format', help='download the best quality video possible', const='0')
1130 video_format.add_option('-m', '--mobile-version',
1131 action='store_const', dest='format', help='alias for -f 17', const='17')
1132 video_format.add_option('-d', '--high-def',
1133 action='store_const', dest='format', help='alias for -f 22', const='22')
1134 parser.add_option_group(video_format)
1136 verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
1137 verbosity.add_option('-q', '--quiet',
1138 action='store_true', dest='quiet', help='activates quiet mode', default=False)
1139 verbosity.add_option('-s', '--simulate',
1140 action='store_true', dest='simulate', help='do not download video', default=False)
1141 verbosity.add_option('-g', '--get-url',
1142 action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
1143 verbosity.add_option('-e', '--get-title',
1144 action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
1145 parser.add_option_group(verbosity)
1147 filesystem = optparse.OptionGroup(parser, 'Filesystem Options')
1148 filesystem.add_option('-t', '--title',
1149 action='store_true', dest='usetitle', help='use title in file name', default=False)
1150 filesystem.add_option('-l', '--literal',
1151 action='store_true', dest='useliteral', help='use literal title in file name', default=False)
1152 filesystem.add_option('-o', '--output',
1153 dest='outtmpl', metavar='TPL', help='output filename template')
1154 filesystem.add_option('-a', '--batch-file',
1155 dest='batchfile', metavar='F', help='file containing URLs to download')
1156 filesystem.add_option('-w', '--no-overwrites',
1157 action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
1158 filesystem.add_option('-c', '--continue',
1159 action='store_true', dest='continue_dl', help='resume partially downloaded files', default=False)
1160 parser.add_option_group(filesystem)
1162 (opts, args) = parser.parse_args()
1164 # Batch file verification
1166 if opts.batchfile is not None:
1168 batchurls = open(opts.batchfile, 'r').readlines()
1169 batchurls = [x.strip() for x in batchurls]
1170 batchurls = [x for x in batchurls if len(x) > 0]
1172 sys.exit(u'ERROR: batch file could not be read')
1173 all_urls = batchurls + args
1175 # Conflicting, missing and erroneous options
1176 if len(all_urls) < 1:
1177 parser.error(u'you must provide at least one URL')
1178 if opts.usenetrc and (opts.username is not None or opts.password is not None):
1179 parser.error(u'using .netrc conflicts with giving username/password')
1180 if opts.password is not None and opts.username is None:
1181 parser.error(u'account username missing')
1182 if opts.outtmpl is not None and (opts.useliteral or opts.usetitle):
1183 parser.error(u'using output template conflicts with using title or literal title')
1184 if opts.usetitle and opts.useliteral:
1185 parser.error(u'using title conflicts with using literal title')
1186 if opts.username is not None and opts.password is None:
1187 opts.password = getpass.getpass(u'Type account password and press return:')
1188 if opts.ratelimit is not None:
1189 numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
1190 if numeric_limit is None:
1191 parser.error(u'invalid rate limit specified')
1192 opts.ratelimit = numeric_limit
1194 # Information extractors
1195 youtube_ie = YoutubeIE()
1196 metacafe_ie = MetacafeIE(youtube_ie)
1197 youtube_pl_ie = YoutubePlaylistIE(youtube_ie)
1198 youtube_search_ie = YoutubeSearchIE(youtube_ie)
1201 fd = FileDownloader({
1202 'usenetrc': opts.usenetrc,
1203 'username': opts.username,
1204 'password': opts.password,
1205 'quiet': (opts.quiet or opts.geturl or opts.gettitle),
1206 'forceurl': opts.geturl,
1207 'forcetitle': opts.gettitle,
1208 'simulate': (opts.simulate or opts.geturl or opts.gettitle),
1209 'format': opts.format,
1210 'outtmpl': ((opts.outtmpl is not None and opts.outtmpl.decode(preferredencoding()))
1211 or (opts.usetitle and u'%(stitle)s-%(id)s.%(ext)s')
1212 or (opts.useliteral and u'%(title)s-%(id)s.%(ext)s')
1213 or u'%(id)s.%(ext)s'),
1214 'ignoreerrors': opts.ignoreerrors,
1215 'ratelimit': opts.ratelimit,
1216 'nooverwrites': opts.nooverwrites,
1217 'continuedl': opts.continue_dl,
1219 fd.add_info_extractor(youtube_search_ie)
1220 fd.add_info_extractor(youtube_pl_ie)
1221 fd.add_info_extractor(metacafe_ie)
1222 fd.add_info_extractor(youtube_ie)
1223 retcode = fd.download(all_urls)
1226 except DownloadError:
1228 except SameFileError:
1229 sys.exit(u'ERROR: fixed output name but more than one file to download')
1230 except KeyboardInterrupt:
1231 sys.exit(u'\nERROR: Interrupted by user')