2 # -*- coding: utf-8 -*-
3 # Author: Ricardo Garcia Gonzalez
4 # Author: Danny Colligan
5 # License: Public domain code
22 'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.0.8) Gecko/2009032609 Firefox/3.0.8',
23 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
24 'Accept': 'text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5',
25 'Accept-Language': 'en-us,en;q=0.5',
28 simple_title_chars = string.ascii_letters.decode('ascii') + string.digits.decode('ascii')
30 class DownloadError(Exception):
31 """Download Error exception.
33 This exception may be thrown by FileDownloader objects if they are not
34 configured to continue on errors. They will contain the appropriate
39 class SameFileError(Exception):
40 """Same File exception.
42 This exception will be thrown by FileDownloader objects if they detect
43 multiple files would have to be downloaded to the same file on disk.
47 class PostProcessingError(Exception):
48 """Post Processing exception.
50 This exception may be raised by PostProcessor's .run() method to
51 indicate an error in the postprocessing task.
55 class UnavailableFormatError(Exception):
56 """Unavailable Format exception.
58 This exception will be thrown when a video is requested
59 in a format that is not available for that video.
63 class ContentTooShortError(Exception):
64 """Content Too Short exception.
66 This exception may be raised by FileDownloader objects when a file they
67 download is too small for what the server announced first, indicating
68 the connection was probably interrupted.
74 def __init__(self, downloaded, expected):
75 self.downloaded = downloaded
76 self.expected = expected
78 class FileDownloader(object):
79 """File Downloader class.
81 File downloader objects are the ones responsible of downloading the
82 actual video file and writing it to disk if the user has requested
83 it, among some other tasks. In most cases there should be one per
84 program. As, given a video URL, the downloader doesn't know how to
85 extract all the needed information, task that InfoExtractors do, it
86 has to pass the URL to one of them.
88 For this, file downloader objects have a method that allows
89 InfoExtractors to be registered in a given order. When it is passed
90 a URL, the file downloader handles it to the first InfoExtractor it
91 finds that reports being able to handle it. The InfoExtractor extracts
92 all the information about the video or videos the URL refers to, and
93 asks the FileDownloader to process the video information, possibly
94 downloading the video.
96 File downloaders accept a lot of parameters. In order not to saturate
97 the object constructor with arguments, it receives a dictionary of
98 options instead. These options are available through the params
99 attribute for the InfoExtractors to use. The FileDownloader also
100 registers itself as the downloader in charge for the InfoExtractors
101 that are added to it, so this is a "mutual registration".
105 username: Username for authentication purposes.
106 password: Password for authentication purposes.
107 usenetrc: Use netrc for authentication instead.
108 quiet: Do not print messages to stdout.
109 forceurl: Force printing final URL.
110 forcetitle: Force printing title.
111 simulate: Do not download the video files.
112 format: Video format code.
113 outtmpl: Template for output names.
114 ignoreerrors: Do not stop on download errors.
115 ratelimit: Download speed limit, in bytes/sec.
116 nooverwrites: Prevent overwriting files.
122 _download_retcode = None
124 def __init__(self, params):
125 """Create a FileDownloader object with the given options."""
128 self._download_retcode = 0
132 def pmkdir(filename):
133 """Create directory components in filename. Similar to Unix "mkdir -p"."""
134 components = filename.split(os.sep)
135 aggregate = [os.sep.join(components[0:x]) for x in xrange(1, len(components))]
136 aggregate = ['%s%s' % (x, os.sep) for x in aggregate] # Finish names with separator
137 for dir in aggregate:
138 if not os.path.exists(dir):
142 def format_bytes(bytes):
148 exponent = long(math.log(float(bytes), 1024.0))
149 suffix = 'bkMGTPEZY'[exponent]
150 converted = float(bytes) / float(1024**exponent)
151 return '%.2f%s' % (converted, suffix)
154 def calc_percent(byte_counter, data_len):
157 return '%6s' % ('%3.1f%%' % (float(byte_counter) / float(data_len) * 100.0))
160 def calc_eta(start, now, total, current):
164 if current == 0 or dif < 0.001: # One millisecond
166 rate = float(current) / dif
167 eta = long((float(total) - float(current)) / rate)
168 (eta_mins, eta_secs) = divmod(eta, 60)
171 return '%02d:%02d' % (eta_mins, eta_secs)
174 def calc_speed(start, now, bytes):
176 if bytes == 0 or dif < 0.001: # One millisecond
177 return '%10s' % '---b/s'
178 return '%10s' % ('%s/s' % FileDownloader.format_bytes(float(bytes) / dif))
181 def best_block_size(elapsed_time, bytes):
182 new_min = max(bytes / 2.0, 1.0)
183 new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
184 if elapsed_time < 0.001:
186 rate = bytes / elapsed_time
194 def parse_bytes(bytestr):
195 """Parse a string indicating a byte quantity into a long integer."""
196 matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
199 number = float(matchobj.group(1))
200 multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
201 return long(round(number * multiplier))
203 def add_info_extractor(self, ie):
204 """Add an InfoExtractor object to the end of the list."""
206 ie.set_downloader(self)
208 def add_post_processor(self, pp):
209 """Add a PostProcessor object to the end of the chain."""
211 pp.set_downloader(self)
213 def to_stdout(self, message, skip_eol=False):
214 """Print message to stdout if not in quiet mode."""
215 if not self.params.get('quiet', False):
216 print (u'%s%s' % (message, [u'\n', u''][skip_eol])).encode(locale.getpreferredencoding()),
219 def to_stderr(self, message):
220 """Print message to stderr."""
221 print >>sys.stderr, message
223 def fixed_template(self):
224 """Checks if the output template is fixed."""
225 return (re.search(ur'(?u)%\(.+?\)s', self.params['outtmpl']) is None)
227 def trouble(self, message=None):
228 """Determine action to take when a download problem appears.
230 Depending on if the downloader has been configured to ignore
231 download errors or not, this method may throw an exception or
232 not when errors are found, after printing the message.
234 if message is not None:
235 self.to_stderr(message)
236 if not self.params.get('ignoreerrors', False):
237 raise DownloadError(message)
238 self._download_retcode = 1
240 def slow_down(self, start_time, byte_counter):
241 """Sleep if the download speed is over the rate limit."""
242 rate_limit = self.params.get('ratelimit', None)
243 if rate_limit is None or byte_counter == 0:
246 elapsed = now - start_time
249 speed = float(byte_counter) / elapsed
250 if speed > rate_limit:
251 time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
253 def report_destination(self, filename):
254 """Report destination filename."""
255 self.to_stdout(u'[download] Destination: %s' % filename)
257 def report_progress(self, percent_str, data_len_str, speed_str, eta_str):
258 """Report download progress."""
259 self.to_stdout(u'\r[download] %s of %s at %s ETA %s' %
260 (percent_str, data_len_str, speed_str, eta_str), skip_eol=True)
262 def report_finish(self):
263 """Report download finished."""
266 def process_info(self, info_dict):
267 """Process a single dictionary returned by an InfoExtractor."""
269 if self.params.get('forcetitle', False):
270 print info_dict['title'].encode(locale.getpreferredencoding())
271 if self.params.get('forceurl', False):
272 print info_dict['url'].encode(locale.getpreferredencoding())
274 # Do nothing else if in simulate mode
275 if self.params.get('simulate', False):
279 template_dict = dict(info_dict)
280 template_dict['epoch'] = unicode(long(time.time()))
281 filename = self.params['outtmpl'] % template_dict
282 self.report_destination(filename)
283 except (ValueError, KeyError), err:
284 self.trouble('ERROR: invalid output template or system charset: %s' % str(err))
285 if self.params['nooverwrites'] and os.path.exists(filename):
286 self.to_stderr('WARNING: file exists: %s; skipping' % filename)
290 self.pmkdir(filename)
291 except (OSError, IOError), err:
292 self.trouble('ERROR: unable to create directories: %s' % str(err))
296 outstream = open(filename, 'wb')
297 except (OSError, IOError), err:
298 self.trouble('ERROR: unable to open for writing: %s' % str(err))
302 self._do_download(outstream, info_dict['url'])
304 except (OSError, IOError), err:
307 raise UnavailableFormatError
308 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
309 self.trouble('ERROR: unable to download video data: %s' % str(err))
311 except (ContentTooShortError, ), err:
312 self.trouble('ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
316 self.post_process(filename, info_dict)
317 except (PostProcessingError), err:
318 self.trouble('ERROR: postprocessing: %s' % str(err))
321 def download(self, url_list):
322 """Download a given list of URLs."""
323 if len(url_list) > 1 and self.fixed_template():
324 raise SameFileError(self.params['outtmpl'])
327 suitable_found = False
329 # Go to next InfoExtractor if not suitable
330 if not ie.suitable(url):
333 # Suitable InfoExtractor found
334 suitable_found = True
336 # Extract information from URL and process it
339 # Suitable InfoExtractor had been found; go to next URL
342 if not suitable_found:
343 self.trouble('ERROR: no suitable InfoExtractor: %s' % url)
345 return self._download_retcode
347 def post_process(self, filename, ie_info):
348 """Run the postprocessing chain on the given file."""
350 info['filepath'] = filename
356 def _do_download(self, stream, url):
357 request = urllib2.Request(url, None, std_headers)
358 data = urllib2.urlopen(request)
359 data_len = data.info().get('Content-length', None)
360 data_len_str = self.format_bytes(data_len)
366 percent_str = self.calc_percent(byte_counter, data_len)
367 eta_str = self.calc_eta(start, time.time(), data_len, byte_counter)
368 speed_str = self.calc_speed(start, time.time(), byte_counter)
369 self.report_progress(percent_str, data_len_str, speed_str, eta_str)
373 data_block = data.read(block_size)
375 data_block_len = len(data_block)
376 if data_block_len == 0:
378 byte_counter += data_block_len
379 stream.write(data_block)
380 block_size = self.best_block_size(after - before, data_block_len)
383 self.slow_down(start, byte_counter)
386 if data_len is not None and str(byte_counter) != data_len:
387 raise ContentTooShortError(byte_counter, long(data_len))
389 class InfoExtractor(object):
390 """Information Extractor class.
392 Information extractors are the classes that, given a URL, extract
393 information from the video (or videos) the URL refers to. This
394 information includes the real video URL, the video title and simplified
395 title, author and others. The information is stored in a dictionary
396 which is then passed to the FileDownloader. The FileDownloader
397 processes this information possibly downloading the video to the file
398 system, among other possible outcomes. The dictionaries must include
399 the following fields:
401 id: Video identifier.
402 url: Final video URL.
403 uploader: Nickname of the video uploader.
404 title: Literal title.
405 stitle: Simplified title.
406 ext: Video filename extension.
408 Subclasses of this one should re-define the _real_initialize() and
409 _real_extract() methods, as well as the suitable() static method.
410 Probably, they should also be instantiated and added to the main
417 def __init__(self, downloader=None):
418 """Constructor. Receives an optional downloader."""
420 self.set_downloader(downloader)
424 """Receives a URL and returns True if suitable for this IE."""
427 def initialize(self):
428 """Initializes an instance (authentication, etc)."""
430 self._real_initialize()
433 def extract(self, url):
434 """Extracts URL information and returns it in list of dicts."""
436 return self._real_extract(url)
438 def set_downloader(self, downloader):
439 """Sets the downloader for this IE."""
440 self._downloader = downloader
442 def _real_initialize(self):
443 """Real initialization process. Redefine in subclasses."""
446 def _real_extract(self, url):
447 """Real extraction process. Redefine in subclasses."""
450 class YoutubeIE(InfoExtractor):
451 """Information extractor for youtube.com."""
453 _VALID_URL = r'^((?:http://)?(?:\w+\.)?youtube\.com/(?:(?:v/)|(?:(?:watch(?:\.php)?)?\?(?:.+&)?v=)))?([0-9A-Za-z_-]+)(?(1).+)?$'
454 _LANG_URL = r'http://uk.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
455 _LOGIN_URL = 'http://www.youtube.com/signup?next=/&gl=US&hl=en'
456 _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
457 _NETRC_MACHINE = 'youtube'
458 _available_formats = ['22', '35', '18', '17', '13'] # listed in order of priority for -b flag
459 _video_extensions = {
468 return (re.match(YoutubeIE._VALID_URL, url) is not None)
471 def htmlentity_transform(matchobj):
472 """Transforms an HTML entity to a Unicode character."""
473 entity = matchobj.group(1)
475 # Known non-numeric HTML entity
476 if entity in htmlentitydefs.name2codepoint:
477 return unichr(htmlentitydefs.name2codepoint[entity])
480 mobj = re.match(ur'(?u)#(x?\d+)', entity)
482 numstr = mobj.group(1)
483 if numstr.startswith(u'x'):
485 numstr = u'0%s' % numstr
488 return unichr(long(numstr, base))
490 # Unknown entity in name, return its literal representation
491 return (u'&%s;' % entity)
493 def report_lang(self):
494 """Report attempt to set language."""
495 self._downloader.to_stdout(u'[youtube] Setting language')
497 def report_login(self):
498 """Report attempt to log in."""
499 self._downloader.to_stdout(u'[youtube] Logging in')
501 def report_age_confirmation(self):
502 """Report attempt to confirm age."""
503 self._downloader.to_stdout(u'[youtube] Confirming age')
505 def report_webpage_download(self, video_id):
506 """Report attempt to download webpage."""
507 self._downloader.to_stdout(u'[youtube] %s: Downloading video webpage' % video_id)
509 def report_information_extraction(self, video_id):
510 """Report attempt to extract video information."""
511 self._downloader.to_stdout(u'[youtube] %s: Extracting video information' % video_id)
513 def report_video_url(self, video_id, video_real_url):
514 """Report extracted video URL."""
515 self._downloader.to_stdout(u'[youtube] %s: URL: %s' % (video_id, video_real_url))
517 def report_unavailable_format(self, video_id, format):
518 """Report extracted video URL."""
519 self._downloader.to_stdout(u'[youtube] %s: Format %s not available' % (video_id, format))
521 def _real_initialize(self):
522 if self._downloader is None:
527 downloader_params = self._downloader.params
529 # Attempt to use provided username and password or .netrc data
530 if downloader_params.get('username', None) is not None:
531 username = downloader_params['username']
532 password = downloader_params['password']
533 elif downloader_params.get('usenetrc', False):
535 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
540 raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
541 except (IOError, netrc.NetrcParseError), err:
542 self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % str(err))
546 request = urllib2.Request(self._LANG_URL, None, std_headers)
549 urllib2.urlopen(request).read()
550 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
551 self._downloader.to_stderr(u'WARNING: unable to set language: %s' % str(err))
554 # No authentication to be performed
560 'current_form': 'loginForm',
562 'action_login': 'Log In',
563 'username': username,
564 'password': password,
566 request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form), std_headers)
569 login_results = urllib2.urlopen(request).read()
570 if re.search(r'(?i)<form[^>]* name="loginForm"', login_results) is not None:
571 self._downloader.to_stderr(u'WARNING: unable to log in: bad username or password')
573 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
574 self._downloader.to_stderr(u'WARNING: unable to log in: %s' % str(err))
580 'action_confirm': 'Confirm',
582 request = urllib2.Request(self._AGE_URL, urllib.urlencode(age_form), std_headers)
584 self.report_age_confirmation()
585 age_results = urllib2.urlopen(request).read()
586 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
587 self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
590 def _real_extract(self, url):
591 # Extract video id from URL
592 mobj = re.match(self._VALID_URL, url)
594 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
596 video_id = mobj.group(2)
598 # Downloader parameters
602 if self._downloader is not None:
603 params = self._downloader.params
604 format_param = params.get('format', None)
605 if format_param == '0':
606 format_param = self._available_formats[quality_index]
612 video_extension = self._video_extensions.get(format_param, 'flv')
614 # Normalize URL, including format
615 normalized_url = 'http://www.youtube.com/watch?v=%s&gl=US&hl=en' % video_id
616 if format_param is not None:
617 normalized_url = '%s&fmt=%s' % (normalized_url, format_param)
618 request = urllib2.Request(normalized_url, None, std_headers)
620 self.report_webpage_download(video_id)
621 video_webpage = urllib2.urlopen(request).read()
622 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
623 self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
625 self.report_information_extraction(video_id)
628 mobj = re.search(r', "t": "([^"]+)"', video_webpage)
630 self._downloader.trouble(u'ERROR: unable to extract "t" parameter')
632 video_real_url = 'http://www.youtube.com/get_video?video_id=%s&t=%s&el=detailpage&ps=' % (video_id, mobj.group(1))
633 if format_param is not None:
634 video_real_url = '%s&fmt=%s' % (video_real_url, format_param)
635 self.report_video_url(video_id, video_real_url)
638 mobj = re.search(r"var watchUsername = '([^']+)';", video_webpage)
640 self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
642 video_uploader = mobj.group(1)
645 mobj = re.search(r'(?im)<title>YouTube - ([^<]*)</title>', video_webpage)
647 self._downloader.trouble(u'ERROR: unable to extract video title')
649 video_title = mobj.group(1).decode('utf-8')
650 video_title = re.sub(ur'(?u)&(.+?);', self.htmlentity_transform, video_title)
651 video_title = video_title.replace(os.sep, u'%')
654 simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
655 simple_title = simple_title.strip(ur'_')
657 # Process video information
658 self._downloader.process_info({
659 'id': video_id.decode('utf-8'),
660 'url': video_real_url.decode('utf-8'),
661 'uploader': video_uploader.decode('utf-8'),
662 'title': video_title,
663 'stitle': simple_title,
664 'ext': video_extension.decode('utf-8'),
669 except UnavailableFormatError, err:
671 if quality_index == len(self._available_formats) - 1:
672 # I don't ever expect this to happen
673 self._downloader.trouble(u'ERROR: no known formats available for video')
676 self.report_unavailable_format(video_id, format_param)
678 format_param = self._available_formats[quality_index]
681 self._downloader.trouble('ERROR: format not available for video')
685 class MetacafeIE(InfoExtractor):
686 """Information Extractor for metacafe.com."""
688 _VALID_URL = r'(?:http://)?(?:www\.)?metacafe\.com/watch/([^/]+)/([^/]+)/.*'
689 _DISCLAIMER = 'http://www.metacafe.com/family_filter/'
690 _FILTER_POST = 'http://www.metacafe.com/f/index.php?inputType=filter&controllerGroup=user'
693 def __init__(self, youtube_ie, downloader=None):
694 InfoExtractor.__init__(self, downloader)
695 self._youtube_ie = youtube_ie
699 return (re.match(MetacafeIE._VALID_URL, url) is not None)
701 def report_disclaimer(self):
702 """Report disclaimer retrieval."""
703 self._downloader.to_stdout(u'[metacafe] Retrieving disclaimer')
705 def report_age_confirmation(self):
706 """Report attempt to confirm age."""
707 self._downloader.to_stdout(u'[metacafe] Confirming age')
709 def report_download_webpage(self, video_id):
710 """Report webpage download."""
711 self._downloader.to_stdout(u'[metacafe] %s: Downloading webpage' % video_id)
713 def report_extraction(self, video_id):
714 """Report information extraction."""
715 self._downloader.to_stdout(u'[metacafe] %s: Extracting information' % video_id)
717 def _real_initialize(self):
718 # Retrieve disclaimer
719 request = urllib2.Request(self._DISCLAIMER, None, std_headers)
721 self.report_disclaimer()
722 disclaimer = urllib2.urlopen(request).read()
723 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
724 self._downloader.trouble(u'ERROR: unable to retrieve disclaimer: %s' % str(err))
730 'submit': "Continue - I'm over 18",
732 request = urllib2.Request(self._FILTER_POST, urllib.urlencode(disclaimer_form), std_headers)
734 self.report_age_confirmation()
735 disclaimer = urllib2.urlopen(request).read()
736 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
737 self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
740 def _real_extract(self, url):
741 # Extract id and simplified title from URL
742 mobj = re.match(self._VALID_URL, url)
744 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
747 video_id = mobj.group(1)
749 # Check if video comes from YouTube
750 mobj2 = re.match(r'^yt-(.*)$', video_id)
751 if mobj2 is not None:
752 self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % mobj2.group(1))
755 simple_title = mobj.group(2).decode('utf-8')
756 video_extension = 'flv'
758 # Retrieve video webpage to extract further information
759 request = urllib2.Request('http://www.metacafe.com/watch/%s/' % video_id)
761 self.report_download_webpage(video_id)
762 webpage = urllib2.urlopen(request).read()
763 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
764 self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % str(err))
767 # Extract URL, uploader and title from webpage
768 self.report_extraction(video_id)
769 mobj = re.search(r'(?m)&mediaURL=(http.*?\.flv)', webpage)
771 self._downloader.trouble(u'ERROR: unable to extract media URL')
773 mediaURL = urllib.unquote(mobj.group(1))
775 mobj = re.search(r'(?m)&gdaKey=(.*?)&', webpage)
777 self._downloader.trouble(u'ERROR: unable to extract gdaKey')
779 gdaKey = mobj.group(1)
781 video_url = '%s?__gda__=%s' % (mediaURL, gdaKey)
783 mobj = re.search(r'(?im)<title>(.*) - Video</title>', webpage)
785 self._downloader.trouble(u'ERROR: unable to extract title')
787 video_title = mobj.group(1).decode('utf-8')
789 mobj = re.search(r'(?ms)<li id="ChnlUsr">.*?Submitter:.*?<a .*?>(.*?)<', webpage)
791 self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
793 video_uploader = mobj.group(1)
796 # Process video information
797 self._downloader.process_info({
798 'id': video_id.decode('utf-8'),
799 'url': video_url.decode('utf-8'),
800 'uploader': video_uploader.decode('utf-8'),
801 'title': video_title,
802 'stitle': simple_title,
803 'ext': video_extension.decode('utf-8'),
805 except UnavailableFormatError:
806 self._downloader.trouble(u'ERROR: format not available for video')
809 class YoutubeSearchIE(InfoExtractor):
810 """Information Extractor for YouTube search queries."""
811 _VALID_QUERY = r'ytsearch(\d+|all)?:[\s\S]+'
812 _TEMPLATE_URL = 'http://www.youtube.com/results?search_query=%s&page=%s&gl=US&hl=en'
813 _VIDEO_INDICATOR = r'href="/watch\?v=.+?"'
814 _MORE_PAGES_INDICATOR = r'>Next</a>'
816 _max_youtube_results = 1000
818 def __init__(self, youtube_ie, downloader=None):
819 InfoExtractor.__init__(self, downloader)
820 self._youtube_ie = youtube_ie
824 return (re.match(YoutubeSearchIE._VALID_QUERY, url) is not None)
826 def report_download_page(self, query, pagenum):
827 """Report attempt to download playlist page with given number."""
828 self._downloader.to_stdout(u'[youtube] query "%s": Downloading page %s' % (query, pagenum))
830 def _real_initialize(self):
831 self._youtube_ie.initialize()
833 def _real_extract(self, query):
834 mobj = re.match(self._VALID_QUERY, query)
836 self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
839 prefix, query = query.split(':')
842 self._download_n_results(query, 1)
844 elif prefix == 'all':
845 self._download_n_results(query, self._max_youtube_results)
851 self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
853 elif n > self._max_youtube_results:
854 self._downloader.to_stderr(u'WARNING: ytsearch returns max %i results (you requested %i)' % (self._max_youtube_results, n))
855 n = self._max_youtube_results
856 self._download_n_results(query, n)
858 except ValueError: # parsing prefix as int fails
859 self._download_n_results(query, 1)
862 def _download_n_results(self, query, n):
863 """Downloads a specified number of results for a query"""
870 self.report_download_page(query, pagenum)
871 result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
872 request = urllib2.Request(result_url, None, std_headers)
874 page = urllib2.urlopen(request).read()
875 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
876 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
879 # Extract video identifiers
880 for mobj in re.finditer(self._VIDEO_INDICATOR, page):
881 video_id = page[mobj.span()[0]:mobj.span()[1]].split('=')[2][:-1]
882 if video_id not in already_seen:
883 video_ids.append(video_id)
884 already_seen.add(video_id)
885 if len(video_ids) == n:
886 # Specified n videos reached
888 self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
891 if self._MORE_PAGES_INDICATOR not in page:
893 self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
896 pagenum = pagenum + 1
898 class YoutubePlaylistIE(InfoExtractor):
899 """Information Extractor for YouTube playlists."""
901 _VALID_URL = r'(?:http://)?(?:\w+\.)?youtube.com/view_play_list\?p=(.+)'
902 _TEMPLATE_URL = 'http://www.youtube.com/view_play_list?p=%s&page=%s&gl=US&hl=en'
903 _VIDEO_INDICATOR = r'/watch\?v=(.+?)&'
904 _MORE_PAGES_INDICATOR = r'/view_play_list?p=%s&page=%s'
907 def __init__(self, youtube_ie, downloader=None):
908 InfoExtractor.__init__(self, downloader)
909 self._youtube_ie = youtube_ie
913 return (re.match(YoutubePlaylistIE._VALID_URL, url) is not None)
915 def report_download_page(self, playlist_id, pagenum):
916 """Report attempt to download playlist page with given number."""
917 self._downloader.to_stdout(u'[youtube] PL %s: Downloading page #%s' % (playlist_id, pagenum))
919 def _real_initialize(self):
920 self._youtube_ie.initialize()
922 def _real_extract(self, url):
923 # Extract playlist id
924 mobj = re.match(self._VALID_URL, url)
926 self._downloader.trouble(u'ERROR: invalid url: %s' % url)
929 # Download playlist pages
930 playlist_id = mobj.group(1)
935 self.report_download_page(playlist_id, pagenum)
936 request = urllib2.Request(self._TEMPLATE_URL % (playlist_id, pagenum), None, std_headers)
938 page = urllib2.urlopen(request).read()
939 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
940 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
943 # Extract video identifiers
945 for mobj in re.finditer(self._VIDEO_INDICATOR, page):
946 if mobj.group(1) not in ids_in_page:
947 ids_in_page.append(mobj.group(1))
948 video_ids.extend(ids_in_page)
950 if (self._MORE_PAGES_INDICATOR % (playlist_id, pagenum + 1)) not in page:
952 pagenum = pagenum + 1
955 self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
958 class PostProcessor(object):
959 """Post Processor class.
961 PostProcessor objects can be added to downloaders with their
962 add_post_processor() method. When the downloader has finished a
963 successful download, it will take its internal chain of PostProcessors
964 and start calling the run() method on each one of them, first with
965 an initial argument and then with the returned value of the previous
968 The chain will be stopped if one of them ever returns None or the end
969 of the chain is reached.
971 PostProcessor objects follow a "mutual registration" process similar
972 to InfoExtractor objects.
977 def __init__(self, downloader=None):
978 self._downloader = downloader
980 def set_downloader(self, downloader):
981 """Sets the downloader for this PP."""
982 self._downloader = downloader
984 def run(self, information):
985 """Run the PostProcessor.
987 The "information" argument is a dictionary like the ones
988 composed by InfoExtractors. The only difference is that this
989 one has an extra field called "filepath" that points to the
992 When this method returns None, the postprocessing chain is
993 stopped. However, this method may return an information
994 dictionary that will be passed to the next postprocessing
995 object in the chain. It can be the one it received after
996 changing some fields.
998 In addition, this method may raise a PostProcessingError
999 exception that will be taken into account by the downloader
1002 return information # by default, do nothing
1004 ### MAIN PROGRAM ###
1005 if __name__ == '__main__':
1007 # Modules needed only when running the main program
1011 # General configuration
1012 urllib2.install_opener(urllib2.build_opener(urllib2.ProxyHandler()))
1013 urllib2.install_opener(urllib2.build_opener(urllib2.HTTPCookieProcessor()))
1014 socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words)
1016 # Parse command line
1017 parser = optparse.OptionParser(
1018 usage='Usage: %prog [options] url...',
1020 conflict_handler='resolve',
1023 parser.add_option('-h', '--help',
1024 action='help', help='print this help text and exit')
1025 parser.add_option('-v', '--version',
1026 action='version', help='print program version and exit')
1027 parser.add_option('-i', '--ignore-errors',
1028 action='store_true', dest='ignoreerrors', help='continue on download errors', default=False)
1029 parser.add_option('-r', '--rate-limit',
1030 dest='ratelimit', metavar='L', help='download rate limit (e.g. 50k or 44.6m)')
1032 authentication = optparse.OptionGroup(parser, 'Authentication Options')
1033 authentication.add_option('-u', '--username',
1034 dest='username', metavar='UN', help='account username')
1035 authentication.add_option('-p', '--password',
1036 dest='password', metavar='PW', help='account password')
1037 authentication.add_option('-n', '--netrc',
1038 action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
1039 parser.add_option_group(authentication)
1041 video_format = optparse.OptionGroup(parser, 'Video Format Options')
1042 video_format.add_option('-f', '--format',
1043 action='append', dest='format', metavar='FMT', help='video format code')
1044 video_format.add_option('-b', '--best-quality',
1045 action='store_const', dest='format', help='download the best quality video possible', const='0')
1046 video_format.add_option('-m', '--mobile-version',
1047 action='store_const', dest='format', help='alias for -f 17', const='17')
1048 video_format.add_option('-d', '--high-def',
1049 action='store_const', dest='format', help='alias for -f 22', const='22')
1050 parser.add_option_group(video_format)
1052 verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
1053 verbosity.add_option('-q', '--quiet',
1054 action='store_true', dest='quiet', help='activates quiet mode', default=False)
1055 verbosity.add_option('-s', '--simulate',
1056 action='store_true', dest='simulate', help='do not download video', default=False)
1057 verbosity.add_option('-g', '--get-url',
1058 action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
1059 verbosity.add_option('-e', '--get-title',
1060 action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
1061 parser.add_option_group(verbosity)
1063 filesystem = optparse.OptionGroup(parser, 'Filesystem Options')
1064 filesystem.add_option('-t', '--title',
1065 action='store_true', dest='usetitle', help='use title in file name', default=False)
1066 filesystem.add_option('-l', '--literal',
1067 action='store_true', dest='useliteral', help='use literal title in file name', default=False)
1068 filesystem.add_option('-o', '--output',
1069 dest='outtmpl', metavar='TPL', help='output filename template')
1070 filesystem.add_option('-a', '--batch-file',
1071 dest='batchfile', metavar='F', help='file containing URLs to download')
1072 filesystem.add_option('-w', '--no-overwrites',
1073 action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
1074 parser.add_option_group(filesystem)
1076 (opts, args) = parser.parse_args()
1078 # Batch file verification
1080 if opts.batchfile is not None:
1082 batchurls = open(opts.batchfile, 'r').readlines()
1083 batchurls = [x.strip() for x in batchurls]
1084 batchurls = [x for x in batchurls if len(x) > 0]
1086 sys.exit(u'ERROR: batch file could not be read')
1087 all_urls = batchurls + args
1089 # Conflicting, missing and erroneous options
1090 if len(all_urls) < 1:
1091 parser.error(u'you must provide at least one URL')
1092 if opts.usenetrc and (opts.username is not None or opts.password is not None):
1093 parser.error(u'using .netrc conflicts with giving username/password')
1094 if opts.password is not None and opts.username is None:
1095 parser.error(u'account username missing')
1096 if opts.outtmpl is not None and (opts.useliteral or opts.usetitle):
1097 parser.error(u'using output template conflicts with using title or literal title')
1098 if opts.usetitle and opts.useliteral:
1099 parser.error(u'using title conflicts with using literal title')
1100 if opts.username is not None and opts.password is None:
1101 opts.password = getpass.getpass(u'Type account password and press return:')
1102 if opts.ratelimit is not None:
1103 numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
1104 if numeric_limit is None:
1105 parser.error(u'invalid rate limit specified')
1106 opts.ratelimit = numeric_limit
1107 if opts.format is not None and len(opts.format) > 1:
1108 parser.error(u'pass at most one of the video format option flags (-f, -b, -m, -d)')
1109 if opts.format is None:
1112 real_format = opts.format[0]
1115 # Information extractors
1116 youtube_ie = YoutubeIE()
1117 metacafe_ie = MetacafeIE(youtube_ie)
1118 youtube_pl_ie = YoutubePlaylistIE(youtube_ie)
1119 youtube_search_ie = YoutubeSearchIE(youtube_ie)
1122 fd = FileDownloader({
1123 'usenetrc': opts.usenetrc,
1124 'username': opts.username,
1125 'password': opts.password,
1126 'quiet': (opts.quiet or opts.geturl or opts.gettitle),
1127 'forceurl': opts.geturl,
1128 'forcetitle': opts.gettitle,
1129 'simulate': (opts.simulate or opts.geturl or opts.gettitle),
1130 'format': real_format,
1131 'outtmpl': ((opts.outtmpl is not None and opts.outtmpl.decode(locale.getpreferredencoding()))
1132 or (opts.usetitle and u'%(stitle)s-%(id)s.%(ext)s')
1133 or (opts.useliteral and u'%(title)s-%(id)s.%(ext)s')
1134 or u'%(id)s.%(ext)s'),
1135 'ignoreerrors': opts.ignoreerrors,
1136 'ratelimit': opts.ratelimit,
1137 'nooverwrites': opts.nooverwrites,
1139 fd.add_info_extractor(youtube_search_ie)
1140 fd.add_info_extractor(youtube_pl_ie)
1141 fd.add_info_extractor(metacafe_ie)
1142 fd.add_info_extractor(youtube_ie)
1143 retcode = fd.download(all_urls)
1146 except DownloadError:
1148 except SameFileError:
1149 sys.exit(u'ERROR: fixed output name but more than one file to download')
1150 except KeyboardInterrupt:
1151 sys.exit(u'\nERROR: Interrupted by user')