2 # -*- coding: utf-8 -*-
5 'Ricardo Garcia Gonzalez',
13 'Philipp Hagemeister',
19 __license__ = 'Public Domain'
20 __version__ = '2011.11.22'
22 UPDATE_URL = 'https://raw.github.com/rg3/youtube-dl/master/youtube-dl'
51 except ImportError: # Python 2.4
54 import cStringIO as StringIO
58 # parse_qs was moved from the cgi module to the urlparse module recently.
60 from urlparse import parse_qs
62 from cgi import parse_qs
70 import xml.etree.ElementTree
71 except ImportError: # Python<2.5: Not officially supported, but let it slip
72 warnings.warn('xml.etree.ElementTree support is missing. Consider upgrading to Python >= 2.5 if you get related errors.')
75 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:5.0.1) Gecko/20100101 Firefox/5.0.1',
76 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
77 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
78 'Accept-Encoding': 'gzip, deflate',
79 'Accept-Language': 'en-us,en;q=0.5',
84 except ImportError: # Python <2.6, use trivialjson (https://github.com/phihag/trivialjson):
90 def raiseError(msg, i):
91 raise ValueError(msg + ' at position ' + str(i) + ' of ' + repr(s) + ': ' + repr(s[i:]))
92 def skipSpace(i, expectMore=True):
93 while i < len(s) and s[i] in ' \t\r\n':
97 raiseError('Premature end', i)
99 def decodeEscape(match):
115 return unichr(int(esc[1:5], 16))
116 if len(esc) == 5+6 and esc[5:7] == '\\u':
117 hi = int(esc[1:5], 16)
118 low = int(esc[7:11], 16)
119 return unichr((hi - 0xd800) * 0x400 + low - 0xdc00 + 0x10000)
120 raise ValueError('Unknown escape ' + str(esc))
127 while s[e-bslashes-1] == '\\':
129 if bslashes % 2 == 1:
133 rexp = re.compile(r'\\(u[dD][89aAbB][0-9a-fA-F]{2}\\u[0-9a-fA-F]{4}|u[0-9a-fA-F]{4}|.|$)')
134 stri = rexp.sub(decodeEscape, s[i:e])
140 if s[i] == '}': # Empty dictionary
144 raiseError('Expected a string object key', i)
145 i,key = parseString(i)
147 if i >= len(s) or s[i] != ':':
148 raiseError('Expected a colon', i)
155 raiseError('Expected comma or closing curly brace', i)
160 if s[i] == ']': # Empty array
165 i = skipSpace(i) # Raise exception if premature end
169 raiseError('Expected a comma or closing bracket', i)
171 def parseDiscrete(i):
172 for k,v in {'true': True, 'false': False, 'null': None}.items():
173 if s.startswith(k, i):
175 raiseError('Not a boolean (or null)', i)
177 mobj = re.match('^(-?(0|[1-9][0-9]*)(\.[0-9]*)?([eE][+-]?[0-9]+)?)', s[i:])
179 raiseError('Not a number', i)
181 if '.' in nums or 'e' in nums or 'E' in nums:
182 return (i+len(nums), float(nums))
183 return (i+len(nums), int(nums))
184 CHARMAP = {'{': parseObj, '[': parseArray, '"': parseString, 't': parseDiscrete, 'f': parseDiscrete, 'n': parseDiscrete}
187 i,res = CHARMAP.get(s[i], parseNumber)(i)
188 i = skipSpace(i, False)
192 raise ValueError('Extra data at end of input (index ' + str(i) + ' of ' + repr(s) + ': ' + repr(s[i:]) + ')')
195 def preferredencoding():
196 """Get preferred encoding.
198 Returns the best encoding scheme for the system, based on
199 locale.getpreferredencoding() and some further tweaks.
201 def yield_preferredencoding():
203 pref = locale.getpreferredencoding()
209 return yield_preferredencoding().next()
212 def htmlentity_transform(matchobj):
213 """Transforms an HTML entity to a Unicode character.
215 This function receives a match object and is intended to be used with
216 the re.sub() function.
218 entity = matchobj.group(1)
220 # Known non-numeric HTML entity
221 if entity in htmlentitydefs.name2codepoint:
222 return unichr(htmlentitydefs.name2codepoint[entity])
225 mobj = re.match(ur'(?u)#(x?\d+)', entity)
227 numstr = mobj.group(1)
228 if numstr.startswith(u'x'):
230 numstr = u'0%s' % numstr
233 return unichr(long(numstr, base))
235 # Unknown entity in name, return its literal representation
236 return (u'&%s;' % entity)
239 def sanitize_title(utitle):
240 """Sanitizes a video title so it could be used as part of a filename."""
241 utitle = re.sub(ur'(?u)&(.+?);', htmlentity_transform, utitle)
242 return utitle.replace(unicode(os.sep), u'%')
245 def sanitize_open(filename, open_mode):
246 """Try to open the given filename, and slightly tweak it if this fails.
248 Attempts to open the given filename. If this fails, it tries to change
249 the filename slightly, step by step, until it's either able to open it
250 or it fails and raises a final exception, like the standard open()
253 It returns the tuple (stream, definitive_file_name).
257 if sys.platform == 'win32':
259 msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY)
260 return (sys.stdout, filename)
261 stream = open(filename, open_mode)
262 return (stream, filename)
263 except (IOError, OSError), err:
264 # In case of error, try to remove win32 forbidden chars
265 filename = re.sub(ur'[/<>:"\|\?\*]', u'#', filename)
267 # An exception here should be caught in the caller
268 stream = open(filename, open_mode)
269 return (stream, filename)
272 def timeconvert(timestr):
273 """Convert RFC 2822 defined time string into system timestamp"""
275 timetuple = email.utils.parsedate_tz(timestr)
276 if timetuple is not None:
277 timestamp = email.utils.mktime_tz(timetuple)
280 def _simplify_title(title):
281 expr = re.compile(ur'[^\w\d_\-]+', flags=re.UNICODE)
282 return expr.sub(u'_', title).strip(u'_')
284 class DownloadError(Exception):
285 """Download Error exception.
287 This exception may be thrown by FileDownloader objects if they are not
288 configured to continue on errors. They will contain the appropriate
294 class SameFileError(Exception):
295 """Same File exception.
297 This exception will be thrown by FileDownloader objects if they detect
298 multiple files would have to be downloaded to the same file on disk.
303 class PostProcessingError(Exception):
304 """Post Processing exception.
306 This exception may be raised by PostProcessor's .run() method to
307 indicate an error in the postprocessing task.
312 class UnavailableVideoError(Exception):
313 """Unavailable Format exception.
315 This exception will be thrown when a video is requested
316 in a format that is not available for that video.
321 class ContentTooShortError(Exception):
322 """Content Too Short exception.
324 This exception may be raised by FileDownloader objects when a file they
325 download is too small for what the server announced first, indicating
326 the connection was probably interrupted.
332 def __init__(self, downloaded, expected):
333 self.downloaded = downloaded
334 self.expected = expected
337 class YoutubeDLHandler(urllib2.HTTPHandler):
338 """Handler for HTTP requests and responses.
340 This class, when installed with an OpenerDirector, automatically adds
341 the standard headers to every HTTP request and handles gzipped and
342 deflated responses from web servers. If compression is to be avoided in
343 a particular request, the original request in the program code only has
344 to include the HTTP header "Youtubedl-No-Compression", which will be
345 removed before making the real request.
347 Part of this code was copied from:
349 http://techknack.net/python-urllib2-handlers/
351 Andrew Rowls, the author of that code, agreed to release it to the
358 return zlib.decompress(data, -zlib.MAX_WBITS)
360 return zlib.decompress(data)
363 def addinfourl_wrapper(stream, headers, url, code):
364 if hasattr(urllib2.addinfourl, 'getcode'):
365 return urllib2.addinfourl(stream, headers, url, code)
366 ret = urllib2.addinfourl(stream, headers, url)
370 def http_request(self, req):
371 for h in std_headers:
374 req.add_header(h, std_headers[h])
375 if 'Youtubedl-no-compression' in req.headers:
376 if 'Accept-encoding' in req.headers:
377 del req.headers['Accept-encoding']
378 del req.headers['Youtubedl-no-compression']
381 def http_response(self, req, resp):
384 if resp.headers.get('Content-encoding', '') == 'gzip':
385 gz = gzip.GzipFile(fileobj=StringIO.StringIO(resp.read()), mode='r')
386 resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
387 resp.msg = old_resp.msg
389 if resp.headers.get('Content-encoding', '') == 'deflate':
390 gz = StringIO.StringIO(self.deflate(resp.read()))
391 resp = self.addinfourl_wrapper(gz, old_resp.headers, old_resp.url, old_resp.code)
392 resp.msg = old_resp.msg
396 class FileDownloader(object):
397 """File Downloader class.
399 File downloader objects are the ones responsible of downloading the
400 actual video file and writing it to disk if the user has requested
401 it, among some other tasks. In most cases there should be one per
402 program. As, given a video URL, the downloader doesn't know how to
403 extract all the needed information, task that InfoExtractors do, it
404 has to pass the URL to one of them.
406 For this, file downloader objects have a method that allows
407 InfoExtractors to be registered in a given order. When it is passed
408 a URL, the file downloader handles it to the first InfoExtractor it
409 finds that reports being able to handle it. The InfoExtractor extracts
410 all the information about the video or videos the URL refers to, and
411 asks the FileDownloader to process the video information, possibly
412 downloading the video.
414 File downloaders accept a lot of parameters. In order not to saturate
415 the object constructor with arguments, it receives a dictionary of
416 options instead. These options are available through the params
417 attribute for the InfoExtractors to use. The FileDownloader also
418 registers itself as the downloader in charge for the InfoExtractors
419 that are added to it, so this is a "mutual registration".
423 username: Username for authentication purposes.
424 password: Password for authentication purposes.
425 usenetrc: Use netrc for authentication instead.
426 quiet: Do not print messages to stdout.
427 forceurl: Force printing final URL.
428 forcetitle: Force printing title.
429 forcethumbnail: Force printing thumbnail URL.
430 forcedescription: Force printing description.
431 forcefilename: Force printing final filename.
432 simulate: Do not download the video files.
433 format: Video format code.
434 format_limit: Highest quality format to try.
435 outtmpl: Template for output names.
436 ignoreerrors: Do not stop on download errors.
437 ratelimit: Download speed limit, in bytes/sec.
438 nooverwrites: Prevent overwriting files.
439 retries: Number of times to retry for HTTP error 5xx
440 continuedl: Try to continue downloads if possible.
441 noprogress: Do not print the progress bar.
442 playliststart: Playlist item to start at.
443 playlistend: Playlist item to end at.
444 matchtitle: Download only matching titles.
445 rejecttitle: Reject downloads for matching titles.
446 logtostderr: Log messages to stderr instead of stdout.
447 consoletitle: Display progress in console window's titlebar.
448 nopart: Do not use temporary .part files.
449 updatetime: Use the Last-modified header to set output file timestamps.
450 writedescription: Write the video description to a .description file
451 writeinfojson: Write the video description to a .info.json file
457 _download_retcode = None
458 _num_downloads = None
461 def __init__(self, params):
462 """Create a FileDownloader object with the given options."""
465 self._download_retcode = 0
466 self._num_downloads = 0
467 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
471 def format_bytes(bytes):
474 if type(bytes) is str:
479 exponent = long(math.log(bytes, 1024.0))
480 suffix = 'bkMGTPEZY'[exponent]
481 converted = float(bytes) / float(1024 ** exponent)
482 return '%.2f%s' % (converted, suffix)
485 def calc_percent(byte_counter, data_len):
488 return '%6s' % ('%3.1f%%' % (float(byte_counter) / float(data_len) * 100.0))
491 def calc_eta(start, now, total, current):
495 if current == 0 or dif < 0.001: # One millisecond
497 rate = float(current) / dif
498 eta = long((float(total) - float(current)) / rate)
499 (eta_mins, eta_secs) = divmod(eta, 60)
502 return '%02d:%02d' % (eta_mins, eta_secs)
505 def calc_speed(start, now, bytes):
507 if bytes == 0 or dif < 0.001: # One millisecond
508 return '%10s' % '---b/s'
509 return '%10s' % ('%s/s' % FileDownloader.format_bytes(float(bytes) / dif))
512 def best_block_size(elapsed_time, bytes):
513 new_min = max(bytes / 2.0, 1.0)
514 new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
515 if elapsed_time < 0.001:
517 rate = bytes / elapsed_time
525 def parse_bytes(bytestr):
526 """Parse a string indicating a byte quantity into a long integer."""
527 matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
530 number = float(matchobj.group(1))
531 multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
532 return long(round(number * multiplier))
534 def add_info_extractor(self, ie):
535 """Add an InfoExtractor object to the end of the list."""
537 ie.set_downloader(self)
539 def add_post_processor(self, pp):
540 """Add a PostProcessor object to the end of the chain."""
542 pp.set_downloader(self)
544 def to_screen(self, message, skip_eol=False, ignore_encoding_errors=False):
545 """Print message to stdout if not in quiet mode."""
547 if not self.params.get('quiet', False):
548 terminator = [u'\n', u''][skip_eol]
549 print >>self._screen_file, (u'%s%s' % (message, terminator)).encode(preferredencoding()),
550 self._screen_file.flush()
551 except (UnicodeEncodeError), err:
552 if not ignore_encoding_errors:
555 def to_stderr(self, message):
556 """Print message to stderr."""
557 print >>sys.stderr, message.encode(preferredencoding())
559 def to_cons_title(self, message):
560 """Set console/terminal window title to message."""
561 if not self.params.get('consoletitle', False):
563 if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
564 # c_wchar_p() might not be necessary if `message` is
565 # already of type unicode()
566 ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
567 elif 'TERM' in os.environ:
568 sys.stderr.write('\033]0;%s\007' % message.encode(preferredencoding()))
570 def fixed_template(self):
571 """Checks if the output template is fixed."""
572 return (re.search(ur'(?u)%\(.+?\)s', self.params['outtmpl']) is None)
574 def trouble(self, message=None):
575 """Determine action to take when a download problem appears.
577 Depending on if the downloader has been configured to ignore
578 download errors or not, this method may throw an exception or
579 not when errors are found, after printing the message.
581 if message is not None:
582 self.to_stderr(message)
583 if not self.params.get('ignoreerrors', False):
584 raise DownloadError(message)
585 self._download_retcode = 1
587 def slow_down(self, start_time, byte_counter):
588 """Sleep if the download speed is over the rate limit."""
589 rate_limit = self.params.get('ratelimit', None)
590 if rate_limit is None or byte_counter == 0:
593 elapsed = now - start_time
596 speed = float(byte_counter) / elapsed
597 if speed > rate_limit:
598 time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
600 def temp_name(self, filename):
601 """Returns a temporary filename for the given filename."""
602 if self.params.get('nopart', False) or filename == u'-' or \
603 (os.path.exists(filename) and not os.path.isfile(filename)):
605 return filename + u'.part'
607 def undo_temp_name(self, filename):
608 if filename.endswith(u'.part'):
609 return filename[:-len(u'.part')]
612 def try_rename(self, old_filename, new_filename):
614 if old_filename == new_filename:
616 os.rename(old_filename, new_filename)
617 except (IOError, OSError), err:
618 self.trouble(u'ERROR: unable to rename file')
620 def try_utime(self, filename, last_modified_hdr):
621 """Try to set the last-modified time of the given file."""
622 if last_modified_hdr is None:
624 if not os.path.isfile(filename):
626 timestr = last_modified_hdr
629 filetime = timeconvert(timestr)
633 os.utime(filename, (time.time(), filetime))
638 def report_writedescription(self, descfn):
639 """ Report that the description file is being written """
640 self.to_screen(u'[info] Writing video description to: %s' % descfn, ignore_encoding_errors=True)
642 def report_writeinfojson(self, infofn):
643 """ Report that the metadata file has been written """
644 self.to_screen(u'[info] Video description metadata as JSON to: %s' % infofn, ignore_encoding_errors=True)
646 def report_destination(self, filename):
647 """Report destination filename."""
648 self.to_screen(u'[download] Destination: %s' % filename, ignore_encoding_errors=True)
650 def report_progress(self, percent_str, data_len_str, speed_str, eta_str):
651 """Report download progress."""
652 if self.params.get('noprogress', False):
654 self.to_screen(u'\r[download] %s of %s at %s ETA %s' %
655 (percent_str, data_len_str, speed_str, eta_str), skip_eol=True)
656 self.to_cons_title(u'youtube-dl - %s of %s at %s ETA %s' %
657 (percent_str.strip(), data_len_str.strip(), speed_str.strip(), eta_str.strip()))
659 def report_resuming_byte(self, resume_len):
660 """Report attempt to resume at given byte."""
661 self.to_screen(u'[download] Resuming download at byte %s' % resume_len)
663 def report_retry(self, count, retries):
664 """Report retry in case of HTTP error 5xx"""
665 self.to_screen(u'[download] Got server HTTP error. Retrying (attempt %d of %d)...' % (count, retries))
667 def report_file_already_downloaded(self, file_name):
668 """Report file has already been fully downloaded."""
670 self.to_screen(u'[download] %s has already been downloaded' % file_name)
671 except (UnicodeEncodeError), err:
672 self.to_screen(u'[download] The file has already been downloaded')
674 def report_unable_to_resume(self):
675 """Report it was impossible to resume download."""
676 self.to_screen(u'[download] Unable to resume')
678 def report_finish(self):
679 """Report download finished."""
680 if self.params.get('noprogress', False):
681 self.to_screen(u'[download] Download completed')
685 def increment_downloads(self):
686 """Increment the ordinal that assigns a number to each file."""
687 self._num_downloads += 1
689 def prepare_filename(self, info_dict):
690 """Generate the output filename."""
692 template_dict = dict(info_dict)
693 template_dict['epoch'] = unicode(long(time.time()))
694 template_dict['autonumber'] = unicode('%05d' % self._num_downloads)
695 filename = self.params['outtmpl'] % template_dict
697 except (ValueError, KeyError), err:
698 self.trouble(u'ERROR: invalid system charset or erroneous output template')
701 def process_info(self, info_dict):
702 """Process a single dictionary returned by an InfoExtractor."""
703 filename = self.prepare_filename(info_dict)
706 if self.params.get('forcetitle', False):
707 print info_dict['title'].encode(preferredencoding(), 'xmlcharrefreplace')
708 if self.params.get('forceurl', False):
709 print info_dict['url'].encode(preferredencoding(), 'xmlcharrefreplace')
710 if self.params.get('forcethumbnail', False) and 'thumbnail' in info_dict:
711 print info_dict['thumbnail'].encode(preferredencoding(), 'xmlcharrefreplace')
712 if self.params.get('forcedescription', False) and 'description' in info_dict:
713 print info_dict['description'].encode(preferredencoding(), 'xmlcharrefreplace')
714 if self.params.get('forcefilename', False) and filename is not None:
715 print filename.encode(preferredencoding(), 'xmlcharrefreplace')
716 if self.params.get('forceformat', False):
717 print info_dict['format'].encode(preferredencoding(), 'xmlcharrefreplace')
719 # Do nothing else if in simulate mode
720 if self.params.get('simulate', False):
726 matchtitle=self.params.get('matchtitle',False)
727 rejecttitle=self.params.get('rejecttitle',False)
728 title=info_dict['title'].encode(preferredencoding(), 'xmlcharrefreplace')
729 if matchtitle and not re.search(matchtitle, title, re.IGNORECASE):
730 self.to_screen(u'[download] "%s" title did not match pattern "%s"' % (title, matchtitle))
732 if rejecttitle and re.search(rejecttitle, title, re.IGNORECASE):
733 self.to_screen(u'[download] "%s" title matched reject pattern "%s"' % (title, rejecttitle))
736 if self.params.get('nooverwrites', False) and os.path.exists(filename):
737 self.to_stderr(u'WARNING: file exists and will be skipped')
741 dn = os.path.dirname(filename)
742 if dn != '' and not os.path.exists(dn):
744 except (OSError, IOError), err:
745 self.trouble(u'ERROR: unable to create directory ' + unicode(err))
748 if self.params.get('writedescription', False):
750 descfn = filename + '.description'
751 self.report_writedescription(descfn)
752 descfile = open(descfn, 'wb')
754 descfile.write(info_dict['description'].encode('utf-8'))
757 except (OSError, IOError):
758 self.trouble(u'ERROR: Cannot write description file ' + descfn)
761 if self.params.get('writeinfojson', False):
762 infofn = filename + '.info.json'
763 self.report_writeinfojson(infofn)
766 except (NameError,AttributeError):
767 self.trouble(u'ERROR: No JSON encoder found. Update to Python 2.6+, setup a json module, or leave out --write-info-json.')
770 infof = open(infofn, 'wb')
772 json_info_dict = dict((k,v) for k,v in info_dict.iteritems() if not k in ('urlhandle',))
773 json.dump(json_info_dict, infof)
776 except (OSError, IOError):
777 self.trouble(u'ERROR: Cannot write metadata to JSON file ' + infofn)
780 if not self.params.get('skip_download', False):
782 success = self._do_download(filename, info_dict)
783 except (OSError, IOError), err:
784 raise UnavailableVideoError
785 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
786 self.trouble(u'ERROR: unable to download video data: %s' % str(err))
788 except (ContentTooShortError, ), err:
789 self.trouble(u'ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
794 self.post_process(filename, info_dict)
795 except (PostProcessingError), err:
796 self.trouble(u'ERROR: postprocessing: %s' % str(err))
799 def download(self, url_list):
800 """Download a given list of URLs."""
801 if len(url_list) > 1 and self.fixed_template():
802 raise SameFileError(self.params['outtmpl'])
805 suitable_found = False
807 # Go to next InfoExtractor if not suitable
808 if not ie.suitable(url):
811 # Suitable InfoExtractor found
812 suitable_found = True
814 # Extract information from URL and process it
817 # Suitable InfoExtractor had been found; go to next URL
820 if not suitable_found:
821 self.trouble(u'ERROR: no suitable InfoExtractor: %s' % url)
823 return self._download_retcode
825 def post_process(self, filename, ie_info):
826 """Run the postprocessing chain on the given file."""
828 info['filepath'] = filename
834 def _download_with_rtmpdump(self, filename, url, player_url):
835 self.report_destination(filename)
836 tmpfilename = self.temp_name(filename)
838 # Check for rtmpdump first
840 subprocess.call(['rtmpdump', '-h'], stdout=(file(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
841 except (OSError, IOError):
842 self.trouble(u'ERROR: RTMP download detected but "rtmpdump" could not be run')
845 # Download using rtmpdump. rtmpdump returns exit code 2 when
846 # the connection was interrumpted and resuming appears to be
847 # possible. This is part of rtmpdump's normal usage, AFAIK.
848 basic_args = ['rtmpdump', '-q'] + [[], ['-W', player_url]][player_url is not None] + ['-r', url, '-o', tmpfilename]
849 retval = subprocess.call(basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)])
850 while retval == 2 or retval == 1:
851 prevsize = os.path.getsize(tmpfilename)
852 self.to_screen(u'\r[rtmpdump] %s bytes' % prevsize, skip_eol=True)
853 time.sleep(5.0) # This seems to be needed
854 retval = subprocess.call(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1])
855 cursize = os.path.getsize(tmpfilename)
856 if prevsize == cursize and retval == 1:
858 # Some rtmp streams seem abort after ~ 99.8%. Don't complain for those
859 if prevsize == cursize and retval == 2 and cursize > 1024:
860 self.to_screen(u'\r[rtmpdump] Could not download the whole video. This can happen for some advertisements.')
864 self.to_screen(u'\r[rtmpdump] %s bytes' % os.path.getsize(tmpfilename))
865 self.try_rename(tmpfilename, filename)
868 self.trouble(u'\nERROR: rtmpdump exited with code %d' % retval)
871 def _do_download(self, filename, info_dict):
872 url = info_dict['url']
873 player_url = info_dict.get('player_url', None)
875 # Check file already present
876 if self.params.get('continuedl', False) and os.path.isfile(filename) and not self.params.get('nopart', False):
877 self.report_file_already_downloaded(filename)
880 # Attempt to download using rtmpdump
881 if url.startswith('rtmp'):
882 return self._download_with_rtmpdump(filename, url, player_url)
884 tmpfilename = self.temp_name(filename)
887 # Do not include the Accept-Encoding header
888 headers = {'Youtubedl-no-compression': 'True'}
889 basic_request = urllib2.Request(url, None, headers)
890 request = urllib2.Request(url, None, headers)
892 # Establish possible resume length
893 if os.path.isfile(tmpfilename):
894 resume_len = os.path.getsize(tmpfilename)
900 if self.params.get('continuedl', False):
901 self.report_resuming_byte(resume_len)
902 request.add_header('Range','bytes=%d-' % resume_len)
908 retries = self.params.get('retries', 0)
909 while count <= retries:
910 # Establish connection
912 if count == 0 and 'urlhandle' in info_dict:
913 data = info_dict['urlhandle']
914 data = urllib2.urlopen(request)
916 except (urllib2.HTTPError, ), err:
917 if (err.code < 500 or err.code >= 600) and err.code != 416:
918 # Unexpected HTTP error
920 elif err.code == 416:
921 # Unable to resume (requested range not satisfiable)
923 # Open the connection again without the range header
924 data = urllib2.urlopen(basic_request)
925 content_length = data.info()['Content-Length']
926 except (urllib2.HTTPError, ), err:
927 if err.code < 500 or err.code >= 600:
930 # Examine the reported length
931 if (content_length is not None and
932 (resume_len - 100 < long(content_length) < resume_len + 100)):
933 # The file had already been fully downloaded.
934 # Explanation to the above condition: in issue #175 it was revealed that
935 # YouTube sometimes adds or removes a few bytes from the end of the file,
936 # changing the file size slightly and causing problems for some users. So
937 # I decided to implement a suggested change and consider the file
938 # completely downloaded if the file size differs less than 100 bytes from
939 # the one in the hard drive.
940 self.report_file_already_downloaded(filename)
941 self.try_rename(tmpfilename, filename)
944 # The length does not match, we start the download over
945 self.report_unable_to_resume()
951 self.report_retry(count, retries)
954 self.trouble(u'ERROR: giving up after %s retries' % retries)
957 data_len = data.info().get('Content-length', None)
958 if data_len is not None:
959 data_len = long(data_len) + resume_len
960 data_len_str = self.format_bytes(data_len)
961 byte_counter = 0 + resume_len
967 data_block = data.read(block_size)
969 if len(data_block) == 0:
971 byte_counter += len(data_block)
973 # Open file just in time
976 (stream, tmpfilename) = sanitize_open(tmpfilename, open_mode)
977 assert stream is not None
978 filename = self.undo_temp_name(tmpfilename)
979 self.report_destination(filename)
980 except (OSError, IOError), err:
981 self.trouble(u'ERROR: unable to open for writing: %s' % str(err))
984 stream.write(data_block)
985 except (IOError, OSError), err:
986 self.trouble(u'\nERROR: unable to write data: %s' % str(err))
988 block_size = self.best_block_size(after - before, len(data_block))
991 speed_str = self.calc_speed(start, time.time(), byte_counter - resume_len)
993 self.report_progress('Unknown %', data_len_str, speed_str, 'Unknown ETA')
995 percent_str = self.calc_percent(byte_counter, data_len)
996 eta_str = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len)
997 self.report_progress(percent_str, data_len_str, speed_str, eta_str)
1000 self.slow_down(start, byte_counter - resume_len)
1003 self.trouble(u'\nERROR: Did not get any data blocks')
1006 self.report_finish()
1007 if data_len is not None and byte_counter != data_len:
1008 raise ContentTooShortError(byte_counter, long(data_len))
1009 self.try_rename(tmpfilename, filename)
1011 # Update file modification time
1012 if self.params.get('updatetime', True):
1013 info_dict['filetime'] = self.try_utime(filename, data.info().get('last-modified', None))
1018 class InfoExtractor(object):
1019 """Information Extractor class.
1021 Information extractors are the classes that, given a URL, extract
1022 information from the video (or videos) the URL refers to. This
1023 information includes the real video URL, the video title and simplified
1024 title, author and others. The information is stored in a dictionary
1025 which is then passed to the FileDownloader. The FileDownloader
1026 processes this information possibly downloading the video to the file
1027 system, among other possible outcomes. The dictionaries must include
1028 the following fields:
1030 id: Video identifier.
1031 url: Final video URL.
1032 uploader: Nickname of the video uploader.
1033 title: Literal title.
1034 stitle: Simplified title.
1035 ext: Video filename extension.
1036 format: Video format.
1037 player_url: SWF Player URL (may be None).
1039 The following fields are optional. Their primary purpose is to allow
1040 youtube-dl to serve as the backend for a video search function, such
1041 as the one in youtube2mp3. They are only used when their respective
1042 forced printing functions are called:
1044 thumbnail: Full URL to a video thumbnail image.
1045 description: One-line video description.
1047 Subclasses of this one should re-define the _real_initialize() and
1048 _real_extract() methods and define a _VALID_URL regexp.
1049 Probably, they should also be added to the list of extractors.
1055 def __init__(self, downloader=None):
1056 """Constructor. Receives an optional downloader."""
1058 self.set_downloader(downloader)
1060 def suitable(self, url):
1061 """Receives a URL and returns True if suitable for this IE."""
1062 return re.match(self._VALID_URL, url) is not None
1064 def initialize(self):
1065 """Initializes an instance (authentication, etc)."""
1067 self._real_initialize()
1070 def extract(self, url):
1071 """Extracts URL information and returns it in list of dicts."""
1073 return self._real_extract(url)
1075 def set_downloader(self, downloader):
1076 """Sets the downloader for this IE."""
1077 self._downloader = downloader
1079 def _real_initialize(self):
1080 """Real initialization process. Redefine in subclasses."""
1083 def _real_extract(self, url):
1084 """Real extraction process. Redefine in subclasses."""
1088 class YoutubeIE(InfoExtractor):
1089 """Information extractor for youtube.com."""
1091 _VALID_URL = r'^((?:https?://)?(?:youtu\.be/|(?:\w+\.)?youtube(?:-nocookie)?\.com/)(?!view_play_list|my_playlists|artist|playlist)(?:(?:(?:v|embed|e)/)|(?:(?:watch(?:_popup)?(?:\.php)?)?(?:\?|#!?)(?:.+&)?v=))?)?([0-9A-Za-z_-]+)(?(1).+)?$'
1092 _LANG_URL = r'http://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
1093 _LOGIN_URL = 'https://www.youtube.com/signup?next=/&gl=US&hl=en'
1094 _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
1095 _NETRC_MACHINE = 'youtube'
1096 # Listed in order of quality
1097 _available_formats = ['38', '37', '22', '45', '35', '44', '34', '18', '43', '6', '5', '17', '13']
1098 _video_extensions = {
1104 '38': 'video', # You actually don't know if this will be MOV, AVI or whatever
1109 _video_dimensions = {
1124 IE_NAME = u'youtube'
1126 def report_lang(self):
1127 """Report attempt to set language."""
1128 self._downloader.to_screen(u'[youtube] Setting language')
1130 def report_login(self):
1131 """Report attempt to log in."""
1132 self._downloader.to_screen(u'[youtube] Logging in')
1134 def report_age_confirmation(self):
1135 """Report attempt to confirm age."""
1136 self._downloader.to_screen(u'[youtube] Confirming age')
1138 def report_video_webpage_download(self, video_id):
1139 """Report attempt to download video webpage."""
1140 self._downloader.to_screen(u'[youtube] %s: Downloading video webpage' % video_id)
1142 def report_video_info_webpage_download(self, video_id):
1143 """Report attempt to download video info webpage."""
1144 self._downloader.to_screen(u'[youtube] %s: Downloading video info webpage' % video_id)
1146 def report_information_extraction(self, video_id):
1147 """Report attempt to extract video information."""
1148 self._downloader.to_screen(u'[youtube] %s: Extracting video information' % video_id)
1150 def report_unavailable_format(self, video_id, format):
1151 """Report extracted video URL."""
1152 self._downloader.to_screen(u'[youtube] %s: Format %s not available' % (video_id, format))
1154 def report_rtmp_download(self):
1155 """Indicate the download will use the RTMP protocol."""
1156 self._downloader.to_screen(u'[youtube] RTMP download detected')
1158 def _print_formats(self, formats):
1159 print 'Available formats:'
1161 print '%s\t:\t%s\t[%s]' %(x, self._video_extensions.get(x, 'flv'), self._video_dimensions.get(x, '???'))
1163 def _real_initialize(self):
1164 if self._downloader is None:
1169 downloader_params = self._downloader.params
1171 # Attempt to use provided username and password or .netrc data
1172 if downloader_params.get('username', None) is not None:
1173 username = downloader_params['username']
1174 password = downloader_params['password']
1175 elif downloader_params.get('usenetrc', False):
1177 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
1178 if info is not None:
1182 raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
1183 except (IOError, netrc.NetrcParseError), err:
1184 self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % str(err))
1188 request = urllib2.Request(self._LANG_URL)
1191 urllib2.urlopen(request).read()
1192 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1193 self._downloader.to_stderr(u'WARNING: unable to set language: %s' % str(err))
1196 # No authentication to be performed
1197 if username is None:
1202 'current_form': 'loginForm',
1204 'action_login': 'Log In',
1205 'username': username,
1206 'password': password,
1208 request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form))
1211 login_results = urllib2.urlopen(request).read()
1212 if re.search(r'(?i)<form[^>]* name="loginForm"', login_results) is not None:
1213 self._downloader.to_stderr(u'WARNING: unable to log in: bad username or password')
1215 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1216 self._downloader.to_stderr(u'WARNING: unable to log in: %s' % str(err))
1222 'action_confirm': 'Confirm',
1224 request = urllib2.Request(self._AGE_URL, urllib.urlencode(age_form))
1226 self.report_age_confirmation()
1227 age_results = urllib2.urlopen(request).read()
1228 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1229 self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
1232 def _real_extract(self, url):
1233 # Extract video id from URL
1234 mobj = re.match(self._VALID_URL, url)
1236 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
1238 video_id = mobj.group(2)
1241 self.report_video_webpage_download(video_id)
1242 request = urllib2.Request('http://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1' % video_id)
1244 video_webpage = urllib2.urlopen(request).read()
1245 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1246 self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
1249 # Attempt to extract SWF player URL
1250 mobj = re.search(r'swfConfig.*?"(http:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)
1251 if mobj is not None:
1252 player_url = re.sub(r'\\(.)', r'\1', mobj.group(1))
1257 self.report_video_info_webpage_download(video_id)
1258 for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']:
1259 video_info_url = ('http://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
1260 % (video_id, el_type))
1261 request = urllib2.Request(video_info_url)
1263 video_info_webpage = urllib2.urlopen(request).read()
1264 video_info = parse_qs(video_info_webpage)
1265 if 'token' in video_info:
1267 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1268 self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % str(err))
1270 if 'token' not in video_info:
1271 if 'reason' in video_info:
1272 self._downloader.trouble(u'ERROR: YouTube said: %s' % video_info['reason'][0].decode('utf-8'))
1274 self._downloader.trouble(u'ERROR: "token" parameter not in video info for unknown reason')
1277 # Start extracting information
1278 self.report_information_extraction(video_id)
1281 if 'author' not in video_info:
1282 self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
1284 video_uploader = urllib.unquote_plus(video_info['author'][0])
1287 if 'title' not in video_info:
1288 self._downloader.trouble(u'ERROR: unable to extract video title')
1290 video_title = urllib.unquote_plus(video_info['title'][0])
1291 video_title = video_title.decode('utf-8')
1292 video_title = sanitize_title(video_title)
1295 simple_title = _simplify_title(video_title)
1298 if 'thumbnail_url' not in video_info:
1299 self._downloader.trouble(u'WARNING: unable to extract video thumbnail')
1300 video_thumbnail = ''
1301 else: # don't panic if we can't find it
1302 video_thumbnail = urllib.unquote_plus(video_info['thumbnail_url'][0])
1306 mobj = re.search(r'id="eow-date.*?>(.*?)</span>', video_webpage, re.DOTALL)
1307 if mobj is not None:
1308 upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())
1309 format_expressions = ['%d %B %Y', '%B %d %Y', '%b %d %Y']
1310 for expression in format_expressions:
1312 upload_date = datetime.datetime.strptime(upload_date, expression).strftime('%Y%m%d')
1320 video_description = u'No description available.'
1321 if self._downloader.params.get('forcedescription', False) or self._downloader.params.get('writedescription', False):
1322 mobj = re.search(r'<meta name="description" content="(.*)"(?:\s*/)?>', video_webpage)
1323 if mobj is not None:
1324 video_description = mobj.group(1).decode('utf-8')
1326 html_parser = lxml.etree.HTMLParser(encoding='utf-8')
1327 vwebpage_doc = lxml.etree.parse(StringIO.StringIO(video_webpage), html_parser)
1328 video_description = u''.join(vwebpage_doc.xpath('id("eow-description")//text()'))
1329 # TODO use another parser
1332 video_token = urllib.unquote_plus(video_info['token'][0])
1334 # Decide which formats to download
1335 req_format = self._downloader.params.get('format', None)
1337 if 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
1338 self.report_rtmp_download()
1339 video_url_list = [(None, video_info['conn'][0])]
1340 elif 'url_encoded_fmt_stream_map' in video_info and len(video_info['url_encoded_fmt_stream_map']) >= 1:
1341 url_data_strs = video_info['url_encoded_fmt_stream_map'][0].split(',')
1342 url_data = [parse_qs(uds) for uds in url_data_strs]
1343 url_data = filter(lambda ud: 'itag' in ud and 'url' in ud, url_data)
1344 url_map = dict((ud['itag'][0], ud['url'][0]) for ud in url_data)
1346 format_limit = self._downloader.params.get('format_limit', None)
1347 if format_limit is not None and format_limit in self._available_formats:
1348 format_list = self._available_formats[self._available_formats.index(format_limit):]
1350 format_list = self._available_formats
1351 existing_formats = [x for x in format_list if x in url_map]
1352 if len(existing_formats) == 0:
1353 self._downloader.trouble(u'ERROR: no known formats available for video')
1355 if self._downloader.params.get('listformats', None):
1356 self._print_formats(existing_formats)
1358 if req_format is None or req_format == 'best':
1359 video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality
1360 elif req_format == 'worst':
1361 video_url_list = [(existing_formats[len(existing_formats)-1], url_map[existing_formats[len(existing_formats)-1]])] # worst quality
1362 elif req_format in ('-1', 'all'):
1363 video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats
1365 # Specific formats. We pick the first in a slash-delimeted sequence.
1366 # For example, if '1/2/3/4' is requested and '2' and '4' are available, we pick '2'.
1367 req_formats = req_format.split('/')
1368 video_url_list = None
1369 for rf in req_formats:
1371 video_url_list = [(rf, url_map[rf])]
1373 if video_url_list is None:
1374 self._downloader.trouble(u'ERROR: requested format not available')
1377 self._downloader.trouble(u'ERROR: no conn or url_encoded_fmt_stream_map information found in video info')
1380 for format_param, video_real_url in video_url_list:
1381 # At this point we have a new video
1382 self._downloader.increment_downloads()
1385 video_extension = self._video_extensions.get(format_param, 'flv')
1388 # Process video information
1389 self._downloader.process_info({
1390 'id': video_id.decode('utf-8'),
1391 'url': video_real_url.decode('utf-8'),
1392 'uploader': video_uploader.decode('utf-8'),
1393 'upload_date': upload_date,
1394 'title': video_title,
1395 'stitle': simple_title,
1396 'ext': video_extension.decode('utf-8'),
1397 'format': (format_param is None and u'NA' or format_param.decode('utf-8')),
1398 'thumbnail': video_thumbnail.decode('utf-8'),
1399 'description': video_description,
1400 'player_url': player_url,
1402 except UnavailableVideoError, err:
1403 self._downloader.trouble(u'\nERROR: unable to download video')
1406 class MetacafeIE(InfoExtractor):
1407 """Information Extractor for metacafe.com."""
1409 _VALID_URL = r'(?:http://)?(?:www\.)?metacafe\.com/watch/([^/]+)/([^/]+)/.*'
1410 _DISCLAIMER = 'http://www.metacafe.com/family_filter/'
1411 _FILTER_POST = 'http://www.metacafe.com/f/index.php?inputType=filter&controllerGroup=user'
1413 IE_NAME = u'metacafe'
1415 def __init__(self, youtube_ie, downloader=None):
1416 InfoExtractor.__init__(self, downloader)
1417 self._youtube_ie = youtube_ie
1419 def report_disclaimer(self):
1420 """Report disclaimer retrieval."""
1421 self._downloader.to_screen(u'[metacafe] Retrieving disclaimer')
1423 def report_age_confirmation(self):
1424 """Report attempt to confirm age."""
1425 self._downloader.to_screen(u'[metacafe] Confirming age')
1427 def report_download_webpage(self, video_id):
1428 """Report webpage download."""
1429 self._downloader.to_screen(u'[metacafe] %s: Downloading webpage' % video_id)
1431 def report_extraction(self, video_id):
1432 """Report information extraction."""
1433 self._downloader.to_screen(u'[metacafe] %s: Extracting information' % video_id)
1435 def _real_initialize(self):
1436 # Retrieve disclaimer
1437 request = urllib2.Request(self._DISCLAIMER)
1439 self.report_disclaimer()
1440 disclaimer = urllib2.urlopen(request).read()
1441 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1442 self._downloader.trouble(u'ERROR: unable to retrieve disclaimer: %s' % str(err))
1448 'submit': "Continue - I'm over 18",
1450 request = urllib2.Request(self._FILTER_POST, urllib.urlencode(disclaimer_form))
1452 self.report_age_confirmation()
1453 disclaimer = urllib2.urlopen(request).read()
1454 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1455 self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
1458 def _real_extract(self, url):
1459 # Extract id and simplified title from URL
1460 mobj = re.match(self._VALID_URL, url)
1462 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
1465 video_id = mobj.group(1)
1467 # Check if video comes from YouTube
1468 mobj2 = re.match(r'^yt-(.*)$', video_id)
1469 if mobj2 is not None:
1470 self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % mobj2.group(1))
1473 # At this point we have a new video
1474 self._downloader.increment_downloads()
1476 simple_title = mobj.group(2).decode('utf-8')
1478 # Retrieve video webpage to extract further information
1479 request = urllib2.Request('http://www.metacafe.com/watch/%s/' % video_id)
1481 self.report_download_webpage(video_id)
1482 webpage = urllib2.urlopen(request).read()
1483 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1484 self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % str(err))
1487 # Extract URL, uploader and title from webpage
1488 self.report_extraction(video_id)
1489 mobj = re.search(r'(?m)&mediaURL=([^&]+)', webpage)
1490 if mobj is not None:
1491 mediaURL = urllib.unquote(mobj.group(1))
1492 video_extension = mediaURL[-3:]
1494 # Extract gdaKey if available
1495 mobj = re.search(r'(?m)&gdaKey=(.*?)&', webpage)
1497 video_url = mediaURL
1499 gdaKey = mobj.group(1)
1500 video_url = '%s?__gda__=%s' % (mediaURL, gdaKey)
1502 mobj = re.search(r' name="flashvars" value="(.*?)"', webpage)
1504 self._downloader.trouble(u'ERROR: unable to extract media URL')
1506 vardict = parse_qs(mobj.group(1))
1507 if 'mediaData' not in vardict:
1508 self._downloader.trouble(u'ERROR: unable to extract media URL')
1510 mobj = re.search(r'"mediaURL":"(http.*?)","key":"(.*?)"', vardict['mediaData'][0])
1512 self._downloader.trouble(u'ERROR: unable to extract media URL')
1514 mediaURL = mobj.group(1).replace('\\/', '/')
1515 video_extension = mediaURL[-3:]
1516 video_url = '%s?__gda__=%s' % (mediaURL, mobj.group(2))
1518 mobj = re.search(r'(?im)<title>(.*) - Video</title>', webpage)
1520 self._downloader.trouble(u'ERROR: unable to extract title')
1522 video_title = mobj.group(1).decode('utf-8')
1523 video_title = sanitize_title(video_title)
1525 mobj = re.search(r'(?ms)By:\s*<a .*?>(.+?)<', webpage)
1527 self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
1529 video_uploader = mobj.group(1)
1532 # Process video information
1533 self._downloader.process_info({
1534 'id': video_id.decode('utf-8'),
1535 'url': video_url.decode('utf-8'),
1536 'uploader': video_uploader.decode('utf-8'),
1537 'upload_date': u'NA',
1538 'title': video_title,
1539 'stitle': simple_title,
1540 'ext': video_extension.decode('utf-8'),
1544 except UnavailableVideoError:
1545 self._downloader.trouble(u'\nERROR: unable to download video')
1548 class DailymotionIE(InfoExtractor):
1549 """Information Extractor for Dailymotion"""
1551 _VALID_URL = r'(?i)(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/video/([^_/]+)_([^/]+)'
1552 IE_NAME = u'dailymotion'
1554 def __init__(self, downloader=None):
1555 InfoExtractor.__init__(self, downloader)
1557 def report_download_webpage(self, video_id):
1558 """Report webpage download."""
1559 self._downloader.to_screen(u'[dailymotion] %s: Downloading webpage' % video_id)
1561 def report_extraction(self, video_id):
1562 """Report information extraction."""
1563 self._downloader.to_screen(u'[dailymotion] %s: Extracting information' % video_id)
1565 def _real_extract(self, url):
1566 # Extract id and simplified title from URL
1567 mobj = re.match(self._VALID_URL, url)
1569 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
1572 # At this point we have a new video
1573 self._downloader.increment_downloads()
1574 video_id = mobj.group(1)
1576 simple_title = mobj.group(2).decode('utf-8')
1577 video_extension = 'flv'
1579 # Retrieve video webpage to extract further information
1580 request = urllib2.Request(url)
1581 request.add_header('Cookie', 'family_filter=off')
1583 self.report_download_webpage(video_id)
1584 webpage = urllib2.urlopen(request).read()
1585 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1586 self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % str(err))
1589 # Extract URL, uploader and title from webpage
1590 self.report_extraction(video_id)
1591 mobj = re.search(r'(?i)addVariable\(\"sequence\"\s*,\s*\"([^\"]+?)\"\)', webpage)
1593 self._downloader.trouble(u'ERROR: unable to extract media URL')
1595 sequence = urllib.unquote(mobj.group(1))
1596 mobj = re.search(r',\"sdURL\"\:\"([^\"]+?)\",', sequence)
1598 self._downloader.trouble(u'ERROR: unable to extract media URL')
1600 mediaURL = urllib.unquote(mobj.group(1)).replace('\\', '')
1602 # if needed add http://www.dailymotion.com/ if relative URL
1604 video_url = mediaURL
1606 mobj = re.search(r'(?im)<title>Dailymotion\s*-\s*(.+)\s*-\s*[^<]+?</title>', webpage)
1608 self._downloader.trouble(u'ERROR: unable to extract title')
1610 video_title = mobj.group(1).decode('utf-8')
1611 video_title = sanitize_title(video_title)
1613 mobj = re.search(r'(?im)<span class="owner[^\"]+?">[^<]+?<a [^>]+?>([^<]+?)</a></span>', webpage)
1615 self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
1617 video_uploader = mobj.group(1)
1620 # Process video information
1621 self._downloader.process_info({
1622 'id': video_id.decode('utf-8'),
1623 'url': video_url.decode('utf-8'),
1624 'uploader': video_uploader.decode('utf-8'),
1625 'upload_date': u'NA',
1626 'title': video_title,
1627 'stitle': simple_title,
1628 'ext': video_extension.decode('utf-8'),
1632 except UnavailableVideoError:
1633 self._downloader.trouble(u'\nERROR: unable to download video')
1636 class GoogleIE(InfoExtractor):
1637 """Information extractor for video.google.com."""
1639 _VALID_URL = r'(?:http://)?video\.google\.(?:com(?:\.au)?|co\.(?:uk|jp|kr|cr)|ca|de|es|fr|it|nl|pl)/videoplay\?docid=([^\&]+).*'
1640 IE_NAME = u'video.google'
1642 def __init__(self, downloader=None):
1643 InfoExtractor.__init__(self, downloader)
1645 def report_download_webpage(self, video_id):
1646 """Report webpage download."""
1647 self._downloader.to_screen(u'[video.google] %s: Downloading webpage' % video_id)
1649 def report_extraction(self, video_id):
1650 """Report information extraction."""
1651 self._downloader.to_screen(u'[video.google] %s: Extracting information' % video_id)
1653 def _real_extract(self, url):
1654 # Extract id from URL
1655 mobj = re.match(self._VALID_URL, url)
1657 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1660 # At this point we have a new video
1661 self._downloader.increment_downloads()
1662 video_id = mobj.group(1)
1664 video_extension = 'mp4'
1666 # Retrieve video webpage to extract further information
1667 request = urllib2.Request('http://video.google.com/videoplay?docid=%s&hl=en&oe=utf-8' % video_id)
1669 self.report_download_webpage(video_id)
1670 webpage = urllib2.urlopen(request).read()
1671 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1672 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1675 # Extract URL, uploader, and title from webpage
1676 self.report_extraction(video_id)
1677 mobj = re.search(r"download_url:'([^']+)'", webpage)
1679 video_extension = 'flv'
1680 mobj = re.search(r"(?i)videoUrl\\x3d(.+?)\\x26", webpage)
1682 self._downloader.trouble(u'ERROR: unable to extract media URL')
1684 mediaURL = urllib.unquote(mobj.group(1))
1685 mediaURL = mediaURL.replace('\\x3d', '\x3d')
1686 mediaURL = mediaURL.replace('\\x26', '\x26')
1688 video_url = mediaURL
1690 mobj = re.search(r'<title>(.*)</title>', webpage)
1692 self._downloader.trouble(u'ERROR: unable to extract title')
1694 video_title = mobj.group(1).decode('utf-8')
1695 video_title = sanitize_title(video_title)
1696 simple_title = _simplify_title(video_title)
1698 # Extract video description
1699 mobj = re.search(r'<span id=short-desc-content>([^<]*)</span>', webpage)
1701 self._downloader.trouble(u'ERROR: unable to extract video description')
1703 video_description = mobj.group(1).decode('utf-8')
1704 if not video_description:
1705 video_description = 'No description available.'
1707 # Extract video thumbnail
1708 if self._downloader.params.get('forcethumbnail', False):
1709 request = urllib2.Request('http://video.google.com/videosearch?q=%s+site:video.google.com&hl=en' % abs(int(video_id)))
1711 webpage = urllib2.urlopen(request).read()
1712 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1713 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1715 mobj = re.search(r'<img class=thumbnail-img (?:.* )?src=(http.*)>', webpage)
1717 self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
1719 video_thumbnail = mobj.group(1)
1720 else: # we need something to pass to process_info
1721 video_thumbnail = ''
1724 # Process video information
1725 self._downloader.process_info({
1726 'id': video_id.decode('utf-8'),
1727 'url': video_url.decode('utf-8'),
1729 'upload_date': u'NA',
1730 'title': video_title,
1731 'stitle': simple_title,
1732 'ext': video_extension.decode('utf-8'),
1736 except UnavailableVideoError:
1737 self._downloader.trouble(u'\nERROR: unable to download video')
1740 class PhotobucketIE(InfoExtractor):
1741 """Information extractor for photobucket.com."""
1743 _VALID_URL = r'(?:http://)?(?:[a-z0-9]+\.)?photobucket\.com/.*[\?\&]current=(.*\.flv)'
1744 IE_NAME = u'photobucket'
1746 def __init__(self, downloader=None):
1747 InfoExtractor.__init__(self, downloader)
1749 def report_download_webpage(self, video_id):
1750 """Report webpage download."""
1751 self._downloader.to_screen(u'[photobucket] %s: Downloading webpage' % video_id)
1753 def report_extraction(self, video_id):
1754 """Report information extraction."""
1755 self._downloader.to_screen(u'[photobucket] %s: Extracting information' % video_id)
1757 def _real_extract(self, url):
1758 # Extract id from URL
1759 mobj = re.match(self._VALID_URL, url)
1761 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1764 # At this point we have a new video
1765 self._downloader.increment_downloads()
1766 video_id = mobj.group(1)
1768 video_extension = 'flv'
1770 # Retrieve video webpage to extract further information
1771 request = urllib2.Request(url)
1773 self.report_download_webpage(video_id)
1774 webpage = urllib2.urlopen(request).read()
1775 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1776 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1779 # Extract URL, uploader, and title from webpage
1780 self.report_extraction(video_id)
1781 mobj = re.search(r'<link rel="video_src" href=".*\?file=([^"]+)" />', webpage)
1783 self._downloader.trouble(u'ERROR: unable to extract media URL')
1785 mediaURL = urllib.unquote(mobj.group(1))
1787 video_url = mediaURL
1789 mobj = re.search(r'<title>(.*) video by (.*) - Photobucket</title>', webpage)
1791 self._downloader.trouble(u'ERROR: unable to extract title')
1793 video_title = mobj.group(1).decode('utf-8')
1794 video_title = sanitize_title(video_title)
1795 simple_title = _simplify_title(vide_title)
1797 video_uploader = mobj.group(2).decode('utf-8')
1800 # Process video information
1801 self._downloader.process_info({
1802 'id': video_id.decode('utf-8'),
1803 'url': video_url.decode('utf-8'),
1804 'uploader': video_uploader,
1805 'upload_date': u'NA',
1806 'title': video_title,
1807 'stitle': simple_title,
1808 'ext': video_extension.decode('utf-8'),
1812 except UnavailableVideoError:
1813 self._downloader.trouble(u'\nERROR: unable to download video')
1816 class YahooIE(InfoExtractor):
1817 """Information extractor for video.yahoo.com."""
1819 # _VALID_URL matches all Yahoo! Video URLs
1820 # _VPAGE_URL matches only the extractable '/watch/' URLs
1821 _VALID_URL = r'(?:http://)?(?:[a-z]+\.)?video\.yahoo\.com/(?:watch|network)/([0-9]+)(?:/|\?v=)([0-9]+)(?:[#\?].*)?'
1822 _VPAGE_URL = r'(?:http://)?video\.yahoo\.com/watch/([0-9]+)/([0-9]+)(?:[#\?].*)?'
1823 IE_NAME = u'video.yahoo'
1825 def __init__(self, downloader=None):
1826 InfoExtractor.__init__(self, downloader)
1828 def report_download_webpage(self, video_id):
1829 """Report webpage download."""
1830 self._downloader.to_screen(u'[video.yahoo] %s: Downloading webpage' % video_id)
1832 def report_extraction(self, video_id):
1833 """Report information extraction."""
1834 self._downloader.to_screen(u'[video.yahoo] %s: Extracting information' % video_id)
1836 def _real_extract(self, url, new_video=True):
1837 # Extract ID from URL
1838 mobj = re.match(self._VALID_URL, url)
1840 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1843 # At this point we have a new video
1844 self._downloader.increment_downloads()
1845 video_id = mobj.group(2)
1846 video_extension = 'flv'
1848 # Rewrite valid but non-extractable URLs as
1849 # extractable English language /watch/ URLs
1850 if re.match(self._VPAGE_URL, url) is None:
1851 request = urllib2.Request(url)
1853 webpage = urllib2.urlopen(request).read()
1854 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1855 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1858 mobj = re.search(r'\("id", "([0-9]+)"\);', webpage)
1860 self._downloader.trouble(u'ERROR: Unable to extract id field')
1862 yahoo_id = mobj.group(1)
1864 mobj = re.search(r'\("vid", "([0-9]+)"\);', webpage)
1866 self._downloader.trouble(u'ERROR: Unable to extract vid field')
1868 yahoo_vid = mobj.group(1)
1870 url = 'http://video.yahoo.com/watch/%s/%s' % (yahoo_vid, yahoo_id)
1871 return self._real_extract(url, new_video=False)
1873 # Retrieve video webpage to extract further information
1874 request = urllib2.Request(url)
1876 self.report_download_webpage(video_id)
1877 webpage = urllib2.urlopen(request).read()
1878 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1879 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1882 # Extract uploader and title from webpage
1883 self.report_extraction(video_id)
1884 mobj = re.search(r'<meta name="title" content="(.*)" />', webpage)
1886 self._downloader.trouble(u'ERROR: unable to extract video title')
1888 video_title = mobj.group(1).decode('utf-8')
1889 simple_title = _simplify_title(video_title)
1891 mobj = re.search(r'<h2 class="ti-5"><a href="http://video\.yahoo\.com/(people|profile)/[0-9]+" beacon=".*">(.*)</a></h2>', webpage)
1893 self._downloader.trouble(u'ERROR: unable to extract video uploader')
1895 video_uploader = mobj.group(1).decode('utf-8')
1897 # Extract video thumbnail
1898 mobj = re.search(r'<link rel="image_src" href="(.*)" />', webpage)
1900 self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
1902 video_thumbnail = mobj.group(1).decode('utf-8')
1904 # Extract video description
1905 mobj = re.search(r'<meta name="description" content="(.*)" />', webpage)
1907 self._downloader.trouble(u'ERROR: unable to extract video description')
1909 video_description = mobj.group(1).decode('utf-8')
1910 if not video_description:
1911 video_description = 'No description available.'
1913 # Extract video height and width
1914 mobj = re.search(r'<meta name="video_height" content="([0-9]+)" />', webpage)
1916 self._downloader.trouble(u'ERROR: unable to extract video height')
1918 yv_video_height = mobj.group(1)
1920 mobj = re.search(r'<meta name="video_width" content="([0-9]+)" />', webpage)
1922 self._downloader.trouble(u'ERROR: unable to extract video width')
1924 yv_video_width = mobj.group(1)
1926 # Retrieve video playlist to extract media URL
1927 # I'm not completely sure what all these options are, but we
1928 # seem to need most of them, otherwise the server sends a 401.
1929 yv_lg = 'R0xx6idZnW2zlrKP8xxAIR' # not sure what this represents
1930 yv_bitrate = '700' # according to Wikipedia this is hard-coded
1931 request = urllib2.Request('http://cosmos.bcst.yahoo.com/up/yep/process/getPlaylistFOP.php?node_id=' + video_id +
1932 '&tech=flash&mode=playlist&lg=' + yv_lg + '&bitrate=' + yv_bitrate + '&vidH=' + yv_video_height +
1933 '&vidW=' + yv_video_width + '&swf=as3&rd=video.yahoo.com&tk=null&adsupported=v1,v2,&eventid=1301797')
1935 self.report_download_webpage(video_id)
1936 webpage = urllib2.urlopen(request).read()
1937 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1938 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
1941 # Extract media URL from playlist XML
1942 mobj = re.search(r'<STREAM APP="(http://.*)" FULLPATH="/?(/.*\.flv\?[^"]*)"', webpage)
1944 self._downloader.trouble(u'ERROR: Unable to extract media URL')
1946 video_url = urllib.unquote(mobj.group(1) + mobj.group(2)).decode('utf-8')
1947 video_url = re.sub(r'(?u)&(.+?);', htmlentity_transform, video_url)
1950 # Process video information
1951 self._downloader.process_info({
1952 'id': video_id.decode('utf-8'),
1954 'uploader': video_uploader,
1955 'upload_date': u'NA',
1956 'title': video_title,
1957 'stitle': simple_title,
1958 'ext': video_extension.decode('utf-8'),
1959 'thumbnail': video_thumbnail.decode('utf-8'),
1960 'description': video_description,
1961 'thumbnail': video_thumbnail,
1964 except UnavailableVideoError:
1965 self._downloader.trouble(u'\nERROR: unable to download video')
1968 class VimeoIE(InfoExtractor):
1969 """Information extractor for vimeo.com."""
1971 # _VALID_URL matches Vimeo URLs
1972 _VALID_URL = r'(?:https?://)?(?:(?:www|player).)?vimeo\.com/(?:groups/[^/]+/)?(?:videos?/)?([0-9]+)'
1975 def __init__(self, downloader=None):
1976 InfoExtractor.__init__(self, downloader)
1978 def report_download_webpage(self, video_id):
1979 """Report webpage download."""
1980 self._downloader.to_screen(u'[vimeo] %s: Downloading webpage' % video_id)
1982 def report_extraction(self, video_id):
1983 """Report information extraction."""
1984 self._downloader.to_screen(u'[vimeo] %s: Extracting information' % video_id)
1986 def _real_extract(self, url, new_video=True):
1987 # Extract ID from URL
1988 mobj = re.match(self._VALID_URL, url)
1990 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1993 # At this point we have a new video
1994 self._downloader.increment_downloads()
1995 video_id = mobj.group(1)
1997 # Retrieve video webpage to extract further information
1998 request = urllib2.Request("http://vimeo.com/moogaloop/load/clip:%s" % video_id, None, std_headers)
2000 self.report_download_webpage(video_id)
2001 webpage = urllib2.urlopen(request).read()
2002 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2003 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
2006 # Now we begin extracting as much information as we can from what we
2007 # retrieved. First we extract the information common to all extractors,
2008 # and latter we extract those that are Vimeo specific.
2009 self.report_extraction(video_id)
2012 mobj = re.search(r'<caption>(.*?)</caption>', webpage)
2014 self._downloader.trouble(u'ERROR: unable to extract video title')
2016 video_title = mobj.group(1).decode('utf-8')
2017 simple_title = _simple_title(video_title)
2020 mobj = re.search(r'<uploader_url>http://vimeo.com/(.*?)</uploader_url>', webpage)
2022 self._downloader.trouble(u'ERROR: unable to extract video uploader')
2024 video_uploader = mobj.group(1).decode('utf-8')
2026 # Extract video thumbnail
2027 mobj = re.search(r'<thumbnail>(.*?)</thumbnail>', webpage)
2029 self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
2031 video_thumbnail = mobj.group(1).decode('utf-8')
2033 # # Extract video description
2034 # mobj = re.search(r'<meta property="og:description" content="(.*)" />', webpage)
2036 # self._downloader.trouble(u'ERROR: unable to extract video description')
2038 # video_description = mobj.group(1).decode('utf-8')
2039 # if not video_description: video_description = 'No description available.'
2040 video_description = 'Foo.'
2042 # Vimeo specific: extract request signature
2043 mobj = re.search(r'<request_signature>(.*?)</request_signature>', webpage)
2045 self._downloader.trouble(u'ERROR: unable to extract request signature')
2047 sig = mobj.group(1).decode('utf-8')
2049 # Vimeo specific: extract video quality information
2050 mobj = re.search(r'<isHD>(\d+)</isHD>', webpage)
2052 self._downloader.trouble(u'ERROR: unable to extract video quality information')
2054 quality = mobj.group(1).decode('utf-8')
2056 if int(quality) == 1:
2061 # Vimeo specific: Extract request signature expiration
2062 mobj = re.search(r'<request_signature_expires>(.*?)</request_signature_expires>', webpage)
2064 self._downloader.trouble(u'ERROR: unable to extract request signature expiration')
2066 sig_exp = mobj.group(1).decode('utf-8')
2068 video_url = "http://vimeo.com/moogaloop/play/clip:%s/%s/%s/?q=%s" % (video_id, sig, sig_exp, quality)
2071 # Process video information
2072 self._downloader.process_info({
2073 'id': video_id.decode('utf-8'),
2075 'uploader': video_uploader,
2076 'upload_date': u'NA',
2077 'title': video_title,
2078 'stitle': simple_title,
2080 'thumbnail': video_thumbnail.decode('utf-8'),
2081 'description': video_description,
2082 'thumbnail': video_thumbnail,
2083 'description': video_description,
2086 except UnavailableVideoError:
2087 self._downloader.trouble(u'ERROR: unable to download video')
2090 class GenericIE(InfoExtractor):
2091 """Generic last-resort information extractor."""
2094 IE_NAME = u'generic'
2096 def __init__(self, downloader=None):
2097 InfoExtractor.__init__(self, downloader)
2099 def report_download_webpage(self, video_id):
2100 """Report webpage download."""
2101 self._downloader.to_screen(u'WARNING: Falling back on generic information extractor.')
2102 self._downloader.to_screen(u'[generic] %s: Downloading webpage' % video_id)
2104 def report_extraction(self, video_id):
2105 """Report information extraction."""
2106 self._downloader.to_screen(u'[generic] %s: Extracting information' % video_id)
2108 def _real_extract(self, url):
2109 # At this point we have a new video
2110 self._downloader.increment_downloads()
2112 video_id = url.split('/')[-1]
2113 request = urllib2.Request(url)
2115 self.report_download_webpage(video_id)
2116 webpage = urllib2.urlopen(request).read()
2117 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2118 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
2120 except ValueError, err:
2121 # since this is the last-resort InfoExtractor, if
2122 # this error is thrown, it'll be thrown here
2123 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
2126 self.report_extraction(video_id)
2127 # Start with something easy: JW Player in SWFObject
2128 mobj = re.search(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage)
2130 # Broaden the search a little bit
2131 mobj = re.search(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage)
2133 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
2136 # It's possible that one of the regexes
2137 # matched, but returned an empty group:
2138 if mobj.group(1) is None:
2139 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
2142 video_url = urllib.unquote(mobj.group(1))
2143 video_id = os.path.basename(video_url)
2145 # here's a fun little line of code for you:
2146 video_extension = os.path.splitext(video_id)[1][1:]
2147 video_id = os.path.splitext(video_id)[0]
2149 # it's tempting to parse this further, but you would
2150 # have to take into account all the variations like
2151 # Video Title - Site Name
2152 # Site Name | Video Title
2153 # Video Title - Tagline | Site Name
2154 # and so on and so forth; it's just not practical
2155 mobj = re.search(r'<title>(.*)</title>', webpage)
2157 self._downloader.trouble(u'ERROR: unable to extract title')
2159 video_title = mobj.group(1).decode('utf-8')
2160 video_title = sanitize_title(video_title)
2161 simple_title = _simplify_title(video_title)
2163 # video uploader is domain name
2164 mobj = re.match(r'(?:https?://)?([^/]*)/.*', url)
2166 self._downloader.trouble(u'ERROR: unable to extract title')
2168 video_uploader = mobj.group(1).decode('utf-8')
2171 # Process video information
2172 self._downloader.process_info({
2173 'id': video_id.decode('utf-8'),
2174 'url': video_url.decode('utf-8'),
2175 'uploader': video_uploader,
2176 'upload_date': u'NA',
2177 'title': video_title,
2178 'stitle': simple_title,
2179 'ext': video_extension.decode('utf-8'),
2183 except UnavailableVideoError, err:
2184 self._downloader.trouble(u'\nERROR: unable to download video')
2187 class YoutubeSearchIE(InfoExtractor):
2188 """Information Extractor for YouTube search queries."""
2189 _VALID_URL = r'ytsearch(\d+|all)?:[\s\S]+'
2190 _TEMPLATE_URL = 'http://www.youtube.com/results?search_query=%s&page=%s&gl=US&hl=en'
2191 _VIDEO_INDICATOR = r'href="/watch\?v=.+?"'
2192 _MORE_PAGES_INDICATOR = r'(?m)>\s*Next\s*</a>'
2194 _max_youtube_results = 1000
2195 IE_NAME = u'youtube:search'
2197 def __init__(self, youtube_ie, downloader=None):
2198 InfoExtractor.__init__(self, downloader)
2199 self._youtube_ie = youtube_ie
2201 def report_download_page(self, query, pagenum):
2202 """Report attempt to download playlist page with given number."""
2203 query = query.decode(preferredencoding())
2204 self._downloader.to_screen(u'[youtube] query "%s": Downloading page %s' % (query, pagenum))
2206 def _real_initialize(self):
2207 self._youtube_ie.initialize()
2209 def _real_extract(self, query):
2210 mobj = re.match(self._VALID_URL, query)
2212 self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
2215 prefix, query = query.split(':')
2217 query = query.encode('utf-8')
2219 self._download_n_results(query, 1)
2221 elif prefix == 'all':
2222 self._download_n_results(query, self._max_youtube_results)
2228 self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
2230 elif n > self._max_youtube_results:
2231 self._downloader.to_stderr(u'WARNING: ytsearch returns max %i results (you requested %i)' % (self._max_youtube_results, n))
2232 n = self._max_youtube_results
2233 self._download_n_results(query, n)
2235 except ValueError: # parsing prefix as integer fails
2236 self._download_n_results(query, 1)
2239 def _download_n_results(self, query, n):
2240 """Downloads a specified number of results for a query"""
2243 already_seen = set()
2247 self.report_download_page(query, pagenum)
2248 result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
2249 request = urllib2.Request(result_url)
2251 page = urllib2.urlopen(request).read()
2252 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2253 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
2256 # Extract video identifiers
2257 for mobj in re.finditer(self._VIDEO_INDICATOR, page):
2258 video_id = page[mobj.span()[0]:mobj.span()[1]].split('=')[2][:-1]
2259 if video_id not in already_seen:
2260 video_ids.append(video_id)
2261 already_seen.add(video_id)
2262 if len(video_ids) == n:
2263 # Specified n videos reached
2264 for id in video_ids:
2265 self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
2268 if re.search(self._MORE_PAGES_INDICATOR, page) is None:
2269 for id in video_ids:
2270 self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
2273 pagenum = pagenum + 1
2276 class GoogleSearchIE(InfoExtractor):
2277 """Information Extractor for Google Video search queries."""
2278 _VALID_URL = r'gvsearch(\d+|all)?:[\s\S]+'
2279 _TEMPLATE_URL = 'http://video.google.com/videosearch?q=%s+site:video.google.com&start=%s&hl=en'
2280 _VIDEO_INDICATOR = r'videoplay\?docid=([^\&>]+)\&'
2281 _MORE_PAGES_INDICATOR = r'<span>Next</span>'
2283 _max_google_results = 1000
2284 IE_NAME = u'video.google:search'
2286 def __init__(self, google_ie, downloader=None):
2287 InfoExtractor.__init__(self, downloader)
2288 self._google_ie = google_ie
2290 def report_download_page(self, query, pagenum):
2291 """Report attempt to download playlist page with given number."""
2292 query = query.decode(preferredencoding())
2293 self._downloader.to_screen(u'[video.google] query "%s": Downloading page %s' % (query, pagenum))
2295 def _real_initialize(self):
2296 self._google_ie.initialize()
2298 def _real_extract(self, query):
2299 mobj = re.match(self._VALID_URL, query)
2301 self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
2304 prefix, query = query.split(':')
2306 query = query.encode('utf-8')
2308 self._download_n_results(query, 1)
2310 elif prefix == 'all':
2311 self._download_n_results(query, self._max_google_results)
2317 self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
2319 elif n > self._max_google_results:
2320 self._downloader.to_stderr(u'WARNING: gvsearch returns max %i results (you requested %i)' % (self._max_google_results, n))
2321 n = self._max_google_results
2322 self._download_n_results(query, n)
2324 except ValueError: # parsing prefix as integer fails
2325 self._download_n_results(query, 1)
2328 def _download_n_results(self, query, n):
2329 """Downloads a specified number of results for a query"""
2332 already_seen = set()
2336 self.report_download_page(query, pagenum)
2337 result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
2338 request = urllib2.Request(result_url)
2340 page = urllib2.urlopen(request).read()
2341 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2342 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
2345 # Extract video identifiers
2346 for mobj in re.finditer(self._VIDEO_INDICATOR, page):
2347 video_id = mobj.group(1)
2348 if video_id not in already_seen:
2349 video_ids.append(video_id)
2350 already_seen.add(video_id)
2351 if len(video_ids) == n:
2352 # Specified n videos reached
2353 for id in video_ids:
2354 self._google_ie.extract('http://video.google.com/videoplay?docid=%s' % id)
2357 if re.search(self._MORE_PAGES_INDICATOR, page) is None:
2358 for id in video_ids:
2359 self._google_ie.extract('http://video.google.com/videoplay?docid=%s' % id)
2362 pagenum = pagenum + 1
2365 class YahooSearchIE(InfoExtractor):
2366 """Information Extractor for Yahoo! Video search queries."""
2367 _VALID_URL = r'yvsearch(\d+|all)?:[\s\S]+'
2368 _TEMPLATE_URL = 'http://video.yahoo.com/search/?p=%s&o=%s'
2369 _VIDEO_INDICATOR = r'href="http://video\.yahoo\.com/watch/([0-9]+/[0-9]+)"'
2370 _MORE_PAGES_INDICATOR = r'\s*Next'
2372 _max_yahoo_results = 1000
2373 IE_NAME = u'video.yahoo:search'
2375 def __init__(self, yahoo_ie, downloader=None):
2376 InfoExtractor.__init__(self, downloader)
2377 self._yahoo_ie = yahoo_ie
2379 def report_download_page(self, query, pagenum):
2380 """Report attempt to download playlist page with given number."""
2381 query = query.decode(preferredencoding())
2382 self._downloader.to_screen(u'[video.yahoo] query "%s": Downloading page %s' % (query, pagenum))
2384 def _real_initialize(self):
2385 self._yahoo_ie.initialize()
2387 def _real_extract(self, query):
2388 mobj = re.match(self._VALID_URL, query)
2390 self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
2393 prefix, query = query.split(':')
2395 query = query.encode('utf-8')
2397 self._download_n_results(query, 1)
2399 elif prefix == 'all':
2400 self._download_n_results(query, self._max_yahoo_results)
2406 self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
2408 elif n > self._max_yahoo_results:
2409 self._downloader.to_stderr(u'WARNING: yvsearch returns max %i results (you requested %i)' % (self._max_yahoo_results, n))
2410 n = self._max_yahoo_results
2411 self._download_n_results(query, n)
2413 except ValueError: # parsing prefix as integer fails
2414 self._download_n_results(query, 1)
2417 def _download_n_results(self, query, n):
2418 """Downloads a specified number of results for a query"""
2421 already_seen = set()
2425 self.report_download_page(query, pagenum)
2426 result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
2427 request = urllib2.Request(result_url)
2429 page = urllib2.urlopen(request).read()
2430 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2431 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
2434 # Extract video identifiers
2435 for mobj in re.finditer(self._VIDEO_INDICATOR, page):
2436 video_id = mobj.group(1)
2437 if video_id not in already_seen:
2438 video_ids.append(video_id)
2439 already_seen.add(video_id)
2440 if len(video_ids) == n:
2441 # Specified n videos reached
2442 for id in video_ids:
2443 self._yahoo_ie.extract('http://video.yahoo.com/watch/%s' % id)
2446 if re.search(self._MORE_PAGES_INDICATOR, page) is None:
2447 for id in video_ids:
2448 self._yahoo_ie.extract('http://video.yahoo.com/watch/%s' % id)
2451 pagenum = pagenum + 1
2454 class YoutubePlaylistIE(InfoExtractor):
2455 """Information Extractor for YouTube playlists."""
2457 _VALID_URL = r'(?:https?://)?(?:\w+\.)?youtube\.com/(?:(?:course|view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)(?:PL)?([0-9A-Za-z-_]+)(?:/.*?/([0-9A-Za-z_-]+))?.*'
2458 _TEMPLATE_URL = 'http://www.youtube.com/%s?%s=%s&page=%s&gl=US&hl=en'
2459 _VIDEO_INDICATOR = r'/watch\?v=(.+?)&'
2460 _MORE_PAGES_INDICATOR = r'(?m)>\s*Next\s*</a>'
2462 IE_NAME = u'youtube:playlist'
2464 def __init__(self, youtube_ie, downloader=None):
2465 InfoExtractor.__init__(self, downloader)
2466 self._youtube_ie = youtube_ie
2468 def report_download_page(self, playlist_id, pagenum):
2469 """Report attempt to download playlist page with given number."""
2470 self._downloader.to_screen(u'[youtube] PL %s: Downloading page #%s' % (playlist_id, pagenum))
2472 def _real_initialize(self):
2473 self._youtube_ie.initialize()
2475 def _real_extract(self, url):
2476 # Extract playlist id
2477 mobj = re.match(self._VALID_URL, url)
2479 self._downloader.trouble(u'ERROR: invalid url: %s' % url)
2483 if mobj.group(3) is not None:
2484 self._youtube_ie.extract(mobj.group(3))
2487 # Download playlist pages
2488 # prefix is 'p' as default for playlists but there are other types that need extra care
2489 playlist_prefix = mobj.group(1)
2490 if playlist_prefix == 'a':
2491 playlist_access = 'artist'
2493 playlist_prefix = 'p'
2494 playlist_access = 'view_play_list'
2495 playlist_id = mobj.group(2)
2500 self.report_download_page(playlist_id, pagenum)
2501 url = self._TEMPLATE_URL % (playlist_access, playlist_prefix, playlist_id, pagenum)
2502 request = urllib2.Request(url)
2504 page = urllib2.urlopen(request).read()
2505 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2506 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
2509 # Extract video identifiers
2511 for mobj in re.finditer(self._VIDEO_INDICATOR, page):
2512 if mobj.group(1) not in ids_in_page:
2513 ids_in_page.append(mobj.group(1))
2514 video_ids.extend(ids_in_page)
2516 if re.search(self._MORE_PAGES_INDICATOR, page) is None:
2518 pagenum = pagenum + 1
2520 playliststart = self._downloader.params.get('playliststart', 1) - 1
2521 playlistend = self._downloader.params.get('playlistend', -1)
2522 video_ids = video_ids[playliststart:playlistend]
2524 for id in video_ids:
2525 self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
2529 class YoutubeUserIE(InfoExtractor):
2530 """Information Extractor for YouTube users."""
2532 _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?youtube\.com/user/)|ytuser:)([A-Za-z0-9_-]+)'
2533 _TEMPLATE_URL = 'http://gdata.youtube.com/feeds/api/users/%s'
2534 _GDATA_PAGE_SIZE = 50
2535 _GDATA_URL = 'http://gdata.youtube.com/feeds/api/users/%s/uploads?max-results=%d&start-index=%d'
2536 _VIDEO_INDICATOR = r'/watch\?v=(.+?)[\<&]'
2538 IE_NAME = u'youtube:user'
2540 def __init__(self, youtube_ie, downloader=None):
2541 InfoExtractor.__init__(self, downloader)
2542 self._youtube_ie = youtube_ie
2544 def report_download_page(self, username, start_index):
2545 """Report attempt to download user page."""
2546 self._downloader.to_screen(u'[youtube] user %s: Downloading video ids from %d to %d' %
2547 (username, start_index, start_index + self._GDATA_PAGE_SIZE))
2549 def _real_initialize(self):
2550 self._youtube_ie.initialize()
2552 def _real_extract(self, url):
2554 mobj = re.match(self._VALID_URL, url)
2556 self._downloader.trouble(u'ERROR: invalid url: %s' % url)
2559 username = mobj.group(1)
2561 # Download video ids using YouTube Data API. Result size per
2562 # query is limited (currently to 50 videos) so we need to query
2563 # page by page until there are no video ids - it means we got
2570 start_index = pagenum * self._GDATA_PAGE_SIZE + 1
2571 self.report_download_page(username, start_index)
2573 request = urllib2.Request(self._GDATA_URL % (username, self._GDATA_PAGE_SIZE, start_index))
2576 page = urllib2.urlopen(request).read()
2577 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2578 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
2581 # Extract video identifiers
2584 for mobj in re.finditer(self._VIDEO_INDICATOR, page):
2585 if mobj.group(1) not in ids_in_page:
2586 ids_in_page.append(mobj.group(1))
2588 video_ids.extend(ids_in_page)
2590 # A little optimization - if current page is not
2591 # "full", ie. does not contain PAGE_SIZE video ids then
2592 # we can assume that this page is the last one - there
2593 # are no more ids on further pages - no need to query
2596 if len(ids_in_page) < self._GDATA_PAGE_SIZE:
2601 all_ids_count = len(video_ids)
2602 playliststart = self._downloader.params.get('playliststart', 1) - 1
2603 playlistend = self._downloader.params.get('playlistend', -1)
2605 if playlistend == -1:
2606 video_ids = video_ids[playliststart:]
2608 video_ids = video_ids[playliststart:playlistend]
2610 self._downloader.to_screen("[youtube] user %s: Collected %d video ids (downloading %d of them)" %
2611 (username, all_ids_count, len(video_ids)))
2613 for video_id in video_ids:
2614 self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % video_id)
2617 class DepositFilesIE(InfoExtractor):
2618 """Information extractor for depositfiles.com"""
2620 _VALID_URL = r'(?:http://)?(?:\w+\.)?depositfiles\.com/(?:../(?#locale))?files/(.+)'
2621 IE_NAME = u'DepositFiles'
2623 def __init__(self, downloader=None):
2624 InfoExtractor.__init__(self, downloader)
2626 def report_download_webpage(self, file_id):
2627 """Report webpage download."""
2628 self._downloader.to_screen(u'[DepositFiles] %s: Downloading webpage' % file_id)
2630 def report_extraction(self, file_id):
2631 """Report information extraction."""
2632 self._downloader.to_screen(u'[DepositFiles] %s: Extracting information' % file_id)
2634 def _real_extract(self, url):
2635 # At this point we have a new file
2636 self._downloader.increment_downloads()
2638 file_id = url.split('/')[-1]
2639 # Rebuild url in english locale
2640 url = 'http://depositfiles.com/en/files/' + file_id
2642 # Retrieve file webpage with 'Free download' button pressed
2643 free_download_indication = { 'gateway_result' : '1' }
2644 request = urllib2.Request(url, urllib.urlencode(free_download_indication))
2646 self.report_download_webpage(file_id)
2647 webpage = urllib2.urlopen(request).read()
2648 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2649 self._downloader.trouble(u'ERROR: Unable to retrieve file webpage: %s' % str(err))
2652 # Search for the real file URL
2653 mobj = re.search(r'<form action="(http://fileshare.+?)"', webpage)
2654 if (mobj is None) or (mobj.group(1) is None):
2655 # Try to figure out reason of the error.
2656 mobj = re.search(r'<strong>(Attention.*?)</strong>', webpage, re.DOTALL)
2657 if (mobj is not None) and (mobj.group(1) is not None):
2658 restriction_message = re.sub('\s+', ' ', mobj.group(1)).strip()
2659 self._downloader.trouble(u'ERROR: %s' % restriction_message)
2661 self._downloader.trouble(u'ERROR: unable to extract download URL from: %s' % url)
2664 file_url = mobj.group(1)
2665 file_extension = os.path.splitext(file_url)[1][1:]
2667 # Search for file title
2668 mobj = re.search(r'<b title="(.*?)">', webpage)
2670 self._downloader.trouble(u'ERROR: unable to extract title')
2672 file_title = mobj.group(1).decode('utf-8')
2675 # Process file information
2676 self._downloader.process_info({
2677 'id': file_id.decode('utf-8'),
2678 'url': file_url.decode('utf-8'),
2680 'upload_date': u'NA',
2681 'title': file_title,
2682 'stitle': file_title,
2683 'ext': file_extension.decode('utf-8'),
2687 except UnavailableVideoError, err:
2688 self._downloader.trouble(u'ERROR: unable to download file')
2691 class FacebookIE(InfoExtractor):
2692 """Information Extractor for Facebook"""
2694 _VALID_URL = r'^(?:https?://)?(?:\w+\.)?facebook\.com/(?:video/video|photo)\.php\?(?:.*?)v=(?P<ID>\d+)(?:.*)'
2695 _LOGIN_URL = 'https://login.facebook.com/login.php?m&next=http%3A%2F%2Fm.facebook.com%2Fhome.php&'
2696 _NETRC_MACHINE = 'facebook'
2697 _available_formats = ['video', 'highqual', 'lowqual']
2698 _video_extensions = {
2703 IE_NAME = u'facebook'
2705 def __init__(self, downloader=None):
2706 InfoExtractor.__init__(self, downloader)
2708 def _reporter(self, message):
2709 """Add header and report message."""
2710 self._downloader.to_screen(u'[facebook] %s' % message)
2712 def report_login(self):
2713 """Report attempt to log in."""
2714 self._reporter(u'Logging in')
2716 def report_video_webpage_download(self, video_id):
2717 """Report attempt to download video webpage."""
2718 self._reporter(u'%s: Downloading video webpage' % video_id)
2720 def report_information_extraction(self, video_id):
2721 """Report attempt to extract video information."""
2722 self._reporter(u'%s: Extracting video information' % video_id)
2724 def _parse_page(self, video_webpage):
2725 """Extract video information from page"""
2727 data = {'title': r'\("video_title", "(.*?)"\)',
2728 'description': r'<div class="datawrap">(.*?)</div>',
2729 'owner': r'\("video_owner_name", "(.*?)"\)',
2730 'thumbnail': r'\("thumb_url", "(?P<THUMB>.*?)"\)',
2733 for piece in data.keys():
2734 mobj = re.search(data[piece], video_webpage)
2735 if mobj is not None:
2736 video_info[piece] = urllib.unquote_plus(mobj.group(1).decode("unicode_escape"))
2740 for fmt in self._available_formats:
2741 mobj = re.search(r'\("%s_src\", "(.+?)"\)' % fmt, video_webpage)
2742 if mobj is not None:
2743 # URL is in a Javascript segment inside an escaped Unicode format within
2744 # the generally utf-8 page
2745 video_urls[fmt] = urllib.unquote_plus(mobj.group(1).decode("unicode_escape"))
2746 video_info['video_urls'] = video_urls
2750 def _real_initialize(self):
2751 if self._downloader is None:
2756 downloader_params = self._downloader.params
2758 # Attempt to use provided username and password or .netrc data
2759 if downloader_params.get('username', None) is not None:
2760 useremail = downloader_params['username']
2761 password = downloader_params['password']
2762 elif downloader_params.get('usenetrc', False):
2764 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
2765 if info is not None:
2769 raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
2770 except (IOError, netrc.NetrcParseError), err:
2771 self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % str(err))
2774 if useremail is None:
2783 request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form))
2786 login_results = urllib2.urlopen(request).read()
2787 if re.search(r'<form(.*)name="login"(.*)</form>', login_results) is not None:
2788 self._downloader.to_stderr(u'WARNING: unable to log in: bad username/password, or exceded login rate limit (~3/min). Check credentials or wait.')
2790 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2791 self._downloader.to_stderr(u'WARNING: unable to log in: %s' % str(err))
2794 def _real_extract(self, url):
2795 mobj = re.match(self._VALID_URL, url)
2797 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2799 video_id = mobj.group('ID')
2802 self.report_video_webpage_download(video_id)
2803 request = urllib2.Request('https://www.facebook.com/video/video.php?v=%s' % video_id)
2805 page = urllib2.urlopen(request)
2806 video_webpage = page.read()
2807 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2808 self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
2811 # Start extracting information
2812 self.report_information_extraction(video_id)
2814 # Extract information
2815 video_info = self._parse_page(video_webpage)
2818 if 'owner' not in video_info:
2819 self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
2821 video_uploader = video_info['owner']
2824 if 'title' not in video_info:
2825 self._downloader.trouble(u'ERROR: unable to extract video title')
2827 video_title = video_info['title']
2828 video_title = video_title.decode('utf-8')
2829 video_title = sanitize_title(video_title)
2831 simple_title = _simplify_title(video_title)
2834 if 'thumbnail' not in video_info:
2835 self._downloader.trouble(u'WARNING: unable to extract video thumbnail')
2836 video_thumbnail = ''
2838 video_thumbnail = video_info['thumbnail']
2842 if 'upload_date' in video_info:
2843 upload_time = video_info['upload_date']
2844 timetuple = email.utils.parsedate_tz(upload_time)
2845 if timetuple is not None:
2847 upload_date = time.strftime('%Y%m%d', timetuple[0:9])
2852 video_description = video_info.get('description', 'No description available.')
2854 url_map = video_info['video_urls']
2855 if len(url_map.keys()) > 0:
2856 # Decide which formats to download
2857 req_format = self._downloader.params.get('format', None)
2858 format_limit = self._downloader.params.get('format_limit', None)
2860 if format_limit is not None and format_limit in self._available_formats:
2861 format_list = self._available_formats[self._available_formats.index(format_limit):]
2863 format_list = self._available_formats
2864 existing_formats = [x for x in format_list if x in url_map]
2865 if len(existing_formats) == 0:
2866 self._downloader.trouble(u'ERROR: no known formats available for video')
2868 if req_format is None:
2869 video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality
2870 elif req_format == 'worst':
2871 video_url_list = [(existing_formats[len(existing_formats)-1], url_map[existing_formats[len(existing_formats)-1]])] # worst quality
2872 elif req_format == '-1':
2873 video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats
2876 if req_format not in url_map:
2877 self._downloader.trouble(u'ERROR: requested format not available')
2879 video_url_list = [(req_format, url_map[req_format])] # Specific format
2881 for format_param, video_real_url in video_url_list:
2883 # At this point we have a new video
2884 self._downloader.increment_downloads()
2887 video_extension = self._video_extensions.get(format_param, 'mp4')
2890 # Process video information
2891 self._downloader.process_info({
2892 'id': video_id.decode('utf-8'),
2893 'url': video_real_url.decode('utf-8'),
2894 'uploader': video_uploader.decode('utf-8'),
2895 'upload_date': upload_date,
2896 'title': video_title,
2897 'stitle': simple_title,
2898 'ext': video_extension.decode('utf-8'),
2899 'format': (format_param is None and u'NA' or format_param.decode('utf-8')),
2900 'thumbnail': video_thumbnail.decode('utf-8'),
2901 'description': video_description.decode('utf-8'),
2904 except UnavailableVideoError, err:
2905 self._downloader.trouble(u'\nERROR: unable to download video')
2907 class BlipTVIE(InfoExtractor):
2908 """Information extractor for blip.tv"""
2910 _VALID_URL = r'^(?:https?://)?(?:\w+\.)?blip\.tv(/.+)$'
2911 _URL_EXT = r'^.*\.([a-z0-9]+)$'
2912 IE_NAME = u'blip.tv'
2914 def report_extraction(self, file_id):
2915 """Report information extraction."""
2916 self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, file_id))
2918 def report_direct_download(self, title):
2919 """Report information extraction."""
2920 self._downloader.to_screen(u'[%s] %s: Direct download detected' % (self.IE_NAME, title))
2922 def _real_extract(self, url):
2923 mobj = re.match(self._VALID_URL, url)
2925 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2932 json_url = url + cchar + 'skin=json&version=2&no_wrap=1'
2933 request = urllib2.Request(json_url)
2934 self.report_extraction(mobj.group(1))
2937 urlh = urllib2.urlopen(request)
2938 if urlh.headers.get('Content-Type', '').startswith('video/'): # Direct download
2939 basename = url.split('/')[-1]
2940 title,ext = os.path.splitext(basename)
2941 title = title.decode('UTF-8')
2942 ext = ext.replace('.', '')
2943 self.report_direct_download(title)
2948 'stitle': _simplify_title(title),
2952 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2953 self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % str(err))
2955 if info is None: # Regular URL
2957 json_code = urlh.read()
2958 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
2959 self._downloader.trouble(u'ERROR: unable to read video info webpage: %s' % str(err))
2963 json_data = json.loads(json_code)
2964 if 'Post' in json_data:
2965 data = json_data['Post']
2969 upload_date = datetime.datetime.strptime(data['datestamp'], '%m-%d-%y %H:%M%p').strftime('%Y%m%d')
2970 video_url = data['media']['url']
2971 umobj = re.match(self._URL_EXT, video_url)
2973 raise ValueError('Can not determine filename extension')
2974 ext = umobj.group(1)
2977 'id': data['item_id'],
2979 'uploader': data['display_name'],
2980 'upload_date': upload_date,
2981 'title': data['title'],
2982 'stitle': _simplify_title(data['title']),
2984 'format': data['media']['mimeType'],
2985 'thumbnail': data['thumbnailUrl'],
2986 'description': data['description'],
2987 'player_url': data['embedUrl']
2989 except (ValueError,KeyError), err:
2990 self._downloader.trouble(u'ERROR: unable to parse video information: %s' % repr(err))
2993 self._downloader.increment_downloads()
2996 self._downloader.process_info(info)
2997 except UnavailableVideoError, err:
2998 self._downloader.trouble(u'\nERROR: unable to download video')
3001 class MyVideoIE(InfoExtractor):
3002 """Information Extractor for myvideo.de."""
3004 _VALID_URL = r'(?:http://)?(?:www\.)?myvideo\.de/watch/([0-9]+)/([^?/]+).*'
3005 IE_NAME = u'myvideo'
3007 def __init__(self, downloader=None):
3008 InfoExtractor.__init__(self, downloader)
3010 def report_download_webpage(self, video_id):
3011 """Report webpage download."""
3012 self._downloader.to_screen(u'[myvideo] %s: Downloading webpage' % video_id)
3014 def report_extraction(self, video_id):
3015 """Report information extraction."""
3016 self._downloader.to_screen(u'[myvideo] %s: Extracting information' % video_id)
3018 def _real_extract(self,url):
3019 mobj = re.match(self._VALID_URL, url)
3021 self._download.trouble(u'ERROR: invalid URL: %s' % url)
3024 video_id = mobj.group(1)
3027 request = urllib2.Request('http://www.myvideo.de/watch/%s' % video_id)
3029 self.report_download_webpage(video_id)
3030 webpage = urllib2.urlopen(request).read()
3031 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
3032 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % str(err))
3035 self.report_extraction(video_id)
3036 mobj = re.search(r'<link rel=\'image_src\' href=\'(http://is[0-9].myvideo\.de/de/movie[0-9]+/[a-f0-9]+)/thumbs/[^.]+\.jpg\' />',
3039 self._downloader.trouble(u'ERROR: unable to extract media URL')
3041 video_url = mobj.group(1) + ('/%s.flv' % video_id)
3043 mobj = re.search('<title>([^<]+)</title>', webpage)
3045 self._downloader.trouble(u'ERROR: unable to extract title')
3048 video_title = mobj.group(1)
3049 video_title = sanitize_title(video_title)
3051 simple_title = _simplify_title(video_title)
3054 self._downloader.process_info({
3058 'upload_date': u'NA',
3059 'title': video_title,
3060 'stitle': simple_title,
3065 except UnavailableVideoError:
3066 self._downloader.trouble(u'\nERROR: Unable to download video')
3068 class ComedyCentralIE(InfoExtractor):
3069 """Information extractor for The Daily Show and Colbert Report """
3071 _VALID_URL = r'^(:(?P<shortname>tds|thedailyshow|cr|colbert|colbertnation|colbertreport))|(https?://)?(www\.)?(?P<showname>thedailyshow|colbertnation)\.com/full-episodes/(?P<episode>.*)$'
3072 IE_NAME = u'comedycentral'
3074 def report_extraction(self, episode_id):
3075 self._downloader.to_screen(u'[comedycentral] %s: Extracting information' % episode_id)
3077 def report_config_download(self, episode_id):
3078 self._downloader.to_screen(u'[comedycentral] %s: Downloading configuration' % episode_id)
3080 def report_index_download(self, episode_id):
3081 self._downloader.to_screen(u'[comedycentral] %s: Downloading show index' % episode_id)
3083 def report_player_url(self, episode_id):
3084 self._downloader.to_screen(u'[comedycentral] %s: Determining player URL' % episode_id)
3086 def _real_extract(self, url):
3087 mobj = re.match(self._VALID_URL, url)
3089 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
3092 if mobj.group('shortname'):
3093 if mobj.group('shortname') in ('tds', 'thedailyshow'):
3094 url = u'http://www.thedailyshow.com/full-episodes/'
3096 url = u'http://www.colbertnation.com/full-episodes/'
3097 mobj = re.match(self._VALID_URL, url)
3098 assert mobj is not None
3100 dlNewest = not mobj.group('episode')
3102 epTitle = mobj.group('showname')
3104 epTitle = mobj.group('episode')
3106 req = urllib2.Request(url)
3107 self.report_extraction(epTitle)
3109 htmlHandle = urllib2.urlopen(req)
3110 html = htmlHandle.read()
3111 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
3112 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % unicode(err))
3115 url = htmlHandle.geturl()
3116 mobj = re.match(self._VALID_URL, url)
3118 self._downloader.trouble(u'ERROR: Invalid redirected URL: ' + url)
3120 if mobj.group('episode') == '':
3121 self._downloader.trouble(u'ERROR: Redirected URL is still not specific: ' + url)
3123 epTitle = mobj.group('episode')
3125 mMovieParams = re.findall('<param name="movie" value="(http://media.mtvnservices.com/([^"]*episode.*?:.*?))"/>', html)
3126 if len(mMovieParams) == 0:
3127 self._downloader.trouble(u'ERROR: unable to find Flash URL in webpage ' + url)
3130 playerUrl_raw = mMovieParams[0][0]
3131 self.report_player_url(epTitle)
3133 urlHandle = urllib2.urlopen(playerUrl_raw)
3134 playerUrl = urlHandle.geturl()
3135 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
3136 self._downloader.trouble(u'ERROR: unable to find out player URL: ' + unicode(err))
3139 uri = mMovieParams[0][1]
3140 indexUrl = 'http://shadow.comedycentral.com/feeds/video_player/mrss/?' + urllib.urlencode({'uri': uri})
3141 self.report_index_download(epTitle)
3143 indexXml = urllib2.urlopen(indexUrl).read()
3144 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
3145 self._downloader.trouble(u'ERROR: unable to download episode index: ' + unicode(err))
3148 idoc = xml.etree.ElementTree.fromstring(indexXml)
3149 itemEls = idoc.findall('.//item')
3150 for itemEl in itemEls:
3151 mediaId = itemEl.findall('./guid')[0].text
3152 shortMediaId = mediaId.split(':')[-1]
3153 showId = mediaId.split(':')[-2].replace('.com', '')
3154 officialTitle = itemEl.findall('./title')[0].text
3155 officialDate = itemEl.findall('./pubDate')[0].text
3157 configUrl = ('http://www.comedycentral.com/global/feeds/entertainment/media/mediaGenEntertainment.jhtml?' +
3158 urllib.urlencode({'uri': mediaId}))
3159 configReq = urllib2.Request(configUrl)
3160 self.report_config_download(epTitle)
3162 configXml = urllib2.urlopen(configReq).read()
3163 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
3164 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % unicode(err))
3167 cdoc = xml.etree.ElementTree.fromstring(configXml)
3169 for rendition in cdoc.findall('.//rendition'):
3170 finfo = (rendition.attrib['bitrate'], rendition.findall('./src')[0].text)
3174 self._downloader.trouble(u'\nERROR: unable to download ' + mediaId + ': No videos found')
3177 # For now, just pick the highest bitrate
3178 format,video_url = turls[-1]
3180 self._downloader.increment_downloads()
3182 effTitle = showId + u'-' + epTitle
3187 'upload_date': officialDate,
3189 'stitle': _simplify_title(effTitle),
3193 'description': officialTitle,
3194 'player_url': playerUrl
3198 self._downloader.process_info(info)
3199 except UnavailableVideoError, err:
3200 self._downloader.trouble(u'\nERROR: unable to download ' + mediaId)
3204 class EscapistIE(InfoExtractor):
3205 """Information extractor for The Escapist """
3207 _VALID_URL = r'^(https?://)?(www\.)?escapistmagazine\.com/videos/view/(?P<showname>[^/]+)/(?P<episode>[^/?]+)[/?]?.*$'
3208 IE_NAME = u'escapist'
3210 def report_extraction(self, showName):
3211 self._downloader.to_screen(u'[escapist] %s: Extracting information' % showName)
3213 def report_config_download(self, showName):
3214 self._downloader.to_screen(u'[escapist] %s: Downloading configuration' % showName)
3216 def _real_extract(self, url):
3217 htmlParser = HTMLParser.HTMLParser()
3219 mobj = re.match(self._VALID_URL, url)
3221 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
3223 showName = mobj.group('showname')
3224 videoId = mobj.group('episode')
3226 self.report_extraction(showName)
3228 webPage = urllib2.urlopen(url).read()
3229 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
3230 self._downloader.trouble(u'ERROR: unable to download webpage: ' + unicode(err))
3233 descMatch = re.search('<meta name="description" content="([^"]*)"', webPage)
3234 description = htmlParser.unescape(descMatch.group(1))
3235 imgMatch = re.search('<meta property="og:image" content="([^"]*)"', webPage)
3236 imgUrl = htmlParser.unescape(imgMatch.group(1))
3237 playerUrlMatch = re.search('<meta property="og:video" content="([^"]*)"', webPage)
3238 playerUrl = htmlParser.unescape(playerUrlMatch.group(1))
3239 configUrlMatch = re.search('config=(.*)$', playerUrl)
3240 configUrl = urllib2.unquote(configUrlMatch.group(1))
3242 self.report_config_download(showName)
3244 configJSON = urllib2.urlopen(configUrl).read()
3245 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
3246 self._downloader.trouble(u'ERROR: unable to download configuration: ' + unicode(err))
3249 # Technically, it's JavaScript, not JSON
3250 configJSON = configJSON.replace("'", '"')
3253 config = json.loads(configJSON)
3254 except (ValueError,), err:
3255 self._downloader.trouble(u'ERROR: Invalid JSON in configuration file: ' + unicode(err))
3258 playlist = config['playlist']
3259 videoUrl = playlist[1]['url']
3261 self._downloader.increment_downloads()
3265 'uploader': showName,
3266 'upload_date': None,
3268 'stitle': _simplify_title(showName),
3271 'thumbnail': imgUrl,
3272 'description': description,
3273 'player_url': playerUrl,
3277 self._downloader.process_info(info)
3278 except UnavailableVideoError, err:
3279 self._downloader.trouble(u'\nERROR: unable to download ' + videoId)
3282 class CollegeHumorIE(InfoExtractor):
3283 """Information extractor for collegehumor.com"""
3285 _VALID_URL = r'^(?:https?://)?(?:www\.)?collegehumor\.com/video/(?P<videoid>[0-9]+)/(?P<shorttitle>.*)$'
3286 IE_NAME = u'collegehumor'
3288 def report_webpage(self, video_id):
3289 """Report information extraction."""
3290 self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
3292 def report_extraction(self, video_id):
3293 """Report information extraction."""
3294 self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
3296 def _real_extract(self, url):
3297 htmlParser = HTMLParser.HTMLParser()
3299 mobj = re.match(self._VALID_URL, url)
3301 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
3303 video_id = mobj.group('videoid')
3305 self.report_webpage(video_id)
3306 request = urllib2.Request(url)
3308 webpage = urllib2.urlopen(request).read()
3309 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
3310 self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
3313 m = re.search(r'id="video:(?P<internalvideoid>[0-9]+)"', webpage)
3315 self._downloader.trouble(u'ERROR: Cannot extract internal video ID')
3317 internal_video_id = m.group('internalvideoid')
3321 'internal_id': internal_video_id,
3324 self.report_extraction(video_id)
3325 xmlUrl = 'http://www.collegehumor.com/moogaloop/video:' + internal_video_id
3327 metaXml = urllib2.urlopen(xmlUrl).read()
3328 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
3329 self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % str(err))
3332 mdoc = xml.etree.ElementTree.fromstring(metaXml)
3334 videoNode = mdoc.findall('./video')[0]
3335 info['description'] = videoNode.findall('./description')[0].text
3336 info['title'] = videoNode.findall('./caption')[0].text
3337 info['stitle'] = _simplify_title(info['title'])
3338 info['url'] = videoNode.findall('./file')[0].text
3339 info['thumbnail'] = videoNode.findall('./thumbnail')[0].text
3340 info['ext'] = info['url'].rpartition('.')[2]
3341 info['format'] = info['ext']
3343 self._downloader.trouble(u'\nERROR: Invalid metadata XML file')
3346 self._downloader.increment_downloads()
3349 self._downloader.process_info(info)
3350 except UnavailableVideoError, err:
3351 self._downloader.trouble(u'\nERROR: unable to download video')
3354 class XVideosIE(InfoExtractor):
3355 """Information extractor for xvideos.com"""
3357 _VALID_URL = r'^(?:https?://)?(?:www\.)?xvideos\.com/video([0-9]+)(?:.*)'
3358 IE_NAME = u'xvideos'
3360 def report_webpage(self, video_id):
3361 """Report information extraction."""
3362 self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
3364 def report_extraction(self, video_id):
3365 """Report information extraction."""
3366 self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
3368 def _real_extract(self, url):
3369 htmlParser = HTMLParser.HTMLParser()
3371 mobj = re.match(self._VALID_URL, url)
3373 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
3375 video_id = mobj.group(1).decode('utf-8')
3377 self.report_webpage(video_id)
3379 request = urllib2.Request(r'http://www.xvideos.com/video' + video_id)
3381 webpage = urllib2.urlopen(request).read()
3382 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
3383 self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
3386 self.report_extraction(video_id)
3390 mobj = re.search(r'flv_url=(.+?)&', webpage)
3392 self._downloader.trouble(u'ERROR: unable to extract video url')
3394 video_url = urllib2.unquote(mobj.group(1).decode('utf-8'))
3398 mobj = re.search(r'<title>(.*?)\s+-\s+XVID', webpage)
3400 self._downloader.trouble(u'ERROR: unable to extract video title')
3402 video_title = mobj.group(1).decode('utf-8')
3405 # Extract video thumbnail
3406 mobj = re.search(r'http://(?:img.*?\.)xvideos.com/videos/thumbs/[a-fA-F0-9]/[a-fA-F0-9]/[a-fA-F0-9]/([a-fA-F0-9.]+jpg)', webpage)
3408 self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
3410 video_thumbnail = mobj.group(1).decode('utf-8')
3414 self._downloader.increment_downloads()
3419 'upload_date': None,
3420 'title': video_title,
3421 'stitle': _simplify_title(video_title),
3424 'thumbnail': video_thumbnail,
3425 'description': None,
3430 self._downloader.process_info(info)
3431 except UnavailableVideoError, err:
3432 self._downloader.trouble(u'\nERROR: unable to download ' + video_id)
3435 class SoundcloudIE(InfoExtractor):
3436 """Information extractor for soundcloud.com
3437 To access the media, the uid of the song and a stream token
3438 must be extracted from the page source and the script must make
3439 a request to media.soundcloud.com/crossdomain.xml. Then
3440 the media can be grabbed by requesting from an url composed
3441 of the stream token and uid
3444 _VALID_URL = r'^(?:https?://)?(?:www\.)?soundcloud\.com/([\w\d-]+)/([\w\d-]+)'
3445 IE_NAME = u'soundcloud'
3447 def __init__(self, downloader=None):
3448 InfoExtractor.__init__(self, downloader)
3450 def report_webpage(self, video_id):
3451 """Report information extraction."""
3452 self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
3454 def report_extraction(self, video_id):
3455 """Report information extraction."""
3456 self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
3458 def _real_extract(self, url):
3459 htmlParser = HTMLParser.HTMLParser()
3461 mobj = re.match(self._VALID_URL, url)
3463 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
3466 # extract uploader (which is in the url)
3467 uploader = mobj.group(1).decode('utf-8')
3468 # extract simple title (uploader + slug of song title)
3469 slug_title = mobj.group(2).decode('utf-8')
3470 simple_title = uploader + '-' + slug_title
3472 self.report_webpage('%s/%s' % (uploader, slug_title))
3474 request = urllib2.Request('http://soundcloud.com/%s/%s' % (uploader, slug_title))
3476 webpage = urllib2.urlopen(request).read()
3477 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
3478 self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
3481 self.report_extraction('%s/%s' % (uploader, slug_title))
3483 # extract uid and stream token that soundcloud hands out for access
3484 mobj = re.search('"uid":"([\w\d]+?)".*?stream_token=([\w\d]+)', webpage)
3486 video_id = mobj.group(1)
3487 stream_token = mobj.group(2)
3489 # extract unsimplified title
3490 mobj = re.search('"title":"(.*?)",', webpage)
3492 title = mobj.group(1)
3494 # construct media url (with uid/token)
3495 mediaURL = "http://media.soundcloud.com/stream/%s?stream_token=%s"
3496 mediaURL = mediaURL % (video_id, stream_token)
3499 description = u'No description available'
3500 mobj = re.search('track-description-value"><p>(.*?)</p>', webpage)
3502 description = mobj.group(1)
3506 mobj = re.search("pretty-date'>on ([\w]+ [\d]+, [\d]+ \d+:\d+)</abbr></h2>", webpage)
3509 upload_date = datetime.datetime.strptime(mobj.group(1), '%B %d, %Y %H:%M').strftime('%Y%m%d')
3510 except Exception, e:
3513 # for soundcloud, a request to a cross domain is required for cookies
3514 request = urllib2.Request('http://media.soundcloud.com/crossdomain.xml', std_headers)
3517 self._downloader.process_info({
3518 'id': video_id.decode('utf-8'),
3520 'uploader': uploader.decode('utf-8'),
3521 'upload_date': upload_date,
3522 'title': simple_title.decode('utf-8'),
3523 'stitle': simple_title.decode('utf-8'),
3527 'description': description.decode('utf-8')
3529 except UnavailableVideoError:
3530 self._downloader.trouble(u'\nERROR: unable to download video')
3533 class InfoQIE(InfoExtractor):
3534 """Information extractor for infoq.com"""
3536 _VALID_URL = r'^(?:https?://)?(?:www\.)?infoq\.com/[^/]+/[^/]+$'
3539 def report_webpage(self, video_id):
3540 """Report information extraction."""
3541 self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
3543 def report_extraction(self, video_id):
3544 """Report information extraction."""
3545 self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
3547 def _real_extract(self, url):
3548 htmlParser = HTMLParser.HTMLParser()
3550 mobj = re.match(self._VALID_URL, url)
3552 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
3555 self.report_webpage(url)
3557 request = urllib2.Request(url)
3559 webpage = urllib2.urlopen(request).read()
3560 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
3561 self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
3564 self.report_extraction(url)
3568 mobj = re.search(r"jsclassref='([^']*)'", webpage)
3570 self._downloader.trouble(u'ERROR: unable to extract video url')
3572 video_url = 'rtmpe://video.infoq.com/cfx/st/' + urllib2.unquote(mobj.group(1).decode('base64'))
3576 mobj = re.search(r'contentTitle = "(.*?)";', webpage)
3578 self._downloader.trouble(u'ERROR: unable to extract video title')
3580 video_title = mobj.group(1).decode('utf-8')
3582 # Extract description
3583 video_description = u'No description available.'
3584 mobj = re.search(r'<meta name="description" content="(.*)"(?:\s*/)?>', webpage)
3585 if mobj is not None:
3586 video_description = mobj.group(1).decode('utf-8')
3588 video_filename = video_url.split('/')[-1]
3589 video_id, extension = video_filename.split('.')
3591 self._downloader.increment_downloads()
3596 'upload_date': None,
3597 'title': video_title,
3598 'stitle': _simplify_title(video_title),
3600 'format': extension, # Extension is always(?) mp4, but seems to be flv
3602 'description': video_description,
3607 self._downloader.process_info(info)
3608 except UnavailableVideoError, err:
3609 self._downloader.trouble(u'\nERROR: unable to download ' + video_url)
3613 class PostProcessor(object):
3614 """Post Processor class.
3616 PostProcessor objects can be added to downloaders with their
3617 add_post_processor() method. When the downloader has finished a
3618 successful download, it will take its internal chain of PostProcessors
3619 and start calling the run() method on each one of them, first with
3620 an initial argument and then with the returned value of the previous
3623 The chain will be stopped if one of them ever returns None or the end
3624 of the chain is reached.
3626 PostProcessor objects follow a "mutual registration" process similar
3627 to InfoExtractor objects.
3632 def __init__(self, downloader=None):
3633 self._downloader = downloader
3635 def set_downloader(self, downloader):
3636 """Sets the downloader for this PP."""
3637 self._downloader = downloader
3639 def run(self, information):
3640 """Run the PostProcessor.
3642 The "information" argument is a dictionary like the ones
3643 composed by InfoExtractors. The only difference is that this
3644 one has an extra field called "filepath" that points to the
3647 When this method returns None, the postprocessing chain is
3648 stopped. However, this method may return an information
3649 dictionary that will be passed to the next postprocessing
3650 object in the chain. It can be the one it received after
3651 changing some fields.
3653 In addition, this method may raise a PostProcessingError
3654 exception that will be taken into account by the downloader
3657 return information # by default, do nothing
3660 class FFmpegExtractAudioPP(PostProcessor):
3662 def __init__(self, downloader=None, preferredcodec=None, preferredquality=None, keepvideo=False):
3663 PostProcessor.__init__(self, downloader)
3664 if preferredcodec is None:
3665 preferredcodec = 'best'
3666 self._preferredcodec = preferredcodec
3667 self._preferredquality = preferredquality
3668 self._keepvideo = keepvideo
3671 def get_audio_codec(path):
3673 cmd = ['ffprobe', '-show_streams', '--', path]
3674 handle = subprocess.Popen(cmd, stderr=file(os.path.devnull, 'w'), stdout=subprocess.PIPE)
3675 output = handle.communicate()[0]
3676 if handle.wait() != 0:
3678 except (IOError, OSError):
3681 for line in output.split('\n'):
3682 if line.startswith('codec_name='):
3683 audio_codec = line.split('=')[1].strip()
3684 elif line.strip() == 'codec_type=audio' and audio_codec is not None:
3689 def run_ffmpeg(path, out_path, codec, more_opts):
3691 cmd = ['ffmpeg', '-y', '-i', path, '-vn', '-acodec', codec] + more_opts + ['--', out_path]
3692 ret = subprocess.call(cmd, stdout=file(os.path.devnull, 'w'), stderr=subprocess.STDOUT)
3694 except (IOError, OSError):
3697 def run(self, information):
3698 path = information['filepath']
3700 filecodec = self.get_audio_codec(path)
3701 if filecodec is None:
3702 self._downloader.to_stderr(u'WARNING: unable to obtain file audio codec with ffprobe')
3706 if self._preferredcodec == 'best' or self._preferredcodec == filecodec:
3707 if filecodec in ['aac', 'mp3', 'vorbis']:
3708 # Lossless if possible
3710 extension = filecodec
3711 if filecodec == 'aac':
3712 more_opts = ['-f', 'adts']
3713 if filecodec == 'vorbis':
3717 acodec = 'libmp3lame'
3720 if self._preferredquality is not None:
3721 more_opts += ['-ab', self._preferredquality]
3723 # We convert the audio (lossy)
3724 acodec = {'mp3': 'libmp3lame', 'aac': 'aac', 'vorbis': 'libvorbis'}[self._preferredcodec]
3725 extension = self._preferredcodec
3727 if self._preferredquality is not None:
3728 more_opts += ['-ab', self._preferredquality]
3729 if self._preferredcodec == 'aac':
3730 more_opts += ['-f', 'adts']
3731 if self._preferredcodec == 'vorbis':
3734 (prefix, ext) = os.path.splitext(path)
3735 new_path = prefix + '.' + extension
3736 self._downloader.to_screen(u'[ffmpeg] Destination: %s' % new_path)
3737 status = self.run_ffmpeg(path, new_path, acodec, more_opts)
3740 self._downloader.to_stderr(u'WARNING: error running ffmpeg')
3743 # Try to update the date time for extracted audio file.
3744 if information.get('filetime') is not None:
3746 os.utime(new_path, (time.time(), information['filetime']))
3748 self._downloader.to_stderr(u'WARNING: Cannot update utime of audio file')
3750 if not self._keepvideo:
3753 except (IOError, OSError):
3754 self._downloader.to_stderr(u'WARNING: Unable to remove downloaded video file')
3757 information['filepath'] = new_path
3761 def updateSelf(downloader, filename):
3762 ''' Update the program file with the latest version from the repository '''
3763 # Note: downloader only used for options
3764 if not os.access(filename, os.W_OK):
3765 sys.exit('ERROR: no write permissions on %s' % filename)
3767 downloader.to_screen('Updating to latest version...')
3771 urlh = urllib.urlopen(UPDATE_URL)
3772 newcontent = urlh.read()
3774 vmatch = re.search("__version__ = '([^']+)'", newcontent)
3775 if vmatch is not None and vmatch.group(1) == __version__:
3776 downloader.to_screen('youtube-dl is up-to-date (' + __version__ + ')')
3780 except (IOError, OSError), err:
3781 sys.exit('ERROR: unable to download latest version')
3784 outf = open(filename, 'wb')
3786 outf.write(newcontent)
3789 except (IOError, OSError), err:
3790 sys.exit('ERROR: unable to overwrite current version')
3792 downloader.to_screen('Updated youtube-dl. Restart youtube-dl to use the new version.')
3799 def _format_option_string(option):
3800 ''' ('-o', '--option') -> -o, --format METAVAR'''
3804 if option._short_opts: opts.append(option._short_opts[0])
3805 if option._long_opts: opts.append(option._long_opts[0])
3806 if len(opts) > 1: opts.insert(1, ', ')
3808 if option.takes_value(): opts.append(' %s' % option.metavar)
3810 return "".join(opts)
3812 def _find_term_columns():
3813 columns = os.environ.get('COLUMNS', None)
3818 sp = subprocess.Popen(['stty', 'size'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
3819 out,err = sp.communicate()
3820 return int(out.split()[1])
3826 max_help_position = 80
3828 # No need to wrap help messages if we're on a wide console
3829 columns = _find_term_columns()
3830 if columns: max_width = columns
3832 fmt = optparse.IndentedHelpFormatter(width=max_width, max_help_position=max_help_position)
3833 fmt.format_option_strings = _format_option_string
3836 'version' : __version__,
3838 'usage' : '%prog [options] url [url...]',
3839 'conflict_handler' : 'resolve',
3842 parser = optparse.OptionParser(**kw)
3845 general = optparse.OptionGroup(parser, 'General Options')
3846 selection = optparse.OptionGroup(parser, 'Video Selection')
3847 authentication = optparse.OptionGroup(parser, 'Authentication Options')
3848 video_format = optparse.OptionGroup(parser, 'Video Format Options')
3849 postproc = optparse.OptionGroup(parser, 'Post-processing Options')
3850 filesystem = optparse.OptionGroup(parser, 'Filesystem Options')
3851 verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
3853 general.add_option('-h', '--help',
3854 action='help', help='print this help text and exit')
3855 general.add_option('-v', '--version',
3856 action='version', help='print program version and exit')
3857 general.add_option('-U', '--update',
3858 action='store_true', dest='update_self', help='update this program to latest version')
3859 general.add_option('-i', '--ignore-errors',
3860 action='store_true', dest='ignoreerrors', help='continue on download errors', default=False)
3861 general.add_option('-r', '--rate-limit',
3862 dest='ratelimit', metavar='LIMIT', help='download rate limit (e.g. 50k or 44.6m)')
3863 general.add_option('-R', '--retries',
3864 dest='retries', metavar='RETRIES', help='number of retries (default is 10)', default=10)
3865 general.add_option('--dump-user-agent',
3866 action='store_true', dest='dump_user_agent',
3867 help='display the current browser identification', default=False)
3868 general.add_option('--list-extractors',
3869 action='store_true', dest='list_extractors',
3870 help='List all supported extractors and the URLs they would handle', default=False)
3872 selection.add_option('--playlist-start',
3873 dest='playliststart', metavar='NUMBER', help='playlist video to start at (default is 1)', default=1)
3874 selection.add_option('--playlist-end',
3875 dest='playlistend', metavar='NUMBER', help='playlist video to end at (default is last)', default=-1)
3876 selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)')
3877 selection.add_option('--reject-title', dest='rejecttitle', metavar='REGEX',help='skip download for matching titles (regex or caseless sub-string)')
3879 authentication.add_option('-u', '--username',
3880 dest='username', metavar='USERNAME', help='account username')
3881 authentication.add_option('-p', '--password',
3882 dest='password', metavar='PASSWORD', help='account password')
3883 authentication.add_option('-n', '--netrc',
3884 action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
3887 video_format.add_option('-f', '--format',
3888 action='store', dest='format', metavar='FORMAT', help='video format code')
3889 video_format.add_option('--all-formats',
3890 action='store_const', dest='format', help='download all available video formats', const='all')
3891 video_format.add_option('--max-quality',
3892 action='store', dest='format_limit', metavar='FORMAT', help='highest quality format to download')
3893 video_format.add_option('-F', '--list-formats',
3894 action='store_true', dest='listformats', help='list all available formats (currently youtube only)')
3897 verbosity.add_option('-q', '--quiet',
3898 action='store_true', dest='quiet', help='activates quiet mode', default=False)
3899 verbosity.add_option('-s', '--simulate',
3900 action='store_true', dest='simulate', help='do not download the video and do not write anything to disk', default=False)
3901 verbosity.add_option('--skip-download',
3902 action='store_true', dest='skip_download', help='do not download the video', default=False)
3903 verbosity.add_option('-g', '--get-url',
3904 action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
3905 verbosity.add_option('-e', '--get-title',
3906 action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
3907 verbosity.add_option('--get-thumbnail',
3908 action='store_true', dest='getthumbnail',
3909 help='simulate, quiet but print thumbnail URL', default=False)
3910 verbosity.add_option('--get-description',
3911 action='store_true', dest='getdescription',
3912 help='simulate, quiet but print video description', default=False)
3913 verbosity.add_option('--get-filename',
3914 action='store_true', dest='getfilename',
3915 help='simulate, quiet but print output filename', default=False)
3916 verbosity.add_option('--get-format',
3917 action='store_true', dest='getformat',
3918 help='simulate, quiet but print output format', default=False)
3919 verbosity.add_option('--no-progress',
3920 action='store_true', dest='noprogress', help='do not print progress bar', default=False)
3921 verbosity.add_option('--console-title',
3922 action='store_true', dest='consoletitle',
3923 help='display progress in console titlebar', default=False)
3926 filesystem.add_option('-t', '--title',
3927 action='store_true', dest='usetitle', help='use title in file name', default=False)
3928 filesystem.add_option('-l', '--literal',
3929 action='store_true', dest='useliteral', help='use literal title in file name', default=False)
3930 filesystem.add_option('-A', '--auto-number',
3931 action='store_true', dest='autonumber',
3932 help='number downloaded files starting from 00000', default=False)
3933 filesystem.add_option('-o', '--output',
3934 dest='outtmpl', metavar='TEMPLATE', help='output filename template. Use %(stitle)s to get the title, %(uploader)s for the uploader name, %(autonumber)s to get an automatically incremented number, %(ext)s for the filename extension, and %% for a literal percent')
3935 filesystem.add_option('-a', '--batch-file',
3936 dest='batchfile', metavar='FILE', help='file containing URLs to download (\'-\' for stdin)')
3937 filesystem.add_option('-w', '--no-overwrites',
3938 action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
3939 filesystem.add_option('-c', '--continue',
3940 action='store_true', dest='continue_dl', help='resume partially downloaded files', default=False)
3941 filesystem.add_option('--no-continue',
3942 action='store_false', dest='continue_dl',
3943 help='do not resume partially downloaded files (restart from beginning)')
3944 filesystem.add_option('--cookies',
3945 dest='cookiefile', metavar='FILE', help='file to read cookies from and dump cookie jar in')
3946 filesystem.add_option('--no-part',
3947 action='store_true', dest='nopart', help='do not use .part files', default=False)
3948 filesystem.add_option('--no-mtime',
3949 action='store_false', dest='updatetime',
3950 help='do not use the Last-modified header to set the file modification time', default=True)
3951 filesystem.add_option('--write-description',
3952 action='store_true', dest='writedescription',
3953 help='write video description to a .description file', default=False)
3954 filesystem.add_option('--write-info-json',
3955 action='store_true', dest='writeinfojson',
3956 help='write video metadata to a .info.json file', default=False)
3959 postproc.add_option('--extract-audio', action='store_true', dest='extractaudio', default=False,
3960 help='convert video files to audio-only files (requires ffmpeg and ffprobe)')
3961 postproc.add_option('--audio-format', metavar='FORMAT', dest='audioformat', default='best',
3962 help='"best", "aac", "vorbis" or "mp3"; best by default')
3963 postproc.add_option('--audio-quality', metavar='QUALITY', dest='audioquality', default='128K',
3964 help='ffmpeg audio bitrate specification, 128k by default')
3965 postproc.add_option('-k', '--keep-video', action='store_true', dest='keepvideo', default=False,
3966 help='keeps the video file on disk after the post-processing; the video is erased by default')
3969 parser.add_option_group(general)
3970 parser.add_option_group(selection)
3971 parser.add_option_group(filesystem)
3972 parser.add_option_group(verbosity)
3973 parser.add_option_group(video_format)
3974 parser.add_option_group(authentication)
3975 parser.add_option_group(postproc)
3977 opts, args = parser.parse_args()
3979 return parser, opts, args
3981 def gen_extractors():
3982 """ Return a list of an instance of every supported extractor.
3983 The order does matter; the first extractor matched is the one handling the URL.
3985 youtube_ie = YoutubeIE()
3986 google_ie = GoogleIE()
3987 yahoo_ie = YahooIE()
3989 YoutubePlaylistIE(youtube_ie),
3990 YoutubeUserIE(youtube_ie),
3991 YoutubeSearchIE(youtube_ie),
3993 MetacafeIE(youtube_ie),
3996 GoogleSearchIE(google_ie),
3999 YahooSearchIE(yahoo_ie),
4016 parser, opts, args = parseOpts()
4018 # Open appropriate CookieJar
4019 if opts.cookiefile is None:
4020 jar = cookielib.CookieJar()
4023 jar = cookielib.MozillaCookieJar(opts.cookiefile)
4024 if os.path.isfile(opts.cookiefile) and os.access(opts.cookiefile, os.R_OK):
4026 except (IOError, OSError), err:
4027 sys.exit(u'ERROR: unable to open cookie file')
4030 if opts.dump_user_agent:
4031 print std_headers['User-Agent']
4034 # Batch file verification
4036 if opts.batchfile is not None:
4038 if opts.batchfile == '-':
4041 batchfd = open(opts.batchfile, 'r')
4042 batchurls = batchfd.readlines()
4043 batchurls = [x.strip() for x in batchurls]
4044 batchurls = [x for x in batchurls if len(x) > 0 and not re.search(r'^[#/;]', x)]
4046 sys.exit(u'ERROR: batch file could not be read')
4047 all_urls = batchurls + args
4049 # General configuration
4050 cookie_processor = urllib2.HTTPCookieProcessor(jar)
4051 opener = urllib2.build_opener(urllib2.ProxyHandler(), cookie_processor, YoutubeDLHandler())
4052 urllib2.install_opener(opener)
4053 socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words)
4055 extractors = gen_extractors()
4057 if opts.list_extractors:
4058 for ie in extractors:
4060 matchedUrls = filter(lambda url: ie.suitable(url), all_urls)
4061 all_urls = filter(lambda url: url not in matchedUrls, all_urls)
4062 for mu in matchedUrls:
4066 # Conflicting, missing and erroneous options
4067 if opts.usenetrc and (opts.username is not None or opts.password is not None):
4068 parser.error(u'using .netrc conflicts with giving username/password')
4069 if opts.password is not None and opts.username is None:
4070 parser.error(u'account username missing')
4071 if opts.outtmpl is not None and (opts.useliteral or opts.usetitle or opts.autonumber):
4072 parser.error(u'using output template conflicts with using title, literal title or auto number')
4073 if opts.usetitle and opts.useliteral:
4074 parser.error(u'using title conflicts with using literal title')
4075 if opts.username is not None and opts.password is None:
4076 opts.password = getpass.getpass(u'Type account password and press return:')
4077 if opts.ratelimit is not None:
4078 numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
4079 if numeric_limit is None:
4080 parser.error(u'invalid rate limit specified')
4081 opts.ratelimit = numeric_limit
4082 if opts.retries is not None:
4084 opts.retries = long(opts.retries)
4085 except (TypeError, ValueError), err:
4086 parser.error(u'invalid retry count specified')
4088 opts.playliststart = int(opts.playliststart)
4089 if opts.playliststart <= 0:
4090 raise ValueError(u'Playlist start must be positive')
4091 except (TypeError, ValueError), err:
4092 parser.error(u'invalid playlist start number specified')
4094 opts.playlistend = int(opts.playlistend)
4095 if opts.playlistend != -1 and (opts.playlistend <= 0 or opts.playlistend < opts.playliststart):
4096 raise ValueError(u'Playlist end must be greater than playlist start')
4097 except (TypeError, ValueError), err:
4098 parser.error(u'invalid playlist end number specified')
4099 if opts.extractaudio:
4100 if opts.audioformat not in ['best', 'aac', 'mp3', 'vorbis']:
4101 parser.error(u'invalid audio format specified')
4104 fd = FileDownloader({
4105 'usenetrc': opts.usenetrc,
4106 'username': opts.username,
4107 'password': opts.password,
4108 'quiet': (opts.quiet or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat),
4109 'forceurl': opts.geturl,
4110 'forcetitle': opts.gettitle,
4111 'forcethumbnail': opts.getthumbnail,
4112 'forcedescription': opts.getdescription,
4113 'forcefilename': opts.getfilename,
4114 'forceformat': opts.getformat,
4115 'simulate': opts.simulate,
4116 'skip_download': (opts.skip_download or opts.simulate or opts.geturl or opts.gettitle or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat),
4117 'format': opts.format,
4118 'format_limit': opts.format_limit,
4119 'listformats': opts.listformats,
4120 'outtmpl': ((opts.outtmpl is not None and opts.outtmpl.decode(preferredencoding()))
4121 or (opts.format == '-1' and opts.usetitle and u'%(stitle)s-%(id)s-%(format)s.%(ext)s')
4122 or (opts.format == '-1' and opts.useliteral and u'%(title)s-%(id)s-%(format)s.%(ext)s')
4123 or (opts.format == '-1' and u'%(id)s-%(format)s.%(ext)s')
4124 or (opts.usetitle and opts.autonumber and u'%(autonumber)s-%(stitle)s-%(id)s.%(ext)s')
4125 or (opts.useliteral and opts.autonumber and u'%(autonumber)s-%(title)s-%(id)s.%(ext)s')
4126 or (opts.usetitle and u'%(stitle)s-%(id)s.%(ext)s')
4127 or (opts.useliteral and u'%(title)s-%(id)s.%(ext)s')
4128 or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s')
4129 or u'%(id)s.%(ext)s'),
4130 'ignoreerrors': opts.ignoreerrors,
4131 'ratelimit': opts.ratelimit,
4132 'nooverwrites': opts.nooverwrites,
4133 'retries': opts.retries,
4134 'continuedl': opts.continue_dl,
4135 'noprogress': opts.noprogress,
4136 'playliststart': opts.playliststart,
4137 'playlistend': opts.playlistend,
4138 'logtostderr': opts.outtmpl == '-',
4139 'consoletitle': opts.consoletitle,
4140 'nopart': opts.nopart,
4141 'updatetime': opts.updatetime,
4142 'writedescription': opts.writedescription,
4143 'writeinfojson': opts.writeinfojson,
4144 'matchtitle': opts.matchtitle,
4145 'rejecttitle': opts.rejecttitle,
4147 for extractor in extractors:
4148 fd.add_info_extractor(extractor)
4151 if opts.extractaudio:
4152 fd.add_post_processor(FFmpegExtractAudioPP(preferredcodec=opts.audioformat, preferredquality=opts.audioquality, keepvideo=opts.keepvideo))
4155 if opts.update_self:
4156 updateSelf(fd, sys.argv[0])
4159 if len(all_urls) < 1:
4160 if not opts.update_self:
4161 parser.error(u'you must provide at least one URL')
4164 retcode = fd.download(all_urls)
4166 # Dump cookie jar if requested
4167 if opts.cookiefile is not None:
4170 except (IOError, OSError), err:
4171 sys.exit(u'ERROR: unable to save cookie jar')
4178 except DownloadError:
4180 except SameFileError:
4181 sys.exit(u'ERROR: fixed output name but more than one file to download')
4182 except KeyboardInterrupt:
4183 sys.exit(u'\nERROR: Interrupted by user')
4185 if __name__ == '__main__':
4188 # vim: set ts=4 sw=4 sts=4 noet ai si filetype=python: