X-Git-Url: http://git.cielonegro.org/gitweb.cgi?a=blobdiff_plain;f=youtube-dl;h=2cddafcf54bb9bb2478928d9675080394cc57e0d;hb=af6a92f4c954d8f0e6628076f751d6ac9935a6d6;hp=29afd930f0782ccfc01b74150e9aefbe885307c7;hpb=25af2bce3ad6a17741388ffa10ae269770c1cfa3;p=youtube-dl.git diff --git a/youtube-dl b/youtube-dl index 29afd930f..2cddafcf5 100755 --- a/youtube-dl +++ b/youtube-dl @@ -1,6 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # Author: Ricardo Garcia Gonzalez +# Author: Danny Colligan # License: Public domain code import htmlentitydefs import httplib @@ -17,7 +18,7 @@ import time import urllib import urllib2 -std_headers = { +std_headers = { 'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.0.5) Gecko/2008120122 Firefox/3.0.5', 'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7', 'Accept': 'text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5', @@ -70,10 +71,10 @@ class FileDownloader(object): File downloaders accept a lot of parameters. In order not to saturate the object constructor with arguments, it receives a dictionary of - options instead. These options are available through the get_params() - method for the InfoExtractors to use. The FileDownloader also registers - itself as the downloader in charge for the InfoExtractors that are - added to it, so this is a "mutual registration". + options instead. These options are available through the params + attribute for the InfoExtractors to use. The FileDownloader also + registers itself as the downloader in charge for the InfoExtractors + that are added to it, so this is a "mutual registration". Available options: @@ -88,9 +89,10 @@ class FileDownloader(object): outtmpl: Template for output names. ignoreerrors: Do not stop on download errors. ratelimit: Download speed limit, in bytes/sec. + nooverwrites: Prevent overwriting files. """ - _params = None + params = None _ies = [] _pps = [] @@ -98,7 +100,7 @@ class FileDownloader(object): """Create a FileDownloader object with the given options.""" self._ies = [] self._pps = [] - self.set_params(params) + self.params = params @staticmethod def pmkdir(filename): @@ -142,7 +144,7 @@ class FileDownloader(object): return '--:--' return '%02d:%02d' % (eta_mins, eta_secs) - @staticmethod + @staticmethod def calc_speed(start, now, bytes): dif = now - start if bytes == 0 or dif < 0.001: # One millisecond @@ -172,16 +174,6 @@ class FileDownloader(object): multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower()) return long(round(number * multiplier)) - def set_params(self, params): - """Sets parameters.""" - if type(params) != dict: - raise ValueError('params: dictionary expected') - self._params = params - - def get_params(self): - """Get parameters.""" - return self._params - def add_info_extractor(self, ie): """Add an InfoExtractor object to the end of the list.""" self._ies.append(ie) @@ -194,7 +186,7 @@ class FileDownloader(object): def to_stdout(self, message, skip_eol=False): """Print message to stdout if not in quiet mode.""" - if not self._params.get('quiet', False): + if not self.params.get('quiet', False): print u'%s%s' % (message, [u'\n', u''][skip_eol]), sys.stdout.flush() @@ -204,7 +196,7 @@ class FileDownloader(object): def fixed_template(self): """Checks if the output template is fixed.""" - return (re.search(ur'(?u)%\(.+?\)s', self._params['outtmpl']) is None) + return (re.search(ur'(?u)%\(.+?\)s', self.params['outtmpl']) is None) def trouble(self, message=None): """Determine action to take when a download problem appears. @@ -217,13 +209,13 @@ class FileDownloader(object): """ if message is not None: self.to_stderr(message) - if not self._params.get('ignoreerrors', False): + if not self.params.get('ignoreerrors', False): raise DownloadError(message) return 1 def slow_down(self, start_time, byte_counter): """Sleep if the download speed is over the rate limit.""" - rate_limit = self._params.get('ratelimit', None) + rate_limit = self.params.get('ratelimit', None) if rate_limit is None or byte_counter == 0: return now = time.time() @@ -251,7 +243,7 @@ class FileDownloader(object): """Download a given list of URLs.""" retcode = 0 if len(url_list) > 1 and self.fixed_template(): - raise SameFileError(self._params['outtmpl']) + raise SameFileError(self.params['outtmpl']) for url in url_list: suitable_found = False @@ -266,25 +258,28 @@ class FileDownloader(object): retcode = self.trouble() if len(results) > 1 and self.fixed_template(): - raise SameFileError(self._params['outtmpl']) + raise SameFileError(self.params['outtmpl']) for result in results: # Forced printings - if self._params.get('forcetitle', False): + if self.params.get('forcetitle', False): print result['title'] - if self._params.get('forceurl', False): + if self.params.get('forceurl', False): print result['url'] # Do nothing else if in simulate mode - if self._params.get('simulate', False): + if self.params.get('simulate', False): continue try: - filename = self._params['outtmpl'] % result + filename = self.params['outtmpl'] % result self.report_destination(filename) except (ValueError, KeyError), err: retcode = self.trouble('ERROR: invalid output template or system charset: %s' % str(err)) continue + if self.params['nooverwrites'] and os.path.exists(filename): + self.to_stderr('WARNING: file exists: %s; skipping' % filename) + continue try: self.pmkdir(filename) except (OSError, IOError), err: @@ -412,7 +407,7 @@ class InfoExtractor(object): def to_stdout(self, message): """Print message to stdout if downloader is not in quiet mode.""" - if self._downloader is None or not self._downloader.get_params().get('quiet', False): + if self._downloader is None or not self._downloader.params.get('quiet', False): print message def to_stderr(self, message): @@ -440,6 +435,29 @@ class YoutubeIE(InfoExtractor): def suitable(url): return (re.match(YoutubeIE._VALID_URL, url) is not None) + @staticmethod + def htmlentity_transform(matchobj): + """Transforms an HTML entity to a Unicode character.""" + entity = matchobj.group(1) + + # Known non-numeric HTML entity + if entity in htmlentitydefs.name2codepoint: + return unichr(htmlentitydefs.name2codepoint[entity]) + + # Unicode character + mobj = re.match(ur'(?u)#(x?\d+)', entity) + if mobj is not None: + numstr = mobj.group(1) + if numstr.startswith(u'x'): + base = 16 + numstr = u'0%s' % numstr + else: + base = 10 + return unichr(long(numstr, base)) + + # Unknown entity in name, return its literal representation + return (u'&%s;' % entity) + def report_lang(self): """Report attempt to set language.""" self.to_stdout(u'[youtube] Setting language') @@ -463,14 +481,14 @@ class YoutubeIE(InfoExtractor): def report_video_url(self, video_id, video_real_url): """Report extracted video URL.""" self.to_stdout(u'[youtube] %s: URL: %s' % (video_id, video_real_url)) - + def _real_initialize(self): if self._downloader is None: return username = None password = None - downloader_params = self._downloader.get_params() + downloader_params = self._downloader.params # Attempt to use provided username and password or .netrc data if downloader_params.get('username', None) is not None: @@ -488,12 +506,8 @@ class YoutubeIE(InfoExtractor): self.to_stderr(u'WARNING: parsing .netrc: %s' % str(err)) return - # No authentication to be performed - if username is None: - return - # Set language - request = urllib2.Request(self._LOGIN_URL, None, std_headers) + request = urllib2.Request(self._LANG_URL, None, std_headers) try: self.report_lang() urllib2.urlopen(request).read() @@ -501,6 +515,10 @@ class YoutubeIE(InfoExtractor): self.to_stderr(u'WARNING: unable to set language: %s' % str(err)) return + # No authentication to be performed + if username is None: + return + # Log in login_form = { 'current_form': 'loginForm', @@ -544,11 +562,15 @@ class YoutubeIE(InfoExtractor): # Downloader parameters format_param = None if self._downloader is not None: - params = self._downloader.get_params() + params = self._downloader.params format_param = params.get('format', None) # Extension - video_extension = {'18': 'mp4', '17': '3gp'}.get(format_param, 'flv') + video_extension = { + '17': '3gp', + '18': 'mp4', + '22': 'mp4', + }.get(format_param, 'flv') # Normalize URL, including format normalized_url = 'http://www.youtube.com/watch?v=%s&gl=US&hl=en' % video_id @@ -586,7 +608,7 @@ class YoutubeIE(InfoExtractor): self.to_stderr(u'ERROR: unable to extract video title') return [None] video_title = mobj.group(1).decode('utf-8') - video_title = re.sub(ur'(?u)&(.+?);', lambda x: unichr(htmlentitydefs.name2codepoint[x.group(1)]), video_title) + video_title = re.sub(ur'(?u)&(.+?);', self.htmlentity_transform, video_title) video_title = video_title.replace(os.sep, u'%') # simplified title @@ -730,7 +752,7 @@ class YoutubeSearchIE(InfoExtractor): _MORE_PAGES_INDICATOR = r'>Next' _youtube_ie = None - def __init__(self, youtube_ie, downloader=None): + def __init__(self, youtube_ie, downloader=None): InfoExtractor.__init__(self, downloader) self._youtube_ie = youtube_ie @@ -753,11 +775,11 @@ class YoutubeSearchIE(InfoExtractor): prefix, query = query.split(':') prefix = prefix[8:] - if prefix == '': + if prefix == '': return self._download_n_results(query, 1) - elif prefix == 'all': + elif prefix == 'all': return self._download_n_results(query, -1) - else: + else: try: n = int(prefix) if n <= 0: @@ -776,7 +798,7 @@ class YoutubeSearchIE(InfoExtractor): while True: self.report_download_page(query, pagenum) - result_url = self._TEMPLATE_URL % (urllib.quote(query.replace(' ', '+')), pagenum) + result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum) request = urllib2.Request(result_url, None, std_headers) try: page = urllib2.urlopen(request).read() @@ -890,7 +912,7 @@ class PostProcessor(object): def to_stdout(self, message): """Print message to stdout if downloader is not in quiet mode.""" - if self._downloader is None or not self._downloader.get_params().get('quiet', False): + if self._downloader is None or not self._downloader.params.get('quiet', False): print message def to_stderr(self, message): @@ -936,7 +958,7 @@ if __name__ == '__main__': # Parse command line parser = optparse.OptionParser( usage='Usage: %prog [options] url...', - version='2009.01.31', + version='INTERNAL', conflict_handler='resolve', ) parser.add_option('-h', '--help', @@ -965,16 +987,18 @@ if __name__ == '__main__': action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False) parser.add_option('-f', '--format', dest='format', metavar='FMT', help='video format code') - parser.add_option('-b', '--best-quality', - action='store_const', dest='format', help='alias for -f 18', const='18') parser.add_option('-m', '--mobile-version', action='store_const', dest='format', help='alias for -f 17', const='17') + parser.add_option('-d', '--high-def', + action='store_const', dest='format', help='alias for -f 22', const='22') parser.add_option('-i', '--ignore-errors', action='store_true', dest='ignoreerrors', help='continue on download errors', default=False) parser.add_option('-r', '--rate-limit', dest='ratelimit', metavar='L', help='download rate limit (e.g. 50k or 44.6m)') parser.add_option('-a', '--batch-file', dest='batchfile', metavar='F', help='file containing URLs to download') + parser.add_option('-w', '--no-overwrites', + action='store_true', dest='nooverwrites', help='do not overwrite files', default=False) (opts, args) = parser.parse_args() # Batch file verification @@ -1030,6 +1054,7 @@ if __name__ == '__main__': or u'%(id)s.%(ext)s'), 'ignoreerrors': opts.ignoreerrors, 'ratelimit': opts.ratelimit, + 'nooverwrites': opts.nooverwrites, }) fd.add_info_extractor(youtube_search_ie) fd.add_info_extractor(youtube_pl_ie)