This exception will be thrown when a video is requested
in a format that is not available for that video.
"""
+ pass
+
+class ContentTooShortError(Exception):
+ """Content Too Short exception.
+
+ This exception may be raised by FileDownloader objects when a file they
+ download is too small for what the server announced first, indicating
+ the connection was probably interrupted.
+ """
+ # Both in bytes
+ downloaded = None
+ expected = None
+
+ def __init__(self, downloaded, expected):
+ self.downloaded = downloaded
+ self.expected = expected
class FileDownloader(object):
"""File Downloader class.
ignoreerrors: Do not stop on download errors.
ratelimit: Download speed limit, in bytes/sec.
nooverwrites: Prevent overwriting files.
+ continuedl: Try to continue downloads if possible.
"""
params = None
new_min = max(bytes / 2.0, 1.0)
new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
if elapsed_time < 0.001:
- return int(new_max)
+ return long(new_max)
rate = bytes / elapsed_time
if rate > new_max:
- return int(new_max)
+ return long(new_max)
if rate < new_min:
- return int(new_min)
- return int(rate)
+ return long(new_min)
+ return long(rate)
@staticmethod
def parse_bytes(bytestr):
multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
return long(round(number * multiplier))
+ @staticmethod
+ def verify_url(url):
+ """Verify a URL is valid and data could be downloaded."""
+ request = urllib2.Request(url, None, std_headers)
+ data = urllib2.urlopen(request)
+ data.read(1)
+ data.close()
+
def add_info_extractor(self, ie):
"""Add an InfoExtractor object to the end of the list."""
self._ies.append(ie)
"""Report download progress."""
self.to_stdout(u'\r[download] %s of %s at %s ETA %s' %
(percent_str, data_len_str, speed_str, eta_str), skip_eol=True)
+
+ def report_resuming_byte(self, resume_len):
+ """Report attemtp to resume at given byte."""
+ self.to_stdout(u'[download] Resuming download at byte %s' % resume_len)
+
+ def report_file_already_downloaded(self, file_name):
+ """Report file has already been fully downloaded."""
+ self.to_stdout(u'[download] %s has already been downloaded' % file_name)
+
+ def report_unable_to_resume(self):
+ """Report it was impossible to resume download."""
+ self.to_stdout(u'[download] Unable to resume')
def report_finish(self):
"""Report download finished."""
def process_info(self, info_dict):
"""Process a single dictionary returned by an InfoExtractor."""
- # Forced printings
- if self.params.get('forcetitle', False):
- print info_dict['title'].encode(locale.getpreferredencoding())
- if self.params.get('forceurl', False):
- print info_dict['url'].encode(locale.getpreferredencoding())
-
# Do nothing else if in simulate mode
if self.params.get('simulate', False):
- return
+ try:
+ self.verify_url(info_dict['url'])
+ except (OSError, IOError, urllib2.URLError, httplib.HTTPException, socket.error), err:
+ raise UnavailableFormatError
+ # Forced printings
+ if self.params.get('forcetitle', False):
+ print info_dict['title'].encode(locale.getpreferredencoding())
+ if self.params.get('forceurl', False):
+ print info_dict['url'].encode(locale.getpreferredencoding())
+
+ return
+
try:
template_dict = dict(info_dict)
template_dict['epoch'] = unicode(long(time.time()))
return
try:
- outstream = open(filename, 'wb')
+ outstream = open(filename, 'ab')
except (OSError, IOError), err:
self.trouble('ERROR: unable to open for writing: %s' % str(err))
return
self._do_download(outstream, info_dict['url'])
outstream.close()
except (OSError, IOError), err:
+ outstream.close()
os.remove(filename)
raise UnavailableFormatError
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
self.trouble('ERROR: unable to download video data: %s' % str(err))
return
+ except (ContentTooShortError, ), err:
+ self.trouble('ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
+ return
try:
self.post_process(filename, info_dict)
break
def _do_download(self, stream, url):
+ basic_request = urllib2.Request(url, None, std_headers)
request = urllib2.Request(url, None, std_headers)
- data = urllib2.urlopen(request)
+
+ # Resume transfer if filesize is non-zero
+ resume_len = stream.tell()
+ if self.params['continuedl'] and resume_len != 0:
+ self.report_resuming_byte(resume_len)
+ request.add_header('Range','bytes=%d-' % resume_len)
+ else:
+ stream.close()
+ stream = open(stream.name,'wb')
+ try:
+ data = urllib2.urlopen(request)
+ except urllib2.HTTPError, e:
+ if not e.code == 416: # 416 is 'Requested range not satisfiable'
+ raise
+ data = urllib2.urlopen(basic_request)
+ content_length = data.info()['Content-Length']
+ if content_length is not None and long(content_length) == resume_len:
+ self.report_file_already_downloaded(stream.name)
+ return
+ else:
+ self.report_unable_to_resume()
+ stream.close()
+ stream = open(stream.name,'wb')
+
data_len = data.info().get('Content-length', None)
data_len_str = self.format_bytes(data_len)
byte_counter = 0
self.report_finish()
if data_len is not None and str(byte_counter) != data_len:
- raise ValueError('Content too short: %s/%s bytes' % (byte_counter, data_len))
+ raise ContentTooShortError(byte_counter, long(data_len))
class InfoExtractor(object):
"""Information Extractor class.
best_quality = True
while True:
+ # Extension
+ video_extension = self._video_extensions.get(format_param, 'flv')
+
+ # Normalize URL, including format
+ normalized_url = 'http://www.youtube.com/watch?v=%s&gl=US&hl=en' % video_id
+ if format_param is not None:
+ normalized_url = '%s&fmt=%s' % (normalized_url, format_param)
+ request = urllib2.Request(normalized_url, None, std_headers)
try:
- # Extension
- video_extension = self._video_extensions.get(format_param, 'flv')
-
- # Normalize URL, including format
- normalized_url = 'http://www.youtube.com/watch?v=%s&gl=US&hl=en' % video_id
- if format_param is not None:
- normalized_url = '%s&fmt=%s' % (normalized_url, format_param)
- request = urllib2.Request(normalized_url, None, std_headers)
- try:
- self.report_webpage_download(video_id)
- video_webpage = urllib2.urlopen(request).read()
- except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
- return
- self.report_information_extraction(video_id)
-
- # "t" param
- mobj = re.search(r', "t": "([^"]+)"', video_webpage)
- if mobj is None:
- self._downloader.trouble(u'ERROR: unable to extract "t" parameter')
- return
- video_real_url = 'http://www.youtube.com/get_video?video_id=%s&t=%s&el=detailpage&ps=' % (video_id, mobj.group(1))
- if format_param is not None:
- video_real_url = '%s&fmt=%s' % (video_real_url, format_param)
- self.report_video_url(video_id, video_real_url)
-
- # uploader
- mobj = re.search(r"var watchUsername = '([^']+)';", video_webpage)
- if mobj is None:
- self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
- return
- video_uploader = mobj.group(1)
+ self.report_webpage_download(video_id)
+ video_webpage = urllib2.urlopen(request).read()
+ except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+ self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % str(err))
+ return
+ self.report_information_extraction(video_id)
+
+ # "t" param
+ mobj = re.search(r', "t": "([^"]+)"', video_webpage)
+ if mobj is None:
+ self._downloader.trouble(u'ERROR: unable to extract "t" parameter')
+ return
+ video_real_url = 'http://www.youtube.com/get_video?video_id=%s&t=%s&el=detailpage&ps=' % (video_id, mobj.group(1))
+ if format_param is not None:
+ video_real_url = '%s&fmt=%s' % (video_real_url, format_param)
+ self.report_video_url(video_id, video_real_url)
+
+ # uploader
+ mobj = re.search(r"var watchUsername = '([^']+)';", video_webpage)
+ if mobj is None:
+ self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
+ return
+ video_uploader = mobj.group(1)
- # title
- mobj = re.search(r'(?im)<title>YouTube - ([^<]*)</title>', video_webpage)
- if mobj is None:
- self._downloader.trouble(u'ERROR: unable to extract video title')
- return
- video_title = mobj.group(1).decode('utf-8')
- video_title = re.sub(ur'(?u)&(.+?);', self.htmlentity_transform, video_title)
- video_title = video_title.replace(os.sep, u'%')
+ # title
+ mobj = re.search(r'(?im)<title>YouTube - ([^<]*)</title>', video_webpage)
+ if mobj is None:
+ self._downloader.trouble(u'ERROR: unable to extract video title')
+ return
+ video_title = mobj.group(1).decode('utf-8')
+ video_title = re.sub(ur'(?u)&(.+?);', self.htmlentity_transform, video_title)
+ video_title = video_title.replace(os.sep, u'%')
- # simplified title
- simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
- simple_title = simple_title.strip(ur'_')
+ # simplified title
+ simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
+ simple_title = simple_title.strip(ur'_')
+ try:
# Process video information
self._downloader.process_info({
'id': video_id.decode('utf-8'),
return
else:
try:
- n = int(prefix)
+ n = long(prefix)
if n <= 0:
self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
return
n = self._max_youtube_results
self._download_n_results(query, n)
return
- except ValueError: # parsing prefix as int fails
+ except ValueError: # parsing prefix as integer fails
self._download_n_results(query, 1)
return
video_format = optparse.OptionGroup(parser, 'Video Format Options')
video_format.add_option('-f', '--format',
- action='append', dest='format', metavar='FMT', help='video format code')
+ action='store', dest='format', metavar='FMT', help='video format code')
video_format.add_option('-b', '--best-quality',
- action='append_const', dest='format', help='download the best quality video possible', const='0')
+ action='store_const', dest='format', help='download the best quality video possible', const='0')
video_format.add_option('-m', '--mobile-version',
- action='append_const', dest='format', help='alias for -f 17', const='17')
+ action='store_const', dest='format', help='alias for -f 17', const='17')
video_format.add_option('-d', '--high-def',
- action='append_const', dest='format', help='alias for -f 22', const='22')
+ action='store_const', dest='format', help='alias for -f 22', const='22')
parser.add_option_group(video_format)
verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
dest='batchfile', metavar='F', help='file containing URLs to download')
filesystem.add_option('-w', '--no-overwrites',
action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
+ filesystem.add_option('-c', '--continue',
+ action='store_true', dest='continue_dl', help='resume partially downloaded files', default=False)
parser.add_option_group(filesystem)
(opts, args) = parser.parse_args()
if numeric_limit is None:
parser.error(u'invalid rate limit specified')
opts.ratelimit = numeric_limit
- if opts.format is not None and len(opts.format) > 1:
- parser.error(u'pass at most one of the video format option flags (-f, -b, -m, -d)')
- if opts.format is None:
- real_format = None
- else:
- real_format = opts.format[0]
-
# Information extractors
youtube_ie = YoutubeIE()
'forceurl': opts.geturl,
'forcetitle': opts.gettitle,
'simulate': (opts.simulate or opts.geturl or opts.gettitle),
- 'format': real_format,
+ 'format': opts.format,
'outtmpl': ((opts.outtmpl is not None and opts.outtmpl.decode(locale.getpreferredencoding()))
or (opts.usetitle and u'%(stitle)s-%(id)s.%(ext)s')
or (opts.useliteral and u'%(title)s-%(id)s.%(ext)s')
'ignoreerrors': opts.ignoreerrors,
'ratelimit': opts.ratelimit,
'nooverwrites': opts.nooverwrites,
+ 'continuedl': opts.continue_dl,
})
fd.add_info_extractor(youtube_search_ie)
fd.add_info_extractor(youtube_pl_ie)
projects / youtube-dl.git / blobdiff