2 # -*- coding: utf-8 -*-
4 from __future__ import absolute_import, unicode_literals
30 compat_urllib_request,
53 UnavailableVideoError,
60 from .extractor import get_info_extractor, gen_extractors
61 from .downloader import get_suitable_downloader
62 from .postprocessor import FFmpegMergerPP
63 from .version import __version__
66 class YoutubeDL(object):
69 YoutubeDL objects are the ones responsible of downloading the
70 actual video file and writing it to disk if the user has requested
71 it, among some other tasks. In most cases there should be one per
72 program. As, given a video URL, the downloader doesn't know how to
73 extract all the needed information, task that InfoExtractors do, it
74 has to pass the URL to one of them.
76 For this, YoutubeDL objects have a method that allows
77 InfoExtractors to be registered in a given order. When it is passed
78 a URL, the YoutubeDL object handles it to the first InfoExtractor it
79 finds that reports being able to handle it. The InfoExtractor extracts
80 all the information about the video or videos the URL refers to, and
81 YoutubeDL process the extracted information, possibly using a File
82 Downloader to download the video.
84 YoutubeDL objects accept a lot of parameters. In order not to saturate
85 the object constructor with arguments, it receives a dictionary of
86 options instead. These options are available through the params
87 attribute for the InfoExtractors to use. The YoutubeDL also
88 registers itself as the downloader in charge for the InfoExtractors
89 that are added to it, so this is a "mutual registration".
93 username: Username for authentication purposes.
94 password: Password for authentication purposes.
95 videopassword: Password for acces a video.
96 usenetrc: Use netrc for authentication instead.
97 verbose: Print additional info to stdout.
98 quiet: Do not print messages to stdout.
99 no_warnings: Do not print out anything for warnings.
100 forceurl: Force printing final URL.
101 forcetitle: Force printing title.
102 forceid: Force printing ID.
103 forcethumbnail: Force printing thumbnail URL.
104 forcedescription: Force printing description.
105 forcefilename: Force printing final filename.
106 forceduration: Force printing duration.
107 forcejson: Force printing info_dict as JSON.
108 simulate: Do not download the video files.
109 format: Video format code.
110 format_limit: Highest quality format to try.
111 outtmpl: Template for output names.
112 restrictfilenames: Do not allow "&" and spaces in file names
113 ignoreerrors: Do not stop on download errors.
114 nooverwrites: Prevent overwriting files.
115 playliststart: Playlist item to start at.
116 playlistend: Playlist item to end at.
117 matchtitle: Download only matching titles.
118 rejecttitle: Reject downloads for matching titles.
119 logger: Log messages to a logging.Logger instance.
120 logtostderr: Log messages to stderr instead of stdout.
121 writedescription: Write the video description to a .description file
122 writeinfojson: Write the video description to a .info.json file
123 writeannotations: Write the video annotations to a .annotations.xml file
124 writethumbnail: Write the thumbnail image to a file
125 writesubtitles: Write the video subtitles to a file
126 writeautomaticsub: Write the automatic subtitles to a file
127 allsubtitles: Downloads all the subtitles of the video
128 (requires writesubtitles or writeautomaticsub)
129 listsubtitles: Lists all available subtitles for the video
130 subtitlesformat: Subtitle format [srt/sbv/vtt] (default=srt)
131 subtitleslangs: List of languages of the subtitles to download
132 keepvideo: Keep the video file after post-processing
133 daterange: A DateRange object, download only if the upload_date is in the range.
134 skip_download: Skip the actual download of the video file
135 cachedir: Location of the cache files in the filesystem.
136 None to disable filesystem cache.
137 noplaylist: Download single video instead of a playlist if in doubt.
138 age_limit: An integer representing the user's age in years.
139 Unsuitable videos for the given age are skipped.
140 min_views: An integer representing the minimum view count the video
141 must have in order to not be skipped.
142 Videos without view count information are always
143 downloaded. None for no limit.
144 max_views: An integer representing the maximum view count.
145 Videos that are more popular than that are not
147 Videos without view count information are always
148 downloaded. None for no limit.
149 download_archive: File name of a file where all downloads are recorded.
150 Videos already present in the file are not downloaded
152 cookiefile: File name where cookies should be read from and dumped to.
153 nocheckcertificate:Do not verify SSL certificates
154 prefer_insecure: Use HTTP instead of HTTPS to retrieve information.
155 At the moment, this is only supported by YouTube.
156 proxy: URL of the proxy server to use
157 socket_timeout: Time to wait for unresponsive hosts, in seconds
158 bidi_workaround: Work around buggy terminals without bidirectional text
159 support, using fridibi
160 debug_printtraffic:Print out sent and received HTTP traffic
161 include_ads: Download ads as well
162 default_search: Prepend this string if an input url is not valid.
163 'auto' for elaborate guessing
164 encoding: Use this encoding instead of the system-specified.
165 extract_flat: Do not resolve URLs, return the immediate result.
167 The following parameters are not used by YoutubeDL itself, they are used by
169 nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
170 noresizebuffer, retries, continuedl, noprogress, consoletitle
172 The following options are used by the post processors:
173 prefer_ffmpeg: If True, use ffmpeg instead of avconv if both are available,
174 otherwise prefer avconv.
175 exec_cmd: Arbitrary command to run after downloading
181 _download_retcode = None
182 _num_downloads = None
185 def __init__(self, params=None):
186 """Create a FileDownloader object with the given options."""
190 self._ies_instances = {}
192 self._progress_hooks = []
193 self._download_retcode = 0
194 self._num_downloads = 0
195 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
196 self._err_file = sys.stderr
199 if params.get('bidi_workaround', False):
202 master, slave = pty.openpty()
203 width = get_term_width()
207 width_args = ['-w', str(width)]
209 stdin=subprocess.PIPE,
211 stderr=self._err_file)
213 self._output_process = subprocess.Popen(
214 ['bidiv'] + width_args, **sp_kwargs
217 self._output_process = subprocess.Popen(
218 ['fribidi', '-c', 'UTF-8'] + width_args, **sp_kwargs)
219 self._output_channel = os.fdopen(master, 'rb')
220 except OSError as ose:
222 self.report_warning('Could not find fribidi executable, ignoring --bidi-workaround . Make sure that fribidi is an executable file in one of the directories in your $PATH.')
226 if (sys.version_info >= (3,) and sys.platform != 'win32' and
227 sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968']
228 and not params['restrictfilenames']):
229 # On Python 3, the Unicode filesystem API will throw errors (#1474)
231 'Assuming --restrict-filenames since file system encoding '
232 'cannot encode all charactes. '
233 'Set the LC_ALL environment variable to fix this.')
234 self.params['restrictfilenames'] = True
236 if '%(stitle)s' in self.params.get('outtmpl', ''):
237 self.report_warning('%(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.')
241 def add_info_extractor(self, ie):
242 """Add an InfoExtractor object to the end of the list."""
244 self._ies_instances[ie.ie_key()] = ie
245 ie.set_downloader(self)
247 def get_info_extractor(self, ie_key):
249 Get an instance of an IE with name ie_key, it will try to get one from
250 the _ies list, if there's no instance it will create a new one and add
251 it to the extractor list.
253 ie = self._ies_instances.get(ie_key)
255 ie = get_info_extractor(ie_key)()
256 self.add_info_extractor(ie)
259 def add_default_info_extractors(self):
261 Add the InfoExtractors returned by gen_extractors to the end of the list
263 for ie in gen_extractors():
264 self.add_info_extractor(ie)
266 def add_post_processor(self, pp):
267 """Add a PostProcessor object to the end of the chain."""
269 pp.set_downloader(self)
271 def add_progress_hook(self, ph):
272 """Add the progress hook (currently only for the file downloader)"""
273 self._progress_hooks.append(ph)
275 def _bidi_workaround(self, message):
276 if not hasattr(self, '_output_channel'):
279 assert hasattr(self, '_output_process')
280 assert isinstance(message, compat_str)
281 line_count = message.count('\n') + 1
282 self._output_process.stdin.write((message + '\n').encode('utf-8'))
283 self._output_process.stdin.flush()
284 res = ''.join(self._output_channel.readline().decode('utf-8')
285 for _ in range(line_count))
286 return res[:-len('\n')]
288 def to_screen(self, message, skip_eol=False):
289 """Print message to stdout if not in quiet mode."""
290 return self.to_stdout(message, skip_eol, check_quiet=True)
292 def _write_string(self, s, out=None):
293 write_string(s, out=out, encoding=self.params.get('encoding'))
295 def to_stdout(self, message, skip_eol=False, check_quiet=False):
296 """Print message to stdout if not in quiet mode."""
297 if self.params.get('logger'):
298 self.params['logger'].debug(message)
299 elif not check_quiet or not self.params.get('quiet', False):
300 message = self._bidi_workaround(message)
301 terminator = ['\n', ''][skip_eol]
302 output = message + terminator
304 self._write_string(output, self._screen_file)
306 def to_stderr(self, message):
307 """Print message to stderr."""
308 assert isinstance(message, compat_str)
309 if self.params.get('logger'):
310 self.params['logger'].error(message)
312 message = self._bidi_workaround(message)
313 output = message + '\n'
314 self._write_string(output, self._err_file)
316 def to_console_title(self, message):
317 if not self.params.get('consoletitle', False):
319 if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
320 # c_wchar_p() might not be necessary if `message` is
321 # already of type unicode()
322 ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
323 elif 'TERM' in os.environ:
324 self._write_string('\033]0;%s\007' % message, self._screen_file)
326 def save_console_title(self):
327 if not self.params.get('consoletitle', False):
329 if 'TERM' in os.environ:
330 # Save the title on stack
331 self._write_string('\033[22;0t', self._screen_file)
333 def restore_console_title(self):
334 if not self.params.get('consoletitle', False):
336 if 'TERM' in os.environ:
337 # Restore the title from stack
338 self._write_string('\033[23;0t', self._screen_file)
341 self.save_console_title()
344 def __exit__(self, *args):
345 self.restore_console_title()
347 if self.params.get('cookiefile') is not None:
348 self.cookiejar.save()
350 def trouble(self, message=None, tb=None):
351 """Determine action to take when a download problem appears.
353 Depending on if the downloader has been configured to ignore
354 download errors or not, this method may throw an exception or
355 not when errors are found, after printing the message.
357 tb, if given, is additional traceback information.
359 if message is not None:
360 self.to_stderr(message)
361 if self.params.get('verbose'):
363 if sys.exc_info()[0]: # if .trouble has been called from an except block
365 if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
366 tb += ''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
367 tb += compat_str(traceback.format_exc())
369 tb_data = traceback.format_list(traceback.extract_stack())
370 tb = ''.join(tb_data)
372 if not self.params.get('ignoreerrors', False):
373 if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
374 exc_info = sys.exc_info()[1].exc_info
376 exc_info = sys.exc_info()
377 raise DownloadError(message, exc_info)
378 self._download_retcode = 1
380 def report_warning(self, message):
382 Print the message to stderr, it will be prefixed with 'WARNING:'
383 If stderr is a tty file the 'WARNING:' will be colored
385 if self.params.get('logger') is not None:
386 self.params['logger'].warning(message)
388 if self.params.get('no_warnings'):
390 if self._err_file.isatty() and os.name != 'nt':
391 _msg_header = '\033[0;33mWARNING:\033[0m'
393 _msg_header = 'WARNING:'
394 warning_message = '%s %s' % (_msg_header, message)
395 self.to_stderr(warning_message)
397 def report_error(self, message, tb=None):
399 Do the same as trouble, but prefixes the message with 'ERROR:', colored
400 in red if stderr is a tty file.
402 if self._err_file.isatty() and os.name != 'nt':
403 _msg_header = '\033[0;31mERROR:\033[0m'
405 _msg_header = 'ERROR:'
406 error_message = '%s %s' % (_msg_header, message)
407 self.trouble(error_message, tb)
409 def report_file_already_downloaded(self, file_name):
410 """Report file has already been fully downloaded."""
412 self.to_screen('[download] %s has already been downloaded' % file_name)
413 except UnicodeEncodeError:
414 self.to_screen('[download] The file has already been downloaded')
416 def prepare_filename(self, info_dict):
417 """Generate the output filename."""
419 template_dict = dict(info_dict)
421 template_dict['epoch'] = int(time.time())
422 autonumber_size = self.params.get('autonumber_size')
423 if autonumber_size is None:
425 autonumber_templ = '%0' + str(autonumber_size) + 'd'
426 template_dict['autonumber'] = autonumber_templ % self._num_downloads
427 if template_dict.get('playlist_index') is not None:
428 template_dict['playlist_index'] = '%05d' % template_dict['playlist_index']
429 if template_dict.get('resolution') is None:
430 if template_dict.get('width') and template_dict.get('height'):
431 template_dict['resolution'] = '%dx%d' % (template_dict['width'], template_dict['height'])
432 elif template_dict.get('height'):
433 template_dict['resolution'] = '%sp' % template_dict['height']
434 elif template_dict.get('width'):
435 template_dict['resolution'] = '?x%d' % template_dict['width']
437 sanitize = lambda k, v: sanitize_filename(
439 restricted=self.params.get('restrictfilenames'),
441 template_dict = dict((k, sanitize(k, v))
442 for k, v in template_dict.items()
444 template_dict = collections.defaultdict(lambda: 'NA', template_dict)
446 outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
447 tmpl = os.path.expanduser(outtmpl)
448 filename = tmpl % template_dict
450 except ValueError as err:
451 self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
454 def _match_entry(self, info_dict):
455 """ Returns None iff the file should be downloaded """
457 video_title = info_dict.get('title', info_dict.get('id', 'video'))
458 if 'title' in info_dict:
459 # This can happen when we're just evaluating the playlist
460 title = info_dict['title']
461 matchtitle = self.params.get('matchtitle', False)
463 if not re.search(matchtitle, title, re.IGNORECASE):
464 return '"' + title + '" title did not match pattern "' + matchtitle + '"'
465 rejecttitle = self.params.get('rejecttitle', False)
467 if re.search(rejecttitle, title, re.IGNORECASE):
468 return '"' + title + '" title matched reject pattern "' + rejecttitle + '"'
469 date = info_dict.get('upload_date', None)
471 dateRange = self.params.get('daterange', DateRange())
472 if date not in dateRange:
473 return '%s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
474 view_count = info_dict.get('view_count', None)
475 if view_count is not None:
476 min_views = self.params.get('min_views')
477 if min_views is not None and view_count < min_views:
478 return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title, view_count, min_views)
479 max_views = self.params.get('max_views')
480 if max_views is not None and view_count > max_views:
481 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views)
482 age_limit = self.params.get('age_limit')
483 if age_limit is not None:
484 actual_age_limit = info_dict.get('age_limit')
485 if actual_age_limit is None:
487 if age_limit < actual_age_limit:
488 return 'Skipping "' + title + '" because it is age restricted'
489 if self.in_download_archive(info_dict):
490 return '%s has already been recorded in archive' % video_title
494 def add_extra_info(info_dict, extra_info):
495 '''Set the keys from extra_info in info dict if they are missing'''
496 for key, value in extra_info.items():
497 info_dict.setdefault(key, value)
499 def extract_info(self, url, download=True, ie_key=None, extra_info={},
502 Returns a list with a dictionary for each video we find.
503 If 'download', also downloads the videos.
504 extra_info is a dict containing the extra values to add to each result
508 ies = [self.get_info_extractor(ie_key)]
513 if not ie.suitable(url):
517 self.report_warning('The program functionality for this site has been marked as broken, '
518 'and will probably not work.')
521 ie_result = ie.extract(url)
522 if ie_result is None: # Finished already (backwards compatibility; listformats and friends should be moved here)
524 if isinstance(ie_result, list):
525 # Backwards compatibility: old IE result format
527 '_type': 'compat_list',
528 'entries': ie_result,
530 self.add_default_extra_info(ie_result, ie, url)
532 return self.process_ie_result(ie_result, download, extra_info)
535 except ExtractorError as de: # An error we somewhat expected
536 self.report_error(compat_str(de), de.format_traceback())
538 except MaxDownloadsReached:
540 except Exception as e:
541 if self.params.get('ignoreerrors', False):
542 self.report_error(compat_str(e), tb=compat_str(traceback.format_exc()))
547 self.report_error('no suitable InfoExtractor for URL %s' % url)
549 def add_default_extra_info(self, ie_result, ie, url):
550 self.add_extra_info(ie_result, {
551 'extractor': ie.IE_NAME,
553 'webpage_url_basename': url_basename(url),
554 'extractor_key': ie.ie_key(),
557 def process_ie_result(self, ie_result, download=True, extra_info={}):
559 Take the result of the ie(may be modified) and resolve all unresolved
560 references (URLs, playlist items).
562 It will also download the videos if 'download'.
563 Returns the resolved ie_result.
566 result_type = ie_result.get('_type', 'video')
568 if self.params.get('extract_flat', False):
569 if result_type in ('url', 'url_transparent'):
572 if result_type == 'video':
573 self.add_extra_info(ie_result, extra_info)
574 return self.process_video_result(ie_result, download=download)
575 elif result_type == 'url':
576 # We have to add extra_info to the results because it may be
577 # contained in a playlist
578 return self.extract_info(ie_result['url'],
580 ie_key=ie_result.get('ie_key'),
581 extra_info=extra_info)
582 elif result_type == 'url_transparent':
583 # Use the information from the embedding page
584 info = self.extract_info(
585 ie_result['url'], ie_key=ie_result.get('ie_key'),
586 extra_info=extra_info, download=False, process=False)
588 def make_result(embedded_info):
589 new_result = ie_result.copy()
590 for f in ('_type', 'url', 'ext', 'player_url', 'formats',
591 'entries', 'ie_key', 'duration',
592 'subtitles', 'annotations', 'format',
593 'thumbnail', 'thumbnails'):
596 if f in embedded_info:
597 new_result[f] = embedded_info[f]
599 new_result = make_result(info)
601 assert new_result.get('_type') != 'url_transparent'
602 if new_result.get('_type') == 'compat_list':
603 new_result['entries'] = [
604 make_result(e) for e in new_result['entries']]
606 return self.process_ie_result(
607 new_result, download=download, extra_info=extra_info)
608 elif result_type == 'playlist':
609 # We process each entry in the playlist
610 playlist = ie_result.get('title', None) or ie_result.get('id', None)
611 self.to_screen('[download] Downloading playlist: %s' % playlist)
613 playlist_results = []
615 playliststart = self.params.get('playliststart', 1) - 1
616 playlistend = self.params.get('playlistend', None)
617 # For backwards compatibility, interpret -1 as whole list
618 if playlistend == -1:
621 if isinstance(ie_result['entries'], list):
622 n_all_entries = len(ie_result['entries'])
623 entries = ie_result['entries'][playliststart:playlistend]
624 n_entries = len(entries)
626 "[%s] playlist %s: Collected %d video ids (downloading %d of them)" %
627 (ie_result['extractor'], playlist, n_all_entries, n_entries))
629 assert isinstance(ie_result['entries'], PagedList)
630 entries = ie_result['entries'].getslice(
631 playliststart, playlistend)
632 n_entries = len(entries)
634 "[%s] playlist %s: Downloading %d videos" %
635 (ie_result['extractor'], playlist, n_entries))
637 for i, entry in enumerate(entries, 1):
638 self.to_screen('[download] Downloading video #%s of %s' % (i, n_entries))
640 'playlist': playlist,
641 'playlist_index': i + playliststart,
642 'extractor': ie_result['extractor'],
643 'webpage_url': ie_result['webpage_url'],
644 'webpage_url_basename': url_basename(ie_result['webpage_url']),
645 'extractor_key': ie_result['extractor_key'],
648 reason = self._match_entry(entry)
649 if reason is not None:
650 self.to_screen('[download] ' + reason)
653 entry_result = self.process_ie_result(entry,
656 playlist_results.append(entry_result)
657 ie_result['entries'] = playlist_results
659 elif result_type == 'compat_list':
661 self.add_extra_info(r,
663 'extractor': ie_result['extractor'],
664 'webpage_url': ie_result['webpage_url'],
665 'webpage_url_basename': url_basename(ie_result['webpage_url']),
666 'extractor_key': ie_result['extractor_key'],
669 ie_result['entries'] = [
670 self.process_ie_result(_fixup(r), download, extra_info)
671 for r in ie_result['entries']
675 raise Exception('Invalid result type: %s' % result_type)
677 def select_format(self, format_spec, available_formats):
678 if format_spec == 'best' or format_spec is None:
679 return available_formats[-1]
680 elif format_spec == 'worst':
681 return available_formats[0]
682 elif format_spec == 'bestaudio':
684 f for f in available_formats
685 if f.get('vcodec') == 'none']
687 return audio_formats[-1]
688 elif format_spec == 'worstaudio':
690 f for f in available_formats
691 if f.get('vcodec') == 'none']
693 return audio_formats[0]
694 elif format_spec == 'bestvideo':
696 f for f in available_formats
697 if f.get('acodec') == 'none']
699 return video_formats[-1]
700 elif format_spec == 'worstvideo':
702 f for f in available_formats
703 if f.get('acodec') == 'none']
705 return video_formats[0]
707 extensions = ['mp4', 'flv', 'webm', '3gp']
708 if format_spec in extensions:
709 filter_f = lambda f: f['ext'] == format_spec
711 filter_f = lambda f: f['format_id'] == format_spec
712 matches = list(filter(filter_f, available_formats))
717 def process_video_result(self, info_dict, download=True):
718 assert info_dict.get('_type', 'video') == 'video'
720 if 'id' not in info_dict:
721 raise ExtractorError('Missing "id" field in extractor result')
722 if 'title' not in info_dict:
723 raise ExtractorError('Missing "title" field in extractor result')
725 if 'playlist' not in info_dict:
726 # It isn't part of a playlist
727 info_dict['playlist'] = None
728 info_dict['playlist_index'] = None
730 thumbnails = info_dict.get('thumbnails')
732 thumbnails.sort(key=lambda t: (
733 t.get('width'), t.get('height'), t.get('url')))
735 if 'width' in t and 'height' in t:
736 t['resolution'] = '%dx%d' % (t['width'], t['height'])
738 if thumbnails and 'thumbnail' not in info_dict:
739 info_dict['thumbnail'] = thumbnails[-1]['url']
741 if 'display_id' not in info_dict and 'id' in info_dict:
742 info_dict['display_id'] = info_dict['id']
744 if info_dict.get('upload_date') is None and info_dict.get('timestamp') is not None:
745 upload_date = datetime.datetime.utcfromtimestamp(
746 info_dict['timestamp'])
747 info_dict['upload_date'] = upload_date.strftime('%Y%m%d')
749 # This extractors handle format selection themselves
750 if info_dict['extractor'] in ['Youku']:
752 self.process_info(info_dict)
755 # We now pick which formats have to be downloaded
756 if info_dict.get('formats') is None:
757 # There's only one format available
758 formats = [info_dict]
760 formats = info_dict['formats']
763 raise ExtractorError('No video formats found!')
765 # We check that all the formats have the format and format_id fields
766 for i, format in enumerate(formats):
767 if 'url' not in format:
768 raise ExtractorError('Missing "url" key in result (index %d)' % i)
770 if format.get('format_id') is None:
771 format['format_id'] = compat_str(i)
772 if format.get('format') is None:
773 format['format'] = '{id} - {res}{note}'.format(
774 id=format['format_id'],
775 res=self.format_resolution(format),
776 note=' ({0})'.format(format['format_note']) if format.get('format_note') is not None else '',
778 # Automatically determine file extension if missing
779 if 'ext' not in format:
780 format['ext'] = determine_ext(format['url']).lower()
782 format_limit = self.params.get('format_limit', None)
784 formats = list(takewhile_inclusive(
785 lambda f: f['format_id'] != format_limit, formats
788 # TODO Central sorting goes here
790 if formats[0] is not info_dict:
791 # only set the 'formats' fields if the original info_dict list them
792 # otherwise we end up with a circular reference, the first (and unique)
793 # element in the 'formats' field in info_dict is info_dict itself,
794 # wich can't be exported to json
795 info_dict['formats'] = formats
796 if self.params.get('listformats', None):
797 self.list_formats(info_dict)
800 req_format = self.params.get('format')
801 if req_format is None:
803 formats_to_download = []
804 # The -1 is for supporting YoutubeIE
805 if req_format in ('-1', 'all'):
806 formats_to_download = formats
808 # We can accept formats requested in the format: 34/5/best, we pick
809 # the first that is available, starting from left
810 req_formats = req_format.split('/')
811 for rf in req_formats:
812 if re.match(r'.+?\+.+?', rf) is not None:
813 # Two formats have been requested like '137+139'
814 format_1, format_2 = rf.split('+')
815 formats_info = (self.select_format(format_1, formats),
816 self.select_format(format_2, formats))
817 if all(formats_info):
819 'requested_formats': formats_info,
821 'ext': formats_info[0]['ext'],
824 selected_format = None
826 selected_format = self.select_format(rf, formats)
827 if selected_format is not None:
828 formats_to_download = [selected_format]
830 if not formats_to_download:
831 raise ExtractorError('requested format not available',
835 if len(formats_to_download) > 1:
836 self.to_screen('[info] %s: downloading video in %s formats' % (info_dict['id'], len(formats_to_download)))
837 for format in formats_to_download:
838 new_info = dict(info_dict)
839 new_info.update(format)
840 self.process_info(new_info)
841 # We update the info dict with the best quality format (backwards compatibility)
842 info_dict.update(formats_to_download[-1])
845 def process_info(self, info_dict):
846 """Process a single resolved IE result."""
848 assert info_dict.get('_type', 'video') == 'video'
850 max_downloads = self.params.get('max_downloads')
851 if max_downloads is not None:
852 if self._num_downloads >= int(max_downloads):
853 raise MaxDownloadsReached()
855 info_dict['fulltitle'] = info_dict['title']
856 if len(info_dict['title']) > 200:
857 info_dict['title'] = info_dict['title'][:197] + '...'
859 # Keep for backwards compatibility
860 info_dict['stitle'] = info_dict['title']
862 if 'format' not in info_dict:
863 info_dict['format'] = info_dict['ext']
865 reason = self._match_entry(info_dict)
866 if reason is not None:
867 self.to_screen('[download] ' + reason)
870 self._num_downloads += 1
872 filename = self.prepare_filename(info_dict)
875 if self.params.get('forcetitle', False):
876 self.to_stdout(info_dict['fulltitle'])
877 if self.params.get('forceid', False):
878 self.to_stdout(info_dict['id'])
879 if self.params.get('forceurl', False):
880 # For RTMP URLs, also include the playpath
881 self.to_stdout(info_dict['url'] + info_dict.get('play_path', ''))
882 if self.params.get('forcethumbnail', False) and info_dict.get('thumbnail') is not None:
883 self.to_stdout(info_dict['thumbnail'])
884 if self.params.get('forcedescription', False) and info_dict.get('description') is not None:
885 self.to_stdout(info_dict['description'])
886 if self.params.get('forcefilename', False) and filename is not None:
887 self.to_stdout(filename)
888 if self.params.get('forceduration', False) and info_dict.get('duration') is not None:
889 self.to_stdout(formatSeconds(info_dict['duration']))
890 if self.params.get('forceformat', False):
891 self.to_stdout(info_dict['format'])
892 if self.params.get('forcejson', False):
893 info_dict['_filename'] = filename
894 self.to_stdout(json.dumps(info_dict))
896 # Do nothing else if in simulate mode
897 if self.params.get('simulate', False):
904 dn = os.path.dirname(encodeFilename(filename))
905 if dn and not os.path.exists(dn):
907 except (OSError, IOError) as err:
908 self.report_error('unable to create directory ' + compat_str(err))
911 if self.params.get('writedescription', False):
912 descfn = filename + '.description'
913 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)):
914 self.to_screen('[info] Video description is already present')
917 self.to_screen('[info] Writing video description to: ' + descfn)
918 with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
919 descfile.write(info_dict['description'])
920 except (KeyError, TypeError):
921 self.report_warning('There\'s no description to write.')
922 except (OSError, IOError):
923 self.report_error('Cannot write description file ' + descfn)
926 if self.params.get('writeannotations', False):
927 annofn = filename + '.annotations.xml'
928 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)):
929 self.to_screen('[info] Video annotations are already present')
932 self.to_screen('[info] Writing video annotations to: ' + annofn)
933 with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile:
934 annofile.write(info_dict['annotations'])
935 except (KeyError, TypeError):
936 self.report_warning('There are no annotations to write.')
937 except (OSError, IOError):
938 self.report_error('Cannot write annotations file: ' + annofn)
941 subtitles_are_requested = any([self.params.get('writesubtitles', False),
942 self.params.get('writeautomaticsub')])
944 if subtitles_are_requested and 'subtitles' in info_dict and info_dict['subtitles']:
945 # subtitles download errors are already managed as troubles in relevant IE
946 # that way it will silently go on when used with unsupporting IE
947 subtitles = info_dict['subtitles']
948 sub_format = self.params.get('subtitlesformat', 'srt')
949 for sub_lang in subtitles.keys():
950 sub = subtitles[sub_lang]
954 sub_filename = subtitles_filename(filename, sub_lang, sub_format)
955 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)):
956 self.to_screen('[info] Video subtitle %s.%s is already_present' % (sub_lang, sub_format))
958 self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
959 with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
961 except (OSError, IOError):
962 self.report_error('Cannot write subtitles file ' + sub_filename)
965 if self.params.get('writeinfojson', False):
966 infofn = os.path.splitext(filename)[0] + '.info.json'
967 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)):
968 self.to_screen('[info] Video description metadata is already present')
970 self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
972 write_json_file(info_dict, encodeFilename(infofn))
973 except (OSError, IOError):
974 self.report_error('Cannot write metadata to JSON file ' + infofn)
977 if self.params.get('writethumbnail', False):
978 if info_dict.get('thumbnail') is not None:
979 thumb_format = determine_ext(info_dict['thumbnail'], 'jpg')
980 thumb_filename = os.path.splitext(filename)[0] + '.' + thumb_format
981 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
982 self.to_screen('[%s] %s: Thumbnail is already present' %
983 (info_dict['extractor'], info_dict['id']))
985 self.to_screen('[%s] %s: Downloading thumbnail ...' %
986 (info_dict['extractor'], info_dict['id']))
988 uf = self.urlopen(info_dict['thumbnail'])
989 with open(thumb_filename, 'wb') as thumbf:
990 shutil.copyfileobj(uf, thumbf)
991 self.to_screen('[%s] %s: Writing thumbnail to: %s' %
992 (info_dict['extractor'], info_dict['id'], thumb_filename))
993 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
994 self.report_warning('Unable to download thumbnail "%s": %s' %
995 (info_dict['thumbnail'], compat_str(err)))
997 if not self.params.get('skip_download', False):
998 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(filename)):
1003 fd = get_suitable_downloader(info)(self, self.params)
1004 for ph in self._progress_hooks:
1005 fd.add_progress_hook(ph)
1006 if self.params.get('verbose'):
1007 self.to_stdout('[debug] Invoking downloader on %r' % info.get('url'))
1008 return fd.download(name, info)
1009 if info_dict.get('requested_formats') is not None:
1012 merger = FFmpegMergerPP(self, not self.params.get('keepvideo'))
1013 if not merger._get_executable():
1015 self.report_warning('You have requested multiple '
1016 'formats but ffmpeg or avconv are not installed.'
1017 ' The formats won\'t be merged')
1019 postprocessors = [merger]
1020 for f in info_dict['requested_formats']:
1021 new_info = dict(info_dict)
1023 fname = self.prepare_filename(new_info)
1024 fname = prepend_extension(fname, 'f%s' % f['format_id'])
1025 downloaded.append(fname)
1026 partial_success = dl(fname, new_info)
1027 success = success and partial_success
1028 info_dict['__postprocessors'] = postprocessors
1029 info_dict['__files_to_merge'] = downloaded
1031 # Just a single file
1032 success = dl(filename, info_dict)
1033 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1034 self.report_error('unable to download video data: %s' % str(err))
1036 except (OSError, IOError) as err:
1037 raise UnavailableVideoError(err)
1038 except (ContentTooShortError, ) as err:
1039 self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
1044 self.post_process(filename, info_dict)
1045 except (PostProcessingError) as err:
1046 self.report_error('postprocessing: %s' % str(err))
1049 self.record_download_archive(info_dict)
1051 def download(self, url_list):
1052 """Download a given list of URLs."""
1053 outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
1054 if (len(url_list) > 1 and
1056 and self.params.get('max_downloads') != 1):
1057 raise SameFileError(outtmpl)
1059 for url in url_list:
1061 #It also downloads the videos
1062 self.extract_info(url)
1063 except UnavailableVideoError:
1064 self.report_error('unable to download video')
1065 except MaxDownloadsReached:
1066 self.to_screen('[info] Maximum number of downloaded files reached.')
1069 return self._download_retcode
1071 def download_with_info_file(self, info_filename):
1072 with io.open(info_filename, 'r', encoding='utf-8') as f:
1075 self.process_ie_result(info, download=True)
1076 except DownloadError:
1077 webpage_url = info.get('webpage_url')
1078 if webpage_url is not None:
1079 self.report_warning('The info failed to download, trying with "%s"' % webpage_url)
1080 return self.download([webpage_url])
1083 return self._download_retcode
1085 def post_process(self, filename, ie_info):
1086 """Run all the postprocessors on the given file."""
1087 info = dict(ie_info)
1088 info['filepath'] = filename
1091 if ie_info.get('__postprocessors') is not None:
1092 pps_chain.extend(ie_info['__postprocessors'])
1093 pps_chain.extend(self._pps)
1094 for pp in pps_chain:
1096 keep_video_wish, new_info = pp.run(info)
1097 if keep_video_wish is not None:
1099 keep_video = keep_video_wish
1100 elif keep_video is None:
1101 # No clear decision yet, let IE decide
1102 keep_video = keep_video_wish
1103 except PostProcessingError as e:
1104 self.report_error(e.msg)
1105 if keep_video is False and not self.params.get('keepvideo', False):
1107 self.to_screen('Deleting original file %s (pass -k to keep)' % filename)
1108 os.remove(encodeFilename(filename))
1109 except (IOError, OSError):
1110 self.report_warning('Unable to remove downloaded video file')
1112 def _make_archive_id(self, info_dict):
1113 # Future-proof against any change in case
1114 # and backwards compatibility with prior versions
1115 extractor = info_dict.get('extractor_key')
1116 if extractor is None:
1117 if 'id' in info_dict:
1118 extractor = info_dict.get('ie_key') # key in a playlist
1119 if extractor is None:
1120 return None # Incomplete video information
1121 return extractor.lower() + ' ' + info_dict['id']
1123 def in_download_archive(self, info_dict):
1124 fn = self.params.get('download_archive')
1128 vid_id = self._make_archive_id(info_dict)
1130 return False # Incomplete video information
1133 with locked_file(fn, 'r', encoding='utf-8') as archive_file:
1134 for line in archive_file:
1135 if line.strip() == vid_id:
1137 except IOError as ioe:
1138 if ioe.errno != errno.ENOENT:
1142 def record_download_archive(self, info_dict):
1143 fn = self.params.get('download_archive')
1146 vid_id = self._make_archive_id(info_dict)
1148 with locked_file(fn, 'a', encoding='utf-8') as archive_file:
1149 archive_file.write(vid_id + '\n')
1152 def format_resolution(format, default='unknown'):
1153 if format.get('vcodec') == 'none':
1155 if format.get('resolution') is not None:
1156 return format['resolution']
1157 if format.get('height') is not None:
1158 if format.get('width') is not None:
1159 res = '%sx%s' % (format['width'], format['height'])
1161 res = '%sp' % format['height']
1162 elif format.get('width') is not None:
1163 res = '?x%d' % format['width']
1168 def _format_note(self, fdict):
1170 if fdict.get('ext') in ['f4f', 'f4m']:
1171 res += '(unsupported) '
1172 if fdict.get('format_note') is not None:
1173 res += fdict['format_note'] + ' '
1174 if fdict.get('tbr') is not None:
1175 res += '%4dk ' % fdict['tbr']
1176 if fdict.get('container') is not None:
1179 res += '%s container' % fdict['container']
1180 if (fdict.get('vcodec') is not None and
1181 fdict.get('vcodec') != 'none'):
1184 res += fdict['vcodec']
1185 if fdict.get('vbr') is not None:
1187 elif fdict.get('vbr') is not None and fdict.get('abr') is not None:
1189 if fdict.get('vbr') is not None:
1190 res += '%4dk' % fdict['vbr']
1191 if fdict.get('acodec') is not None:
1194 if fdict['acodec'] == 'none':
1197 res += '%-5s' % fdict['acodec']
1198 elif fdict.get('abr') is not None:
1202 if fdict.get('abr') is not None:
1203 res += '@%3dk' % fdict['abr']
1204 if fdict.get('asr') is not None:
1205 res += ' (%5dHz)' % fdict['asr']
1206 if fdict.get('filesize') is not None:
1209 res += format_bytes(fdict['filesize'])
1210 elif fdict.get('filesize_approx') is not None:
1213 res += '~' + format_bytes(fdict['filesize_approx'])
1216 def list_formats(self, info_dict):
1217 def line(format, idlen=20):
1218 return (('%-' + compat_str(idlen + 1) + 's%-10s%-12s%s') % (
1219 format['format_id'],
1221 self.format_resolution(format),
1222 self._format_note(format),
1225 formats = info_dict.get('formats', [info_dict])
1226 idlen = max(len('format code'),
1227 max(len(f['format_id']) for f in formats))
1228 formats_s = [line(f, idlen) for f in formats]
1229 if len(formats) > 1:
1230 formats_s[0] += (' ' if self._format_note(formats[0]) else '') + '(worst)'
1231 formats_s[-1] += (' ' if self._format_note(formats[-1]) else '') + '(best)'
1233 header_line = line({
1234 'format_id': 'format code', 'ext': 'extension',
1235 'resolution': 'resolution', 'format_note': 'note'}, idlen=idlen)
1236 self.to_screen('[info] Available formats for %s:\n%s\n%s' %
1237 (info_dict['id'], header_line, '\n'.join(formats_s)))
1239 def urlopen(self, req):
1240 """ Start an HTTP download """
1241 return self._opener.open(req, timeout=self._socket_timeout)
1243 def print_debug_header(self):
1244 if not self.params.get('verbose'):
1247 if type('') is not compat_str:
1248 # Python 2.6 on SLES11 SP1 (https://github.com/rg3/youtube-dl/issues/3326)
1249 self.report_warning(
1250 'Your Python is broken! Update to a newer and supported version')
1253 '[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % (
1254 locale.getpreferredencoding(),
1255 sys.getfilesystemencoding(),
1256 sys.stdout.encoding,
1257 self.get_encoding()))
1258 write_string(encoding_str, encoding=None)
1260 self._write_string('[debug] youtube-dl version ' + __version__ + '\n')
1262 sp = subprocess.Popen(
1263 ['git', 'rev-parse', '--short', 'HEAD'],
1264 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
1265 cwd=os.path.dirname(os.path.abspath(__file__)))
1266 out, err = sp.communicate()
1267 out = out.decode().strip()
1268 if re.match('[0-9a-f]+', out):
1269 self._write_string('[debug] Git HEAD: ' + out + '\n')
1275 self._write_string('[debug] Python version %s - %s' %
1276 (platform.python_version(), platform_name()) + '\n')
1279 for handler in self._opener.handlers:
1280 if hasattr(handler, 'proxies'):
1281 proxy_map.update(handler.proxies)
1282 self._write_string('[debug] Proxy map: ' + compat_str(proxy_map) + '\n')
1284 def _setup_opener(self):
1285 timeout_val = self.params.get('socket_timeout')
1286 self._socket_timeout = 600 if timeout_val is None else float(timeout_val)
1288 opts_cookiefile = self.params.get('cookiefile')
1289 opts_proxy = self.params.get('proxy')
1291 if opts_cookiefile is None:
1292 self.cookiejar = compat_cookiejar.CookieJar()
1294 self.cookiejar = compat_cookiejar.MozillaCookieJar(
1296 if os.access(opts_cookiefile, os.R_OK):
1297 self.cookiejar.load()
1299 cookie_processor = compat_urllib_request.HTTPCookieProcessor(
1301 if opts_proxy is not None:
1302 if opts_proxy == '':
1305 proxies = {'http': opts_proxy, 'https': opts_proxy}
1307 proxies = compat_urllib_request.getproxies()
1308 # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
1309 if 'http' in proxies and 'https' not in proxies:
1310 proxies['https'] = proxies['http']
1311 proxy_handler = compat_urllib_request.ProxyHandler(proxies)
1313 debuglevel = 1 if self.params.get('debug_printtraffic') else 0
1314 https_handler = make_HTTPS_handler(
1315 self.params.get('nocheckcertificate', False), debuglevel=debuglevel)
1316 ydlh = YoutubeDLHandler(debuglevel=debuglevel)
1317 opener = compat_urllib_request.build_opener(
1318 https_handler, proxy_handler, cookie_processor, ydlh)
1319 # Delete the default user-agent header, which would otherwise apply in
1320 # cases where our custom HTTP handler doesn't come into play
1321 # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
1322 opener.addheaders = []
1323 self._opener = opener
1325 def encode(self, s):
1326 if isinstance(s, bytes):
1327 return s # Already encoded
1330 return s.encode(self.get_encoding())
1331 except UnicodeEncodeError as err:
1332 err.reason = err.reason + '. Check your system encoding configuration or use the --encoding option.'
1335 def get_encoding(self):
1336 encoding = self.params.get('encoding')
1337 if encoding is None:
1338 encoding = preferredencoding()