2 # -*- coding: utf-8 -*-
4 from __future__ import absolute_import, unicode_literals
33 compat_get_terminal_size,
38 compat_urllib_request,
58 PerRequestProxyHandler,
68 UnavailableVideoError,
79 from .cache import Cache
80 from .extractor import get_info_extractor, gen_extractors
81 from .downloader import get_suitable_downloader
82 from .downloader.rtmp import rtmpdump_version
83 from .postprocessor import (
85 FFmpegFixupStretchedPP,
90 from .version import __version__
93 class YoutubeDL(object):
96 YoutubeDL objects are the ones responsible of downloading the
97 actual video file and writing it to disk if the user has requested
98 it, among some other tasks. In most cases there should be one per
99 program. As, given a video URL, the downloader doesn't know how to
100 extract all the needed information, task that InfoExtractors do, it
101 has to pass the URL to one of them.
103 For this, YoutubeDL objects have a method that allows
104 InfoExtractors to be registered in a given order. When it is passed
105 a URL, the YoutubeDL object handles it to the first InfoExtractor it
106 finds that reports being able to handle it. The InfoExtractor extracts
107 all the information about the video or videos the URL refers to, and
108 YoutubeDL process the extracted information, possibly using a File
109 Downloader to download the video.
111 YoutubeDL objects accept a lot of parameters. In order not to saturate
112 the object constructor with arguments, it receives a dictionary of
113 options instead. These options are available through the params
114 attribute for the InfoExtractors to use. The YoutubeDL also
115 registers itself as the downloader in charge for the InfoExtractors
116 that are added to it, so this is a "mutual registration".
120 username: Username for authentication purposes.
121 password: Password for authentication purposes.
122 videopassword: Password for accessing a video.
123 usenetrc: Use netrc for authentication instead.
124 verbose: Print additional info to stdout.
125 quiet: Do not print messages to stdout.
126 no_warnings: Do not print out anything for warnings.
127 forceurl: Force printing final URL.
128 forcetitle: Force printing title.
129 forceid: Force printing ID.
130 forcethumbnail: Force printing thumbnail URL.
131 forcedescription: Force printing description.
132 forcefilename: Force printing final filename.
133 forceduration: Force printing duration.
134 forcejson: Force printing info_dict as JSON.
135 dump_single_json: Force printing the info_dict of the whole playlist
136 (or video) as a single JSON line.
137 simulate: Do not download the video files.
138 format: Video format code. See options.py for more information.
139 outtmpl: Template for output names.
140 restrictfilenames: Do not allow "&" and spaces in file names
141 ignoreerrors: Do not stop on download errors.
142 force_generic_extractor: Force downloader to use the generic extractor
143 nooverwrites: Prevent overwriting files.
144 playliststart: Playlist item to start at.
145 playlistend: Playlist item to end at.
146 playlist_items: Specific indices of playlist to download.
147 playlistreverse: Download playlist items in reverse order.
148 matchtitle: Download only matching titles.
149 rejecttitle: Reject downloads for matching titles.
150 logger: Log messages to a logging.Logger instance.
151 logtostderr: Log messages to stderr instead of stdout.
152 writedescription: Write the video description to a .description file
153 writeinfojson: Write the video description to a .info.json file
154 writeannotations: Write the video annotations to a .annotations.xml file
155 writethumbnail: Write the thumbnail image to a file
156 write_all_thumbnails: Write all thumbnail formats to files
157 writesubtitles: Write the video subtitles to a file
158 writeautomaticsub: Write the automatic subtitles to a file
159 allsubtitles: Downloads all the subtitles of the video
160 (requires writesubtitles or writeautomaticsub)
161 listsubtitles: Lists all available subtitles for the video
162 subtitlesformat: The format code for subtitles
163 subtitleslangs: List of languages of the subtitles to download
164 keepvideo: Keep the video file after post-processing
165 daterange: A DateRange object, download only if the upload_date is in the range.
166 skip_download: Skip the actual download of the video file
167 cachedir: Location of the cache files in the filesystem.
168 False to disable filesystem cache.
169 noplaylist: Download single video instead of a playlist if in doubt.
170 age_limit: An integer representing the user's age in years.
171 Unsuitable videos for the given age are skipped.
172 min_views: An integer representing the minimum view count the video
173 must have in order to not be skipped.
174 Videos without view count information are always
175 downloaded. None for no limit.
176 max_views: An integer representing the maximum view count.
177 Videos that are more popular than that are not
179 Videos without view count information are always
180 downloaded. None for no limit.
181 download_archive: File name of a file where all downloads are recorded.
182 Videos already present in the file are not downloaded
184 cookiefile: File name where cookies should be read from and dumped to.
185 nocheckcertificate:Do not verify SSL certificates
186 prefer_insecure: Use HTTP instead of HTTPS to retrieve information.
187 At the moment, this is only supported by YouTube.
188 proxy: URL of the proxy server to use
189 cn_verification_proxy: URL of the proxy to use for IP address verification
190 on Chinese sites. (Experimental)
191 socket_timeout: Time to wait for unresponsive hosts, in seconds
192 bidi_workaround: Work around buggy terminals without bidirectional text
193 support, using fridibi
194 debug_printtraffic:Print out sent and received HTTP traffic
195 include_ads: Download ads as well
196 default_search: Prepend this string if an input url is not valid.
197 'auto' for elaborate guessing
198 encoding: Use this encoding instead of the system-specified.
199 extract_flat: Do not resolve URLs, return the immediate result.
200 Pass in 'in_playlist' to only show this behavior for
202 postprocessors: A list of dictionaries, each with an entry
203 * key: The name of the postprocessor. See
204 youtube_dl/postprocessor/__init__.py for a list.
205 as well as any further keyword arguments for the
207 progress_hooks: A list of functions that get called on download
208 progress, with a dictionary with the entries
209 * status: One of "downloading", "error", or "finished".
210 Check this first and ignore unknown values.
212 If status is one of "downloading", or "finished", the
213 following properties may also be present:
214 * filename: The final filename (always present)
215 * tmpfilename: The filename we're currently writing to
216 * downloaded_bytes: Bytes on disk
217 * total_bytes: Size of the whole file, None if unknown
218 * total_bytes_estimate: Guess of the eventual file size,
220 * elapsed: The number of seconds since download started.
221 * eta: The estimated time in seconds, None if unknown
222 * speed: The download speed in bytes/second, None if
224 * fragment_index: The counter of the currently
225 downloaded video fragment.
226 * fragment_count: The number of fragments (= individual
227 files that will be merged)
229 Progress hooks are guaranteed to be called at least once
230 (with status "finished") if the download is successful.
231 merge_output_format: Extension to use when merging formats.
232 fixup: Automatically correct known faults of the file.
234 - "never": do nothing
235 - "warn": only emit a warning
236 - "detect_or_warn": check whether we can do anything
237 about it, warn otherwise (default)
238 source_address: (Experimental) Client-side IP address to bind to.
239 call_home: Boolean, true iff we are allowed to contact the
240 youtube-dl servers for debugging.
241 sleep_interval: Number of seconds to sleep before each download.
242 listformats: Print an overview of available video formats and exit.
243 list_thumbnails: Print a table of all thumbnails and exit.
244 match_filter: A function that gets called with the info_dict of
246 If it returns a message, the video is ignored.
247 If it returns None, the video is downloaded.
248 match_filter_func in utils.py is one example for this.
249 no_color: Do not emit color codes in output.
251 The following options determine which downloader is picked:
252 external_downloader: Executable of the external downloader to call.
253 None or unset for standard (built-in) downloader.
254 hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv.
256 The following parameters are not used by YoutubeDL itself, they are used by
257 the downloader (see youtube_dl/downloader/common.py):
258 nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
259 noresizebuffer, retries, continuedl, noprogress, consoletitle,
260 xattr_set_filesize, external_downloader_args.
262 The following options are used by the post processors:
263 prefer_ffmpeg: If True, use ffmpeg instead of avconv if both are available,
264 otherwise prefer avconv.
270 _download_retcode = None
271 _num_downloads = None
274 def __init__(self, params=None, auto_init=True):
275 """Create a FileDownloader object with the given options."""
279 self._ies_instances = {}
281 self._progress_hooks = []
282 self._download_retcode = 0
283 self._num_downloads = 0
284 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
285 self._err_file = sys.stderr
287 self.cache = Cache(self)
289 if params.get('bidi_workaround', False):
292 master, slave = pty.openpty()
293 width = compat_get_terminal_size().columns
297 width_args = ['-w', str(width)]
299 stdin=subprocess.PIPE,
301 stderr=self._err_file)
303 self._output_process = subprocess.Popen(
304 ['bidiv'] + width_args, **sp_kwargs
307 self._output_process = subprocess.Popen(
308 ['fribidi', '-c', 'UTF-8'] + width_args, **sp_kwargs)
309 self._output_channel = os.fdopen(master, 'rb')
310 except OSError as ose:
312 self.report_warning('Could not find fribidi executable, ignoring --bidi-workaround . Make sure that fribidi is an executable file in one of the directories in your $PATH.')
316 if (sys.version_info >= (3,) and sys.platform != 'win32' and
317 sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968'] and
318 not params.get('restrictfilenames', False)):
319 # On Python 3, the Unicode filesystem API will throw errors (#1474)
321 'Assuming --restrict-filenames since file system encoding '
322 'cannot encode all characters. '
323 'Set the LC_ALL environment variable to fix this.')
324 self.params['restrictfilenames'] = True
326 if isinstance(params.get('outtmpl'), bytes):
328 'Parameter outtmpl is bytes, but should be a unicode string. '
329 'Put from __future__ import unicode_literals at the top of your code file or consider switching to Python 3.x.')
334 self.print_debug_header()
335 self.add_default_info_extractors()
337 for pp_def_raw in self.params.get('postprocessors', []):
338 pp_class = get_postprocessor(pp_def_raw['key'])
339 pp_def = dict(pp_def_raw)
341 pp = pp_class(self, **compat_kwargs(pp_def))
342 self.add_post_processor(pp)
344 for ph in self.params.get('progress_hooks', []):
345 self.add_progress_hook(ph)
347 def warn_if_short_id(self, argv):
348 # short YouTube ID starting with dash?
350 i for i, a in enumerate(argv)
351 if re.match(r'^-[0-9A-Za-z_-]{10}$', a)]
355 [a for i, a in enumerate(argv) if i not in idxs] +
356 ['--'] + [argv[i] for i in idxs]
359 'Long argument string detected. '
360 'Use -- to separate parameters and URLs, like this:\n%s\n' %
361 args_to_str(correct_argv))
363 def add_info_extractor(self, ie):
364 """Add an InfoExtractor object to the end of the list."""
366 self._ies_instances[ie.ie_key()] = ie
367 ie.set_downloader(self)
369 def get_info_extractor(self, ie_key):
371 Get an instance of an IE with name ie_key, it will try to get one from
372 the _ies list, if there's no instance it will create a new one and add
373 it to the extractor list.
375 ie = self._ies_instances.get(ie_key)
377 ie = get_info_extractor(ie_key)()
378 self.add_info_extractor(ie)
381 def add_default_info_extractors(self):
383 Add the InfoExtractors returned by gen_extractors to the end of the list
385 for ie in gen_extractors():
386 self.add_info_extractor(ie)
388 def add_post_processor(self, pp):
389 """Add a PostProcessor object to the end of the chain."""
391 pp.set_downloader(self)
393 def add_progress_hook(self, ph):
394 """Add the progress hook (currently only for the file downloader)"""
395 self._progress_hooks.append(ph)
397 def _bidi_workaround(self, message):
398 if not hasattr(self, '_output_channel'):
401 assert hasattr(self, '_output_process')
402 assert isinstance(message, compat_str)
403 line_count = message.count('\n') + 1
404 self._output_process.stdin.write((message + '\n').encode('utf-8'))
405 self._output_process.stdin.flush()
406 res = ''.join(self._output_channel.readline().decode('utf-8')
407 for _ in range(line_count))
408 return res[:-len('\n')]
410 def to_screen(self, message, skip_eol=False):
411 """Print message to stdout if not in quiet mode."""
412 return self.to_stdout(message, skip_eol, check_quiet=True)
414 def _write_string(self, s, out=None):
415 write_string(s, out=out, encoding=self.params.get('encoding'))
417 def to_stdout(self, message, skip_eol=False, check_quiet=False):
418 """Print message to stdout if not in quiet mode."""
419 if self.params.get('logger'):
420 self.params['logger'].debug(message)
421 elif not check_quiet or not self.params.get('quiet', False):
422 message = self._bidi_workaround(message)
423 terminator = ['\n', ''][skip_eol]
424 output = message + terminator
426 self._write_string(output, self._screen_file)
428 def to_stderr(self, message):
429 """Print message to stderr."""
430 assert isinstance(message, compat_str)
431 if self.params.get('logger'):
432 self.params['logger'].error(message)
434 message = self._bidi_workaround(message)
435 output = message + '\n'
436 self._write_string(output, self._err_file)
438 def to_console_title(self, message):
439 if not self.params.get('consoletitle', False):
441 if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
442 # c_wchar_p() might not be necessary if `message` is
443 # already of type unicode()
444 ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
445 elif 'TERM' in os.environ:
446 self._write_string('\033]0;%s\007' % message, self._screen_file)
448 def save_console_title(self):
449 if not self.params.get('consoletitle', False):
451 if 'TERM' in os.environ:
452 # Save the title on stack
453 self._write_string('\033[22;0t', self._screen_file)
455 def restore_console_title(self):
456 if not self.params.get('consoletitle', False):
458 if 'TERM' in os.environ:
459 # Restore the title from stack
460 self._write_string('\033[23;0t', self._screen_file)
463 self.save_console_title()
466 def __exit__(self, *args):
467 self.restore_console_title()
469 if self.params.get('cookiefile') is not None:
470 self.cookiejar.save()
472 def trouble(self, message=None, tb=None):
473 """Determine action to take when a download problem appears.
475 Depending on if the downloader has been configured to ignore
476 download errors or not, this method may throw an exception or
477 not when errors are found, after printing the message.
479 tb, if given, is additional traceback information.
481 if message is not None:
482 self.to_stderr(message)
483 if self.params.get('verbose'):
485 if sys.exc_info()[0]: # if .trouble has been called from an except block
487 if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
488 tb += ''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
489 tb += compat_str(traceback.format_exc())
491 tb_data = traceback.format_list(traceback.extract_stack())
492 tb = ''.join(tb_data)
494 if not self.params.get('ignoreerrors', False):
495 if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
496 exc_info = sys.exc_info()[1].exc_info
498 exc_info = sys.exc_info()
499 raise DownloadError(message, exc_info)
500 self._download_retcode = 1
502 def report_warning(self, message):
504 Print the message to stderr, it will be prefixed with 'WARNING:'
505 If stderr is a tty file the 'WARNING:' will be colored
507 if self.params.get('logger') is not None:
508 self.params['logger'].warning(message)
510 if self.params.get('no_warnings'):
512 if not self.params.get('no_color') and self._err_file.isatty() and os.name != 'nt':
513 _msg_header = '\033[0;33mWARNING:\033[0m'
515 _msg_header = 'WARNING:'
516 warning_message = '%s %s' % (_msg_header, message)
517 self.to_stderr(warning_message)
519 def report_error(self, message, tb=None):
521 Do the same as trouble, but prefixes the message with 'ERROR:', colored
522 in red if stderr is a tty file.
524 if not self.params.get('no_color') and self._err_file.isatty() and os.name != 'nt':
525 _msg_header = '\033[0;31mERROR:\033[0m'
527 _msg_header = 'ERROR:'
528 error_message = '%s %s' % (_msg_header, message)
529 self.trouble(error_message, tb)
531 def report_file_already_downloaded(self, file_name):
532 """Report file has already been fully downloaded."""
534 self.to_screen('[download] %s has already been downloaded' % file_name)
535 except UnicodeEncodeError:
536 self.to_screen('[download] The file has already been downloaded')
538 def prepare_filename(self, info_dict):
539 """Generate the output filename."""
541 template_dict = dict(info_dict)
543 template_dict['epoch'] = int(time.time())
544 autonumber_size = self.params.get('autonumber_size')
545 if autonumber_size is None:
547 autonumber_templ = '%0' + str(autonumber_size) + 'd'
548 template_dict['autonumber'] = autonumber_templ % self._num_downloads
549 if template_dict.get('playlist_index') is not None:
550 template_dict['playlist_index'] = '%0*d' % (len(str(template_dict['n_entries'])), template_dict['playlist_index'])
551 if template_dict.get('resolution') is None:
552 if template_dict.get('width') and template_dict.get('height'):
553 template_dict['resolution'] = '%dx%d' % (template_dict['width'], template_dict['height'])
554 elif template_dict.get('height'):
555 template_dict['resolution'] = '%sp' % template_dict['height']
556 elif template_dict.get('width'):
557 template_dict['resolution'] = '?x%d' % template_dict['width']
559 sanitize = lambda k, v: sanitize_filename(
561 restricted=self.params.get('restrictfilenames'),
563 template_dict = dict((k, sanitize(k, v))
564 for k, v in template_dict.items()
566 template_dict = collections.defaultdict(lambda: 'NA', template_dict)
568 outtmpl = sanitize_path(self.params.get('outtmpl', DEFAULT_OUTTMPL))
569 tmpl = compat_expanduser(outtmpl)
570 filename = tmpl % template_dict
571 # Temporary fix for #4787
572 # 'Treat' all problem characters by passing filename through preferredencoding
573 # to workaround encoding issues with subprocess on python2 @ Windows
574 if sys.version_info < (3, 0) and sys.platform == 'win32':
575 filename = encodeFilename(filename, True).decode(preferredencoding())
577 except ValueError as err:
578 self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
581 def _match_entry(self, info_dict, incomplete):
582 """ Returns None iff the file should be downloaded """
584 video_title = info_dict.get('title', info_dict.get('id', 'video'))
585 if 'title' in info_dict:
586 # This can happen when we're just evaluating the playlist
587 title = info_dict['title']
588 matchtitle = self.params.get('matchtitle', False)
590 if not re.search(matchtitle, title, re.IGNORECASE):
591 return '"' + title + '" title did not match pattern "' + matchtitle + '"'
592 rejecttitle = self.params.get('rejecttitle', False)
594 if re.search(rejecttitle, title, re.IGNORECASE):
595 return '"' + title + '" title matched reject pattern "' + rejecttitle + '"'
596 date = info_dict.get('upload_date', None)
598 dateRange = self.params.get('daterange', DateRange())
599 if date not in dateRange:
600 return '%s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
601 view_count = info_dict.get('view_count', None)
602 if view_count is not None:
603 min_views = self.params.get('min_views')
604 if min_views is not None and view_count < min_views:
605 return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title, view_count, min_views)
606 max_views = self.params.get('max_views')
607 if max_views is not None and view_count > max_views:
608 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views)
609 if age_restricted(info_dict.get('age_limit'), self.params.get('age_limit')):
610 return 'Skipping "%s" because it is age restricted' % video_title
611 if self.in_download_archive(info_dict):
612 return '%s has already been recorded in archive' % video_title
615 match_filter = self.params.get('match_filter')
616 if match_filter is not None:
617 ret = match_filter(info_dict)
624 def add_extra_info(info_dict, extra_info):
625 '''Set the keys from extra_info in info dict if they are missing'''
626 for key, value in extra_info.items():
627 info_dict.setdefault(key, value)
629 def extract_info(self, url, download=True, ie_key=None, extra_info={},
630 process=True, force_generic_extractor=False):
632 Returns a list with a dictionary for each video we find.
633 If 'download', also downloads the videos.
634 extra_info is a dict containing the extra values to add to each result
637 if not ie_key and force_generic_extractor:
641 ies = [self.get_info_extractor(ie_key)]
646 if not ie.suitable(url):
650 self.report_warning('The program functionality for this site has been marked as broken, '
651 'and will probably not work.')
654 ie_result = ie.extract(url)
655 if ie_result is None: # Finished already (backwards compatibility; listformats and friends should be moved here)
657 if isinstance(ie_result, list):
658 # Backwards compatibility: old IE result format
660 '_type': 'compat_list',
661 'entries': ie_result,
663 self.add_default_extra_info(ie_result, ie, url)
665 return self.process_ie_result(ie_result, download, extra_info)
668 except ExtractorError as de: # An error we somewhat expected
669 self.report_error(compat_str(de), de.format_traceback())
671 except MaxDownloadsReached:
673 except Exception as e:
674 if self.params.get('ignoreerrors', False):
675 self.report_error(compat_str(e), tb=compat_str(traceback.format_exc()))
680 self.report_error('no suitable InfoExtractor for URL %s' % url)
682 def add_default_extra_info(self, ie_result, ie, url):
683 self.add_extra_info(ie_result, {
684 'extractor': ie.IE_NAME,
686 'webpage_url_basename': url_basename(url),
687 'extractor_key': ie.ie_key(),
690 def process_ie_result(self, ie_result, download=True, extra_info={}):
692 Take the result of the ie(may be modified) and resolve all unresolved
693 references (URLs, playlist items).
695 It will also download the videos if 'download'.
696 Returns the resolved ie_result.
699 result_type = ie_result.get('_type', 'video')
701 if result_type in ('url', 'url_transparent'):
702 extract_flat = self.params.get('extract_flat', False)
703 if ((extract_flat == 'in_playlist' and 'playlist' in extra_info) or
704 extract_flat is True):
705 if self.params.get('forcejson', False):
706 self.to_stdout(json.dumps(ie_result))
709 if result_type == 'video':
710 self.add_extra_info(ie_result, extra_info)
711 return self.process_video_result(ie_result, download=download)
712 elif result_type == 'url':
713 # We have to add extra_info to the results because it may be
714 # contained in a playlist
715 return self.extract_info(ie_result['url'],
717 ie_key=ie_result.get('ie_key'),
718 extra_info=extra_info)
719 elif result_type == 'url_transparent':
720 # Use the information from the embedding page
721 info = self.extract_info(
722 ie_result['url'], ie_key=ie_result.get('ie_key'),
723 extra_info=extra_info, download=False, process=False)
725 force_properties = dict(
726 (k, v) for k, v in ie_result.items() if v is not None)
727 for f in ('_type', 'url'):
728 if f in force_properties:
729 del force_properties[f]
730 new_result = info.copy()
731 new_result.update(force_properties)
733 assert new_result.get('_type') != 'url_transparent'
735 return self.process_ie_result(
736 new_result, download=download, extra_info=extra_info)
737 elif result_type == 'playlist' or result_type == 'multi_video':
738 # We process each entry in the playlist
739 playlist = ie_result.get('title', None) or ie_result.get('id', None)
740 self.to_screen('[download] Downloading playlist: %s' % playlist)
742 playlist_results = []
744 playliststart = self.params.get('playliststart', 1) - 1
745 playlistend = self.params.get('playlistend', None)
746 # For backwards compatibility, interpret -1 as whole list
747 if playlistend == -1:
750 playlistitems_str = self.params.get('playlist_items', None)
752 if playlistitems_str is not None:
753 def iter_playlistitems(format):
754 for string_segment in format.split(','):
755 if '-' in string_segment:
756 start, end = string_segment.split('-')
757 for item in range(int(start), int(end) + 1):
760 yield int(string_segment)
761 playlistitems = iter_playlistitems(playlistitems_str)
763 ie_entries = ie_result['entries']
764 if isinstance(ie_entries, list):
765 n_all_entries = len(ie_entries)
768 ie_entries[i - 1] for i in playlistitems
769 if -n_all_entries <= i - 1 < n_all_entries]
771 entries = ie_entries[playliststart:playlistend]
772 n_entries = len(entries)
774 "[%s] playlist %s: Collected %d video ids (downloading %d of them)" %
775 (ie_result['extractor'], playlist, n_all_entries, n_entries))
776 elif isinstance(ie_entries, PagedList):
779 for item in playlistitems:
780 entries.extend(ie_entries.getslice(
784 entries = ie_entries.getslice(
785 playliststart, playlistend)
786 n_entries = len(entries)
788 "[%s] playlist %s: Downloading %d videos" %
789 (ie_result['extractor'], playlist, n_entries))
792 entry_list = list(ie_entries)
793 entries = [entry_list[i - 1] for i in playlistitems]
795 entries = list(itertools.islice(
796 ie_entries, playliststart, playlistend))
797 n_entries = len(entries)
799 "[%s] playlist %s: Downloading %d videos" %
800 (ie_result['extractor'], playlist, n_entries))
802 if self.params.get('playlistreverse', False):
803 entries = entries[::-1]
805 for i, entry in enumerate(entries, 1):
806 self.to_screen('[download] Downloading video %s of %s' % (i, n_entries))
808 'n_entries': n_entries,
809 'playlist': playlist,
810 'playlist_id': ie_result.get('id'),
811 'playlist_title': ie_result.get('title'),
812 'playlist_index': i + playliststart,
813 'extractor': ie_result['extractor'],
814 'webpage_url': ie_result['webpage_url'],
815 'webpage_url_basename': url_basename(ie_result['webpage_url']),
816 'extractor_key': ie_result['extractor_key'],
819 reason = self._match_entry(entry, incomplete=True)
820 if reason is not None:
821 self.to_screen('[download] ' + reason)
824 entry_result = self.process_ie_result(entry,
827 playlist_results.append(entry_result)
828 ie_result['entries'] = playlist_results
830 elif result_type == 'compat_list':
832 'Extractor %s returned a compat_list result. '
833 'It needs to be updated.' % ie_result.get('extractor'))
839 'extractor': ie_result['extractor'],
840 'webpage_url': ie_result['webpage_url'],
841 'webpage_url_basename': url_basename(ie_result['webpage_url']),
842 'extractor_key': ie_result['extractor_key'],
846 ie_result['entries'] = [
847 self.process_ie_result(_fixup(r), download, extra_info)
848 for r in ie_result['entries']
852 raise Exception('Invalid result type: %s' % result_type)
854 def _apply_format_filter(self, format_spec, available_formats):
855 " Returns a tuple of the remaining format_spec and filtered formats "
865 operator_rex = re.compile(r'''(?x)\s*\[
866 (?P<key>width|height|tbr|abr|vbr|asr|filesize|fps)
867 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
868 (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
870 ''' % '|'.join(map(re.escape, OPERATORS.keys())))
871 m = operator_rex.search(format_spec)
874 comparison_value = int(m.group('value'))
876 comparison_value = parse_filesize(m.group('value'))
877 if comparison_value is None:
878 comparison_value = parse_filesize(m.group('value') + 'B')
879 if comparison_value is None:
881 'Invalid value %r in format specification %r' % (
882 m.group('value'), format_spec))
883 op = OPERATORS[m.group('op')]
890 str_operator_rex = re.compile(r'''(?x)\s*\[
891 \s*(?P<key>ext|acodec|vcodec|container|protocol)
892 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?
893 \s*(?P<value>[a-zA-Z0-9_-]+)
895 ''' % '|'.join(map(re.escape, STR_OPERATORS.keys())))
896 m = str_operator_rex.search(format_spec)
898 comparison_value = m.group('value')
899 op = STR_OPERATORS[m.group('op')]
902 raise ValueError('Invalid format specification %r' % format_spec)
905 actual_value = f.get(m.group('key'))
906 if actual_value is None:
907 return m.group('none_inclusive')
908 return op(actual_value, comparison_value)
909 new_formats = [f for f in available_formats if _filter(f)]
911 new_format_spec = format_spec[:-len(m.group(0))]
912 if not new_format_spec:
913 new_format_spec = 'best'
915 return (new_format_spec, new_formats)
917 def select_format(self, format_spec, available_formats):
918 while format_spec.endswith(']'):
919 format_spec, available_formats = self._apply_format_filter(
920 format_spec, available_formats)
921 if not available_formats:
924 if format_spec in ['best', 'worst', None]:
925 format_idx = 0 if format_spec == 'worst' else -1
926 audiovideo_formats = [
927 f for f in available_formats
928 if f.get('vcodec') != 'none' and f.get('acodec') != 'none']
929 if audiovideo_formats:
930 return audiovideo_formats[format_idx]
931 # for audio only (soundcloud) or video only (imgur) urls, select the best/worst audio format
932 elif (all(f.get('acodec') != 'none' for f in available_formats) or
933 all(f.get('vcodec') != 'none' for f in available_formats)):
934 return available_formats[format_idx]
935 elif format_spec == 'bestaudio':
937 f for f in available_formats
938 if f.get('vcodec') == 'none']
940 return audio_formats[-1]
941 elif format_spec == 'worstaudio':
943 f for f in available_formats
944 if f.get('vcodec') == 'none']
946 return audio_formats[0]
947 elif format_spec == 'bestvideo':
949 f for f in available_formats
950 if f.get('acodec') == 'none']
952 return video_formats[-1]
953 elif format_spec == 'worstvideo':
955 f for f in available_formats
956 if f.get('acodec') == 'none']
958 return video_formats[0]
960 extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav']
961 if format_spec in extensions:
962 filter_f = lambda f: f['ext'] == format_spec
964 filter_f = lambda f: f['format_id'] == format_spec
965 matches = list(filter(filter_f, available_formats))
970 def _calc_headers(self, info_dict):
971 res = std_headers.copy()
973 add_headers = info_dict.get('http_headers')
975 res.update(add_headers)
977 cookies = self._calc_cookies(info_dict)
979 res['Cookie'] = cookies
983 def _calc_cookies(self, info_dict):
984 pr = compat_urllib_request.Request(info_dict['url'])
985 self.cookiejar.add_cookie_header(pr)
986 return pr.get_header('Cookie')
988 def process_video_result(self, info_dict, download=True):
989 assert info_dict.get('_type', 'video') == 'video'
991 if 'id' not in info_dict:
992 raise ExtractorError('Missing "id" field in extractor result')
993 if 'title' not in info_dict:
994 raise ExtractorError('Missing "title" field in extractor result')
996 if 'playlist' not in info_dict:
997 # It isn't part of a playlist
998 info_dict['playlist'] = None
999 info_dict['playlist_index'] = None
1001 thumbnails = info_dict.get('thumbnails')
1002 if thumbnails is None:
1003 thumbnail = info_dict.get('thumbnail')
1005 info_dict['thumbnails'] = thumbnails = [{'url': thumbnail}]
1007 thumbnails.sort(key=lambda t: (
1008 t.get('preference'), t.get('width'), t.get('height'),
1009 t.get('id'), t.get('url')))
1010 for i, t in enumerate(thumbnails):
1011 if 'width' in t and 'height' in t:
1012 t['resolution'] = '%dx%d' % (t['width'], t['height'])
1013 if t.get('id') is None:
1016 if thumbnails and 'thumbnail' not in info_dict:
1017 info_dict['thumbnail'] = thumbnails[-1]['url']
1019 if 'display_id' not in info_dict and 'id' in info_dict:
1020 info_dict['display_id'] = info_dict['id']
1022 if info_dict.get('upload_date') is None and info_dict.get('timestamp') is not None:
1023 # Working around out-of-range timestamp values (e.g. negative ones on Windows,
1024 # see http://bugs.python.org/issue1646728)
1026 upload_date = datetime.datetime.utcfromtimestamp(info_dict['timestamp'])
1027 info_dict['upload_date'] = upload_date.strftime('%Y%m%d')
1028 except (ValueError, OverflowError, OSError):
1031 if self.params.get('listsubtitles', False):
1032 if 'automatic_captions' in info_dict:
1033 self.list_subtitles(info_dict['id'], info_dict.get('automatic_captions'), 'automatic captions')
1034 self.list_subtitles(info_dict['id'], info_dict.get('subtitles'), 'subtitles')
1036 info_dict['requested_subtitles'] = self.process_subtitles(
1037 info_dict['id'], info_dict.get('subtitles'),
1038 info_dict.get('automatic_captions'))
1040 # We now pick which formats have to be downloaded
1041 if info_dict.get('formats') is None:
1042 # There's only one format available
1043 formats = [info_dict]
1045 formats = info_dict['formats']
1048 raise ExtractorError('No video formats found!')
1052 # We check that all the formats have the format and format_id fields
1053 for i, format in enumerate(formats):
1054 if 'url' not in format:
1055 raise ExtractorError('Missing "url" key in result (index %d)' % i)
1057 if format.get('format_id') is None:
1058 format['format_id'] = compat_str(i)
1059 format_id = format['format_id']
1060 if format_id not in formats_dict:
1061 formats_dict[format_id] = []
1062 formats_dict[format_id].append(format)
1064 # Make sure all formats have unique format_id
1065 for format_id, ambiguous_formats in formats_dict.items():
1066 if len(ambiguous_formats) > 1:
1067 for i, format in enumerate(ambiguous_formats):
1068 format['format_id'] = '%s-%d' % (format_id, i)
1070 for i, format in enumerate(formats):
1071 if format.get('format') is None:
1072 format['format'] = '{id} - {res}{note}'.format(
1073 id=format['format_id'],
1074 res=self.format_resolution(format),
1075 note=' ({0})'.format(format['format_note']) if format.get('format_note') is not None else '',
1077 # Automatically determine file extension if missing
1078 if 'ext' not in format:
1079 format['ext'] = determine_ext(format['url']).lower()
1080 # Add HTTP headers, so that external programs can use them from the
1082 full_format_info = info_dict.copy()
1083 full_format_info.update(format)
1084 format['http_headers'] = self._calc_headers(full_format_info)
1086 # TODO Central sorting goes here
1088 if formats[0] is not info_dict:
1089 # only set the 'formats' fields if the original info_dict list them
1090 # otherwise we end up with a circular reference, the first (and unique)
1091 # element in the 'formats' field in info_dict is info_dict itself,
1092 # wich can't be exported to json
1093 info_dict['formats'] = formats
1094 if self.params.get('listformats'):
1095 self.list_formats(info_dict)
1097 if self.params.get('list_thumbnails'):
1098 self.list_thumbnails(info_dict)
1101 req_format = self.params.get('format')
1102 if req_format is None:
1103 req_format_list = []
1104 if (self.params.get('outtmpl', DEFAULT_OUTTMPL) != '-' and
1105 info_dict['extractor'] in ['youtube', 'ted']):
1106 merger = FFmpegMergerPP(self)
1107 if merger.available and merger.can_merge():
1108 req_format_list.append('bestvideo+bestaudio')
1109 req_format_list.append('best')
1110 req_format = '/'.join(req_format_list)
1111 formats_to_download = []
1112 if req_format == 'all':
1113 formats_to_download = formats
1115 for rfstr in req_format.split(','):
1116 # We can accept formats requested in the format: 34/5/best, we pick
1117 # the first that is available, starting from left
1118 req_formats = rfstr.split('/')
1119 for rf in req_formats:
1120 if re.match(r'.+?\+.+?', rf) is not None:
1121 # Two formats have been requested like '137+139'
1122 format_1, format_2 = rf.split('+')
1123 formats_info = (self.select_format(format_1, formats),
1124 self.select_format(format_2, formats))
1125 if all(formats_info):
1126 # The first format must contain the video and the
1128 if formats_info[0].get('vcodec') == 'none':
1129 self.report_error('The first format must '
1130 'contain the video, try using '
1131 '"-f %s+%s"' % (format_2, format_1))
1134 formats_info[0]['ext']
1135 if self.params.get('merge_output_format') is None
1136 else self.params['merge_output_format'])
1138 'requested_formats': formats_info,
1139 'format': '%s+%s' % (formats_info[0].get('format'),
1140 formats_info[1].get('format')),
1141 'format_id': '%s+%s' % (formats_info[0].get('format_id'),
1142 formats_info[1].get('format_id')),
1143 'width': formats_info[0].get('width'),
1144 'height': formats_info[0].get('height'),
1145 'resolution': formats_info[0].get('resolution'),
1146 'fps': formats_info[0].get('fps'),
1147 'vcodec': formats_info[0].get('vcodec'),
1148 'vbr': formats_info[0].get('vbr'),
1149 'stretched_ratio': formats_info[0].get('stretched_ratio'),
1150 'acodec': formats_info[1].get('acodec'),
1151 'abr': formats_info[1].get('abr'),
1155 selected_format = None
1157 selected_format = self.select_format(rf, formats)
1158 if selected_format is not None:
1159 formats_to_download.append(selected_format)
1161 if not formats_to_download:
1162 raise ExtractorError('requested format not available',
1166 if len(formats_to_download) > 1:
1167 self.to_screen('[info] %s: downloading video in %s formats' % (info_dict['id'], len(formats_to_download)))
1168 for format in formats_to_download:
1169 new_info = dict(info_dict)
1170 new_info.update(format)
1171 self.process_info(new_info)
1172 # We update the info dict with the best quality format (backwards compatibility)
1173 info_dict.update(formats_to_download[-1])
1176 def process_subtitles(self, video_id, normal_subtitles, automatic_captions):
1177 """Select the requested subtitles and their format"""
1179 if normal_subtitles and self.params.get('writesubtitles'):
1180 available_subs.update(normal_subtitles)
1181 if automatic_captions and self.params.get('writeautomaticsub'):
1182 for lang, cap_info in automatic_captions.items():
1183 if lang not in available_subs:
1184 available_subs[lang] = cap_info
1186 if (not self.params.get('writesubtitles') and not
1187 self.params.get('writeautomaticsub') or not
1191 if self.params.get('allsubtitles', False):
1192 requested_langs = available_subs.keys()
1194 if self.params.get('subtitleslangs', False):
1195 requested_langs = self.params.get('subtitleslangs')
1196 elif 'en' in available_subs:
1197 requested_langs = ['en']
1199 requested_langs = [list(available_subs.keys())[0]]
1201 formats_query = self.params.get('subtitlesformat', 'best')
1202 formats_preference = formats_query.split('/') if formats_query else []
1204 for lang in requested_langs:
1205 formats = available_subs.get(lang)
1207 self.report_warning('%s subtitles not available for %s' % (lang, video_id))
1209 for ext in formats_preference:
1213 matches = list(filter(lambda f: f['ext'] == ext, formats))
1219 self.report_warning(
1220 'No subtitle format found matching "%s" for language %s, '
1221 'using %s' % (formats_query, lang, f['ext']))
1225 def process_info(self, info_dict):
1226 """Process a single resolved IE result."""
1228 assert info_dict.get('_type', 'video') == 'video'
1230 max_downloads = self.params.get('max_downloads')
1231 if max_downloads is not None:
1232 if self._num_downloads >= int(max_downloads):
1233 raise MaxDownloadsReached()
1235 info_dict['fulltitle'] = info_dict['title']
1236 if len(info_dict['title']) > 200:
1237 info_dict['title'] = info_dict['title'][:197] + '...'
1239 if 'format' not in info_dict:
1240 info_dict['format'] = info_dict['ext']
1242 reason = self._match_entry(info_dict, incomplete=False)
1243 if reason is not None:
1244 self.to_screen('[download] ' + reason)
1247 self._num_downloads += 1
1249 info_dict['_filename'] = filename = self.prepare_filename(info_dict)
1252 if self.params.get('forcetitle', False):
1253 self.to_stdout(info_dict['fulltitle'])
1254 if self.params.get('forceid', False):
1255 self.to_stdout(info_dict['id'])
1256 if self.params.get('forceurl', False):
1257 if info_dict.get('requested_formats') is not None:
1258 for f in info_dict['requested_formats']:
1259 self.to_stdout(f['url'] + f.get('play_path', ''))
1261 # For RTMP URLs, also include the playpath
1262 self.to_stdout(info_dict['url'] + info_dict.get('play_path', ''))
1263 if self.params.get('forcethumbnail', False) and info_dict.get('thumbnail') is not None:
1264 self.to_stdout(info_dict['thumbnail'])
1265 if self.params.get('forcedescription', False) and info_dict.get('description') is not None:
1266 self.to_stdout(info_dict['description'])
1267 if self.params.get('forcefilename', False) and filename is not None:
1268 self.to_stdout(filename)
1269 if self.params.get('forceduration', False) and info_dict.get('duration') is not None:
1270 self.to_stdout(formatSeconds(info_dict['duration']))
1271 if self.params.get('forceformat', False):
1272 self.to_stdout(info_dict['format'])
1273 if self.params.get('forcejson', False):
1274 self.to_stdout(json.dumps(info_dict))
1276 # Do nothing else if in simulate mode
1277 if self.params.get('simulate', False):
1280 if filename is None:
1284 dn = os.path.dirname(sanitize_path(encodeFilename(filename)))
1285 if dn and not os.path.exists(dn):
1287 except (OSError, IOError) as err:
1288 self.report_error('unable to create directory ' + compat_str(err))
1291 if self.params.get('writedescription', False):
1292 descfn = replace_extension(filename, 'description', info_dict.get('ext'))
1293 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)):
1294 self.to_screen('[info] Video description is already present')
1295 elif info_dict.get('description') is None:
1296 self.report_warning('There\'s no description to write.')
1299 self.to_screen('[info] Writing video description to: ' + descfn)
1300 with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
1301 descfile.write(info_dict['description'])
1302 except (OSError, IOError):
1303 self.report_error('Cannot write description file ' + descfn)
1306 if self.params.get('writeannotations', False):
1307 annofn = replace_extension(filename, 'annotations.xml', info_dict.get('ext'))
1308 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)):
1309 self.to_screen('[info] Video annotations are already present')
1312 self.to_screen('[info] Writing video annotations to: ' + annofn)
1313 with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile:
1314 annofile.write(info_dict['annotations'])
1315 except (KeyError, TypeError):
1316 self.report_warning('There are no annotations to write.')
1317 except (OSError, IOError):
1318 self.report_error('Cannot write annotations file: ' + annofn)
1321 subtitles_are_requested = any([self.params.get('writesubtitles', False),
1322 self.params.get('writeautomaticsub')])
1324 if subtitles_are_requested and info_dict.get('requested_subtitles'):
1325 # subtitles download errors are already managed as troubles in relevant IE
1326 # that way it will silently go on when used with unsupporting IE
1327 subtitles = info_dict['requested_subtitles']
1328 ie = self.get_info_extractor(info_dict['extractor_key'])
1329 for sub_lang, sub_info in subtitles.items():
1330 sub_format = sub_info['ext']
1331 if sub_info.get('data') is not None:
1332 sub_data = sub_info['data']
1335 sub_data = ie._download_webpage(
1336 sub_info['url'], info_dict['id'], note=False)
1337 except ExtractorError as err:
1338 self.report_warning('Unable to download subtitle for "%s": %s' %
1339 (sub_lang, compat_str(err.cause)))
1342 sub_filename = subtitles_filename(filename, sub_lang, sub_format)
1343 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)):
1344 self.to_screen('[info] Video subtitle %s.%s is already_present' % (sub_lang, sub_format))
1346 self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
1347 with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
1348 subfile.write(sub_data)
1349 except (OSError, IOError):
1350 self.report_error('Cannot write subtitles file ' + sub_filename)
1353 if self.params.get('writeinfojson', False):
1354 infofn = replace_extension(filename, 'info.json', info_dict.get('ext'))
1355 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)):
1356 self.to_screen('[info] Video description metadata is already present')
1358 self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
1360 write_json_file(self.filter_requested_info(info_dict), infofn)
1361 except (OSError, IOError):
1362 self.report_error('Cannot write metadata to JSON file ' + infofn)
1365 self._write_thumbnails(info_dict, filename)
1367 if not self.params.get('skip_download', False):
1370 fd = get_suitable_downloader(info, self.params)(self, self.params)
1371 for ph in self._progress_hooks:
1372 fd.add_progress_hook(ph)
1373 if self.params.get('verbose'):
1374 self.to_stdout('[debug] Invoking downloader on %r' % info.get('url'))
1375 return fd.download(name, info)
1377 if info_dict.get('requested_formats') is not None:
1380 merger = FFmpegMergerPP(self)
1381 if not merger.available:
1383 self.report_warning('You have requested multiple '
1384 'formats but ffmpeg or avconv are not installed.'
1385 ' The formats won\'t be merged.')
1387 postprocessors = [merger]
1389 def compatible_formats(formats):
1390 video, audio = formats
1392 video_ext, audio_ext = audio.get('ext'), video.get('ext')
1393 if video_ext and audio_ext:
1395 ('mp3', 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v'),
1398 for exts in COMPATIBLE_EXTS:
1399 if video_ext in exts and audio_ext in exts:
1401 # TODO: Check acodec/vcodec
1404 filename_real_ext = os.path.splitext(filename)[1][1:]
1406 os.path.splitext(filename)[0]
1407 if filename_real_ext == info_dict['ext']
1409 requested_formats = info_dict['requested_formats']
1410 if self.params.get('merge_output_format') is None and not compatible_formats(requested_formats):
1411 info_dict['ext'] = 'mkv'
1412 self.report_warning(
1413 'Requested formats are incompatible for merge and will be merged into mkv.')
1414 # Ensure filename always has a correct extension for successful merge
1415 filename = '%s.%s' % (filename_wo_ext, info_dict['ext'])
1416 if os.path.exists(encodeFilename(filename)):
1418 '[download] %s has already been downloaded and '
1419 'merged' % filename)
1421 for f in requested_formats:
1422 new_info = dict(info_dict)
1424 fname = self.prepare_filename(new_info)
1425 fname = prepend_extension(fname, 'f%s' % f['format_id'], new_info['ext'])
1426 downloaded.append(fname)
1427 partial_success = dl(fname, new_info)
1428 success = success and partial_success
1429 info_dict['__postprocessors'] = postprocessors
1430 info_dict['__files_to_merge'] = downloaded
1432 # Just a single file
1433 success = dl(filename, info_dict)
1434 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1435 self.report_error('unable to download video data: %s' % str(err))
1437 except (OSError, IOError) as err:
1438 raise UnavailableVideoError(err)
1439 except (ContentTooShortError, ) as err:
1440 self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
1445 fixup_policy = self.params.get('fixup')
1446 if fixup_policy is None:
1447 fixup_policy = 'detect_or_warn'
1449 stretched_ratio = info_dict.get('stretched_ratio')
1450 if stretched_ratio is not None and stretched_ratio != 1:
1451 if fixup_policy == 'warn':
1452 self.report_warning('%s: Non-uniform pixel ratio (%s)' % (
1453 info_dict['id'], stretched_ratio))
1454 elif fixup_policy == 'detect_or_warn':
1455 stretched_pp = FFmpegFixupStretchedPP(self)
1456 if stretched_pp.available:
1457 info_dict.setdefault('__postprocessors', [])
1458 info_dict['__postprocessors'].append(stretched_pp)
1460 self.report_warning(
1461 '%s: Non-uniform pixel ratio (%s). Install ffmpeg or avconv to fix this automatically.' % (
1462 info_dict['id'], stretched_ratio))
1464 assert fixup_policy in ('ignore', 'never')
1466 if info_dict.get('requested_formats') is None and info_dict.get('container') == 'm4a_dash':
1467 if fixup_policy == 'warn':
1468 self.report_warning('%s: writing DASH m4a. Only some players support this container.' % (
1470 elif fixup_policy == 'detect_or_warn':
1471 fixup_pp = FFmpegFixupM4aPP(self)
1472 if fixup_pp.available:
1473 info_dict.setdefault('__postprocessors', [])
1474 info_dict['__postprocessors'].append(fixup_pp)
1476 self.report_warning(
1477 '%s: writing DASH m4a. Only some players support this container. Install ffmpeg or avconv to fix this automatically.' % (
1480 assert fixup_policy in ('ignore', 'never')
1483 self.post_process(filename, info_dict)
1484 except (PostProcessingError) as err:
1485 self.report_error('postprocessing: %s' % str(err))
1487 self.record_download_archive(info_dict)
1489 def download(self, url_list):
1490 """Download a given list of URLs."""
1491 outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
1492 if (len(url_list) > 1 and
1493 '%' not in outtmpl and
1494 self.params.get('max_downloads') != 1):
1495 raise SameFileError(outtmpl)
1497 for url in url_list:
1499 # It also downloads the videos
1500 res = self.extract_info(
1501 url, force_generic_extractor=self.params.get('force_generic_extractor', False))
1502 except UnavailableVideoError:
1503 self.report_error('unable to download video')
1504 except MaxDownloadsReached:
1505 self.to_screen('[info] Maximum number of downloaded files reached.')
1508 if self.params.get('dump_single_json', False):
1509 self.to_stdout(json.dumps(res))
1511 return self._download_retcode
1513 def download_with_info_file(self, info_filename):
1514 with contextlib.closing(fileinput.FileInput(
1515 [info_filename], mode='r',
1516 openhook=fileinput.hook_encoded('utf-8'))) as f:
1517 # FileInput doesn't have a read method, we can't call json.load
1518 info = self.filter_requested_info(json.loads('\n'.join(f)))
1520 self.process_ie_result(info, download=True)
1521 except DownloadError:
1522 webpage_url = info.get('webpage_url')
1523 if webpage_url is not None:
1524 self.report_warning('The info failed to download, trying with "%s"' % webpage_url)
1525 return self.download([webpage_url])
1528 return self._download_retcode
1531 def filter_requested_info(info_dict):
1533 (k, v) for k, v in info_dict.items()
1534 if k not in ['requested_formats', 'requested_subtitles'])
1536 def post_process(self, filename, ie_info):
1537 """Run all the postprocessors on the given file."""
1538 info = dict(ie_info)
1539 info['filepath'] = filename
1541 if ie_info.get('__postprocessors') is not None:
1542 pps_chain.extend(ie_info['__postprocessors'])
1543 pps_chain.extend(self._pps)
1544 for pp in pps_chain:
1545 files_to_delete = []
1547 files_to_delete, info = pp.run(info)
1548 except PostProcessingError as e:
1549 self.report_error(e.msg)
1550 if files_to_delete and not self.params.get('keepvideo', False):
1551 for old_filename in files_to_delete:
1552 self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename)
1554 os.remove(encodeFilename(old_filename))
1555 except (IOError, OSError):
1556 self.report_warning('Unable to remove downloaded original file')
1558 def _make_archive_id(self, info_dict):
1559 # Future-proof against any change in case
1560 # and backwards compatibility with prior versions
1561 extractor = info_dict.get('extractor_key')
1562 if extractor is None:
1563 if 'id' in info_dict:
1564 extractor = info_dict.get('ie_key') # key in a playlist
1565 if extractor is None:
1566 return None # Incomplete video information
1567 return extractor.lower() + ' ' + info_dict['id']
1569 def in_download_archive(self, info_dict):
1570 fn = self.params.get('download_archive')
1574 vid_id = self._make_archive_id(info_dict)
1576 return False # Incomplete video information
1579 with locked_file(fn, 'r', encoding='utf-8') as archive_file:
1580 for line in archive_file:
1581 if line.strip() == vid_id:
1583 except IOError as ioe:
1584 if ioe.errno != errno.ENOENT:
1588 def record_download_archive(self, info_dict):
1589 fn = self.params.get('download_archive')
1592 vid_id = self._make_archive_id(info_dict)
1594 with locked_file(fn, 'a', encoding='utf-8') as archive_file:
1595 archive_file.write(vid_id + '\n')
1598 def format_resolution(format, default='unknown'):
1599 if format.get('vcodec') == 'none':
1601 if format.get('resolution') is not None:
1602 return format['resolution']
1603 if format.get('height') is not None:
1604 if format.get('width') is not None:
1605 res = '%sx%s' % (format['width'], format['height'])
1607 res = '%sp' % format['height']
1608 elif format.get('width') is not None:
1609 res = '?x%d' % format['width']
1614 def _format_note(self, fdict):
1616 if fdict.get('ext') in ['f4f', 'f4m']:
1617 res += '(unsupported) '
1618 if fdict.get('format_note') is not None:
1619 res += fdict['format_note'] + ' '
1620 if fdict.get('tbr') is not None:
1621 res += '%4dk ' % fdict['tbr']
1622 if fdict.get('container') is not None:
1625 res += '%s container' % fdict['container']
1626 if (fdict.get('vcodec') is not None and
1627 fdict.get('vcodec') != 'none'):
1630 res += fdict['vcodec']
1631 if fdict.get('vbr') is not None:
1633 elif fdict.get('vbr') is not None and fdict.get('abr') is not None:
1635 if fdict.get('vbr') is not None:
1636 res += '%4dk' % fdict['vbr']
1637 if fdict.get('fps') is not None:
1638 res += ', %sfps' % fdict['fps']
1639 if fdict.get('acodec') is not None:
1642 if fdict['acodec'] == 'none':
1645 res += '%-5s' % fdict['acodec']
1646 elif fdict.get('abr') is not None:
1650 if fdict.get('abr') is not None:
1651 res += '@%3dk' % fdict['abr']
1652 if fdict.get('asr') is not None:
1653 res += ' (%5dHz)' % fdict['asr']
1654 if fdict.get('filesize') is not None:
1657 res += format_bytes(fdict['filesize'])
1658 elif fdict.get('filesize_approx') is not None:
1661 res += '~' + format_bytes(fdict['filesize_approx'])
1664 def list_formats(self, info_dict):
1665 formats = info_dict.get('formats', [info_dict])
1667 [f['format_id'], f['ext'], self.format_resolution(f), self._format_note(f)]
1669 if f.get('preference') is None or f['preference'] >= -1000]
1670 if len(formats) > 1:
1671 table[-1][-1] += (' ' if table[-1][-1] else '') + '(best)'
1673 header_line = ['format code', 'extension', 'resolution', 'note']
1675 '[info] Available formats for %s:\n%s' %
1676 (info_dict['id'], render_table(header_line, table)))
1678 def list_thumbnails(self, info_dict):
1679 thumbnails = info_dict.get('thumbnails')
1681 tn_url = info_dict.get('thumbnail')
1683 thumbnails = [{'id': '0', 'url': tn_url}]
1686 '[info] No thumbnails present for %s' % info_dict['id'])
1690 '[info] Thumbnails for %s:' % info_dict['id'])
1691 self.to_screen(render_table(
1692 ['ID', 'width', 'height', 'URL'],
1693 [[t['id'], t.get('width', 'unknown'), t.get('height', 'unknown'), t['url']] for t in thumbnails]))
1695 def list_subtitles(self, video_id, subtitles, name='subtitles'):
1697 self.to_screen('%s has no %s' % (video_id, name))
1700 'Available %s for %s:' % (name, video_id))
1701 self.to_screen(render_table(
1702 ['Language', 'formats'],
1703 [[lang, ', '.join(f['ext'] for f in reversed(formats))]
1704 for lang, formats in subtitles.items()]))
1706 def urlopen(self, req):
1707 """ Start an HTTP download """
1709 # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
1710 # always respected by websites, some tend to give out URLs with non percent-encoded
1711 # non-ASCII characters (see telemb.py, ard.py [#3412])
1712 # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
1713 # To work around aforementioned issue we will replace request's original URL with
1714 # percent-encoded one
1715 req_is_string = isinstance(req, compat_basestring)
1716 url = req if req_is_string else req.get_full_url()
1717 url_escaped = escape_url(url)
1719 # Substitute URL if any change after escaping
1720 if url != url_escaped:
1724 req_type = HEADRequest if req.get_method() == 'HEAD' else compat_urllib_request.Request
1726 url_escaped, data=req.data, headers=req.headers,
1727 origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
1729 return self._opener.open(req, timeout=self._socket_timeout)
1731 def print_debug_header(self):
1732 if not self.params.get('verbose'):
1735 if type('') is not compat_str:
1736 # Python 2.6 on SLES11 SP1 (https://github.com/rg3/youtube-dl/issues/3326)
1737 self.report_warning(
1738 'Your Python is broken! Update to a newer and supported version')
1740 stdout_encoding = getattr(
1741 sys.stdout, 'encoding', 'missing (%s)' % type(sys.stdout).__name__)
1743 '[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % (
1744 locale.getpreferredencoding(),
1745 sys.getfilesystemencoding(),
1747 self.get_encoding()))
1748 write_string(encoding_str, encoding=None)
1750 self._write_string('[debug] youtube-dl version ' + __version__ + '\n')
1752 sp = subprocess.Popen(
1753 ['git', 'rev-parse', '--short', 'HEAD'],
1754 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
1755 cwd=os.path.dirname(os.path.abspath(__file__)))
1756 out, err = sp.communicate()
1757 out = out.decode().strip()
1758 if re.match('[0-9a-f]+', out):
1759 self._write_string('[debug] Git HEAD: ' + out + '\n')
1765 self._write_string('[debug] Python version %s - %s\n' % (
1766 platform.python_version(), platform_name()))
1768 exe_versions = FFmpegPostProcessor.get_versions(self)
1769 exe_versions['rtmpdump'] = rtmpdump_version()
1770 exe_str = ', '.join(
1772 for exe, v in sorted(exe_versions.items())
1777 self._write_string('[debug] exe versions: %s\n' % exe_str)
1780 for handler in self._opener.handlers:
1781 if hasattr(handler, 'proxies'):
1782 proxy_map.update(handler.proxies)
1783 self._write_string('[debug] Proxy map: ' + compat_str(proxy_map) + '\n')
1785 if self.params.get('call_home', False):
1786 ipaddr = self.urlopen('https://yt-dl.org/ip').read().decode('utf-8')
1787 self._write_string('[debug] Public IP address: %s\n' % ipaddr)
1788 latest_version = self.urlopen(
1789 'https://yt-dl.org/latest/version').read().decode('utf-8')
1790 if version_tuple(latest_version) > version_tuple(__version__):
1791 self.report_warning(
1792 'You are using an outdated version (newest version: %s)! '
1793 'See https://yt-dl.org/update if you need help updating.' %
1796 def _setup_opener(self):
1797 timeout_val = self.params.get('socket_timeout')
1798 self._socket_timeout = 600 if timeout_val is None else float(timeout_val)
1800 opts_cookiefile = self.params.get('cookiefile')
1801 opts_proxy = self.params.get('proxy')
1803 if opts_cookiefile is None:
1804 self.cookiejar = compat_cookiejar.CookieJar()
1806 self.cookiejar = compat_cookiejar.MozillaCookieJar(
1808 if os.access(opts_cookiefile, os.R_OK):
1809 self.cookiejar.load()
1811 cookie_processor = compat_urllib_request.HTTPCookieProcessor(
1813 if opts_proxy is not None:
1814 if opts_proxy == '':
1817 proxies = {'http': opts_proxy, 'https': opts_proxy}
1819 proxies = compat_urllib_request.getproxies()
1820 # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
1821 if 'http' in proxies and 'https' not in proxies:
1822 proxies['https'] = proxies['http']
1823 proxy_handler = PerRequestProxyHandler(proxies)
1825 debuglevel = 1 if self.params.get('debug_printtraffic') else 0
1826 https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
1827 ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
1828 opener = compat_urllib_request.build_opener(
1829 proxy_handler, https_handler, cookie_processor, ydlh)
1831 # Delete the default user-agent header, which would otherwise apply in
1832 # cases where our custom HTTP handler doesn't come into play
1833 # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
1834 opener.addheaders = []
1835 self._opener = opener
1837 def encode(self, s):
1838 if isinstance(s, bytes):
1839 return s # Already encoded
1842 return s.encode(self.get_encoding())
1843 except UnicodeEncodeError as err:
1844 err.reason = err.reason + '. Check your system encoding configuration or use the --encoding option.'
1847 def get_encoding(self):
1848 encoding = self.params.get('encoding')
1849 if encoding is None:
1850 encoding = preferredencoding()
1853 def _write_thumbnails(self, info_dict, filename):
1854 if self.params.get('writethumbnail', False):
1855 thumbnails = info_dict.get('thumbnails')
1857 thumbnails = [thumbnails[-1]]
1858 elif self.params.get('write_all_thumbnails', False):
1859 thumbnails = info_dict.get('thumbnails')
1864 # No thumbnails present, so return immediately
1867 for t in thumbnails:
1868 thumb_ext = determine_ext(t['url'], 'jpg')
1869 suffix = '_%s' % t['id'] if len(thumbnails) > 1 else ''
1870 thumb_display_id = '%s ' % t['id'] if len(thumbnails) > 1 else ''
1871 t['filename'] = thumb_filename = os.path.splitext(filename)[0] + suffix + '.' + thumb_ext
1873 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
1874 self.to_screen('[%s] %s: Thumbnail %sis already present' %
1875 (info_dict['extractor'], info_dict['id'], thumb_display_id))
1877 self.to_screen('[%s] %s: Downloading thumbnail %s...' %
1878 (info_dict['extractor'], info_dict['id'], thumb_display_id))
1880 uf = self.urlopen(t['url'])
1881 with open(thumb_filename, 'wb') as thumbf:
1882 shutil.copyfileobj(uf, thumbf)
1883 self.to_screen('[%s] %s: Writing thumbnail %sto: %s' %
1884 (info_dict['extractor'], info_dict['id'], thumb_display_id, thumb_filename))
1885 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1886 self.report_warning('Unable to download thumbnail "%s": %s' %
1887 (t['url'], compat_str(err)))