2 # -*- coding: utf-8 -*-
4 from __future__ import absolute_import, unicode_literals
33 compat_get_terminal_size,
38 compat_urllib_request,
57 PerRequestProxyHandler,
68 UnavailableVideoError,
78 from .cache import Cache
79 from .extractor import get_info_extractor, gen_extractors
80 from .downloader import get_suitable_downloader
81 from .downloader.rtmp import rtmpdump_version
82 from .postprocessor import (
84 FFmpegFixupStretchedPP,
89 from .version import __version__
92 class YoutubeDL(object):
95 YoutubeDL objects are the ones responsible of downloading the
96 actual video file and writing it to disk if the user has requested
97 it, among some other tasks. In most cases there should be one per
98 program. As, given a video URL, the downloader doesn't know how to
99 extract all the needed information, task that InfoExtractors do, it
100 has to pass the URL to one of them.
102 For this, YoutubeDL objects have a method that allows
103 InfoExtractors to be registered in a given order. When it is passed
104 a URL, the YoutubeDL object handles it to the first InfoExtractor it
105 finds that reports being able to handle it. The InfoExtractor extracts
106 all the information about the video or videos the URL refers to, and
107 YoutubeDL process the extracted information, possibly using a File
108 Downloader to download the video.
110 YoutubeDL objects accept a lot of parameters. In order not to saturate
111 the object constructor with arguments, it receives a dictionary of
112 options instead. These options are available through the params
113 attribute for the InfoExtractors to use. The YoutubeDL also
114 registers itself as the downloader in charge for the InfoExtractors
115 that are added to it, so this is a "mutual registration".
119 username: Username for authentication purposes.
120 password: Password for authentication purposes.
121 videopassword: Password for acces a video.
122 usenetrc: Use netrc for authentication instead.
123 verbose: Print additional info to stdout.
124 quiet: Do not print messages to stdout.
125 no_warnings: Do not print out anything for warnings.
126 forceurl: Force printing final URL.
127 forcetitle: Force printing title.
128 forceid: Force printing ID.
129 forcethumbnail: Force printing thumbnail URL.
130 forcedescription: Force printing description.
131 forcefilename: Force printing final filename.
132 forceduration: Force printing duration.
133 forcejson: Force printing info_dict as JSON.
134 dump_single_json: Force printing the info_dict of the whole playlist
135 (or video) as a single JSON line.
136 simulate: Do not download the video files.
137 format: Video format code. See options.py for more information.
138 format_limit: Highest quality format to try.
139 outtmpl: Template for output names.
140 restrictfilenames: Do not allow "&" and spaces in file names
141 ignoreerrors: Do not stop on download errors.
142 nooverwrites: Prevent overwriting files.
143 playliststart: Playlist item to start at.
144 playlistend: Playlist item to end at.
145 playlist_items: Specific indices of playlist to download.
146 playlistreverse: Download playlist items in reverse order.
147 matchtitle: Download only matching titles.
148 rejecttitle: Reject downloads for matching titles.
149 logger: Log messages to a logging.Logger instance.
150 logtostderr: Log messages to stderr instead of stdout.
151 writedescription: Write the video description to a .description file
152 writeinfojson: Write the video description to a .info.json file
153 writeannotations: Write the video annotations to a .annotations.xml file
154 writethumbnail: Write the thumbnail image to a file
155 write_all_thumbnails: Write all thumbnail formats to files
156 writesubtitles: Write the video subtitles to a file
157 writeautomaticsub: Write the automatic subtitles to a file
158 allsubtitles: Downloads all the subtitles of the video
159 (requires writesubtitles or writeautomaticsub)
160 listsubtitles: Lists all available subtitles for the video
161 subtitlesformat: The format code for subtitles
162 subtitleslangs: List of languages of the subtitles to download
163 keepvideo: Keep the video file after post-processing
164 daterange: A DateRange object, download only if the upload_date is in the range.
165 skip_download: Skip the actual download of the video file
166 cachedir: Location of the cache files in the filesystem.
167 False to disable filesystem cache.
168 noplaylist: Download single video instead of a playlist if in doubt.
169 age_limit: An integer representing the user's age in years.
170 Unsuitable videos for the given age are skipped.
171 min_views: An integer representing the minimum view count the video
172 must have in order to not be skipped.
173 Videos without view count information are always
174 downloaded. None for no limit.
175 max_views: An integer representing the maximum view count.
176 Videos that are more popular than that are not
178 Videos without view count information are always
179 downloaded. None for no limit.
180 download_archive: File name of a file where all downloads are recorded.
181 Videos already present in the file are not downloaded
183 cookiefile: File name where cookies should be read from and dumped to.
184 nocheckcertificate:Do not verify SSL certificates
185 prefer_insecure: Use HTTP instead of HTTPS to retrieve information.
186 At the moment, this is only supported by YouTube.
187 proxy: URL of the proxy server to use
188 cn_verification_proxy: URL of the proxy to use for IP address verification
189 on Chinese sites. (Experimental)
190 socket_timeout: Time to wait for unresponsive hosts, in seconds
191 bidi_workaround: Work around buggy terminals without bidirectional text
192 support, using fridibi
193 debug_printtraffic:Print out sent and received HTTP traffic
194 include_ads: Download ads as well
195 default_search: Prepend this string if an input url is not valid.
196 'auto' for elaborate guessing
197 encoding: Use this encoding instead of the system-specified.
198 extract_flat: Do not resolve URLs, return the immediate result.
199 Pass in 'in_playlist' to only show this behavior for
201 postprocessors: A list of dictionaries, each with an entry
202 * key: The name of the postprocessor. See
203 youtube_dl/postprocessor/__init__.py for a list.
204 as well as any further keyword arguments for the
206 progress_hooks: A list of functions that get called on download
207 progress, with a dictionary with the entries
208 * status: One of "downloading", "error", or "finished".
209 Check this first and ignore unknown values.
211 If status is one of "downloading", or "finished", the
212 following properties may also be present:
213 * filename: The final filename (always present)
214 * tmpfilename: The filename we're currently writing to
215 * downloaded_bytes: Bytes on disk
216 * total_bytes: Size of the whole file, None if unknown
217 * total_bytes_estimate: Guess of the eventual file size,
219 * elapsed: The number of seconds since download started.
220 * eta: The estimated time in seconds, None if unknown
221 * speed: The download speed in bytes/second, None if
223 * fragment_index: The counter of the currently
224 downloaded video fragment.
225 * fragment_count: The number of fragments (= individual
226 files that will be merged)
228 Progress hooks are guaranteed to be called at least once
229 (with status "finished") if the download is successful.
230 merge_output_format: Extension to use when merging formats.
231 fixup: Automatically correct known faults of the file.
233 - "never": do nothing
234 - "warn": only emit a warning
235 - "detect_or_warn": check whether we can do anything
236 about it, warn otherwise (default)
237 source_address: (Experimental) Client-side IP address to bind to.
238 call_home: Boolean, true iff we are allowed to contact the
239 youtube-dl servers for debugging.
240 sleep_interval: Number of seconds to sleep before each download.
241 listformats: Print an overview of available video formats and exit.
242 list_thumbnails: Print a table of all thumbnails and exit.
243 match_filter: A function that gets called with the info_dict of
245 If it returns a message, the video is ignored.
246 If it returns None, the video is downloaded.
247 match_filter_func in utils.py is one example for this.
248 no_color: Do not emit color codes in output.
250 The following options determine which downloader is picked:
251 external_downloader: Executable of the external downloader to call.
252 None or unset for standard (built-in) downloader.
253 hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv.
255 The following parameters are not used by YoutubeDL itself, they are used by
256 the downloader (see youtube_dl/downloader/common.py):
257 nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
258 noresizebuffer, retries, continuedl, noprogress, consoletitle,
259 xattr_set_filesize, external_downloader_args.
261 The following options are used by the post processors:
262 prefer_ffmpeg: If True, use ffmpeg instead of avconv if both are available,
263 otherwise prefer avconv.
264 exec_cmd: Arbitrary command to run after downloading
270 _download_retcode = None
271 _num_downloads = None
274 def __init__(self, params=None, auto_init=True):
275 """Create a FileDownloader object with the given options."""
279 self._ies_instances = {}
281 self._progress_hooks = []
282 self._download_retcode = 0
283 self._num_downloads = 0
284 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
285 self._err_file = sys.stderr
287 self.cache = Cache(self)
289 if params.get('bidi_workaround', False):
292 master, slave = pty.openpty()
293 width = compat_get_terminal_size().columns
297 width_args = ['-w', str(width)]
299 stdin=subprocess.PIPE,
301 stderr=self._err_file)
303 self._output_process = subprocess.Popen(
304 ['bidiv'] + width_args, **sp_kwargs
307 self._output_process = subprocess.Popen(
308 ['fribidi', '-c', 'UTF-8'] + width_args, **sp_kwargs)
309 self._output_channel = os.fdopen(master, 'rb')
310 except OSError as ose:
312 self.report_warning('Could not find fribidi executable, ignoring --bidi-workaround . Make sure that fribidi is an executable file in one of the directories in your $PATH.')
316 if (sys.version_info >= (3,) and sys.platform != 'win32' and
317 sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968'] and
318 not params.get('restrictfilenames', False)):
319 # On Python 3, the Unicode filesystem API will throw errors (#1474)
321 'Assuming --restrict-filenames since file system encoding '
322 'cannot encode all characters. '
323 'Set the LC_ALL environment variable to fix this.')
324 self.params['restrictfilenames'] = True
326 if '%(stitle)s' in self.params.get('outtmpl', ''):
327 self.report_warning('%(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.')
332 self.print_debug_header()
333 self.add_default_info_extractors()
335 for pp_def_raw in self.params.get('postprocessors', []):
336 pp_class = get_postprocessor(pp_def_raw['key'])
337 pp_def = dict(pp_def_raw)
339 pp = pp_class(self, **compat_kwargs(pp_def))
340 self.add_post_processor(pp)
342 for ph in self.params.get('progress_hooks', []):
343 self.add_progress_hook(ph)
345 def warn_if_short_id(self, argv):
346 # short YouTube ID starting with dash?
348 i for i, a in enumerate(argv)
349 if re.match(r'^-[0-9A-Za-z_-]{10}$', a)]
353 [a for i, a in enumerate(argv) if i not in idxs] +
354 ['--'] + [argv[i] for i in idxs]
357 'Long argument string detected. '
358 'Use -- to separate parameters and URLs, like this:\n%s\n' %
359 args_to_str(correct_argv))
361 def add_info_extractor(self, ie):
362 """Add an InfoExtractor object to the end of the list."""
364 self._ies_instances[ie.ie_key()] = ie
365 ie.set_downloader(self)
367 def get_info_extractor(self, ie_key):
369 Get an instance of an IE with name ie_key, it will try to get one from
370 the _ies list, if there's no instance it will create a new one and add
371 it to the extractor list.
373 ie = self._ies_instances.get(ie_key)
375 ie = get_info_extractor(ie_key)()
376 self.add_info_extractor(ie)
379 def add_default_info_extractors(self):
381 Add the InfoExtractors returned by gen_extractors to the end of the list
383 for ie in gen_extractors():
384 self.add_info_extractor(ie)
386 def add_post_processor(self, pp):
387 """Add a PostProcessor object to the end of the chain."""
389 pp.set_downloader(self)
391 def add_progress_hook(self, ph):
392 """Add the progress hook (currently only for the file downloader)"""
393 self._progress_hooks.append(ph)
395 def _bidi_workaround(self, message):
396 if not hasattr(self, '_output_channel'):
399 assert hasattr(self, '_output_process')
400 assert isinstance(message, compat_str)
401 line_count = message.count('\n') + 1
402 self._output_process.stdin.write((message + '\n').encode('utf-8'))
403 self._output_process.stdin.flush()
404 res = ''.join(self._output_channel.readline().decode('utf-8')
405 for _ in range(line_count))
406 return res[:-len('\n')]
408 def to_screen(self, message, skip_eol=False):
409 """Print message to stdout if not in quiet mode."""
410 return self.to_stdout(message, skip_eol, check_quiet=True)
412 def _write_string(self, s, out=None):
413 write_string(s, out=out, encoding=self.params.get('encoding'))
415 def to_stdout(self, message, skip_eol=False, check_quiet=False):
416 """Print message to stdout if not in quiet mode."""
417 if self.params.get('logger'):
418 self.params['logger'].debug(message)
419 elif not check_quiet or not self.params.get('quiet', False):
420 message = self._bidi_workaround(message)
421 terminator = ['\n', ''][skip_eol]
422 output = message + terminator
424 self._write_string(output, self._screen_file)
426 def to_stderr(self, message):
427 """Print message to stderr."""
428 assert isinstance(message, compat_str)
429 if self.params.get('logger'):
430 self.params['logger'].error(message)
432 message = self._bidi_workaround(message)
433 output = message + '\n'
434 self._write_string(output, self._err_file)
436 def to_console_title(self, message):
437 if not self.params.get('consoletitle', False):
439 if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
440 # c_wchar_p() might not be necessary if `message` is
441 # already of type unicode()
442 ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
443 elif 'TERM' in os.environ:
444 self._write_string('\033]0;%s\007' % message, self._screen_file)
446 def save_console_title(self):
447 if not self.params.get('consoletitle', False):
449 if 'TERM' in os.environ:
450 # Save the title on stack
451 self._write_string('\033[22;0t', self._screen_file)
453 def restore_console_title(self):
454 if not self.params.get('consoletitle', False):
456 if 'TERM' in os.environ:
457 # Restore the title from stack
458 self._write_string('\033[23;0t', self._screen_file)
461 self.save_console_title()
464 def __exit__(self, *args):
465 self.restore_console_title()
467 if self.params.get('cookiefile') is not None:
468 self.cookiejar.save()
470 def trouble(self, message=None, tb=None):
471 """Determine action to take when a download problem appears.
473 Depending on if the downloader has been configured to ignore
474 download errors or not, this method may throw an exception or
475 not when errors are found, after printing the message.
477 tb, if given, is additional traceback information.
479 if message is not None:
480 self.to_stderr(message)
481 if self.params.get('verbose'):
483 if sys.exc_info()[0]: # if .trouble has been called from an except block
485 if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
486 tb += ''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
487 tb += compat_str(traceback.format_exc())
489 tb_data = traceback.format_list(traceback.extract_stack())
490 tb = ''.join(tb_data)
492 if not self.params.get('ignoreerrors', False):
493 if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
494 exc_info = sys.exc_info()[1].exc_info
496 exc_info = sys.exc_info()
497 raise DownloadError(message, exc_info)
498 self._download_retcode = 1
500 def report_warning(self, message):
502 Print the message to stderr, it will be prefixed with 'WARNING:'
503 If stderr is a tty file the 'WARNING:' will be colored
505 if self.params.get('logger') is not None:
506 self.params['logger'].warning(message)
508 if self.params.get('no_warnings'):
510 if not self.params.get('no_color') and self._err_file.isatty() and os.name != 'nt':
511 _msg_header = '\033[0;33mWARNING:\033[0m'
513 _msg_header = 'WARNING:'
514 warning_message = '%s %s' % (_msg_header, message)
515 self.to_stderr(warning_message)
517 def report_error(self, message, tb=None):
519 Do the same as trouble, but prefixes the message with 'ERROR:', colored
520 in red if stderr is a tty file.
522 if not self.params.get('no_color') and self._err_file.isatty() and os.name != 'nt':
523 _msg_header = '\033[0;31mERROR:\033[0m'
525 _msg_header = 'ERROR:'
526 error_message = '%s %s' % (_msg_header, message)
527 self.trouble(error_message, tb)
529 def report_file_already_downloaded(self, file_name):
530 """Report file has already been fully downloaded."""
532 self.to_screen('[download] %s has already been downloaded' % file_name)
533 except UnicodeEncodeError:
534 self.to_screen('[download] The file has already been downloaded')
536 def prepare_filename(self, info_dict):
537 """Generate the output filename."""
539 template_dict = dict(info_dict)
541 template_dict['epoch'] = int(time.time())
542 autonumber_size = self.params.get('autonumber_size')
543 if autonumber_size is None:
545 autonumber_templ = '%0' + str(autonumber_size) + 'd'
546 template_dict['autonumber'] = autonumber_templ % self._num_downloads
547 if template_dict.get('playlist_index') is not None:
548 template_dict['playlist_index'] = '%0*d' % (len(str(template_dict['n_entries'])), template_dict['playlist_index'])
549 if template_dict.get('resolution') is None:
550 if template_dict.get('width') and template_dict.get('height'):
551 template_dict['resolution'] = '%dx%d' % (template_dict['width'], template_dict['height'])
552 elif template_dict.get('height'):
553 template_dict['resolution'] = '%sp' % template_dict['height']
554 elif template_dict.get('width'):
555 template_dict['resolution'] = '?x%d' % template_dict['width']
557 sanitize = lambda k, v: sanitize_filename(
559 restricted=self.params.get('restrictfilenames'),
561 template_dict = dict((k, sanitize(k, v))
562 for k, v in template_dict.items()
564 template_dict = collections.defaultdict(lambda: 'NA', template_dict)
566 outtmpl = sanitize_path(self.params.get('outtmpl', DEFAULT_OUTTMPL))
567 tmpl = compat_expanduser(outtmpl)
568 filename = tmpl % template_dict
569 # Temporary fix for #4787
570 # 'Treat' all problem characters by passing filename through preferredencoding
571 # to workaround encoding issues with subprocess on python2 @ Windows
572 if sys.version_info < (3, 0) and sys.platform == 'win32':
573 filename = encodeFilename(filename, True).decode(preferredencoding())
575 except ValueError as err:
576 self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
579 def _match_entry(self, info_dict, incomplete):
580 """ Returns None iff the file should be downloaded """
582 video_title = info_dict.get('title', info_dict.get('id', 'video'))
583 if 'title' in info_dict:
584 # This can happen when we're just evaluating the playlist
585 title = info_dict['title']
586 matchtitle = self.params.get('matchtitle', False)
588 if not re.search(matchtitle, title, re.IGNORECASE):
589 return '"' + title + '" title did not match pattern "' + matchtitle + '"'
590 rejecttitle = self.params.get('rejecttitle', False)
592 if re.search(rejecttitle, title, re.IGNORECASE):
593 return '"' + title + '" title matched reject pattern "' + rejecttitle + '"'
594 date = info_dict.get('upload_date', None)
596 dateRange = self.params.get('daterange', DateRange())
597 if date not in dateRange:
598 return '%s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
599 view_count = info_dict.get('view_count', None)
600 if view_count is not None:
601 min_views = self.params.get('min_views')
602 if min_views is not None and view_count < min_views:
603 return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title, view_count, min_views)
604 max_views = self.params.get('max_views')
605 if max_views is not None and view_count > max_views:
606 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views)
607 if age_restricted(info_dict.get('age_limit'), self.params.get('age_limit')):
608 return 'Skipping "%s" because it is age restricted' % video_title
609 if self.in_download_archive(info_dict):
610 return '%s has already been recorded in archive' % video_title
613 match_filter = self.params.get('match_filter')
614 if match_filter is not None:
615 ret = match_filter(info_dict)
622 def add_extra_info(info_dict, extra_info):
623 '''Set the keys from extra_info in info dict if they are missing'''
624 for key, value in extra_info.items():
625 info_dict.setdefault(key, value)
627 def extract_info(self, url, download=True, ie_key=None, extra_info={},
630 Returns a list with a dictionary for each video we find.
631 If 'download', also downloads the videos.
632 extra_info is a dict containing the extra values to add to each result
636 ies = [self.get_info_extractor(ie_key)]
641 if not ie.suitable(url):
645 self.report_warning('The program functionality for this site has been marked as broken, '
646 'and will probably not work.')
649 ie_result = ie.extract(url)
650 if ie_result is None: # Finished already (backwards compatibility; listformats and friends should be moved here)
652 if isinstance(ie_result, list):
653 # Backwards compatibility: old IE result format
655 '_type': 'compat_list',
656 'entries': ie_result,
658 self.add_default_extra_info(ie_result, ie, url)
660 return self.process_ie_result(ie_result, download, extra_info)
663 except ExtractorError as de: # An error we somewhat expected
664 self.report_error(compat_str(de), de.format_traceback())
666 except MaxDownloadsReached:
668 except Exception as e:
669 if self.params.get('ignoreerrors', False):
670 self.report_error(compat_str(e), tb=compat_str(traceback.format_exc()))
675 self.report_error('no suitable InfoExtractor for URL %s' % url)
677 def add_default_extra_info(self, ie_result, ie, url):
678 self.add_extra_info(ie_result, {
679 'extractor': ie.IE_NAME,
681 'webpage_url_basename': url_basename(url),
682 'extractor_key': ie.ie_key(),
685 def process_ie_result(self, ie_result, download=True, extra_info={}):
687 Take the result of the ie(may be modified) and resolve all unresolved
688 references (URLs, playlist items).
690 It will also download the videos if 'download'.
691 Returns the resolved ie_result.
694 result_type = ie_result.get('_type', 'video')
696 if result_type in ('url', 'url_transparent'):
697 extract_flat = self.params.get('extract_flat', False)
698 if ((extract_flat == 'in_playlist' and 'playlist' in extra_info) or
699 extract_flat is True):
700 if self.params.get('forcejson', False):
701 self.to_stdout(json.dumps(ie_result))
704 if result_type == 'video':
705 self.add_extra_info(ie_result, extra_info)
706 return self.process_video_result(ie_result, download=download)
707 elif result_type == 'url':
708 # We have to add extra_info to the results because it may be
709 # contained in a playlist
710 return self.extract_info(ie_result['url'],
712 ie_key=ie_result.get('ie_key'),
713 extra_info=extra_info)
714 elif result_type == 'url_transparent':
715 # Use the information from the embedding page
716 info = self.extract_info(
717 ie_result['url'], ie_key=ie_result.get('ie_key'),
718 extra_info=extra_info, download=False, process=False)
720 force_properties = dict(
721 (k, v) for k, v in ie_result.items() if v is not None)
722 for f in ('_type', 'url'):
723 if f in force_properties:
724 del force_properties[f]
725 new_result = info.copy()
726 new_result.update(force_properties)
728 assert new_result.get('_type') != 'url_transparent'
730 return self.process_ie_result(
731 new_result, download=download, extra_info=extra_info)
732 elif result_type == 'playlist' or result_type == 'multi_video':
733 # We process each entry in the playlist
734 playlist = ie_result.get('title', None) or ie_result.get('id', None)
735 self.to_screen('[download] Downloading playlist: %s' % playlist)
737 playlist_results = []
739 playliststart = self.params.get('playliststart', 1) - 1
740 playlistend = self.params.get('playlistend', None)
741 # For backwards compatibility, interpret -1 as whole list
742 if playlistend == -1:
745 playlistitems_str = self.params.get('playlist_items', None)
747 if playlistitems_str is not None:
748 def iter_playlistitems(format):
749 for string_segment in format.split(','):
750 if '-' in string_segment:
751 start, end = string_segment.split('-')
752 for item in range(int(start), int(end) + 1):
755 yield int(string_segment)
756 playlistitems = iter_playlistitems(playlistitems_str)
758 ie_entries = ie_result['entries']
759 if isinstance(ie_entries, list):
760 n_all_entries = len(ie_entries)
762 entries = [ie_entries[i - 1] for i in playlistitems]
764 entries = ie_entries[playliststart:playlistend]
765 n_entries = len(entries)
767 "[%s] playlist %s: Collected %d video ids (downloading %d of them)" %
768 (ie_result['extractor'], playlist, n_all_entries, n_entries))
769 elif isinstance(ie_entries, PagedList):
772 for item in playlistitems:
773 entries.extend(ie_entries.getslice(
777 entries = ie_entries.getslice(
778 playliststart, playlistend)
779 n_entries = len(entries)
781 "[%s] playlist %s: Downloading %d videos" %
782 (ie_result['extractor'], playlist, n_entries))
785 entry_list = list(ie_entries)
786 entries = [entry_list[i - 1] for i in playlistitems]
788 entries = list(itertools.islice(
789 ie_entries, playliststart, playlistend))
790 n_entries = len(entries)
792 "[%s] playlist %s: Downloading %d videos" %
793 (ie_result['extractor'], playlist, n_entries))
795 if self.params.get('playlistreverse', False):
796 entries = entries[::-1]
798 for i, entry in enumerate(entries, 1):
799 self.to_screen('[download] Downloading video %s of %s' % (i, n_entries))
801 'n_entries': n_entries,
802 'playlist': playlist,
803 'playlist_id': ie_result.get('id'),
804 'playlist_title': ie_result.get('title'),
805 'playlist_index': i + playliststart,
806 'extractor': ie_result['extractor'],
807 'webpage_url': ie_result['webpage_url'],
808 'webpage_url_basename': url_basename(ie_result['webpage_url']),
809 'extractor_key': ie_result['extractor_key'],
812 reason = self._match_entry(entry, incomplete=True)
813 if reason is not None:
814 self.to_screen('[download] ' + reason)
817 entry_result = self.process_ie_result(entry,
820 playlist_results.append(entry_result)
821 ie_result['entries'] = playlist_results
823 elif result_type == 'compat_list':
825 'Extractor %s returned a compat_list result. '
826 'It needs to be updated.' % ie_result.get('extractor'))
832 'extractor': ie_result['extractor'],
833 'webpage_url': ie_result['webpage_url'],
834 'webpage_url_basename': url_basename(ie_result['webpage_url']),
835 'extractor_key': ie_result['extractor_key'],
839 ie_result['entries'] = [
840 self.process_ie_result(_fixup(r), download, extra_info)
841 for r in ie_result['entries']
845 raise Exception('Invalid result type: %s' % result_type)
847 def _apply_format_filter(self, format_spec, available_formats):
848 " Returns a tuple of the remaining format_spec and filtered formats "
858 operator_rex = re.compile(r'''(?x)\s*\[
859 (?P<key>width|height|tbr|abr|vbr|asr|filesize|fps)
860 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
861 (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
863 ''' % '|'.join(map(re.escape, OPERATORS.keys())))
864 m = operator_rex.search(format_spec)
867 comparison_value = int(m.group('value'))
869 comparison_value = parse_filesize(m.group('value'))
870 if comparison_value is None:
871 comparison_value = parse_filesize(m.group('value') + 'B')
872 if comparison_value is None:
874 'Invalid value %r in format specification %r' % (
875 m.group('value'), format_spec))
876 op = OPERATORS[m.group('op')]
883 str_operator_rex = re.compile(r'''(?x)\s*\[
884 \s*(?P<key>ext|acodec|vcodec|container|protocol)
885 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?
886 \s*(?P<value>[a-zA-Z0-9_-]+)
888 ''' % '|'.join(map(re.escape, STR_OPERATORS.keys())))
889 m = str_operator_rex.search(format_spec)
891 comparison_value = m.group('value')
892 op = STR_OPERATORS[m.group('op')]
895 raise ValueError('Invalid format specification %r' % format_spec)
898 actual_value = f.get(m.group('key'))
899 if actual_value is None:
900 return m.group('none_inclusive')
901 return op(actual_value, comparison_value)
902 new_formats = [f for f in available_formats if _filter(f)]
904 new_format_spec = format_spec[:-len(m.group(0))]
905 if not new_format_spec:
906 new_format_spec = 'best'
908 return (new_format_spec, new_formats)
910 def select_format(self, format_spec, available_formats):
911 while format_spec.endswith(']'):
912 format_spec, available_formats = self._apply_format_filter(
913 format_spec, available_formats)
914 if not available_formats:
917 if format_spec == 'best' or format_spec is None:
918 return available_formats[-1]
919 elif format_spec == 'worst':
920 return available_formats[0]
921 elif format_spec == 'bestaudio':
923 f for f in available_formats
924 if f.get('vcodec') == 'none']
926 return audio_formats[-1]
927 elif format_spec == 'worstaudio':
929 f for f in available_formats
930 if f.get('vcodec') == 'none']
932 return audio_formats[0]
933 elif format_spec == 'bestvideo':
935 f for f in available_formats
936 if f.get('acodec') == 'none']
938 return video_formats[-1]
939 elif format_spec == 'worstvideo':
941 f for f in available_formats
942 if f.get('acodec') == 'none']
944 return video_formats[0]
946 extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav']
947 if format_spec in extensions:
948 filter_f = lambda f: f['ext'] == format_spec
950 filter_f = lambda f: f['format_id'] == format_spec
951 matches = list(filter(filter_f, available_formats))
956 def _calc_headers(self, info_dict):
957 res = std_headers.copy()
959 add_headers = info_dict.get('http_headers')
961 res.update(add_headers)
963 cookies = self._calc_cookies(info_dict)
965 res['Cookie'] = cookies
969 def _calc_cookies(self, info_dict):
970 pr = compat_urllib_request.Request(info_dict['url'])
971 self.cookiejar.add_cookie_header(pr)
972 return pr.get_header('Cookie')
974 def process_video_result(self, info_dict, download=True):
975 assert info_dict.get('_type', 'video') == 'video'
977 if 'id' not in info_dict:
978 raise ExtractorError('Missing "id" field in extractor result')
979 if 'title' not in info_dict:
980 raise ExtractorError('Missing "title" field in extractor result')
982 if 'playlist' not in info_dict:
983 # It isn't part of a playlist
984 info_dict['playlist'] = None
985 info_dict['playlist_index'] = None
987 thumbnails = info_dict.get('thumbnails')
988 if thumbnails is None:
989 thumbnail = info_dict.get('thumbnail')
991 info_dict['thumbnails'] = thumbnails = [{'url': thumbnail}]
993 thumbnails.sort(key=lambda t: (
994 t.get('preference'), t.get('width'), t.get('height'),
995 t.get('id'), t.get('url')))
996 for i, t in enumerate(thumbnails):
997 if 'width' in t and 'height' in t:
998 t['resolution'] = '%dx%d' % (t['width'], t['height'])
999 if t.get('id') is None:
1002 if thumbnails and 'thumbnail' not in info_dict:
1003 info_dict['thumbnail'] = thumbnails[-1]['url']
1005 if 'display_id' not in info_dict and 'id' in info_dict:
1006 info_dict['display_id'] = info_dict['id']
1008 if info_dict.get('upload_date') is None and info_dict.get('timestamp') is not None:
1009 # Working around negative timestamps in Windows
1010 # (see http://bugs.python.org/issue1646728)
1011 if info_dict['timestamp'] < 0 and os.name == 'nt':
1012 info_dict['timestamp'] = 0
1013 upload_date = datetime.datetime.utcfromtimestamp(
1014 info_dict['timestamp'])
1015 info_dict['upload_date'] = upload_date.strftime('%Y%m%d')
1017 if self.params.get('listsubtitles', False):
1018 if 'automatic_captions' in info_dict:
1019 self.list_subtitles(info_dict['id'], info_dict.get('automatic_captions'), 'automatic captions')
1020 self.list_subtitles(info_dict['id'], info_dict.get('subtitles'), 'subtitles')
1022 info_dict['requested_subtitles'] = self.process_subtitles(
1023 info_dict['id'], info_dict.get('subtitles'),
1024 info_dict.get('automatic_captions'))
1026 # This extractors handle format selection themselves
1027 if info_dict['extractor'] in ['Youku']:
1029 self.process_info(info_dict)
1032 # We now pick which formats have to be downloaded
1033 if info_dict.get('formats') is None:
1034 # There's only one format available
1035 formats = [info_dict]
1037 formats = info_dict['formats']
1040 raise ExtractorError('No video formats found!')
1042 # We check that all the formats have the format and format_id fields
1043 for i, format in enumerate(formats):
1044 if 'url' not in format:
1045 raise ExtractorError('Missing "url" key in result (index %d)' % i)
1047 if format.get('format_id') is None:
1048 format['format_id'] = compat_str(i)
1049 if format.get('format') is None:
1050 format['format'] = '{id} - {res}{note}'.format(
1051 id=format['format_id'],
1052 res=self.format_resolution(format),
1053 note=' ({0})'.format(format['format_note']) if format.get('format_note') is not None else '',
1055 # Automatically determine file extension if missing
1056 if 'ext' not in format:
1057 format['ext'] = determine_ext(format['url']).lower()
1058 # Add HTTP headers, so that external programs can use them from the
1060 full_format_info = info_dict.copy()
1061 full_format_info.update(format)
1062 format['http_headers'] = self._calc_headers(full_format_info)
1064 format_limit = self.params.get('format_limit', None)
1066 formats = list(takewhile_inclusive(
1067 lambda f: f['format_id'] != format_limit, formats
1070 # TODO Central sorting goes here
1072 if formats[0] is not info_dict:
1073 # only set the 'formats' fields if the original info_dict list them
1074 # otherwise we end up with a circular reference, the first (and unique)
1075 # element in the 'formats' field in info_dict is info_dict itself,
1076 # wich can't be exported to json
1077 info_dict['formats'] = formats
1078 if self.params.get('listformats'):
1079 self.list_formats(info_dict)
1081 if self.params.get('list_thumbnails'):
1082 self.list_thumbnails(info_dict)
1085 req_format = self.params.get('format')
1086 if req_format is None:
1088 formats_to_download = []
1089 # The -1 is for supporting YoutubeIE
1090 if req_format in ('-1', 'all'):
1091 formats_to_download = formats
1093 for rfstr in req_format.split(','):
1094 # We can accept formats requested in the format: 34/5/best, we pick
1095 # the first that is available, starting from left
1096 req_formats = rfstr.split('/')
1097 for rf in req_formats:
1098 if re.match(r'.+?\+.+?', rf) is not None:
1099 # Two formats have been requested like '137+139'
1100 format_1, format_2 = rf.split('+')
1101 formats_info = (self.select_format(format_1, formats),
1102 self.select_format(format_2, formats))
1103 if all(formats_info):
1104 # The first format must contain the video and the
1106 if formats_info[0].get('vcodec') == 'none':
1107 self.report_error('The first format must '
1108 'contain the video, try using '
1109 '"-f %s+%s"' % (format_2, format_1))
1112 formats_info[0]['ext']
1113 if self.params.get('merge_output_format') is None
1114 else self.params['merge_output_format'])
1116 'requested_formats': formats_info,
1117 'format': '%s+%s' % (formats_info[0].get('format'),
1118 formats_info[1].get('format')),
1119 'format_id': '%s+%s' % (formats_info[0].get('format_id'),
1120 formats_info[1].get('format_id')),
1121 'width': formats_info[0].get('width'),
1122 'height': formats_info[0].get('height'),
1123 'resolution': formats_info[0].get('resolution'),
1124 'fps': formats_info[0].get('fps'),
1125 'vcodec': formats_info[0].get('vcodec'),
1126 'vbr': formats_info[0].get('vbr'),
1127 'stretched_ratio': formats_info[0].get('stretched_ratio'),
1128 'acodec': formats_info[1].get('acodec'),
1129 'abr': formats_info[1].get('abr'),
1133 selected_format = None
1135 selected_format = self.select_format(rf, formats)
1136 if selected_format is not None:
1137 formats_to_download.append(selected_format)
1139 if not formats_to_download:
1140 raise ExtractorError('requested format not available',
1144 if len(formats_to_download) > 1:
1145 self.to_screen('[info] %s: downloading video in %s formats' % (info_dict['id'], len(formats_to_download)))
1146 for format in formats_to_download:
1147 new_info = dict(info_dict)
1148 new_info.update(format)
1149 self.process_info(new_info)
1150 # We update the info dict with the best quality format (backwards compatibility)
1151 info_dict.update(formats_to_download[-1])
1154 def process_subtitles(self, video_id, normal_subtitles, automatic_captions):
1155 """Select the requested subtitles and their format"""
1157 if normal_subtitles and self.params.get('writesubtitles'):
1158 available_subs.update(normal_subtitles)
1159 if automatic_captions and self.params.get('writeautomaticsub'):
1160 for lang, cap_info in automatic_captions.items():
1161 if lang not in available_subs:
1162 available_subs[lang] = cap_info
1164 if (not self.params.get('writesubtitles') and not
1165 self.params.get('writeautomaticsub') or not
1169 if self.params.get('allsubtitles', False):
1170 requested_langs = available_subs.keys()
1172 if self.params.get('subtitleslangs', False):
1173 requested_langs = self.params.get('subtitleslangs')
1174 elif 'en' in available_subs:
1175 requested_langs = ['en']
1177 requested_langs = [list(available_subs.keys())[0]]
1179 formats_query = self.params.get('subtitlesformat', 'best')
1180 formats_preference = formats_query.split('/') if formats_query else []
1182 for lang in requested_langs:
1183 formats = available_subs.get(lang)
1185 self.report_warning('%s subtitles not available for %s' % (lang, video_id))
1187 for ext in formats_preference:
1191 matches = list(filter(lambda f: f['ext'] == ext, formats))
1197 self.report_warning(
1198 'No subtitle format found matching "%s" for language %s, '
1199 'using %s' % (formats_query, lang, f['ext']))
1203 def process_info(self, info_dict):
1204 """Process a single resolved IE result."""
1206 assert info_dict.get('_type', 'video') == 'video'
1208 max_downloads = self.params.get('max_downloads')
1209 if max_downloads is not None:
1210 if self._num_downloads >= int(max_downloads):
1211 raise MaxDownloadsReached()
1213 info_dict['fulltitle'] = info_dict['title']
1214 if len(info_dict['title']) > 200:
1215 info_dict['title'] = info_dict['title'][:197] + '...'
1217 # Keep for backwards compatibility
1218 info_dict['stitle'] = info_dict['title']
1220 if 'format' not in info_dict:
1221 info_dict['format'] = info_dict['ext']
1223 reason = self._match_entry(info_dict, incomplete=False)
1224 if reason is not None:
1225 self.to_screen('[download] ' + reason)
1228 self._num_downloads += 1
1230 info_dict['_filename'] = filename = self.prepare_filename(info_dict)
1233 if self.params.get('forcetitle', False):
1234 self.to_stdout(info_dict['fulltitle'])
1235 if self.params.get('forceid', False):
1236 self.to_stdout(info_dict['id'])
1237 if self.params.get('forceurl', False):
1238 if info_dict.get('requested_formats') is not None:
1239 for f in info_dict['requested_formats']:
1240 self.to_stdout(f['url'] + f.get('play_path', ''))
1242 # For RTMP URLs, also include the playpath
1243 self.to_stdout(info_dict['url'] + info_dict.get('play_path', ''))
1244 if self.params.get('forcethumbnail', False) and info_dict.get('thumbnail') is not None:
1245 self.to_stdout(info_dict['thumbnail'])
1246 if self.params.get('forcedescription', False) and info_dict.get('description') is not None:
1247 self.to_stdout(info_dict['description'])
1248 if self.params.get('forcefilename', False) and filename is not None:
1249 self.to_stdout(filename)
1250 if self.params.get('forceduration', False) and info_dict.get('duration') is not None:
1251 self.to_stdout(formatSeconds(info_dict['duration']))
1252 if self.params.get('forceformat', False):
1253 self.to_stdout(info_dict['format'])
1254 if self.params.get('forcejson', False):
1255 self.to_stdout(json.dumps(info_dict))
1257 # Do nothing else if in simulate mode
1258 if self.params.get('simulate', False):
1261 if filename is None:
1265 dn = os.path.dirname(encodeFilename(filename))
1266 if dn and not os.path.exists(dn):
1268 except (OSError, IOError) as err:
1269 self.report_error('unable to create directory ' + compat_str(err))
1272 if self.params.get('writedescription', False):
1273 descfn = filename + '.description'
1274 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)):
1275 self.to_screen('[info] Video description is already present')
1276 elif info_dict.get('description') is None:
1277 self.report_warning('There\'s no description to write.')
1280 self.to_screen('[info] Writing video description to: ' + descfn)
1281 with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
1282 descfile.write(info_dict['description'])
1283 except (OSError, IOError):
1284 self.report_error('Cannot write description file ' + descfn)
1287 if self.params.get('writeannotations', False):
1288 annofn = filename + '.annotations.xml'
1289 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)):
1290 self.to_screen('[info] Video annotations are already present')
1293 self.to_screen('[info] Writing video annotations to: ' + annofn)
1294 with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile:
1295 annofile.write(info_dict['annotations'])
1296 except (KeyError, TypeError):
1297 self.report_warning('There are no annotations to write.')
1298 except (OSError, IOError):
1299 self.report_error('Cannot write annotations file: ' + annofn)
1302 subtitles_are_requested = any([self.params.get('writesubtitles', False),
1303 self.params.get('writeautomaticsub')])
1305 if subtitles_are_requested and info_dict.get('requested_subtitles'):
1306 # subtitles download errors are already managed as troubles in relevant IE
1307 # that way it will silently go on when used with unsupporting IE
1308 subtitles = info_dict['requested_subtitles']
1309 ie = self.get_info_extractor(info_dict['extractor_key'])
1310 for sub_lang, sub_info in subtitles.items():
1311 sub_format = sub_info['ext']
1312 if sub_info.get('data') is not None:
1313 sub_data = sub_info['data']
1316 sub_data = ie._download_webpage(
1317 sub_info['url'], info_dict['id'], note=False)
1318 except ExtractorError as err:
1319 self.report_warning('Unable to download subtitle for "%s": %s' %
1320 (sub_lang, compat_str(err.cause)))
1323 sub_filename = subtitles_filename(filename, sub_lang, sub_format)
1324 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)):
1325 self.to_screen('[info] Video subtitle %s.%s is already_present' % (sub_lang, sub_format))
1327 self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
1328 with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
1329 subfile.write(sub_data)
1330 except (OSError, IOError):
1331 self.report_error('Cannot write subtitles file ' + sub_filename)
1334 if self.params.get('writeinfojson', False):
1335 infofn = os.path.splitext(filename)[0] + '.info.json'
1336 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)):
1337 self.to_screen('[info] Video description metadata is already present')
1339 self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
1341 write_json_file(info_dict, infofn)
1342 except (OSError, IOError):
1343 self.report_error('Cannot write metadata to JSON file ' + infofn)
1346 self._write_thumbnails(info_dict, filename)
1348 if not self.params.get('skip_download', False):
1351 fd = get_suitable_downloader(info, self.params)(self, self.params)
1352 for ph in self._progress_hooks:
1353 fd.add_progress_hook(ph)
1354 if self.params.get('verbose'):
1355 self.to_stdout('[debug] Invoking downloader on %r' % info.get('url'))
1356 return fd.download(name, info)
1358 if info_dict.get('requested_formats') is not None:
1361 merger = FFmpegMergerPP(self, not self.params.get('keepvideo'))
1362 if not merger.available:
1364 self.report_warning('You have requested multiple '
1365 'formats but ffmpeg or avconv are not installed.'
1366 ' The formats won\'t be merged')
1368 postprocessors = [merger]
1369 for f in info_dict['requested_formats']:
1370 new_info = dict(info_dict)
1372 fname = self.prepare_filename(new_info)
1373 fname = prepend_extension(fname, 'f%s' % f['format_id'])
1374 downloaded.append(fname)
1375 partial_success = dl(fname, new_info)
1376 success = success and partial_success
1377 info_dict['__postprocessors'] = postprocessors
1378 info_dict['__files_to_merge'] = downloaded
1380 # Just a single file
1381 success = dl(filename, info_dict)
1382 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1383 self.report_error('unable to download video data: %s' % str(err))
1385 except (OSError, IOError) as err:
1386 raise UnavailableVideoError(err)
1387 except (ContentTooShortError, ) as err:
1388 self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
1393 fixup_policy = self.params.get('fixup')
1394 if fixup_policy is None:
1395 fixup_policy = 'detect_or_warn'
1397 stretched_ratio = info_dict.get('stretched_ratio')
1398 if stretched_ratio is not None and stretched_ratio != 1:
1399 if fixup_policy == 'warn':
1400 self.report_warning('%s: Non-uniform pixel ratio (%s)' % (
1401 info_dict['id'], stretched_ratio))
1402 elif fixup_policy == 'detect_or_warn':
1403 stretched_pp = FFmpegFixupStretchedPP(self)
1404 if stretched_pp.available:
1405 info_dict.setdefault('__postprocessors', [])
1406 info_dict['__postprocessors'].append(stretched_pp)
1408 self.report_warning(
1409 '%s: Non-uniform pixel ratio (%s). Install ffmpeg or avconv to fix this automatically.' % (
1410 info_dict['id'], stretched_ratio))
1412 assert fixup_policy in ('ignore', 'never')
1414 if info_dict.get('requested_formats') is None and info_dict.get('container') == 'm4a_dash':
1415 if fixup_policy == 'warn':
1416 self.report_warning('%s: writing DASH m4a. Only some players support this container.' % (
1418 elif fixup_policy == 'detect_or_warn':
1419 fixup_pp = FFmpegFixupM4aPP(self)
1420 if fixup_pp.available:
1421 info_dict.setdefault('__postprocessors', [])
1422 info_dict['__postprocessors'].append(fixup_pp)
1424 self.report_warning(
1425 '%s: writing DASH m4a. Only some players support this container. Install ffmpeg or avconv to fix this automatically.' % (
1428 assert fixup_policy in ('ignore', 'never')
1431 self.post_process(filename, info_dict)
1432 except (PostProcessingError) as err:
1433 self.report_error('postprocessing: %s' % str(err))
1435 self.record_download_archive(info_dict)
1437 def download(self, url_list):
1438 """Download a given list of URLs."""
1439 outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
1440 if (len(url_list) > 1 and
1441 '%' not in outtmpl and
1442 self.params.get('max_downloads') != 1):
1443 raise SameFileError(outtmpl)
1445 for url in url_list:
1447 # It also downloads the videos
1448 res = self.extract_info(url)
1449 except UnavailableVideoError:
1450 self.report_error('unable to download video')
1451 except MaxDownloadsReached:
1452 self.to_screen('[info] Maximum number of downloaded files reached.')
1455 if self.params.get('dump_single_json', False):
1456 self.to_stdout(json.dumps(res))
1458 return self._download_retcode
1460 def download_with_info_file(self, info_filename):
1461 with contextlib.closing(fileinput.FileInput(
1462 [info_filename], mode='r',
1463 openhook=fileinput.hook_encoded('utf-8'))) as f:
1464 # FileInput doesn't have a read method, we can't call json.load
1465 info = json.loads('\n'.join(f))
1467 self.process_ie_result(info, download=True)
1468 except DownloadError:
1469 webpage_url = info.get('webpage_url')
1470 if webpage_url is not None:
1471 self.report_warning('The info failed to download, trying with "%s"' % webpage_url)
1472 return self.download([webpage_url])
1475 return self._download_retcode
1477 def post_process(self, filename, ie_info):
1478 """Run all the postprocessors on the given file."""
1479 info = dict(ie_info)
1480 info['filepath'] = filename
1482 if ie_info.get('__postprocessors') is not None:
1483 pps_chain.extend(ie_info['__postprocessors'])
1484 pps_chain.extend(self._pps)
1485 for pp in pps_chain:
1487 old_filename = info['filepath']
1489 keep_video_wish, info = pp.run(info)
1490 if keep_video_wish is not None:
1492 keep_video = keep_video_wish
1493 elif keep_video is None:
1494 # No clear decision yet, let IE decide
1495 keep_video = keep_video_wish
1496 except PostProcessingError as e:
1497 self.report_error(e.msg)
1498 if keep_video is False and not self.params.get('keepvideo', False):
1500 self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename)
1501 os.remove(encodeFilename(old_filename))
1502 except (IOError, OSError):
1503 self.report_warning('Unable to remove downloaded video file')
1505 def _make_archive_id(self, info_dict):
1506 # Future-proof against any change in case
1507 # and backwards compatibility with prior versions
1508 extractor = info_dict.get('extractor_key')
1509 if extractor is None:
1510 if 'id' in info_dict:
1511 extractor = info_dict.get('ie_key') # key in a playlist
1512 if extractor is None:
1513 return None # Incomplete video information
1514 return extractor.lower() + ' ' + info_dict['id']
1516 def in_download_archive(self, info_dict):
1517 fn = self.params.get('download_archive')
1521 vid_id = self._make_archive_id(info_dict)
1523 return False # Incomplete video information
1526 with locked_file(fn, 'r', encoding='utf-8') as archive_file:
1527 for line in archive_file:
1528 if line.strip() == vid_id:
1530 except IOError as ioe:
1531 if ioe.errno != errno.ENOENT:
1535 def record_download_archive(self, info_dict):
1536 fn = self.params.get('download_archive')
1539 vid_id = self._make_archive_id(info_dict)
1541 with locked_file(fn, 'a', encoding='utf-8') as archive_file:
1542 archive_file.write(vid_id + '\n')
1545 def format_resolution(format, default='unknown'):
1546 if format.get('vcodec') == 'none':
1548 if format.get('resolution') is not None:
1549 return format['resolution']
1550 if format.get('height') is not None:
1551 if format.get('width') is not None:
1552 res = '%sx%s' % (format['width'], format['height'])
1554 res = '%sp' % format['height']
1555 elif format.get('width') is not None:
1556 res = '?x%d' % format['width']
1561 def _format_note(self, fdict):
1563 if fdict.get('ext') in ['f4f', 'f4m']:
1564 res += '(unsupported) '
1565 if fdict.get('format_note') is not None:
1566 res += fdict['format_note'] + ' '
1567 if fdict.get('tbr') is not None:
1568 res += '%4dk ' % fdict['tbr']
1569 if fdict.get('container') is not None:
1572 res += '%s container' % fdict['container']
1573 if (fdict.get('vcodec') is not None and
1574 fdict.get('vcodec') != 'none'):
1577 res += fdict['vcodec']
1578 if fdict.get('vbr') is not None:
1580 elif fdict.get('vbr') is not None and fdict.get('abr') is not None:
1582 if fdict.get('vbr') is not None:
1583 res += '%4dk' % fdict['vbr']
1584 if fdict.get('fps') is not None:
1585 res += ', %sfps' % fdict['fps']
1586 if fdict.get('acodec') is not None:
1589 if fdict['acodec'] == 'none':
1592 res += '%-5s' % fdict['acodec']
1593 elif fdict.get('abr') is not None:
1597 if fdict.get('abr') is not None:
1598 res += '@%3dk' % fdict['abr']
1599 if fdict.get('asr') is not None:
1600 res += ' (%5dHz)' % fdict['asr']
1601 if fdict.get('filesize') is not None:
1604 res += format_bytes(fdict['filesize'])
1605 elif fdict.get('filesize_approx') is not None:
1608 res += '~' + format_bytes(fdict['filesize_approx'])
1611 def list_formats(self, info_dict):
1612 formats = info_dict.get('formats', [info_dict])
1614 [f['format_id'], f['ext'], self.format_resolution(f), self._format_note(f)]
1616 if f.get('preference') is None or f['preference'] >= -1000]
1617 if len(formats) > 1:
1618 table[-1][-1] += (' ' if table[-1][-1] else '') + '(best)'
1620 header_line = ['format code', 'extension', 'resolution', 'note']
1622 '[info] Available formats for %s:\n%s' %
1623 (info_dict['id'], render_table(header_line, table)))
1625 def list_thumbnails(self, info_dict):
1626 thumbnails = info_dict.get('thumbnails')
1628 tn_url = info_dict.get('thumbnail')
1630 thumbnails = [{'id': '0', 'url': tn_url}]
1633 '[info] No thumbnails present for %s' % info_dict['id'])
1637 '[info] Thumbnails for %s:' % info_dict['id'])
1638 self.to_screen(render_table(
1639 ['ID', 'width', 'height', 'URL'],
1640 [[t['id'], t.get('width', 'unknown'), t.get('height', 'unknown'), t['url']] for t in thumbnails]))
1642 def list_subtitles(self, video_id, subtitles, name='subtitles'):
1644 self.to_screen('%s has no %s' % (video_id, name))
1647 'Available %s for %s:' % (name, video_id))
1648 self.to_screen(render_table(
1649 ['Language', 'formats'],
1650 [[lang, ', '.join(f['ext'] for f in reversed(formats))]
1651 for lang, formats in subtitles.items()]))
1653 def urlopen(self, req):
1654 """ Start an HTTP download """
1656 # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
1657 # always respected by websites, some tend to give out URLs with non percent-encoded
1658 # non-ASCII characters (see telemb.py, ard.py [#3412])
1659 # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
1660 # To work around aforementioned issue we will replace request's original URL with
1661 # percent-encoded one
1662 req_is_string = isinstance(req, compat_basestring)
1663 url = req if req_is_string else req.get_full_url()
1664 url_escaped = escape_url(url)
1666 # Substitute URL if any change after escaping
1667 if url != url_escaped:
1671 req = compat_urllib_request.Request(
1672 url_escaped, data=req.data, headers=req.headers,
1673 origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
1675 return self._opener.open(req, timeout=self._socket_timeout)
1677 def print_debug_header(self):
1678 if not self.params.get('verbose'):
1681 if type('') is not compat_str:
1682 # Python 2.6 on SLES11 SP1 (https://github.com/rg3/youtube-dl/issues/3326)
1683 self.report_warning(
1684 'Your Python is broken! Update to a newer and supported version')
1686 stdout_encoding = getattr(
1687 sys.stdout, 'encoding', 'missing (%s)' % type(sys.stdout).__name__)
1689 '[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % (
1690 locale.getpreferredencoding(),
1691 sys.getfilesystemencoding(),
1693 self.get_encoding()))
1694 write_string(encoding_str, encoding=None)
1696 self._write_string('[debug] youtube-dl version ' + __version__ + '\n')
1698 sp = subprocess.Popen(
1699 ['git', 'rev-parse', '--short', 'HEAD'],
1700 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
1701 cwd=os.path.dirname(os.path.abspath(__file__)))
1702 out, err = sp.communicate()
1703 out = out.decode().strip()
1704 if re.match('[0-9a-f]+', out):
1705 self._write_string('[debug] Git HEAD: ' + out + '\n')
1711 self._write_string('[debug] Python version %s - %s\n' % (
1712 platform.python_version(), platform_name()))
1714 exe_versions = FFmpegPostProcessor.get_versions(self)
1715 exe_versions['rtmpdump'] = rtmpdump_version()
1716 exe_str = ', '.join(
1718 for exe, v in sorted(exe_versions.items())
1723 self._write_string('[debug] exe versions: %s\n' % exe_str)
1726 for handler in self._opener.handlers:
1727 if hasattr(handler, 'proxies'):
1728 proxy_map.update(handler.proxies)
1729 self._write_string('[debug] Proxy map: ' + compat_str(proxy_map) + '\n')
1731 if self.params.get('call_home', False):
1732 ipaddr = self.urlopen('https://yt-dl.org/ip').read().decode('utf-8')
1733 self._write_string('[debug] Public IP address: %s\n' % ipaddr)
1734 latest_version = self.urlopen(
1735 'https://yt-dl.org/latest/version').read().decode('utf-8')
1736 if version_tuple(latest_version) > version_tuple(__version__):
1737 self.report_warning(
1738 'You are using an outdated version (newest version: %s)! '
1739 'See https://yt-dl.org/update if you need help updating.' %
1742 def _setup_opener(self):
1743 timeout_val = self.params.get('socket_timeout')
1744 self._socket_timeout = 600 if timeout_val is None else float(timeout_val)
1746 opts_cookiefile = self.params.get('cookiefile')
1747 opts_proxy = self.params.get('proxy')
1749 if opts_cookiefile is None:
1750 self.cookiejar = compat_cookiejar.CookieJar()
1752 self.cookiejar = compat_cookiejar.MozillaCookieJar(
1754 if os.access(opts_cookiefile, os.R_OK):
1755 self.cookiejar.load()
1757 cookie_processor = compat_urllib_request.HTTPCookieProcessor(
1759 if opts_proxy is not None:
1760 if opts_proxy == '':
1763 proxies = {'http': opts_proxy, 'https': opts_proxy}
1765 proxies = compat_urllib_request.getproxies()
1766 # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
1767 if 'http' in proxies and 'https' not in proxies:
1768 proxies['https'] = proxies['http']
1769 proxy_handler = PerRequestProxyHandler(proxies)
1771 debuglevel = 1 if self.params.get('debug_printtraffic') else 0
1772 https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
1773 ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
1774 opener = compat_urllib_request.build_opener(
1775 proxy_handler, https_handler, cookie_processor, ydlh)
1777 # Delete the default user-agent header, which would otherwise apply in
1778 # cases where our custom HTTP handler doesn't come into play
1779 # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
1780 opener.addheaders = []
1781 self._opener = opener
1783 def encode(self, s):
1784 if isinstance(s, bytes):
1785 return s # Already encoded
1788 return s.encode(self.get_encoding())
1789 except UnicodeEncodeError as err:
1790 err.reason = err.reason + '. Check your system encoding configuration or use the --encoding option.'
1793 def get_encoding(self):
1794 encoding = self.params.get('encoding')
1795 if encoding is None:
1796 encoding = preferredencoding()
1799 def _write_thumbnails(self, info_dict, filename):
1800 if self.params.get('writethumbnail', False):
1801 thumbnails = info_dict.get('thumbnails')
1803 thumbnails = [thumbnails[-1]]
1804 elif self.params.get('write_all_thumbnails', False):
1805 thumbnails = info_dict.get('thumbnails')
1810 # No thumbnails present, so return immediately
1813 for t in thumbnails:
1814 thumb_ext = determine_ext(t['url'], 'jpg')
1815 suffix = '_%s' % t['id'] if len(thumbnails) > 1 else ''
1816 thumb_display_id = '%s ' % t['id'] if len(thumbnails) > 1 else ''
1817 thumb_filename = os.path.splitext(filename)[0] + suffix + '.' + thumb_ext
1819 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
1820 self.to_screen('[%s] %s: Thumbnail %sis already present' %
1821 (info_dict['extractor'], info_dict['id'], thumb_display_id))
1823 self.to_screen('[%s] %s: Downloading thumbnail %s...' %
1824 (info_dict['extractor'], info_dict['id'], thumb_display_id))
1826 uf = self.urlopen(t['url'])
1827 with open(thumb_filename, 'wb') as thumbf:
1828 shutil.copyfileobj(uf, thumbf)
1829 self.to_screen('[%s] %s: Writing thumbnail %sto: %s' %
1830 (info_dict['extractor'], info_dict['id'], thumb_display_id, thumb_filename))
1831 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1832 self.report_warning('Unable to download thumbnail "%s": %s' %
1833 (t['url'], compat_str(err)))