2 # -*- coding: utf-8 -*-
4 from __future__ import absolute_import, unicode_literals
35 compat_urllib_request,
64 UnavailableVideoError,
74 from .cache import Cache
75 from .extractor import get_info_extractor, gen_extractors
76 from .downloader import get_suitable_downloader
77 from .downloader.rtmp import rtmpdump_version
78 from .postprocessor import (
80 FFmpegFixupStretchedPP,
85 from .version import __version__
88 class YoutubeDL(object):
91 YoutubeDL objects are the ones responsible of downloading the
92 actual video file and writing it to disk if the user has requested
93 it, among some other tasks. In most cases there should be one per
94 program. As, given a video URL, the downloader doesn't know how to
95 extract all the needed information, task that InfoExtractors do, it
96 has to pass the URL to one of them.
98 For this, YoutubeDL objects have a method that allows
99 InfoExtractors to be registered in a given order. When it is passed
100 a URL, the YoutubeDL object handles it to the first InfoExtractor it
101 finds that reports being able to handle it. The InfoExtractor extracts
102 all the information about the video or videos the URL refers to, and
103 YoutubeDL process the extracted information, possibly using a File
104 Downloader to download the video.
106 YoutubeDL objects accept a lot of parameters. In order not to saturate
107 the object constructor with arguments, it receives a dictionary of
108 options instead. These options are available through the params
109 attribute for the InfoExtractors to use. The YoutubeDL also
110 registers itself as the downloader in charge for the InfoExtractors
111 that are added to it, so this is a "mutual registration".
115 username: Username for authentication purposes.
116 password: Password for authentication purposes.
117 videopassword: Password for acces a video.
118 usenetrc: Use netrc for authentication instead.
119 verbose: Print additional info to stdout.
120 quiet: Do not print messages to stdout.
121 no_warnings: Do not print out anything for warnings.
122 forceurl: Force printing final URL.
123 forcetitle: Force printing title.
124 forceid: Force printing ID.
125 forcethumbnail: Force printing thumbnail URL.
126 forcedescription: Force printing description.
127 forcefilename: Force printing final filename.
128 forceduration: Force printing duration.
129 forcejson: Force printing info_dict as JSON.
130 dump_single_json: Force printing the info_dict of the whole playlist
131 (or video) as a single JSON line.
132 simulate: Do not download the video files.
133 format: Video format code. See options.py for more information.
134 format_limit: Highest quality format to try.
135 outtmpl: Template for output names.
136 restrictfilenames: Do not allow "&" and spaces in file names
137 ignoreerrors: Do not stop on download errors.
138 nooverwrites: Prevent overwriting files.
139 playliststart: Playlist item to start at.
140 playlistend: Playlist item to end at.
141 playlist_items: Specific indices of playlist to download.
142 playlistreverse: Download playlist items in reverse order.
143 matchtitle: Download only matching titles.
144 rejecttitle: Reject downloads for matching titles.
145 logger: Log messages to a logging.Logger instance.
146 logtostderr: Log messages to stderr instead of stdout.
147 writedescription: Write the video description to a .description file
148 writeinfojson: Write the video description to a .info.json file
149 writeannotations: Write the video annotations to a .annotations.xml file
150 writethumbnail: Write the thumbnail image to a file
151 write_all_thumbnails: Write all thumbnail formats to files
152 writesubtitles: Write the video subtitles to a file
153 writeautomaticsub: Write the automatic subtitles to a file
154 allsubtitles: Downloads all the subtitles of the video
155 (requires writesubtitles or writeautomaticsub)
156 listsubtitles: Lists all available subtitles for the video
157 subtitlesformat: The format code for subtitles
158 subtitleslangs: List of languages of the subtitles to download
159 keepvideo: Keep the video file after post-processing
160 daterange: A DateRange object, download only if the upload_date is in the range.
161 skip_download: Skip the actual download of the video file
162 cachedir: Location of the cache files in the filesystem.
163 False to disable filesystem cache.
164 noplaylist: Download single video instead of a playlist if in doubt.
165 age_limit: An integer representing the user's age in years.
166 Unsuitable videos for the given age are skipped.
167 min_views: An integer representing the minimum view count the video
168 must have in order to not be skipped.
169 Videos without view count information are always
170 downloaded. None for no limit.
171 max_views: An integer representing the maximum view count.
172 Videos that are more popular than that are not
174 Videos without view count information are always
175 downloaded. None for no limit.
176 download_archive: File name of a file where all downloads are recorded.
177 Videos already present in the file are not downloaded
179 cookiefile: File name where cookies should be read from and dumped to.
180 nocheckcertificate:Do not verify SSL certificates
181 prefer_insecure: Use HTTP instead of HTTPS to retrieve information.
182 At the moment, this is only supported by YouTube.
183 proxy: URL of the proxy server to use
184 socket_timeout: Time to wait for unresponsive hosts, in seconds
185 bidi_workaround: Work around buggy terminals without bidirectional text
186 support, using fridibi
187 debug_printtraffic:Print out sent and received HTTP traffic
188 include_ads: Download ads as well
189 default_search: Prepend this string if an input url is not valid.
190 'auto' for elaborate guessing
191 encoding: Use this encoding instead of the system-specified.
192 extract_flat: Do not resolve URLs, return the immediate result.
193 Pass in 'in_playlist' to only show this behavior for
195 postprocessors: A list of dictionaries, each with an entry
196 * key: The name of the postprocessor. See
197 youtube_dl/postprocessor/__init__.py for a list.
198 as well as any further keyword arguments for the
200 progress_hooks: A list of functions that get called on download
201 progress, with a dictionary with the entries
202 * status: One of "downloading" and "finished".
203 Check this first and ignore unknown values.
205 If status is one of "downloading" or "finished", the
206 following properties may also be present:
207 * filename: The final filename (always present)
208 * downloaded_bytes: Bytes on disk
209 * total_bytes: Size of the whole file, None if unknown
210 * tmpfilename: The filename we're currently writing to
211 * eta: The estimated time in seconds, None if unknown
212 * speed: The download speed in bytes/second, None if
215 Progress hooks are guaranteed to be called at least once
216 (with status "finished") if the download is successful.
217 merge_output_format: Extension to use when merging formats.
218 fixup: Automatically correct known faults of the file.
220 - "never": do nothing
221 - "warn": only emit a warning
222 - "detect_or_warn": check whether we can do anything
223 about it, warn otherwise (default)
224 source_address: (Experimental) Client-side IP address to bind to.
225 call_home: Boolean, true iff we are allowed to contact the
226 youtube-dl servers for debugging.
227 sleep_interval: Number of seconds to sleep before each download.
228 external_downloader: Executable of the external downloader to call.
229 listformats: Print an overview of available video formats and exit.
230 list_thumbnails: Print a table of all thumbnails and exit.
231 match_filter: A function that gets called with the info_dict of
233 If it returns a message, the video is ignored.
234 If it returns None, the video is downloaded.
235 match_filter_func in utils.py is one example for this.
236 no_color: Do not emit color codes in output.
239 The following parameters are not used by YoutubeDL itself, they are used by
241 nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
242 noresizebuffer, retries, continuedl, noprogress, consoletitle,
245 The following options are used by the post processors:
246 prefer_ffmpeg: If True, use ffmpeg instead of avconv if both are available,
247 otherwise prefer avconv.
248 exec_cmd: Arbitrary command to run after downloading
254 _download_retcode = None
255 _num_downloads = None
258 def __init__(self, params=None, auto_init=True):
259 """Create a FileDownloader object with the given options."""
263 self._ies_instances = {}
265 self._progress_hooks = []
266 self._download_retcode = 0
267 self._num_downloads = 0
268 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
269 self._err_file = sys.stderr
271 self.cache = Cache(self)
273 if params.get('bidi_workaround', False):
276 master, slave = pty.openpty()
277 width = get_term_width()
281 width_args = ['-w', str(width)]
283 stdin=subprocess.PIPE,
285 stderr=self._err_file)
287 self._output_process = subprocess.Popen(
288 ['bidiv'] + width_args, **sp_kwargs
291 self._output_process = subprocess.Popen(
292 ['fribidi', '-c', 'UTF-8'] + width_args, **sp_kwargs)
293 self._output_channel = os.fdopen(master, 'rb')
294 except OSError as ose:
296 self.report_warning('Could not find fribidi executable, ignoring --bidi-workaround . Make sure that fribidi is an executable file in one of the directories in your $PATH.')
300 if (sys.version_info >= (3,) and sys.platform != 'win32' and
301 sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968']
302 and not params.get('restrictfilenames', False)):
303 # On Python 3, the Unicode filesystem API will throw errors (#1474)
305 'Assuming --restrict-filenames since file system encoding '
306 'cannot encode all characters. '
307 'Set the LC_ALL environment variable to fix this.')
308 self.params['restrictfilenames'] = True
310 if '%(stitle)s' in self.params.get('outtmpl', ''):
311 self.report_warning('%(stitle)s is deprecated. Use the %(title)s and the --restrict-filenames flag(which also secures %(uploader)s et al) instead.')
316 self.print_debug_header()
317 self.add_default_info_extractors()
319 for pp_def_raw in self.params.get('postprocessors', []):
320 pp_class = get_postprocessor(pp_def_raw['key'])
321 pp_def = dict(pp_def_raw)
323 pp = pp_class(self, **compat_kwargs(pp_def))
324 self.add_post_processor(pp)
326 for ph in self.params.get('progress_hooks', []):
327 self.add_progress_hook(ph)
329 def warn_if_short_id(self, argv):
330 # short YouTube ID starting with dash?
332 i for i, a in enumerate(argv)
333 if re.match(r'^-[0-9A-Za-z_-]{10}$', a)]
337 [a for i, a in enumerate(argv) if i not in idxs] +
338 ['--'] + [argv[i] for i in idxs]
341 'Long argument string detected. '
342 'Use -- to separate parameters and URLs, like this:\n%s\n' %
343 args_to_str(correct_argv))
345 def add_info_extractor(self, ie):
346 """Add an InfoExtractor object to the end of the list."""
348 self._ies_instances[ie.ie_key()] = ie
349 ie.set_downloader(self)
351 def get_info_extractor(self, ie_key):
353 Get an instance of an IE with name ie_key, it will try to get one from
354 the _ies list, if there's no instance it will create a new one and add
355 it to the extractor list.
357 ie = self._ies_instances.get(ie_key)
359 ie = get_info_extractor(ie_key)()
360 self.add_info_extractor(ie)
363 def add_default_info_extractors(self):
365 Add the InfoExtractors returned by gen_extractors to the end of the list
367 for ie in gen_extractors():
368 self.add_info_extractor(ie)
370 def add_post_processor(self, pp):
371 """Add a PostProcessor object to the end of the chain."""
373 pp.set_downloader(self)
375 def add_progress_hook(self, ph):
376 """Add the progress hook (currently only for the file downloader)"""
377 self._progress_hooks.append(ph)
379 def _bidi_workaround(self, message):
380 if not hasattr(self, '_output_channel'):
383 assert hasattr(self, '_output_process')
384 assert isinstance(message, compat_str)
385 line_count = message.count('\n') + 1
386 self._output_process.stdin.write((message + '\n').encode('utf-8'))
387 self._output_process.stdin.flush()
388 res = ''.join(self._output_channel.readline().decode('utf-8')
389 for _ in range(line_count))
390 return res[:-len('\n')]
392 def to_screen(self, message, skip_eol=False):
393 """Print message to stdout if not in quiet mode."""
394 return self.to_stdout(message, skip_eol, check_quiet=True)
396 def _write_string(self, s, out=None):
397 write_string(s, out=out, encoding=self.params.get('encoding'))
399 def to_stdout(self, message, skip_eol=False, check_quiet=False):
400 """Print message to stdout if not in quiet mode."""
401 if self.params.get('logger'):
402 self.params['logger'].debug(message)
403 elif not check_quiet or not self.params.get('quiet', False):
404 message = self._bidi_workaround(message)
405 terminator = ['\n', ''][skip_eol]
406 output = message + terminator
408 self._write_string(output, self._screen_file)
410 def to_stderr(self, message):
411 """Print message to stderr."""
412 assert isinstance(message, compat_str)
413 if self.params.get('logger'):
414 self.params['logger'].error(message)
416 message = self._bidi_workaround(message)
417 output = message + '\n'
418 self._write_string(output, self._err_file)
420 def to_console_title(self, message):
421 if not self.params.get('consoletitle', False):
423 if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
424 # c_wchar_p() might not be necessary if `message` is
425 # already of type unicode()
426 ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
427 elif 'TERM' in os.environ:
428 self._write_string('\033]0;%s\007' % message, self._screen_file)
430 def save_console_title(self):
431 if not self.params.get('consoletitle', False):
433 if 'TERM' in os.environ:
434 # Save the title on stack
435 self._write_string('\033[22;0t', self._screen_file)
437 def restore_console_title(self):
438 if not self.params.get('consoletitle', False):
440 if 'TERM' in os.environ:
441 # Restore the title from stack
442 self._write_string('\033[23;0t', self._screen_file)
445 self.save_console_title()
448 def __exit__(self, *args):
449 self.restore_console_title()
451 if self.params.get('cookiefile') is not None:
452 self.cookiejar.save()
454 def trouble(self, message=None, tb=None):
455 """Determine action to take when a download problem appears.
457 Depending on if the downloader has been configured to ignore
458 download errors or not, this method may throw an exception or
459 not when errors are found, after printing the message.
461 tb, if given, is additional traceback information.
463 if message is not None:
464 self.to_stderr(message)
465 if self.params.get('verbose'):
467 if sys.exc_info()[0]: # if .trouble has been called from an except block
469 if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
470 tb += ''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
471 tb += compat_str(traceback.format_exc())
473 tb_data = traceback.format_list(traceback.extract_stack())
474 tb = ''.join(tb_data)
476 if not self.params.get('ignoreerrors', False):
477 if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
478 exc_info = sys.exc_info()[1].exc_info
480 exc_info = sys.exc_info()
481 raise DownloadError(message, exc_info)
482 self._download_retcode = 1
484 def report_warning(self, message):
486 Print the message to stderr, it will be prefixed with 'WARNING:'
487 If stderr is a tty file the 'WARNING:' will be colored
489 if self.params.get('logger') is not None:
490 self.params['logger'].warning(message)
492 if self.params.get('no_warnings'):
494 if not self.params.get('no_color') and self._err_file.isatty() and os.name != 'nt':
495 _msg_header = '\033[0;33mWARNING:\033[0m'
497 _msg_header = 'WARNING:'
498 warning_message = '%s %s' % (_msg_header, message)
499 self.to_stderr(warning_message)
501 def report_error(self, message, tb=None):
503 Do the same as trouble, but prefixes the message with 'ERROR:', colored
504 in red if stderr is a tty file.
506 if not self.params.get('no_color') and self._err_file.isatty() and os.name != 'nt':
507 _msg_header = '\033[0;31mERROR:\033[0m'
509 _msg_header = 'ERROR:'
510 error_message = '%s %s' % (_msg_header, message)
511 self.trouble(error_message, tb)
513 def report_file_already_downloaded(self, file_name):
514 """Report file has already been fully downloaded."""
516 self.to_screen('[download] %s has already been downloaded' % file_name)
517 except UnicodeEncodeError:
518 self.to_screen('[download] The file has already been downloaded')
520 def prepare_filename(self, info_dict):
521 """Generate the output filename."""
523 template_dict = dict(info_dict)
525 template_dict['epoch'] = int(time.time())
526 autonumber_size = self.params.get('autonumber_size')
527 if autonumber_size is None:
529 autonumber_templ = '%0' + str(autonumber_size) + 'd'
530 template_dict['autonumber'] = autonumber_templ % self._num_downloads
531 if template_dict.get('playlist_index') is not None:
532 template_dict['playlist_index'] = '%0*d' % (len(str(template_dict['n_entries'])), template_dict['playlist_index'])
533 if template_dict.get('resolution') is None:
534 if template_dict.get('width') and template_dict.get('height'):
535 template_dict['resolution'] = '%dx%d' % (template_dict['width'], template_dict['height'])
536 elif template_dict.get('height'):
537 template_dict['resolution'] = '%sp' % template_dict['height']
538 elif template_dict.get('width'):
539 template_dict['resolution'] = '?x%d' % template_dict['width']
541 sanitize = lambda k, v: sanitize_filename(
543 restricted=self.params.get('restrictfilenames'),
545 template_dict = dict((k, sanitize(k, v))
546 for k, v in template_dict.items()
548 template_dict = collections.defaultdict(lambda: 'NA', template_dict)
550 outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
551 tmpl = compat_expanduser(outtmpl)
552 filename = tmpl % template_dict
553 # Temporary fix for #4787
554 # 'Treat' all problem characters by passing filename through preferredencoding
555 # to workaround encoding issues with subprocess on python2 @ Windows
556 if sys.version_info < (3, 0) and sys.platform == 'win32':
557 filename = encodeFilename(filename, True).decode(preferredencoding())
559 except ValueError as err:
560 self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
563 def _match_entry(self, info_dict, incomplete):
564 """ Returns None iff the file should be downloaded """
566 video_title = info_dict.get('title', info_dict.get('id', 'video'))
567 if 'title' in info_dict:
568 # This can happen when we're just evaluating the playlist
569 title = info_dict['title']
570 matchtitle = self.params.get('matchtitle', False)
572 if not re.search(matchtitle, title, re.IGNORECASE):
573 return '"' + title + '" title did not match pattern "' + matchtitle + '"'
574 rejecttitle = self.params.get('rejecttitle', False)
576 if re.search(rejecttitle, title, re.IGNORECASE):
577 return '"' + title + '" title matched reject pattern "' + rejecttitle + '"'
578 date = info_dict.get('upload_date', None)
580 dateRange = self.params.get('daterange', DateRange())
581 if date not in dateRange:
582 return '%s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
583 view_count = info_dict.get('view_count', None)
584 if view_count is not None:
585 min_views = self.params.get('min_views')
586 if min_views is not None and view_count < min_views:
587 return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title, view_count, min_views)
588 max_views = self.params.get('max_views')
589 if max_views is not None and view_count > max_views:
590 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views)
591 if age_restricted(info_dict.get('age_limit'), self.params.get('age_limit')):
592 return 'Skipping "%s" because it is age restricted' % video_title
593 if self.in_download_archive(info_dict):
594 return '%s has already been recorded in archive' % video_title
597 match_filter = self.params.get('match_filter')
598 if match_filter is not None:
599 ret = match_filter(info_dict)
606 def add_extra_info(info_dict, extra_info):
607 '''Set the keys from extra_info in info dict if they are missing'''
608 for key, value in extra_info.items():
609 info_dict.setdefault(key, value)
611 def extract_info(self, url, download=True, ie_key=None, extra_info={},
614 Returns a list with a dictionary for each video we find.
615 If 'download', also downloads the videos.
616 extra_info is a dict containing the extra values to add to each result
620 ies = [self.get_info_extractor(ie_key)]
625 if not ie.suitable(url):
629 self.report_warning('The program functionality for this site has been marked as broken, '
630 'and will probably not work.')
633 ie_result = ie.extract(url)
634 if ie_result is None: # Finished already (backwards compatibility; listformats and friends should be moved here)
636 if isinstance(ie_result, list):
637 # Backwards compatibility: old IE result format
639 '_type': 'compat_list',
640 'entries': ie_result,
642 self.add_default_extra_info(ie_result, ie, url)
644 return self.process_ie_result(ie_result, download, extra_info)
647 except ExtractorError as de: # An error we somewhat expected
648 self.report_error(compat_str(de), de.format_traceback())
650 except MaxDownloadsReached:
652 except Exception as e:
653 if self.params.get('ignoreerrors', False):
654 self.report_error(compat_str(e), tb=compat_str(traceback.format_exc()))
659 self.report_error('no suitable InfoExtractor for URL %s' % url)
661 def add_default_extra_info(self, ie_result, ie, url):
662 self.add_extra_info(ie_result, {
663 'extractor': ie.IE_NAME,
665 'webpage_url_basename': url_basename(url),
666 'extractor_key': ie.ie_key(),
669 def process_ie_result(self, ie_result, download=True, extra_info={}):
671 Take the result of the ie(may be modified) and resolve all unresolved
672 references (URLs, playlist items).
674 It will also download the videos if 'download'.
675 Returns the resolved ie_result.
678 result_type = ie_result.get('_type', 'video')
680 if result_type in ('url', 'url_transparent'):
681 extract_flat = self.params.get('extract_flat', False)
682 if ((extract_flat == 'in_playlist' and 'playlist' in extra_info) or
683 extract_flat is True):
684 if self.params.get('forcejson', False):
685 self.to_stdout(json.dumps(ie_result))
688 if result_type == 'video':
689 self.add_extra_info(ie_result, extra_info)
690 return self.process_video_result(ie_result, download=download)
691 elif result_type == 'url':
692 # We have to add extra_info to the results because it may be
693 # contained in a playlist
694 return self.extract_info(ie_result['url'],
696 ie_key=ie_result.get('ie_key'),
697 extra_info=extra_info)
698 elif result_type == 'url_transparent':
699 # Use the information from the embedding page
700 info = self.extract_info(
701 ie_result['url'], ie_key=ie_result.get('ie_key'),
702 extra_info=extra_info, download=False, process=False)
704 force_properties = dict(
705 (k, v) for k, v in ie_result.items() if v is not None)
706 for f in ('_type', 'url'):
707 if f in force_properties:
708 del force_properties[f]
709 new_result = info.copy()
710 new_result.update(force_properties)
712 assert new_result.get('_type') != 'url_transparent'
714 return self.process_ie_result(
715 new_result, download=download, extra_info=extra_info)
716 elif result_type == 'playlist' or result_type == 'multi_video':
717 # We process each entry in the playlist
718 playlist = ie_result.get('title', None) or ie_result.get('id', None)
719 self.to_screen('[download] Downloading playlist: %s' % playlist)
721 playlist_results = []
723 playliststart = self.params.get('playliststart', 1) - 1
724 playlistend = self.params.get('playlistend', None)
725 # For backwards compatibility, interpret -1 as whole list
726 if playlistend == -1:
729 playlistitems_str = self.params.get('playlist_items', None)
731 if playlistitems_str is not None:
732 def iter_playlistitems(format):
733 for string_segment in format.split(','):
734 if '-' in string_segment:
735 start, end = string_segment.split('-')
736 for item in range(int(start), int(end) + 1):
739 yield int(string_segment)
740 playlistitems = iter_playlistitems(playlistitems_str)
742 ie_entries = ie_result['entries']
743 if isinstance(ie_entries, list):
744 n_all_entries = len(ie_entries)
746 entries = [ie_entries[i - 1] for i in playlistitems]
748 entries = ie_entries[playliststart:playlistend]
749 n_entries = len(entries)
751 "[%s] playlist %s: Collected %d video ids (downloading %d of them)" %
752 (ie_result['extractor'], playlist, n_all_entries, n_entries))
753 elif isinstance(ie_entries, PagedList):
756 for item in playlistitems:
757 entries.extend(ie_entries.getslice(
761 entries = ie_entries.getslice(
762 playliststart, playlistend)
763 n_entries = len(entries)
765 "[%s] playlist %s: Downloading %d videos" %
766 (ie_result['extractor'], playlist, n_entries))
769 entry_list = list(ie_entries)
770 entries = [entry_list[i - 1] for i in playlistitems]
772 entries = list(itertools.islice(
773 ie_entries, playliststart, playlistend))
774 n_entries = len(entries)
776 "[%s] playlist %s: Downloading %d videos" %
777 (ie_result['extractor'], playlist, n_entries))
779 if self.params.get('playlistreverse', False):
780 entries = entries[::-1]
782 for i, entry in enumerate(entries, 1):
783 self.to_screen('[download] Downloading video %s of %s' % (i, n_entries))
785 'n_entries': n_entries,
786 'playlist': playlist,
787 'playlist_id': ie_result.get('id'),
788 'playlist_title': ie_result.get('title'),
789 'playlist_index': i + playliststart,
790 'extractor': ie_result['extractor'],
791 'webpage_url': ie_result['webpage_url'],
792 'webpage_url_basename': url_basename(ie_result['webpage_url']),
793 'extractor_key': ie_result['extractor_key'],
796 reason = self._match_entry(entry, incomplete=True)
797 if reason is not None:
798 self.to_screen('[download] ' + reason)
801 entry_result = self.process_ie_result(entry,
804 playlist_results.append(entry_result)
805 ie_result['entries'] = playlist_results
807 elif result_type == 'compat_list':
809 'Extractor %s returned a compat_list result. '
810 'It needs to be updated.' % ie_result.get('extractor'))
816 'extractor': ie_result['extractor'],
817 'webpage_url': ie_result['webpage_url'],
818 'webpage_url_basename': url_basename(ie_result['webpage_url']),
819 'extractor_key': ie_result['extractor_key'],
823 ie_result['entries'] = [
824 self.process_ie_result(_fixup(r), download, extra_info)
825 for r in ie_result['entries']
829 raise Exception('Invalid result type: %s' % result_type)
831 def _apply_format_filter(self, format_spec, available_formats):
832 " Returns a tuple of the remaining format_spec and filtered formats "
842 operator_rex = re.compile(r'''(?x)\s*\[
843 (?P<key>width|height|tbr|abr|vbr|asr|filesize|fps)
844 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
845 (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
847 ''' % '|'.join(map(re.escape, OPERATORS.keys())))
848 m = operator_rex.search(format_spec)
851 comparison_value = int(m.group('value'))
853 comparison_value = parse_filesize(m.group('value'))
854 if comparison_value is None:
855 comparison_value = parse_filesize(m.group('value') + 'B')
856 if comparison_value is None:
858 'Invalid value %r in format specification %r' % (
859 m.group('value'), format_spec))
860 op = OPERATORS[m.group('op')]
867 str_operator_rex = re.compile(r'''(?x)\s*\[
868 \s*(?P<key>ext|acodec|vcodec|container|protocol)
869 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?
870 \s*(?P<value>[a-zA-Z0-9_-]+)
872 ''' % '|'.join(map(re.escape, STR_OPERATORS.keys())))
873 m = str_operator_rex.search(format_spec)
875 comparison_value = m.group('value')
876 op = STR_OPERATORS[m.group('op')]
879 raise ValueError('Invalid format specification %r' % format_spec)
882 actual_value = f.get(m.group('key'))
883 if actual_value is None:
884 return m.group('none_inclusive')
885 return op(actual_value, comparison_value)
886 new_formats = [f for f in available_formats if _filter(f)]
888 new_format_spec = format_spec[:-len(m.group(0))]
889 if not new_format_spec:
890 new_format_spec = 'best'
892 return (new_format_spec, new_formats)
894 def select_format(self, format_spec, available_formats):
895 while format_spec.endswith(']'):
896 format_spec, available_formats = self._apply_format_filter(
897 format_spec, available_formats)
898 if not available_formats:
901 if format_spec == 'best' or format_spec is None:
902 return available_formats[-1]
903 elif format_spec == 'worst':
904 return available_formats[0]
905 elif format_spec == 'bestaudio':
907 f for f in available_formats
908 if f.get('vcodec') == 'none']
910 return audio_formats[-1]
911 elif format_spec == 'worstaudio':
913 f for f in available_formats
914 if f.get('vcodec') == 'none']
916 return audio_formats[0]
917 elif format_spec == 'bestvideo':
919 f for f in available_formats
920 if f.get('acodec') == 'none']
922 return video_formats[-1]
923 elif format_spec == 'worstvideo':
925 f for f in available_formats
926 if f.get('acodec') == 'none']
928 return video_formats[0]
930 extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav']
931 if format_spec in extensions:
932 filter_f = lambda f: f['ext'] == format_spec
934 filter_f = lambda f: f['format_id'] == format_spec
935 matches = list(filter(filter_f, available_formats))
940 def _calc_headers(self, info_dict):
941 res = std_headers.copy()
943 add_headers = info_dict.get('http_headers')
945 res.update(add_headers)
947 cookies = self._calc_cookies(info_dict)
949 res['Cookie'] = cookies
953 def _calc_cookies(self, info_dict):
954 class _PseudoRequest(object):
955 def __init__(self, url):
958 self.unverifiable = False
960 def add_unredirected_header(self, k, v):
963 def get_full_url(self):
966 def is_unverifiable(self):
967 return self.unverifiable
969 def has_header(self, h):
970 return h in self.headers
972 def get_header(self, h, default=None):
973 return self.headers.get(h, default)
975 pr = _PseudoRequest(info_dict['url'])
976 self.cookiejar.add_cookie_header(pr)
977 return pr.headers.get('Cookie')
979 def process_video_result(self, info_dict, download=True):
980 assert info_dict.get('_type', 'video') == 'video'
982 if 'id' not in info_dict:
983 raise ExtractorError('Missing "id" field in extractor result')
984 if 'title' not in info_dict:
985 raise ExtractorError('Missing "title" field in extractor result')
987 if 'playlist' not in info_dict:
988 # It isn't part of a playlist
989 info_dict['playlist'] = None
990 info_dict['playlist_index'] = None
992 thumbnails = info_dict.get('thumbnails')
993 if thumbnails is None:
994 thumbnail = info_dict.get('thumbnail')
996 info_dict['thumbnails'] = thumbnails = [{'url': thumbnail}]
998 thumbnails.sort(key=lambda t: (
999 t.get('preference'), t.get('width'), t.get('height'),
1000 t.get('id'), t.get('url')))
1001 for i, t in enumerate(thumbnails):
1002 if 'width' in t and 'height' in t:
1003 t['resolution'] = '%dx%d' % (t['width'], t['height'])
1004 if t.get('id') is None:
1007 if thumbnails and 'thumbnail' not in info_dict:
1008 info_dict['thumbnail'] = thumbnails[-1]['url']
1010 if 'display_id' not in info_dict and 'id' in info_dict:
1011 info_dict['display_id'] = info_dict['id']
1013 if info_dict.get('upload_date') is None and info_dict.get('timestamp') is not None:
1014 # Working around negative timestamps in Windows
1015 # (see http://bugs.python.org/issue1646728)
1016 if info_dict['timestamp'] < 0 and os.name == 'nt':
1017 info_dict['timestamp'] = 0
1018 upload_date = datetime.datetime.utcfromtimestamp(
1019 info_dict['timestamp'])
1020 info_dict['upload_date'] = upload_date.strftime('%Y%m%d')
1022 if self.params.get('listsubtitles', False):
1023 if 'automatic_captions' in info_dict:
1024 self.list_subtitles(info_dict['id'], info_dict.get('automatic_captions'), 'automatic captions')
1025 self.list_subtitles(info_dict['id'], info_dict.get('subtitles'), 'subtitles')
1027 info_dict['requested_subtitles'] = self.process_subtitles(
1028 info_dict['id'], info_dict.get('subtitles'),
1029 info_dict.get('automatic_captions'))
1031 # This extractors handle format selection themselves
1032 if info_dict['extractor'] in ['Youku']:
1034 self.process_info(info_dict)
1037 # We now pick which formats have to be downloaded
1038 if info_dict.get('formats') is None:
1039 # There's only one format available
1040 formats = [info_dict]
1042 formats = info_dict['formats']
1045 raise ExtractorError('No video formats found!')
1047 # We check that all the formats have the format and format_id fields
1048 for i, format in enumerate(formats):
1049 if 'url' not in format:
1050 raise ExtractorError('Missing "url" key in result (index %d)' % i)
1052 if format.get('format_id') is None:
1053 format['format_id'] = compat_str(i)
1054 if format.get('format') is None:
1055 format['format'] = '{id} - {res}{note}'.format(
1056 id=format['format_id'],
1057 res=self.format_resolution(format),
1058 note=' ({0})'.format(format['format_note']) if format.get('format_note') is not None else '',
1060 # Automatically determine file extension if missing
1061 if 'ext' not in format:
1062 format['ext'] = determine_ext(format['url']).lower()
1063 # Add HTTP headers, so that external programs can use them from the
1065 full_format_info = info_dict.copy()
1066 full_format_info.update(format)
1067 format['http_headers'] = self._calc_headers(full_format_info)
1069 format_limit = self.params.get('format_limit', None)
1071 formats = list(takewhile_inclusive(
1072 lambda f: f['format_id'] != format_limit, formats
1075 # TODO Central sorting goes here
1077 if formats[0] is not info_dict:
1078 # only set the 'formats' fields if the original info_dict list them
1079 # otherwise we end up with a circular reference, the first (and unique)
1080 # element in the 'formats' field in info_dict is info_dict itself,
1081 # wich can't be exported to json
1082 info_dict['formats'] = formats
1083 if self.params.get('listformats'):
1084 self.list_formats(info_dict)
1086 if self.params.get('list_thumbnails'):
1087 self.list_thumbnails(info_dict)
1090 req_format = self.params.get('format')
1091 if req_format is None:
1093 formats_to_download = []
1094 # The -1 is for supporting YoutubeIE
1095 if req_format in ('-1', 'all'):
1096 formats_to_download = formats
1098 for rfstr in req_format.split(','):
1099 # We can accept formats requested in the format: 34/5/best, we pick
1100 # the first that is available, starting from left
1101 req_formats = rfstr.split('/')
1102 for rf in req_formats:
1103 if re.match(r'.+?\+.+?', rf) is not None:
1104 # Two formats have been requested like '137+139'
1105 format_1, format_2 = rf.split('+')
1106 formats_info = (self.select_format(format_1, formats),
1107 self.select_format(format_2, formats))
1108 if all(formats_info):
1109 # The first format must contain the video and the
1111 if formats_info[0].get('vcodec') == 'none':
1112 self.report_error('The first format must '
1113 'contain the video, try using '
1114 '"-f %s+%s"' % (format_2, format_1))
1117 formats_info[0]['ext']
1118 if self.params.get('merge_output_format') is None
1119 else self.params['merge_output_format'])
1121 'requested_formats': formats_info,
1122 'format': '%s+%s' % (formats_info[0].get('format'),
1123 formats_info[1].get('format')),
1124 'format_id': '%s+%s' % (formats_info[0].get('format_id'),
1125 formats_info[1].get('format_id')),
1126 'width': formats_info[0].get('width'),
1127 'height': formats_info[0].get('height'),
1128 'resolution': formats_info[0].get('resolution'),
1129 'fps': formats_info[0].get('fps'),
1130 'vcodec': formats_info[0].get('vcodec'),
1131 'vbr': formats_info[0].get('vbr'),
1132 'stretched_ratio': formats_info[0].get('stretched_ratio'),
1133 'acodec': formats_info[1].get('acodec'),
1134 'abr': formats_info[1].get('abr'),
1138 selected_format = None
1140 selected_format = self.select_format(rf, formats)
1141 if selected_format is not None:
1142 formats_to_download.append(selected_format)
1144 if not formats_to_download:
1145 raise ExtractorError('requested format not available',
1149 if len(formats_to_download) > 1:
1150 self.to_screen('[info] %s: downloading video in %s formats' % (info_dict['id'], len(formats_to_download)))
1151 for format in formats_to_download:
1152 new_info = dict(info_dict)
1153 new_info.update(format)
1154 self.process_info(new_info)
1155 # We update the info dict with the best quality format (backwards compatibility)
1156 info_dict.update(formats_to_download[-1])
1159 def process_subtitles(self, video_id, available_subs, available_autocaps):
1160 """Select the requested subtitles and their format"""
1161 if available_autocaps and self.params.get('writeautomaticsub'):
1162 available_subs = available_subs.copy()
1163 for lang, cap_info in available_autocaps.items():
1164 if lang not in available_subs:
1165 available_subs[lang] = cap_info
1167 if not available_subs:
1168 return available_subs
1170 if self.params.get('allsubtitles', False):
1171 requested_langs = available_subs.keys()
1173 if self.params.get('subtitleslangs', False):
1174 requested_langs = self.params.get('subtitleslangs')
1175 elif 'en' in available_subs:
1176 requested_langs = ['en']
1178 requested_langs = [list(available_subs.keys())[0]]
1180 formats_query = self.params.get('subtitlesformat', 'best')
1181 formats_preference = formats_query.split('/') if formats_query else []
1183 for lang in requested_langs:
1184 formats = available_subs.get(lang)
1186 self.report_warning('%s subtitles not available for %s' % (lang, video_id))
1188 if isinstance(formats, compat_str):
1189 # TODO: convert all IE with subtitles support to the new format
1192 'ext': formats_preference[0],
1196 for ext in formats_preference:
1200 matches = list(filter(lambda f: f['ext'] == ext, formats))
1206 self.report_warning(
1207 'No subtitle format found matching "%s" for language %s, '
1208 'using %s' % (formats_query, lang, f['ext']))
1212 def process_info(self, info_dict):
1213 """Process a single resolved IE result."""
1215 assert info_dict.get('_type', 'video') == 'video'
1217 max_downloads = self.params.get('max_downloads')
1218 if max_downloads is not None:
1219 if self._num_downloads >= int(max_downloads):
1220 raise MaxDownloadsReached()
1222 info_dict['fulltitle'] = info_dict['title']
1223 if len(info_dict['title']) > 200:
1224 info_dict['title'] = info_dict['title'][:197] + '...'
1226 # Keep for backwards compatibility
1227 info_dict['stitle'] = info_dict['title']
1229 if 'format' not in info_dict:
1230 info_dict['format'] = info_dict['ext']
1232 reason = self._match_entry(info_dict, incomplete=False)
1233 if reason is not None:
1234 self.to_screen('[download] ' + reason)
1237 self._num_downloads += 1
1239 info_dict['_filename'] = filename = self.prepare_filename(info_dict)
1242 if self.params.get('forcetitle', False):
1243 self.to_stdout(info_dict['fulltitle'])
1244 if self.params.get('forceid', False):
1245 self.to_stdout(info_dict['id'])
1246 if self.params.get('forceurl', False):
1247 if info_dict.get('requested_formats') is not None:
1248 for f in info_dict['requested_formats']:
1249 self.to_stdout(f['url'] + f.get('play_path', ''))
1251 # For RTMP URLs, also include the playpath
1252 self.to_stdout(info_dict['url'] + info_dict.get('play_path', ''))
1253 if self.params.get('forcethumbnail', False) and info_dict.get('thumbnail') is not None:
1254 self.to_stdout(info_dict['thumbnail'])
1255 if self.params.get('forcedescription', False) and info_dict.get('description') is not None:
1256 self.to_stdout(info_dict['description'])
1257 if self.params.get('forcefilename', False) and filename is not None:
1258 self.to_stdout(filename)
1259 if self.params.get('forceduration', False) and info_dict.get('duration') is not None:
1260 self.to_stdout(formatSeconds(info_dict['duration']))
1261 if self.params.get('forceformat', False):
1262 self.to_stdout(info_dict['format'])
1263 if self.params.get('forcejson', False):
1264 self.to_stdout(json.dumps(info_dict))
1266 # Do nothing else if in simulate mode
1267 if self.params.get('simulate', False):
1270 if filename is None:
1274 dn = os.path.dirname(encodeFilename(filename))
1275 if dn and not os.path.exists(dn):
1277 except (OSError, IOError) as err:
1278 self.report_error('unable to create directory ' + compat_str(err))
1281 if self.params.get('writedescription', False):
1282 descfn = filename + '.description'
1283 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)):
1284 self.to_screen('[info] Video description is already present')
1285 elif info_dict.get('description') is None:
1286 self.report_warning('There\'s no description to write.')
1289 self.to_screen('[info] Writing video description to: ' + descfn)
1290 with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
1291 descfile.write(info_dict['description'])
1292 except (OSError, IOError):
1293 self.report_error('Cannot write description file ' + descfn)
1296 if self.params.get('writeannotations', False):
1297 annofn = filename + '.annotations.xml'
1298 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)):
1299 self.to_screen('[info] Video annotations are already present')
1302 self.to_screen('[info] Writing video annotations to: ' + annofn)
1303 with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile:
1304 annofile.write(info_dict['annotations'])
1305 except (KeyError, TypeError):
1306 self.report_warning('There are no annotations to write.')
1307 except (OSError, IOError):
1308 self.report_error('Cannot write annotations file: ' + annofn)
1311 subtitles_are_requested = any([self.params.get('writesubtitles', False),
1312 self.params.get('writeautomaticsub')])
1314 if subtitles_are_requested and info_dict.get('requested_subtitles'):
1315 # subtitles download errors are already managed as troubles in relevant IE
1316 # that way it will silently go on when used with unsupporting IE
1317 subtitles = info_dict['requested_subtitles']
1318 for sub_lang, sub_info in subtitles.items():
1319 sub_format = sub_info['ext']
1320 if sub_info.get('data') is not None:
1321 sub_data = sub_info['data']
1324 uf = self.urlopen(sub_info['url'])
1325 sub_data = uf.read().decode('utf-8')
1326 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1327 self.report_warning('Unable to download subtitle for "%s": %s' %
1328 (sub_lang, compat_str(err)))
1331 sub_filename = subtitles_filename(filename, sub_lang, sub_format)
1332 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)):
1333 self.to_screen('[info] Video subtitle %s.%s is already_present' % (sub_lang, sub_format))
1335 self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
1336 with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
1337 subfile.write(sub_data)
1338 except (OSError, IOError):
1339 self.report_error('Cannot write subtitles file ' + sub_filename)
1342 if self.params.get('writeinfojson', False):
1343 infofn = os.path.splitext(filename)[0] + '.info.json'
1344 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)):
1345 self.to_screen('[info] Video description metadata is already present')
1347 self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
1349 write_json_file(info_dict, infofn)
1350 except (OSError, IOError):
1351 self.report_error('Cannot write metadata to JSON file ' + infofn)
1354 self._write_thumbnails(info_dict, filename)
1356 if not self.params.get('skip_download', False):
1359 fd = get_suitable_downloader(info, self.params)(self, self.params)
1360 for ph in self._progress_hooks:
1361 fd.add_progress_hook(ph)
1362 if self.params.get('verbose'):
1363 self.to_stdout('[debug] Invoking downloader on %r' % info.get('url'))
1364 return fd.download(name, info)
1366 if info_dict.get('requested_formats') is not None:
1369 merger = FFmpegMergerPP(self, not self.params.get('keepvideo'))
1370 if not merger._executable:
1372 self.report_warning('You have requested multiple '
1373 'formats but ffmpeg or avconv are not installed.'
1374 ' The formats won\'t be merged')
1376 postprocessors = [merger]
1377 for f in info_dict['requested_formats']:
1378 new_info = dict(info_dict)
1380 fname = self.prepare_filename(new_info)
1381 fname = prepend_extension(fname, 'f%s' % f['format_id'])
1382 downloaded.append(fname)
1383 partial_success = dl(fname, new_info)
1384 success = success and partial_success
1385 info_dict['__postprocessors'] = postprocessors
1386 info_dict['__files_to_merge'] = downloaded
1388 # Just a single file
1389 success = dl(filename, info_dict)
1390 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1391 self.report_error('unable to download video data: %s' % str(err))
1393 except (OSError, IOError) as err:
1394 raise UnavailableVideoError(err)
1395 except (ContentTooShortError, ) as err:
1396 self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
1401 fixup_policy = self.params.get('fixup')
1402 if fixup_policy is None:
1403 fixup_policy = 'detect_or_warn'
1405 stretched_ratio = info_dict.get('stretched_ratio')
1406 if stretched_ratio is not None and stretched_ratio != 1:
1407 if fixup_policy == 'warn':
1408 self.report_warning('%s: Non-uniform pixel ratio (%s)' % (
1409 info_dict['id'], stretched_ratio))
1410 elif fixup_policy == 'detect_or_warn':
1411 stretched_pp = FFmpegFixupStretchedPP(self)
1412 if stretched_pp.available:
1413 info_dict.setdefault('__postprocessors', [])
1414 info_dict['__postprocessors'].append(stretched_pp)
1416 self.report_warning(
1417 '%s: Non-uniform pixel ratio (%s). Install ffmpeg or avconv to fix this automatically.' % (
1418 info_dict['id'], stretched_ratio))
1420 assert fixup_policy in ('ignore', 'never')
1422 if info_dict.get('requested_formats') is None and info_dict.get('container') == 'm4a_dash':
1423 if fixup_policy == 'warn':
1424 self.report_warning('%s: writing DASH m4a. Only some players support this container.' % (
1426 elif fixup_policy == 'detect_or_warn':
1427 fixup_pp = FFmpegFixupM4aPP(self)
1428 if fixup_pp.available:
1429 info_dict.setdefault('__postprocessors', [])
1430 info_dict['__postprocessors'].append(fixup_pp)
1432 self.report_warning(
1433 '%s: writing DASH m4a. Only some players support this container. Install ffmpeg or avconv to fix this automatically.' % (
1436 assert fixup_policy in ('ignore', 'never')
1439 self.post_process(filename, info_dict)
1440 except (PostProcessingError) as err:
1441 self.report_error('postprocessing: %s' % str(err))
1443 self.record_download_archive(info_dict)
1445 def download(self, url_list):
1446 """Download a given list of URLs."""
1447 outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
1448 if (len(url_list) > 1 and
1450 and self.params.get('max_downloads') != 1):
1451 raise SameFileError(outtmpl)
1453 for url in url_list:
1455 # It also downloads the videos
1456 res = self.extract_info(url)
1457 except UnavailableVideoError:
1458 self.report_error('unable to download video')
1459 except MaxDownloadsReached:
1460 self.to_screen('[info] Maximum number of downloaded files reached.')
1463 if self.params.get('dump_single_json', False):
1464 self.to_stdout(json.dumps(res))
1466 return self._download_retcode
1468 def download_with_info_file(self, info_filename):
1469 with io.open(info_filename, 'r', encoding='utf-8') as f:
1472 self.process_ie_result(info, download=True)
1473 except DownloadError:
1474 webpage_url = info.get('webpage_url')
1475 if webpage_url is not None:
1476 self.report_warning('The info failed to download, trying with "%s"' % webpage_url)
1477 return self.download([webpage_url])
1480 return self._download_retcode
1482 def post_process(self, filename, ie_info):
1483 """Run all the postprocessors on the given file."""
1484 info = dict(ie_info)
1485 info['filepath'] = filename
1487 if ie_info.get('__postprocessors') is not None:
1488 pps_chain.extend(ie_info['__postprocessors'])
1489 pps_chain.extend(self._pps)
1490 for pp in pps_chain:
1492 old_filename = info['filepath']
1494 keep_video_wish, info = pp.run(info)
1495 if keep_video_wish is not None:
1497 keep_video = keep_video_wish
1498 elif keep_video is None:
1499 # No clear decision yet, let IE decide
1500 keep_video = keep_video_wish
1501 except PostProcessingError as e:
1502 self.report_error(e.msg)
1503 if keep_video is False and not self.params.get('keepvideo', False):
1505 self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename)
1506 os.remove(encodeFilename(old_filename))
1507 except (IOError, OSError):
1508 self.report_warning('Unable to remove downloaded video file')
1510 def _make_archive_id(self, info_dict):
1511 # Future-proof against any change in case
1512 # and backwards compatibility with prior versions
1513 extractor = info_dict.get('extractor_key')
1514 if extractor is None:
1515 if 'id' in info_dict:
1516 extractor = info_dict.get('ie_key') # key in a playlist
1517 if extractor is None:
1518 return None # Incomplete video information
1519 return extractor.lower() + ' ' + info_dict['id']
1521 def in_download_archive(self, info_dict):
1522 fn = self.params.get('download_archive')
1526 vid_id = self._make_archive_id(info_dict)
1528 return False # Incomplete video information
1531 with locked_file(fn, 'r', encoding='utf-8') as archive_file:
1532 for line in archive_file:
1533 if line.strip() == vid_id:
1535 except IOError as ioe:
1536 if ioe.errno != errno.ENOENT:
1540 def record_download_archive(self, info_dict):
1541 fn = self.params.get('download_archive')
1544 vid_id = self._make_archive_id(info_dict)
1546 with locked_file(fn, 'a', encoding='utf-8') as archive_file:
1547 archive_file.write(vid_id + '\n')
1550 def format_resolution(format, default='unknown'):
1551 if format.get('vcodec') == 'none':
1553 if format.get('resolution') is not None:
1554 return format['resolution']
1555 if format.get('height') is not None:
1556 if format.get('width') is not None:
1557 res = '%sx%s' % (format['width'], format['height'])
1559 res = '%sp' % format['height']
1560 elif format.get('width') is not None:
1561 res = '?x%d' % format['width']
1566 def _format_note(self, fdict):
1568 if fdict.get('ext') in ['f4f', 'f4m']:
1569 res += '(unsupported) '
1570 if fdict.get('format_note') is not None:
1571 res += fdict['format_note'] + ' '
1572 if fdict.get('tbr') is not None:
1573 res += '%4dk ' % fdict['tbr']
1574 if fdict.get('container') is not None:
1577 res += '%s container' % fdict['container']
1578 if (fdict.get('vcodec') is not None and
1579 fdict.get('vcodec') != 'none'):
1582 res += fdict['vcodec']
1583 if fdict.get('vbr') is not None:
1585 elif fdict.get('vbr') is not None and fdict.get('abr') is not None:
1587 if fdict.get('vbr') is not None:
1588 res += '%4dk' % fdict['vbr']
1589 if fdict.get('fps') is not None:
1590 res += ', %sfps' % fdict['fps']
1591 if fdict.get('acodec') is not None:
1594 if fdict['acodec'] == 'none':
1597 res += '%-5s' % fdict['acodec']
1598 elif fdict.get('abr') is not None:
1602 if fdict.get('abr') is not None:
1603 res += '@%3dk' % fdict['abr']
1604 if fdict.get('asr') is not None:
1605 res += ' (%5dHz)' % fdict['asr']
1606 if fdict.get('filesize') is not None:
1609 res += format_bytes(fdict['filesize'])
1610 elif fdict.get('filesize_approx') is not None:
1613 res += '~' + format_bytes(fdict['filesize_approx'])
1616 def list_formats(self, info_dict):
1617 def line(format, idlen=20):
1618 return (('%-' + compat_str(idlen + 1) + 's%-10s%-12s%s') % (
1619 format['format_id'],
1621 self.format_resolution(format),
1622 self._format_note(format),
1625 formats = info_dict.get('formats', [info_dict])
1626 idlen = max(len('format code'),
1627 max(len(f['format_id']) for f in formats))
1629 line(f, idlen) for f in formats
1630 if f.get('preference') is None or f['preference'] >= -1000]
1631 if len(formats) > 1:
1632 formats_s[-1] += (' ' if self._format_note(formats[-1]) else '') + '(best)'
1634 header_line = line({
1635 'format_id': 'format code', 'ext': 'extension',
1636 'resolution': 'resolution', 'format_note': 'note'}, idlen=idlen)
1638 '[info] Available formats for %s:\n%s\n%s' %
1639 (info_dict['id'], header_line, '\n'.join(formats_s)))
1641 def list_thumbnails(self, info_dict):
1642 thumbnails = info_dict.get('thumbnails')
1644 tn_url = info_dict.get('thumbnail')
1646 thumbnails = [{'id': '0', 'url': tn_url}]
1649 '[info] No thumbnails present for %s' % info_dict['id'])
1653 '[info] Thumbnails for %s:' % info_dict['id'])
1654 self.to_screen(render_table(
1655 ['ID', 'width', 'height', 'URL'],
1656 [[t['id'], t.get('width', 'unknown'), t.get('height', 'unknown'), t['url']] for t in thumbnails]))
1658 def list_subtitles(self, video_id, subtitles, name='subtitles'):
1660 self.to_screen('%s has no %s' % (video_id, name))
1663 'Available %s for %s:' % (name, video_id))
1664 self.to_screen(render_table(
1665 ['Language', 'formats'],
1666 [[lang, ', '.join(f['ext'] for f in reversed(formats))]
1667 for lang, formats in subtitles.items()]))
1669 def urlopen(self, req):
1670 """ Start an HTTP download """
1672 # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
1673 # always respected by websites, some tend to give out URLs with non percent-encoded
1674 # non-ASCII characters (see telemb.py, ard.py [#3412])
1675 # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
1676 # To work around aforementioned issue we will replace request's original URL with
1677 # percent-encoded one
1678 req_is_string = isinstance(req, compat_basestring)
1679 url = req if req_is_string else req.get_full_url()
1680 url_escaped = escape_url(url)
1682 # Substitute URL if any change after escaping
1683 if url != url_escaped:
1687 req = compat_urllib_request.Request(
1688 url_escaped, data=req.data, headers=req.headers,
1689 origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
1691 return self._opener.open(req, timeout=self._socket_timeout)
1693 def print_debug_header(self):
1694 if not self.params.get('verbose'):
1697 if type('') is not compat_str:
1698 # Python 2.6 on SLES11 SP1 (https://github.com/rg3/youtube-dl/issues/3326)
1699 self.report_warning(
1700 'Your Python is broken! Update to a newer and supported version')
1702 stdout_encoding = getattr(
1703 sys.stdout, 'encoding', 'missing (%s)' % type(sys.stdout).__name__)
1705 '[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % (
1706 locale.getpreferredencoding(),
1707 sys.getfilesystemencoding(),
1709 self.get_encoding()))
1710 write_string(encoding_str, encoding=None)
1712 self._write_string('[debug] youtube-dl version ' + __version__ + '\n')
1714 sp = subprocess.Popen(
1715 ['git', 'rev-parse', '--short', 'HEAD'],
1716 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
1717 cwd=os.path.dirname(os.path.abspath(__file__)))
1718 out, err = sp.communicate()
1719 out = out.decode().strip()
1720 if re.match('[0-9a-f]+', out):
1721 self._write_string('[debug] Git HEAD: ' + out + '\n')
1727 self._write_string('[debug] Python version %s - %s\n' % (
1728 platform.python_version(), platform_name()))
1730 exe_versions = FFmpegPostProcessor.get_versions()
1731 exe_versions['rtmpdump'] = rtmpdump_version()
1732 exe_str = ', '.join(
1734 for exe, v in sorted(exe_versions.items())
1739 self._write_string('[debug] exe versions: %s\n' % exe_str)
1742 for handler in self._opener.handlers:
1743 if hasattr(handler, 'proxies'):
1744 proxy_map.update(handler.proxies)
1745 self._write_string('[debug] Proxy map: ' + compat_str(proxy_map) + '\n')
1747 if self.params.get('call_home', False):
1748 ipaddr = self.urlopen('https://yt-dl.org/ip').read().decode('utf-8')
1749 self._write_string('[debug] Public IP address: %s\n' % ipaddr)
1750 latest_version = self.urlopen(
1751 'https://yt-dl.org/latest/version').read().decode('utf-8')
1752 if version_tuple(latest_version) > version_tuple(__version__):
1753 self.report_warning(
1754 'You are using an outdated version (newest version: %s)! '
1755 'See https://yt-dl.org/update if you need help updating.' %
1758 def _setup_opener(self):
1759 timeout_val = self.params.get('socket_timeout')
1760 self._socket_timeout = 600 if timeout_val is None else float(timeout_val)
1762 opts_cookiefile = self.params.get('cookiefile')
1763 opts_proxy = self.params.get('proxy')
1765 if opts_cookiefile is None:
1766 self.cookiejar = compat_cookiejar.CookieJar()
1768 self.cookiejar = compat_cookiejar.MozillaCookieJar(
1770 if os.access(opts_cookiefile, os.R_OK):
1771 self.cookiejar.load()
1773 cookie_processor = compat_urllib_request.HTTPCookieProcessor(
1775 if opts_proxy is not None:
1776 if opts_proxy == '':
1779 proxies = {'http': opts_proxy, 'https': opts_proxy}
1781 proxies = compat_urllib_request.getproxies()
1782 # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
1783 if 'http' in proxies and 'https' not in proxies:
1784 proxies['https'] = proxies['http']
1785 proxy_handler = compat_urllib_request.ProxyHandler(proxies)
1787 debuglevel = 1 if self.params.get('debug_printtraffic') else 0
1788 https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
1789 ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
1790 opener = compat_urllib_request.build_opener(
1791 https_handler, proxy_handler, cookie_processor, ydlh)
1792 # Delete the default user-agent header, which would otherwise apply in
1793 # cases where our custom HTTP handler doesn't come into play
1794 # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
1795 opener.addheaders = []
1796 self._opener = opener
1798 def encode(self, s):
1799 if isinstance(s, bytes):
1800 return s # Already encoded
1803 return s.encode(self.get_encoding())
1804 except UnicodeEncodeError as err:
1805 err.reason = err.reason + '. Check your system encoding configuration or use the --encoding option.'
1808 def get_encoding(self):
1809 encoding = self.params.get('encoding')
1810 if encoding is None:
1811 encoding = preferredencoding()
1814 def _write_thumbnails(self, info_dict, filename):
1815 if self.params.get('writethumbnail', False):
1816 thumbnails = info_dict.get('thumbnails')
1818 thumbnails = [thumbnails[-1]]
1819 elif self.params.get('write_all_thumbnails', False):
1820 thumbnails = info_dict.get('thumbnails')
1825 # No thumbnails present, so return immediately
1828 for t in thumbnails:
1829 thumb_ext = determine_ext(t['url'], 'jpg')
1830 suffix = '_%s' % t['id'] if len(thumbnails) > 1 else ''
1831 thumb_display_id = '%s ' % t['id'] if len(thumbnails) > 1 else ''
1832 thumb_filename = os.path.splitext(filename)[0] + suffix + '.' + thumb_ext
1834 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
1835 self.to_screen('[%s] %s: Thumbnail %sis already present' %
1836 (info_dict['extractor'], info_dict['id'], thumb_display_id))
1838 self.to_screen('[%s] %s: Downloading thumbnail %s...' %
1839 (info_dict['extractor'], info_dict['id'], thumb_display_id))
1841 uf = self.urlopen(t['url'])
1842 with open(thumb_filename, 'wb') as thumbf:
1843 shutil.copyfileobj(uf, thumbf)
1844 self.to_screen('[%s] %s: Writing thumbnail %sto: %s' %
1845 (info_dict['extractor'], info_dict['id'], thumb_display_id, thumb_filename))
1846 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1847 self.report_warning('Unable to download thumbnail "%s": %s' %
1848 (t['url'], compat_str(err)))