2 # -*- coding: utf-8 -*-
4 from __future__ import absolute_import, unicode_literals
33 compat_get_terminal_size,
37 compat_tokenize_tokenize,
39 compat_urllib_request,
57 PerRequestProxyHandler,
67 UnavailableVideoError,
72 YoutubeDLCookieProcessor,
79 from .cache import Cache
80 from .extractor import get_info_extractor, gen_extractors
81 from .downloader import get_suitable_downloader
82 from .downloader.rtmp import rtmpdump_version
83 from .postprocessor import (
85 FFmpegFixupStretchedPP,
90 from .version import __version__
93 class YoutubeDL(object):
96 YoutubeDL objects are the ones responsible of downloading the
97 actual video file and writing it to disk if the user has requested
98 it, among some other tasks. In most cases there should be one per
99 program. As, given a video URL, the downloader doesn't know how to
100 extract all the needed information, task that InfoExtractors do, it
101 has to pass the URL to one of them.
103 For this, YoutubeDL objects have a method that allows
104 InfoExtractors to be registered in a given order. When it is passed
105 a URL, the YoutubeDL object handles it to the first InfoExtractor it
106 finds that reports being able to handle it. The InfoExtractor extracts
107 all the information about the video or videos the URL refers to, and
108 YoutubeDL process the extracted information, possibly using a File
109 Downloader to download the video.
111 YoutubeDL objects accept a lot of parameters. In order not to saturate
112 the object constructor with arguments, it receives a dictionary of
113 options instead. These options are available through the params
114 attribute for the InfoExtractors to use. The YoutubeDL also
115 registers itself as the downloader in charge for the InfoExtractors
116 that are added to it, so this is a "mutual registration".
120 username: Username for authentication purposes.
121 password: Password for authentication purposes.
122 videopassword: Password for accessing a video.
123 usenetrc: Use netrc for authentication instead.
124 verbose: Print additional info to stdout.
125 quiet: Do not print messages to stdout.
126 no_warnings: Do not print out anything for warnings.
127 forceurl: Force printing final URL.
128 forcetitle: Force printing title.
129 forceid: Force printing ID.
130 forcethumbnail: Force printing thumbnail URL.
131 forcedescription: Force printing description.
132 forcefilename: Force printing final filename.
133 forceduration: Force printing duration.
134 forcejson: Force printing info_dict as JSON.
135 dump_single_json: Force printing the info_dict of the whole playlist
136 (or video) as a single JSON line.
137 simulate: Do not download the video files.
138 format: Video format code. See options.py for more information.
139 outtmpl: Template for output names.
140 restrictfilenames: Do not allow "&" and spaces in file names
141 ignoreerrors: Do not stop on download errors.
142 force_generic_extractor: Force downloader to use the generic extractor
143 nooverwrites: Prevent overwriting files.
144 playliststart: Playlist item to start at.
145 playlistend: Playlist item to end at.
146 playlist_items: Specific indices of playlist to download.
147 playlistreverse: Download playlist items in reverse order.
148 matchtitle: Download only matching titles.
149 rejecttitle: Reject downloads for matching titles.
150 logger: Log messages to a logging.Logger instance.
151 logtostderr: Log messages to stderr instead of stdout.
152 writedescription: Write the video description to a .description file
153 writeinfojson: Write the video description to a .info.json file
154 writeannotations: Write the video annotations to a .annotations.xml file
155 writethumbnail: Write the thumbnail image to a file
156 write_all_thumbnails: Write all thumbnail formats to files
157 writesubtitles: Write the video subtitles to a file
158 writeautomaticsub: Write the automatic subtitles to a file
159 allsubtitles: Downloads all the subtitles of the video
160 (requires writesubtitles or writeautomaticsub)
161 listsubtitles: Lists all available subtitles for the video
162 subtitlesformat: The format code for subtitles
163 subtitleslangs: List of languages of the subtitles to download
164 keepvideo: Keep the video file after post-processing
165 daterange: A DateRange object, download only if the upload_date is in the range.
166 skip_download: Skip the actual download of the video file
167 cachedir: Location of the cache files in the filesystem.
168 False to disable filesystem cache.
169 noplaylist: Download single video instead of a playlist if in doubt.
170 age_limit: An integer representing the user's age in years.
171 Unsuitable videos for the given age are skipped.
172 min_views: An integer representing the minimum view count the video
173 must have in order to not be skipped.
174 Videos without view count information are always
175 downloaded. None for no limit.
176 max_views: An integer representing the maximum view count.
177 Videos that are more popular than that are not
179 Videos without view count information are always
180 downloaded. None for no limit.
181 download_archive: File name of a file where all downloads are recorded.
182 Videos already present in the file are not downloaded
184 cookiefile: File name where cookies should be read from and dumped to.
185 nocheckcertificate:Do not verify SSL certificates
186 prefer_insecure: Use HTTP instead of HTTPS to retrieve information.
187 At the moment, this is only supported by YouTube.
188 proxy: URL of the proxy server to use
189 cn_verification_proxy: URL of the proxy to use for IP address verification
190 on Chinese sites. (Experimental)
191 socket_timeout: Time to wait for unresponsive hosts, in seconds
192 bidi_workaround: Work around buggy terminals without bidirectional text
193 support, using fridibi
194 debug_printtraffic:Print out sent and received HTTP traffic
195 include_ads: Download ads as well
196 default_search: Prepend this string if an input url is not valid.
197 'auto' for elaborate guessing
198 encoding: Use this encoding instead of the system-specified.
199 extract_flat: Do not resolve URLs, return the immediate result.
200 Pass in 'in_playlist' to only show this behavior for
202 postprocessors: A list of dictionaries, each with an entry
203 * key: The name of the postprocessor. See
204 youtube_dl/postprocessor/__init__.py for a list.
205 as well as any further keyword arguments for the
207 progress_hooks: A list of functions that get called on download
208 progress, with a dictionary with the entries
209 * status: One of "downloading", "error", or "finished".
210 Check this first and ignore unknown values.
212 If status is one of "downloading", or "finished", the
213 following properties may also be present:
214 * filename: The final filename (always present)
215 * tmpfilename: The filename we're currently writing to
216 * downloaded_bytes: Bytes on disk
217 * total_bytes: Size of the whole file, None if unknown
218 * total_bytes_estimate: Guess of the eventual file size,
220 * elapsed: The number of seconds since download started.
221 * eta: The estimated time in seconds, None if unknown
222 * speed: The download speed in bytes/second, None if
224 * fragment_index: The counter of the currently
225 downloaded video fragment.
226 * fragment_count: The number of fragments (= individual
227 files that will be merged)
229 Progress hooks are guaranteed to be called at least once
230 (with status "finished") if the download is successful.
231 merge_output_format: Extension to use when merging formats.
232 fixup: Automatically correct known faults of the file.
234 - "never": do nothing
235 - "warn": only emit a warning
236 - "detect_or_warn": check whether we can do anything
237 about it, warn otherwise (default)
238 source_address: (Experimental) Client-side IP address to bind to.
239 call_home: Boolean, true iff we are allowed to contact the
240 youtube-dl servers for debugging.
241 sleep_interval: Number of seconds to sleep before each download.
242 listformats: Print an overview of available video formats and exit.
243 list_thumbnails: Print a table of all thumbnails and exit.
244 match_filter: A function that gets called with the info_dict of
246 If it returns a message, the video is ignored.
247 If it returns None, the video is downloaded.
248 match_filter_func in utils.py is one example for this.
249 no_color: Do not emit color codes in output.
251 The following options determine which downloader is picked:
252 external_downloader: Executable of the external downloader to call.
253 None or unset for standard (built-in) downloader.
254 hls_prefer_native: Use the native HLS downloader instead of ffmpeg/avconv.
256 The following parameters are not used by YoutubeDL itself, they are used by
257 the downloader (see youtube_dl/downloader/common.py):
258 nopart, updatetime, buffersize, ratelimit, min_filesize, max_filesize, test,
259 noresizebuffer, retries, continuedl, noprogress, consoletitle,
260 xattr_set_filesize, external_downloader_args.
262 The following options are used by the post processors:
263 prefer_ffmpeg: If True, use ffmpeg instead of avconv if both are available,
264 otherwise prefer avconv.
265 postprocessor_args: A list of additional command-line arguments for the
272 _download_retcode = None
273 _num_downloads = None
276 def __init__(self, params=None, auto_init=True):
277 """Create a FileDownloader object with the given options."""
281 self._ies_instances = {}
283 self._progress_hooks = []
284 self._download_retcode = 0
285 self._num_downloads = 0
286 self._screen_file = [sys.stdout, sys.stderr][params.get('logtostderr', False)]
287 self._err_file = sys.stderr
290 'nocheckcertificate': False,
292 self.params.update(params)
293 self.cache = Cache(self)
295 if params.get('bidi_workaround', False):
298 master, slave = pty.openpty()
299 width = compat_get_terminal_size().columns
303 width_args = ['-w', str(width)]
305 stdin=subprocess.PIPE,
307 stderr=self._err_file)
309 self._output_process = subprocess.Popen(
310 ['bidiv'] + width_args, **sp_kwargs
313 self._output_process = subprocess.Popen(
314 ['fribidi', '-c', 'UTF-8'] + width_args, **sp_kwargs)
315 self._output_channel = os.fdopen(master, 'rb')
316 except OSError as ose:
318 self.report_warning('Could not find fribidi executable, ignoring --bidi-workaround . Make sure that fribidi is an executable file in one of the directories in your $PATH.')
322 if (sys.version_info >= (3,) and sys.platform != 'win32' and
323 sys.getfilesystemencoding() in ['ascii', 'ANSI_X3.4-1968'] and
324 not params.get('restrictfilenames', False)):
325 # On Python 3, the Unicode filesystem API will throw errors (#1474)
327 'Assuming --restrict-filenames since file system encoding '
328 'cannot encode all characters. '
329 'Set the LC_ALL environment variable to fix this.')
330 self.params['restrictfilenames'] = True
332 if isinstance(params.get('outtmpl'), bytes):
334 'Parameter outtmpl is bytes, but should be a unicode string. '
335 'Put from __future__ import unicode_literals at the top of your code file or consider switching to Python 3.x.')
340 self.print_debug_header()
341 self.add_default_info_extractors()
343 for pp_def_raw in self.params.get('postprocessors', []):
344 pp_class = get_postprocessor(pp_def_raw['key'])
345 pp_def = dict(pp_def_raw)
347 pp = pp_class(self, **compat_kwargs(pp_def))
348 self.add_post_processor(pp)
350 for ph in self.params.get('progress_hooks', []):
351 self.add_progress_hook(ph)
353 def warn_if_short_id(self, argv):
354 # short YouTube ID starting with dash?
356 i for i, a in enumerate(argv)
357 if re.match(r'^-[0-9A-Za-z_-]{10}$', a)]
361 [a for i, a in enumerate(argv) if i not in idxs] +
362 ['--'] + [argv[i] for i in idxs]
365 'Long argument string detected. '
366 'Use -- to separate parameters and URLs, like this:\n%s\n' %
367 args_to_str(correct_argv))
369 def add_info_extractor(self, ie):
370 """Add an InfoExtractor object to the end of the list."""
372 self._ies_instances[ie.ie_key()] = ie
373 ie.set_downloader(self)
375 def get_info_extractor(self, ie_key):
377 Get an instance of an IE with name ie_key, it will try to get one from
378 the _ies list, if there's no instance it will create a new one and add
379 it to the extractor list.
381 ie = self._ies_instances.get(ie_key)
383 ie = get_info_extractor(ie_key)()
384 self.add_info_extractor(ie)
387 def add_default_info_extractors(self):
389 Add the InfoExtractors returned by gen_extractors to the end of the list
391 for ie in gen_extractors():
392 self.add_info_extractor(ie)
394 def add_post_processor(self, pp):
395 """Add a PostProcessor object to the end of the chain."""
397 pp.set_downloader(self)
399 def add_progress_hook(self, ph):
400 """Add the progress hook (currently only for the file downloader)"""
401 self._progress_hooks.append(ph)
403 def _bidi_workaround(self, message):
404 if not hasattr(self, '_output_channel'):
407 assert hasattr(self, '_output_process')
408 assert isinstance(message, compat_str)
409 line_count = message.count('\n') + 1
410 self._output_process.stdin.write((message + '\n').encode('utf-8'))
411 self._output_process.stdin.flush()
412 res = ''.join(self._output_channel.readline().decode('utf-8')
413 for _ in range(line_count))
414 return res[:-len('\n')]
416 def to_screen(self, message, skip_eol=False):
417 """Print message to stdout if not in quiet mode."""
418 return self.to_stdout(message, skip_eol, check_quiet=True)
420 def _write_string(self, s, out=None):
421 write_string(s, out=out, encoding=self.params.get('encoding'))
423 def to_stdout(self, message, skip_eol=False, check_quiet=False):
424 """Print message to stdout if not in quiet mode."""
425 if self.params.get('logger'):
426 self.params['logger'].debug(message)
427 elif not check_quiet or not self.params.get('quiet', False):
428 message = self._bidi_workaround(message)
429 terminator = ['\n', ''][skip_eol]
430 output = message + terminator
432 self._write_string(output, self._screen_file)
434 def to_stderr(self, message):
435 """Print message to stderr."""
436 assert isinstance(message, compat_str)
437 if self.params.get('logger'):
438 self.params['logger'].error(message)
440 message = self._bidi_workaround(message)
441 output = message + '\n'
442 self._write_string(output, self._err_file)
444 def to_console_title(self, message):
445 if not self.params.get('consoletitle', False):
447 if os.name == 'nt' and ctypes.windll.kernel32.GetConsoleWindow():
448 # c_wchar_p() might not be necessary if `message` is
449 # already of type unicode()
450 ctypes.windll.kernel32.SetConsoleTitleW(ctypes.c_wchar_p(message))
451 elif 'TERM' in os.environ:
452 self._write_string('\033]0;%s\007' % message, self._screen_file)
454 def save_console_title(self):
455 if not self.params.get('consoletitle', False):
457 if 'TERM' in os.environ:
458 # Save the title on stack
459 self._write_string('\033[22;0t', self._screen_file)
461 def restore_console_title(self):
462 if not self.params.get('consoletitle', False):
464 if 'TERM' in os.environ:
465 # Restore the title from stack
466 self._write_string('\033[23;0t', self._screen_file)
469 self.save_console_title()
472 def __exit__(self, *args):
473 self.restore_console_title()
475 if self.params.get('cookiefile') is not None:
476 self.cookiejar.save()
478 def trouble(self, message=None, tb=None):
479 """Determine action to take when a download problem appears.
481 Depending on if the downloader has been configured to ignore
482 download errors or not, this method may throw an exception or
483 not when errors are found, after printing the message.
485 tb, if given, is additional traceback information.
487 if message is not None:
488 self.to_stderr(message)
489 if self.params.get('verbose'):
491 if sys.exc_info()[0]: # if .trouble has been called from an except block
493 if hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
494 tb += ''.join(traceback.format_exception(*sys.exc_info()[1].exc_info))
495 tb += compat_str(traceback.format_exc())
497 tb_data = traceback.format_list(traceback.extract_stack())
498 tb = ''.join(tb_data)
500 if not self.params.get('ignoreerrors', False):
501 if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
502 exc_info = sys.exc_info()[1].exc_info
504 exc_info = sys.exc_info()
505 raise DownloadError(message, exc_info)
506 self._download_retcode = 1
508 def report_warning(self, message):
510 Print the message to stderr, it will be prefixed with 'WARNING:'
511 If stderr is a tty file the 'WARNING:' will be colored
513 if self.params.get('logger') is not None:
514 self.params['logger'].warning(message)
516 if self.params.get('no_warnings'):
518 if not self.params.get('no_color') and self._err_file.isatty() and os.name != 'nt':
519 _msg_header = '\033[0;33mWARNING:\033[0m'
521 _msg_header = 'WARNING:'
522 warning_message = '%s %s' % (_msg_header, message)
523 self.to_stderr(warning_message)
525 def report_error(self, message, tb=None):
527 Do the same as trouble, but prefixes the message with 'ERROR:', colored
528 in red if stderr is a tty file.
530 if not self.params.get('no_color') and self._err_file.isatty() and os.name != 'nt':
531 _msg_header = '\033[0;31mERROR:\033[0m'
533 _msg_header = 'ERROR:'
534 error_message = '%s %s' % (_msg_header, message)
535 self.trouble(error_message, tb)
537 def report_file_already_downloaded(self, file_name):
538 """Report file has already been fully downloaded."""
540 self.to_screen('[download] %s has already been downloaded' % file_name)
541 except UnicodeEncodeError:
542 self.to_screen('[download] The file has already been downloaded')
544 def prepare_filename(self, info_dict):
545 """Generate the output filename."""
547 template_dict = dict(info_dict)
549 template_dict['epoch'] = int(time.time())
550 autonumber_size = self.params.get('autonumber_size')
551 if autonumber_size is None:
553 autonumber_templ = '%0' + str(autonumber_size) + 'd'
554 template_dict['autonumber'] = autonumber_templ % self._num_downloads
555 if template_dict.get('playlist_index') is not None:
556 template_dict['playlist_index'] = '%0*d' % (len(str(template_dict['n_entries'])), template_dict['playlist_index'])
557 if template_dict.get('resolution') is None:
558 if template_dict.get('width') and template_dict.get('height'):
559 template_dict['resolution'] = '%dx%d' % (template_dict['width'], template_dict['height'])
560 elif template_dict.get('height'):
561 template_dict['resolution'] = '%sp' % template_dict['height']
562 elif template_dict.get('width'):
563 template_dict['resolution'] = '?x%d' % template_dict['width']
565 sanitize = lambda k, v: sanitize_filename(
567 restricted=self.params.get('restrictfilenames'),
569 template_dict = dict((k, sanitize(k, v))
570 for k, v in template_dict.items()
572 template_dict = collections.defaultdict(lambda: 'NA', template_dict)
574 outtmpl = sanitize_path(self.params.get('outtmpl', DEFAULT_OUTTMPL))
575 tmpl = compat_expanduser(outtmpl)
576 filename = tmpl % template_dict
577 # Temporary fix for #4787
578 # 'Treat' all problem characters by passing filename through preferredencoding
579 # to workaround encoding issues with subprocess on python2 @ Windows
580 if sys.version_info < (3, 0) and sys.platform == 'win32':
581 filename = encodeFilename(filename, True).decode(preferredencoding())
583 except ValueError as err:
584 self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
587 def _match_entry(self, info_dict, incomplete):
588 """ Returns None iff the file should be downloaded """
590 video_title = info_dict.get('title', info_dict.get('id', 'video'))
591 if 'title' in info_dict:
592 # This can happen when we're just evaluating the playlist
593 title = info_dict['title']
594 matchtitle = self.params.get('matchtitle', False)
596 if not re.search(matchtitle, title, re.IGNORECASE):
597 return '"' + title + '" title did not match pattern "' + matchtitle + '"'
598 rejecttitle = self.params.get('rejecttitle', False)
600 if re.search(rejecttitle, title, re.IGNORECASE):
601 return '"' + title + '" title matched reject pattern "' + rejecttitle + '"'
602 date = info_dict.get('upload_date', None)
604 dateRange = self.params.get('daterange', DateRange())
605 if date not in dateRange:
606 return '%s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
607 view_count = info_dict.get('view_count', None)
608 if view_count is not None:
609 min_views = self.params.get('min_views')
610 if min_views is not None and view_count < min_views:
611 return 'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title, view_count, min_views)
612 max_views = self.params.get('max_views')
613 if max_views is not None and view_count > max_views:
614 return 'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views)
615 if age_restricted(info_dict.get('age_limit'), self.params.get('age_limit')):
616 return 'Skipping "%s" because it is age restricted' % video_title
617 if self.in_download_archive(info_dict):
618 return '%s has already been recorded in archive' % video_title
621 match_filter = self.params.get('match_filter')
622 if match_filter is not None:
623 ret = match_filter(info_dict)
630 def add_extra_info(info_dict, extra_info):
631 '''Set the keys from extra_info in info dict if they are missing'''
632 for key, value in extra_info.items():
633 info_dict.setdefault(key, value)
635 def extract_info(self, url, download=True, ie_key=None, extra_info={},
636 process=True, force_generic_extractor=False):
638 Returns a list with a dictionary for each video we find.
639 If 'download', also downloads the videos.
640 extra_info is a dict containing the extra values to add to each result
643 if not ie_key and force_generic_extractor:
647 ies = [self.get_info_extractor(ie_key)]
652 if not ie.suitable(url):
656 self.report_warning('The program functionality for this site has been marked as broken, '
657 'and will probably not work.')
660 ie_result = ie.extract(url)
661 if ie_result is None: # Finished already (backwards compatibility; listformats and friends should be moved here)
663 if isinstance(ie_result, list):
664 # Backwards compatibility: old IE result format
666 '_type': 'compat_list',
667 'entries': ie_result,
669 self.add_default_extra_info(ie_result, ie, url)
671 return self.process_ie_result(ie_result, download, extra_info)
674 except ExtractorError as de: # An error we somewhat expected
675 self.report_error(compat_str(de), de.format_traceback())
677 except MaxDownloadsReached:
679 except Exception as e:
680 if self.params.get('ignoreerrors', False):
681 self.report_error(compat_str(e), tb=compat_str(traceback.format_exc()))
686 self.report_error('no suitable InfoExtractor for URL %s' % url)
688 def add_default_extra_info(self, ie_result, ie, url):
689 self.add_extra_info(ie_result, {
690 'extractor': ie.IE_NAME,
692 'webpage_url_basename': url_basename(url),
693 'extractor_key': ie.ie_key(),
696 def process_ie_result(self, ie_result, download=True, extra_info={}):
698 Take the result of the ie(may be modified) and resolve all unresolved
699 references (URLs, playlist items).
701 It will also download the videos if 'download'.
702 Returns the resolved ie_result.
705 result_type = ie_result.get('_type', 'video')
707 if result_type in ('url', 'url_transparent'):
708 extract_flat = self.params.get('extract_flat', False)
709 if ((extract_flat == 'in_playlist' and 'playlist' in extra_info) or
710 extract_flat is True):
711 if self.params.get('forcejson', False):
712 self.to_stdout(json.dumps(ie_result))
715 if result_type == 'video':
716 self.add_extra_info(ie_result, extra_info)
717 return self.process_video_result(ie_result, download=download)
718 elif result_type == 'url':
719 # We have to add extra_info to the results because it may be
720 # contained in a playlist
721 return self.extract_info(ie_result['url'],
723 ie_key=ie_result.get('ie_key'),
724 extra_info=extra_info)
725 elif result_type == 'url_transparent':
726 # Use the information from the embedding page
727 info = self.extract_info(
728 ie_result['url'], ie_key=ie_result.get('ie_key'),
729 extra_info=extra_info, download=False, process=False)
731 force_properties = dict(
732 (k, v) for k, v in ie_result.items() if v is not None)
733 for f in ('_type', 'url'):
734 if f in force_properties:
735 del force_properties[f]
736 new_result = info.copy()
737 new_result.update(force_properties)
739 assert new_result.get('_type') != 'url_transparent'
741 return self.process_ie_result(
742 new_result, download=download, extra_info=extra_info)
743 elif result_type == 'playlist' or result_type == 'multi_video':
744 # We process each entry in the playlist
745 playlist = ie_result.get('title', None) or ie_result.get('id', None)
746 self.to_screen('[download] Downloading playlist: %s' % playlist)
748 playlist_results = []
750 playliststart = self.params.get('playliststart', 1) - 1
751 playlistend = self.params.get('playlistend', None)
752 # For backwards compatibility, interpret -1 as whole list
753 if playlistend == -1:
756 playlistitems_str = self.params.get('playlist_items', None)
758 if playlistitems_str is not None:
759 def iter_playlistitems(format):
760 for string_segment in format.split(','):
761 if '-' in string_segment:
762 start, end = string_segment.split('-')
763 for item in range(int(start), int(end) + 1):
766 yield int(string_segment)
767 playlistitems = iter_playlistitems(playlistitems_str)
769 ie_entries = ie_result['entries']
770 if isinstance(ie_entries, list):
771 n_all_entries = len(ie_entries)
774 ie_entries[i - 1] for i in playlistitems
775 if -n_all_entries <= i - 1 < n_all_entries]
777 entries = ie_entries[playliststart:playlistend]
778 n_entries = len(entries)
780 "[%s] playlist %s: Collected %d video ids (downloading %d of them)" %
781 (ie_result['extractor'], playlist, n_all_entries, n_entries))
782 elif isinstance(ie_entries, PagedList):
785 for item in playlistitems:
786 entries.extend(ie_entries.getslice(
790 entries = ie_entries.getslice(
791 playliststart, playlistend)
792 n_entries = len(entries)
794 "[%s] playlist %s: Downloading %d videos" %
795 (ie_result['extractor'], playlist, n_entries))
798 entry_list = list(ie_entries)
799 entries = [entry_list[i - 1] for i in playlistitems]
801 entries = list(itertools.islice(
802 ie_entries, playliststart, playlistend))
803 n_entries = len(entries)
805 "[%s] playlist %s: Downloading %d videos" %
806 (ie_result['extractor'], playlist, n_entries))
808 if self.params.get('playlistreverse', False):
809 entries = entries[::-1]
811 for i, entry in enumerate(entries, 1):
812 self.to_screen('[download] Downloading video %s of %s' % (i, n_entries))
814 'n_entries': n_entries,
815 'playlist': playlist,
816 'playlist_id': ie_result.get('id'),
817 'playlist_title': ie_result.get('title'),
818 'playlist_index': i + playliststart,
819 'extractor': ie_result['extractor'],
820 'webpage_url': ie_result['webpage_url'],
821 'webpage_url_basename': url_basename(ie_result['webpage_url']),
822 'extractor_key': ie_result['extractor_key'],
825 reason = self._match_entry(entry, incomplete=True)
826 if reason is not None:
827 self.to_screen('[download] ' + reason)
830 entry_result = self.process_ie_result(entry,
833 playlist_results.append(entry_result)
834 ie_result['entries'] = playlist_results
836 elif result_type == 'compat_list':
838 'Extractor %s returned a compat_list result. '
839 'It needs to be updated.' % ie_result.get('extractor'))
845 'extractor': ie_result['extractor'],
846 'webpage_url': ie_result['webpage_url'],
847 'webpage_url_basename': url_basename(ie_result['webpage_url']),
848 'extractor_key': ie_result['extractor_key'],
852 ie_result['entries'] = [
853 self.process_ie_result(_fixup(r), download, extra_info)
854 for r in ie_result['entries']
858 raise Exception('Invalid result type: %s' % result_type)
860 def _build_format_filter(self, filter_spec):
861 " Returns a function to filter the formats according to the filter_spec "
871 operator_rex = re.compile(r'''(?x)\s*
872 (?P<key>width|height|tbr|abr|vbr|asr|filesize|fps)
873 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?\s*
874 (?P<value>[0-9.]+(?:[kKmMgGtTpPeEzZyY]i?[Bb]?)?)
876 ''' % '|'.join(map(re.escape, OPERATORS.keys())))
877 m = operator_rex.search(filter_spec)
880 comparison_value = int(m.group('value'))
882 comparison_value = parse_filesize(m.group('value'))
883 if comparison_value is None:
884 comparison_value = parse_filesize(m.group('value') + 'B')
885 if comparison_value is None:
887 'Invalid value %r in format specification %r' % (
888 m.group('value'), filter_spec))
889 op = OPERATORS[m.group('op')]
896 str_operator_rex = re.compile(r'''(?x)
897 \s*(?P<key>ext|acodec|vcodec|container|protocol)
898 \s*(?P<op>%s)(?P<none_inclusive>\s*\?)?
899 \s*(?P<value>[a-zA-Z0-9_-]+)
901 ''' % '|'.join(map(re.escape, STR_OPERATORS.keys())))
902 m = str_operator_rex.search(filter_spec)
904 comparison_value = m.group('value')
905 op = STR_OPERATORS[m.group('op')]
908 raise ValueError('Invalid filter specification %r' % filter_spec)
911 actual_value = f.get(m.group('key'))
912 if actual_value is None:
913 return m.group('none_inclusive')
914 return op(actual_value, comparison_value)
917 def build_format_selector(self, format_spec):
918 def syntax_error(note, start):
920 'Invalid format specification: '
921 '{0}\n\t{1}\n\t{2}^'.format(note, format_spec, ' ' * start[1]))
922 return SyntaxError(message)
924 PICKFIRST = 'PICKFIRST'
928 FormatSelector = collections.namedtuple('FormatSelector', ['type', 'selector', 'filters'])
930 def _parse_filter(tokens):
932 for type, string, start, _, _ in tokens:
933 if type == tokenize.OP and string == ']':
934 return ''.join(filter_parts)
936 filter_parts.append(string)
938 def _remove_unused_ops(tokens):
939 # Remove operators that we don't use and join them with the sourrounding strings
940 # for example: 'mp4' '-' 'baseline' '-' '16x9' is converted to 'mp4-baseline-16x9'
941 ALLOWED_OPS = ('/', '+', ',', '(', ')')
942 last_string, last_start, last_end, last_line = None, None, None, None
943 for type, string, start, end, line in tokens:
944 if type == tokenize.OP and string == '[':
946 yield tokenize.NAME, last_string, last_start, last_end, last_line
948 yield type, string, start, end, line
949 # everything inside brackets will be handled by _parse_filter
950 for type, string, start, end, line in tokens:
951 yield type, string, start, end, line
952 if type == tokenize.OP and string == ']':
954 elif type == tokenize.OP and string in ALLOWED_OPS:
956 yield tokenize.NAME, last_string, last_start, last_end, last_line
958 yield type, string, start, end, line
959 elif type in [tokenize.NAME, tokenize.NUMBER, tokenize.OP]:
965 last_string += string
967 yield tokenize.NAME, last_string, last_start, last_end, last_line
969 def _parse_format_selection(tokens, inside_merge=False, inside_choice=False, inside_group=False):
971 current_selector = None
972 for type, string, start, _, _ in tokens:
973 # ENCODING is only defined in python 3.x
974 if type == getattr(tokenize, 'ENCODING', None):
976 elif type in [tokenize.NAME, tokenize.NUMBER]:
977 current_selector = FormatSelector(SINGLE, string, [])
978 elif type == tokenize.OP:
981 # ')' will be handled by the parentheses group
982 tokens.restore_last_token()
984 elif inside_merge and string in ['/', ',']:
985 tokens.restore_last_token()
987 elif inside_choice and string == ',':
988 tokens.restore_last_token()
991 if not current_selector:
992 raise syntax_error('"," must follow a format selector', start)
993 selectors.append(current_selector)
994 current_selector = None
996 if not current_selector:
997 raise syntax_error('"/" must follow a format selector', start)
998 first_choice = current_selector
999 second_choice = _parse_format_selection(tokens, inside_choice=True)
1000 current_selector = FormatSelector(PICKFIRST, (first_choice, second_choice), [])
1002 if not current_selector:
1003 current_selector = FormatSelector(SINGLE, 'best', [])
1004 format_filter = _parse_filter(tokens)
1005 current_selector.filters.append(format_filter)
1007 if current_selector:
1008 raise syntax_error('Unexpected "("', start)
1009 group = _parse_format_selection(tokens, inside_group=True)
1010 current_selector = FormatSelector(GROUP, group, [])
1012 video_selector = current_selector
1013 audio_selector = _parse_format_selection(tokens, inside_merge=True)
1014 if not video_selector or not audio_selector:
1015 raise syntax_error('"+" must be between two format selectors', start)
1016 current_selector = FormatSelector(MERGE, (video_selector, audio_selector), [])
1018 raise syntax_error('Operator not recognized: "{0}"'.format(string), start)
1019 elif type == tokenize.ENDMARKER:
1021 if current_selector:
1022 selectors.append(current_selector)
1025 def _build_selector_function(selector):
1026 if isinstance(selector, list):
1027 fs = [_build_selector_function(s) for s in selector]
1029 def selector_function(formats):
1031 for format in f(formats):
1033 return selector_function
1034 elif selector.type == GROUP:
1035 selector_function = _build_selector_function(selector.selector)
1036 elif selector.type == PICKFIRST:
1037 fs = [_build_selector_function(s) for s in selector.selector]
1039 def selector_function(formats):
1041 picked_formats = list(f(formats))
1043 return picked_formats
1045 elif selector.type == SINGLE:
1046 format_spec = selector.selector
1048 def selector_function(formats):
1049 formats = list(formats)
1052 if format_spec == 'all':
1055 elif format_spec in ['best', 'worst', None]:
1056 format_idx = 0 if format_spec == 'worst' else -1
1057 audiovideo_formats = [
1059 if f.get('vcodec') != 'none' and f.get('acodec') != 'none']
1060 if audiovideo_formats:
1061 yield audiovideo_formats[format_idx]
1062 # for audio only (soundcloud) or video only (imgur) urls, select the best/worst audio format
1063 elif (all(f.get('acodec') != 'none' for f in formats) or
1064 all(f.get('vcodec') != 'none' for f in formats)):
1065 yield formats[format_idx]
1066 elif format_spec == 'bestaudio':
1069 if f.get('vcodec') == 'none']
1071 yield audio_formats[-1]
1072 elif format_spec == 'worstaudio':
1075 if f.get('vcodec') == 'none']
1077 yield audio_formats[0]
1078 elif format_spec == 'bestvideo':
1081 if f.get('acodec') == 'none']
1083 yield video_formats[-1]
1084 elif format_spec == 'worstvideo':
1087 if f.get('acodec') == 'none']
1089 yield video_formats[0]
1091 extensions = ['mp4', 'flv', 'webm', '3gp', 'm4a', 'mp3', 'ogg', 'aac', 'wav']
1092 if format_spec in extensions:
1093 filter_f = lambda f: f['ext'] == format_spec
1095 filter_f = lambda f: f['format_id'] == format_spec
1096 matches = list(filter(filter_f, formats))
1099 elif selector.type == MERGE:
1100 def _merge(formats_info):
1101 format_1, format_2 = [f['format_id'] for f in formats_info]
1102 # The first format must contain the video and the
1104 if formats_info[0].get('vcodec') == 'none':
1105 self.report_error('The first format must '
1106 'contain the video, try using '
1107 '"-f %s+%s"' % (format_2, format_1))
1110 formats_info[0]['ext']
1111 if self.params.get('merge_output_format') is None
1112 else self.params['merge_output_format'])
1114 'requested_formats': formats_info,
1115 'format': '%s+%s' % (formats_info[0].get('format'),
1116 formats_info[1].get('format')),
1117 'format_id': '%s+%s' % (formats_info[0].get('format_id'),
1118 formats_info[1].get('format_id')),
1119 'width': formats_info[0].get('width'),
1120 'height': formats_info[0].get('height'),
1121 'resolution': formats_info[0].get('resolution'),
1122 'fps': formats_info[0].get('fps'),
1123 'vcodec': formats_info[0].get('vcodec'),
1124 'vbr': formats_info[0].get('vbr'),
1125 'stretched_ratio': formats_info[0].get('stretched_ratio'),
1126 'acodec': formats_info[1].get('acodec'),
1127 'abr': formats_info[1].get('abr'),
1130 video_selector, audio_selector = map(_build_selector_function, selector.selector)
1132 def selector_function(formats):
1133 formats = list(formats)
1134 for pair in itertools.product(video_selector(formats), audio_selector(formats)):
1137 filters = [self._build_format_filter(f) for f in selector.filters]
1139 def final_selector(formats):
1140 for _filter in filters:
1141 formats = list(filter(_filter, formats))
1142 return selector_function(formats)
1143 return final_selector
1145 stream = io.BytesIO(format_spec.encode('utf-8'))
1147 tokens = list(_remove_unused_ops(compat_tokenize_tokenize(stream.readline)))
1148 except tokenize.TokenError:
1149 raise syntax_error('Missing closing/opening brackets or parenthesis', (0, len(format_spec)))
1151 class TokenIterator(object):
1152 def __init__(self, tokens):
1153 self.tokens = tokens
1160 if self.counter >= len(self.tokens):
1161 raise StopIteration()
1162 value = self.tokens[self.counter]
1168 def restore_last_token(self):
1171 parsed_selector = _parse_format_selection(iter(TokenIterator(tokens)))
1172 return _build_selector_function(parsed_selector)
1174 def _calc_headers(self, info_dict):
1175 res = std_headers.copy()
1177 add_headers = info_dict.get('http_headers')
1179 res.update(add_headers)
1181 cookies = self._calc_cookies(info_dict)
1183 res['Cookie'] = cookies
1187 def _calc_cookies(self, info_dict):
1188 pr = compat_urllib_request.Request(info_dict['url'])
1189 self.cookiejar.add_cookie_header(pr)
1190 return pr.get_header('Cookie')
1192 def process_video_result(self, info_dict, download=True):
1193 assert info_dict.get('_type', 'video') == 'video'
1195 if 'id' not in info_dict:
1196 raise ExtractorError('Missing "id" field in extractor result')
1197 if 'title' not in info_dict:
1198 raise ExtractorError('Missing "title" field in extractor result')
1200 if 'playlist' not in info_dict:
1201 # It isn't part of a playlist
1202 info_dict['playlist'] = None
1203 info_dict['playlist_index'] = None
1205 thumbnails = info_dict.get('thumbnails')
1206 if thumbnails is None:
1207 thumbnail = info_dict.get('thumbnail')
1209 info_dict['thumbnails'] = thumbnails = [{'url': thumbnail}]
1211 thumbnails.sort(key=lambda t: (
1212 t.get('preference'), t.get('width'), t.get('height'),
1213 t.get('id'), t.get('url')))
1214 for i, t in enumerate(thumbnails):
1215 if t.get('width') and t.get('height'):
1216 t['resolution'] = '%dx%d' % (t['width'], t['height'])
1217 if t.get('id') is None:
1220 if thumbnails and 'thumbnail' not in info_dict:
1221 info_dict['thumbnail'] = thumbnails[-1]['url']
1223 if 'display_id' not in info_dict and 'id' in info_dict:
1224 info_dict['display_id'] = info_dict['id']
1226 if info_dict.get('upload_date') is None and info_dict.get('timestamp') is not None:
1227 # Working around out-of-range timestamp values (e.g. negative ones on Windows,
1228 # see http://bugs.python.org/issue1646728)
1230 upload_date = datetime.datetime.utcfromtimestamp(info_dict['timestamp'])
1231 info_dict['upload_date'] = upload_date.strftime('%Y%m%d')
1232 except (ValueError, OverflowError, OSError):
1235 if self.params.get('listsubtitles', False):
1236 if 'automatic_captions' in info_dict:
1237 self.list_subtitles(info_dict['id'], info_dict.get('automatic_captions'), 'automatic captions')
1238 self.list_subtitles(info_dict['id'], info_dict.get('subtitles'), 'subtitles')
1240 info_dict['requested_subtitles'] = self.process_subtitles(
1241 info_dict['id'], info_dict.get('subtitles'),
1242 info_dict.get('automatic_captions'))
1244 # We now pick which formats have to be downloaded
1245 if info_dict.get('formats') is None:
1246 # There's only one format available
1247 formats = [info_dict]
1249 formats = info_dict['formats']
1252 raise ExtractorError('No video formats found!')
1256 # We check that all the formats have the format and format_id fields
1257 for i, format in enumerate(formats):
1258 if 'url' not in format:
1259 raise ExtractorError('Missing "url" key in result (index %d)' % i)
1261 if format.get('format_id') is None:
1262 format['format_id'] = compat_str(i)
1263 format_id = format['format_id']
1264 if format_id not in formats_dict:
1265 formats_dict[format_id] = []
1266 formats_dict[format_id].append(format)
1268 # Make sure all formats have unique format_id
1269 for format_id, ambiguous_formats in formats_dict.items():
1270 if len(ambiguous_formats) > 1:
1271 for i, format in enumerate(ambiguous_formats):
1272 format['format_id'] = '%s-%d' % (format_id, i)
1274 for i, format in enumerate(formats):
1275 if format.get('format') is None:
1276 format['format'] = '{id} - {res}{note}'.format(
1277 id=format['format_id'],
1278 res=self.format_resolution(format),
1279 note=' ({0})'.format(format['format_note']) if format.get('format_note') is not None else '',
1281 # Automatically determine file extension if missing
1282 if 'ext' not in format:
1283 format['ext'] = determine_ext(format['url']).lower()
1284 # Add HTTP headers, so that external programs can use them from the
1286 full_format_info = info_dict.copy()
1287 full_format_info.update(format)
1288 format['http_headers'] = self._calc_headers(full_format_info)
1290 # TODO Central sorting goes here
1292 if formats[0] is not info_dict:
1293 # only set the 'formats' fields if the original info_dict list them
1294 # otherwise we end up with a circular reference, the first (and unique)
1295 # element in the 'formats' field in info_dict is info_dict itself,
1296 # wich can't be exported to json
1297 info_dict['formats'] = formats
1298 if self.params.get('listformats'):
1299 self.list_formats(info_dict)
1301 if self.params.get('list_thumbnails'):
1302 self.list_thumbnails(info_dict)
1305 req_format = self.params.get('format')
1306 if req_format is None:
1307 req_format_list = []
1308 if (self.params.get('outtmpl', DEFAULT_OUTTMPL) != '-' and
1309 info_dict['extractor'] in ['youtube', 'ted'] and
1310 not info_dict.get('is_live')):
1311 merger = FFmpegMergerPP(self)
1312 if merger.available and merger.can_merge():
1313 req_format_list.append('bestvideo+bestaudio')
1314 req_format_list.append('best')
1315 req_format = '/'.join(req_format_list)
1316 format_selector = self.build_format_selector(req_format)
1317 formats_to_download = list(format_selector(formats))
1318 if not formats_to_download:
1319 raise ExtractorError('requested format not available',
1323 if len(formats_to_download) > 1:
1324 self.to_screen('[info] %s: downloading video in %s formats' % (info_dict['id'], len(formats_to_download)))
1325 for format in formats_to_download:
1326 new_info = dict(info_dict)
1327 new_info.update(format)
1328 self.process_info(new_info)
1329 # We update the info dict with the best quality format (backwards compatibility)
1330 info_dict.update(formats_to_download[-1])
1333 def process_subtitles(self, video_id, normal_subtitles, automatic_captions):
1334 """Select the requested subtitles and their format"""
1336 if normal_subtitles and self.params.get('writesubtitles'):
1337 available_subs.update(normal_subtitles)
1338 if automatic_captions and self.params.get('writeautomaticsub'):
1339 for lang, cap_info in automatic_captions.items():
1340 if lang not in available_subs:
1341 available_subs[lang] = cap_info
1343 if (not self.params.get('writesubtitles') and not
1344 self.params.get('writeautomaticsub') or not
1348 if self.params.get('allsubtitles', False):
1349 requested_langs = available_subs.keys()
1351 if self.params.get('subtitleslangs', False):
1352 requested_langs = self.params.get('subtitleslangs')
1353 elif 'en' in available_subs:
1354 requested_langs = ['en']
1356 requested_langs = [list(available_subs.keys())[0]]
1358 formats_query = self.params.get('subtitlesformat', 'best')
1359 formats_preference = formats_query.split('/') if formats_query else []
1361 for lang in requested_langs:
1362 formats = available_subs.get(lang)
1364 self.report_warning('%s subtitles not available for %s' % (lang, video_id))
1366 for ext in formats_preference:
1370 matches = list(filter(lambda f: f['ext'] == ext, formats))
1376 self.report_warning(
1377 'No subtitle format found matching "%s" for language %s, '
1378 'using %s' % (formats_query, lang, f['ext']))
1382 def process_info(self, info_dict):
1383 """Process a single resolved IE result."""
1385 assert info_dict.get('_type', 'video') == 'video'
1387 max_downloads = self.params.get('max_downloads')
1388 if max_downloads is not None:
1389 if self._num_downloads >= int(max_downloads):
1390 raise MaxDownloadsReached()
1392 info_dict['fulltitle'] = info_dict['title']
1393 if len(info_dict['title']) > 200:
1394 info_dict['title'] = info_dict['title'][:197] + '...'
1396 if 'format' not in info_dict:
1397 info_dict['format'] = info_dict['ext']
1399 reason = self._match_entry(info_dict, incomplete=False)
1400 if reason is not None:
1401 self.to_screen('[download] ' + reason)
1404 self._num_downloads += 1
1406 info_dict['_filename'] = filename = self.prepare_filename(info_dict)
1409 if self.params.get('forcetitle', False):
1410 self.to_stdout(info_dict['fulltitle'])
1411 if self.params.get('forceid', False):
1412 self.to_stdout(info_dict['id'])
1413 if self.params.get('forceurl', False):
1414 if info_dict.get('requested_formats') is not None:
1415 for f in info_dict['requested_formats']:
1416 self.to_stdout(f['url'] + f.get('play_path', ''))
1418 # For RTMP URLs, also include the playpath
1419 self.to_stdout(info_dict['url'] + info_dict.get('play_path', ''))
1420 if self.params.get('forcethumbnail', False) and info_dict.get('thumbnail') is not None:
1421 self.to_stdout(info_dict['thumbnail'])
1422 if self.params.get('forcedescription', False) and info_dict.get('description') is not None:
1423 self.to_stdout(info_dict['description'])
1424 if self.params.get('forcefilename', False) and filename is not None:
1425 self.to_stdout(filename)
1426 if self.params.get('forceduration', False) and info_dict.get('duration') is not None:
1427 self.to_stdout(formatSeconds(info_dict['duration']))
1428 if self.params.get('forceformat', False):
1429 self.to_stdout(info_dict['format'])
1430 if self.params.get('forcejson', False):
1431 self.to_stdout(json.dumps(info_dict))
1433 # Do nothing else if in simulate mode
1434 if self.params.get('simulate', False):
1437 if filename is None:
1441 dn = os.path.dirname(sanitize_path(encodeFilename(filename)))
1442 if dn and not os.path.exists(dn):
1444 except (OSError, IOError) as err:
1445 self.report_error('unable to create directory ' + compat_str(err))
1448 if self.params.get('writedescription', False):
1449 descfn = replace_extension(filename, 'description', info_dict.get('ext'))
1450 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)):
1451 self.to_screen('[info] Video description is already present')
1452 elif info_dict.get('description') is None:
1453 self.report_warning('There\'s no description to write.')
1456 self.to_screen('[info] Writing video description to: ' + descfn)
1457 with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
1458 descfile.write(info_dict['description'])
1459 except (OSError, IOError):
1460 self.report_error('Cannot write description file ' + descfn)
1463 if self.params.get('writeannotations', False):
1464 annofn = replace_extension(filename, 'annotations.xml', info_dict.get('ext'))
1465 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)):
1466 self.to_screen('[info] Video annotations are already present')
1469 self.to_screen('[info] Writing video annotations to: ' + annofn)
1470 with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile:
1471 annofile.write(info_dict['annotations'])
1472 except (KeyError, TypeError):
1473 self.report_warning('There are no annotations to write.')
1474 except (OSError, IOError):
1475 self.report_error('Cannot write annotations file: ' + annofn)
1478 subtitles_are_requested = any([self.params.get('writesubtitles', False),
1479 self.params.get('writeautomaticsub')])
1481 if subtitles_are_requested and info_dict.get('requested_subtitles'):
1482 # subtitles download errors are already managed as troubles in relevant IE
1483 # that way it will silently go on when used with unsupporting IE
1484 subtitles = info_dict['requested_subtitles']
1485 ie = self.get_info_extractor(info_dict['extractor_key'])
1486 for sub_lang, sub_info in subtitles.items():
1487 sub_format = sub_info['ext']
1488 if sub_info.get('data') is not None:
1489 sub_data = sub_info['data']
1492 sub_data = ie._download_webpage(
1493 sub_info['url'], info_dict['id'], note=False)
1494 except ExtractorError as err:
1495 self.report_warning('Unable to download subtitle for "%s": %s' %
1496 (sub_lang, compat_str(err.cause)))
1499 sub_filename = subtitles_filename(filename, sub_lang, sub_format)
1500 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)):
1501 self.to_screen('[info] Video subtitle %s.%s is already_present' % (sub_lang, sub_format))
1503 self.to_screen('[info] Writing video subtitles to: ' + sub_filename)
1504 with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile:
1505 subfile.write(sub_data)
1506 except (OSError, IOError):
1507 self.report_error('Cannot write subtitles file ' + sub_filename)
1510 if self.params.get('writeinfojson', False):
1511 infofn = replace_extension(filename, 'info.json', info_dict.get('ext'))
1512 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)):
1513 self.to_screen('[info] Video description metadata is already present')
1515 self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
1517 write_json_file(self.filter_requested_info(info_dict), infofn)
1518 except (OSError, IOError):
1519 self.report_error('Cannot write metadata to JSON file ' + infofn)
1522 self._write_thumbnails(info_dict, filename)
1524 if not self.params.get('skip_download', False):
1527 fd = get_suitable_downloader(info, self.params)(self, self.params)
1528 for ph in self._progress_hooks:
1529 fd.add_progress_hook(ph)
1530 if self.params.get('verbose'):
1531 self.to_stdout('[debug] Invoking downloader on %r' % info.get('url'))
1532 return fd.download(name, info)
1534 if info_dict.get('requested_formats') is not None:
1537 merger = FFmpegMergerPP(self)
1538 if not merger.available:
1540 self.report_warning('You have requested multiple '
1541 'formats but ffmpeg or avconv are not installed.'
1542 ' The formats won\'t be merged.')
1544 postprocessors = [merger]
1546 def compatible_formats(formats):
1547 video, audio = formats
1549 video_ext, audio_ext = audio.get('ext'), video.get('ext')
1550 if video_ext and audio_ext:
1552 ('mp3', 'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v'),
1555 for exts in COMPATIBLE_EXTS:
1556 if video_ext in exts and audio_ext in exts:
1558 # TODO: Check acodec/vcodec
1561 filename_real_ext = os.path.splitext(filename)[1][1:]
1563 os.path.splitext(filename)[0]
1564 if filename_real_ext == info_dict['ext']
1566 requested_formats = info_dict['requested_formats']
1567 if self.params.get('merge_output_format') is None and not compatible_formats(requested_formats):
1568 info_dict['ext'] = 'mkv'
1569 self.report_warning(
1570 'Requested formats are incompatible for merge and will be merged into mkv.')
1571 # Ensure filename always has a correct extension for successful merge
1572 filename = '%s.%s' % (filename_wo_ext, info_dict['ext'])
1573 if os.path.exists(encodeFilename(filename)):
1575 '[download] %s has already been downloaded and '
1576 'merged' % filename)
1578 for f in requested_formats:
1579 new_info = dict(info_dict)
1581 fname = self.prepare_filename(new_info)
1582 fname = prepend_extension(fname, 'f%s' % f['format_id'], new_info['ext'])
1583 downloaded.append(fname)
1584 partial_success = dl(fname, new_info)
1585 success = success and partial_success
1586 info_dict['__postprocessors'] = postprocessors
1587 info_dict['__files_to_merge'] = downloaded
1589 # Just a single file
1590 success = dl(filename, info_dict)
1591 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1592 self.report_error('unable to download video data: %s' % str(err))
1594 except (OSError, IOError) as err:
1595 raise UnavailableVideoError(err)
1596 except (ContentTooShortError, ) as err:
1597 self.report_error('content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
1602 fixup_policy = self.params.get('fixup')
1603 if fixup_policy is None:
1604 fixup_policy = 'detect_or_warn'
1606 stretched_ratio = info_dict.get('stretched_ratio')
1607 if stretched_ratio is not None and stretched_ratio != 1:
1608 if fixup_policy == 'warn':
1609 self.report_warning('%s: Non-uniform pixel ratio (%s)' % (
1610 info_dict['id'], stretched_ratio))
1611 elif fixup_policy == 'detect_or_warn':
1612 stretched_pp = FFmpegFixupStretchedPP(self)
1613 if stretched_pp.available:
1614 info_dict.setdefault('__postprocessors', [])
1615 info_dict['__postprocessors'].append(stretched_pp)
1617 self.report_warning(
1618 '%s: Non-uniform pixel ratio (%s). Install ffmpeg or avconv to fix this automatically.' % (
1619 info_dict['id'], stretched_ratio))
1621 assert fixup_policy in ('ignore', 'never')
1623 if info_dict.get('requested_formats') is None and info_dict.get('container') == 'm4a_dash':
1624 if fixup_policy == 'warn':
1625 self.report_warning('%s: writing DASH m4a. Only some players support this container.' % (
1627 elif fixup_policy == 'detect_or_warn':
1628 fixup_pp = FFmpegFixupM4aPP(self)
1629 if fixup_pp.available:
1630 info_dict.setdefault('__postprocessors', [])
1631 info_dict['__postprocessors'].append(fixup_pp)
1633 self.report_warning(
1634 '%s: writing DASH m4a. Only some players support this container. Install ffmpeg or avconv to fix this automatically.' % (
1637 assert fixup_policy in ('ignore', 'never')
1640 self.post_process(filename, info_dict)
1641 except (PostProcessingError) as err:
1642 self.report_error('postprocessing: %s' % str(err))
1644 self.record_download_archive(info_dict)
1646 def download(self, url_list):
1647 """Download a given list of URLs."""
1648 outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
1649 if (len(url_list) > 1 and
1650 '%' not in outtmpl and
1651 self.params.get('max_downloads') != 1):
1652 raise SameFileError(outtmpl)
1654 for url in url_list:
1656 # It also downloads the videos
1657 res = self.extract_info(
1658 url, force_generic_extractor=self.params.get('force_generic_extractor', False))
1659 except UnavailableVideoError:
1660 self.report_error('unable to download video')
1661 except MaxDownloadsReached:
1662 self.to_screen('[info] Maximum number of downloaded files reached.')
1665 if self.params.get('dump_single_json', False):
1666 self.to_stdout(json.dumps(res))
1668 return self._download_retcode
1670 def download_with_info_file(self, info_filename):
1671 with contextlib.closing(fileinput.FileInput(
1672 [info_filename], mode='r',
1673 openhook=fileinput.hook_encoded('utf-8'))) as f:
1674 # FileInput doesn't have a read method, we can't call json.load
1675 info = self.filter_requested_info(json.loads('\n'.join(f)))
1677 self.process_ie_result(info, download=True)
1678 except DownloadError:
1679 webpage_url = info.get('webpage_url')
1680 if webpage_url is not None:
1681 self.report_warning('The info failed to download, trying with "%s"' % webpage_url)
1682 return self.download([webpage_url])
1685 return self._download_retcode
1688 def filter_requested_info(info_dict):
1690 (k, v) for k, v in info_dict.items()
1691 if k not in ['requested_formats', 'requested_subtitles'])
1693 def post_process(self, filename, ie_info):
1694 """Run all the postprocessors on the given file."""
1695 info = dict(ie_info)
1696 info['filepath'] = filename
1698 if ie_info.get('__postprocessors') is not None:
1699 pps_chain.extend(ie_info['__postprocessors'])
1700 pps_chain.extend(self._pps)
1701 for pp in pps_chain:
1702 files_to_delete = []
1704 files_to_delete, info = pp.run(info)
1705 except PostProcessingError as e:
1706 self.report_error(e.msg)
1707 if files_to_delete and not self.params.get('keepvideo', False):
1708 for old_filename in files_to_delete:
1709 self.to_screen('Deleting original file %s (pass -k to keep)' % old_filename)
1711 os.remove(encodeFilename(old_filename))
1712 except (IOError, OSError):
1713 self.report_warning('Unable to remove downloaded original file')
1715 def _make_archive_id(self, info_dict):
1716 # Future-proof against any change in case
1717 # and backwards compatibility with prior versions
1718 extractor = info_dict.get('extractor_key')
1719 if extractor is None:
1720 if 'id' in info_dict:
1721 extractor = info_dict.get('ie_key') # key in a playlist
1722 if extractor is None:
1723 return None # Incomplete video information
1724 return extractor.lower() + ' ' + info_dict['id']
1726 def in_download_archive(self, info_dict):
1727 fn = self.params.get('download_archive')
1731 vid_id = self._make_archive_id(info_dict)
1733 return False # Incomplete video information
1736 with locked_file(fn, 'r', encoding='utf-8') as archive_file:
1737 for line in archive_file:
1738 if line.strip() == vid_id:
1740 except IOError as ioe:
1741 if ioe.errno != errno.ENOENT:
1745 def record_download_archive(self, info_dict):
1746 fn = self.params.get('download_archive')
1749 vid_id = self._make_archive_id(info_dict)
1751 with locked_file(fn, 'a', encoding='utf-8') as archive_file:
1752 archive_file.write(vid_id + '\n')
1755 def format_resolution(format, default='unknown'):
1756 if format.get('vcodec') == 'none':
1758 if format.get('resolution') is not None:
1759 return format['resolution']
1760 if format.get('height') is not None:
1761 if format.get('width') is not None:
1762 res = '%sx%s' % (format['width'], format['height'])
1764 res = '%sp' % format['height']
1765 elif format.get('width') is not None:
1766 res = '?x%d' % format['width']
1771 def _format_note(self, fdict):
1773 if fdict.get('ext') in ['f4f', 'f4m']:
1774 res += '(unsupported) '
1775 if fdict.get('format_note') is not None:
1776 res += fdict['format_note'] + ' '
1777 if fdict.get('tbr') is not None:
1778 res += '%4dk ' % fdict['tbr']
1779 if fdict.get('container') is not None:
1782 res += '%s container' % fdict['container']
1783 if (fdict.get('vcodec') is not None and
1784 fdict.get('vcodec') != 'none'):
1787 res += fdict['vcodec']
1788 if fdict.get('vbr') is not None:
1790 elif fdict.get('vbr') is not None and fdict.get('abr') is not None:
1792 if fdict.get('vbr') is not None:
1793 res += '%4dk' % fdict['vbr']
1794 if fdict.get('fps') is not None:
1795 res += ', %sfps' % fdict['fps']
1796 if fdict.get('acodec') is not None:
1799 if fdict['acodec'] == 'none':
1802 res += '%-5s' % fdict['acodec']
1803 elif fdict.get('abr') is not None:
1807 if fdict.get('abr') is not None:
1808 res += '@%3dk' % fdict['abr']
1809 if fdict.get('asr') is not None:
1810 res += ' (%5dHz)' % fdict['asr']
1811 if fdict.get('filesize') is not None:
1814 res += format_bytes(fdict['filesize'])
1815 elif fdict.get('filesize_approx') is not None:
1818 res += '~' + format_bytes(fdict['filesize_approx'])
1821 def list_formats(self, info_dict):
1822 formats = info_dict.get('formats', [info_dict])
1824 [f['format_id'], f['ext'], self.format_resolution(f), self._format_note(f)]
1826 if f.get('preference') is None or f['preference'] >= -1000]
1827 if len(formats) > 1:
1828 table[-1][-1] += (' ' if table[-1][-1] else '') + '(best)'
1830 header_line = ['format code', 'extension', 'resolution', 'note']
1832 '[info] Available formats for %s:\n%s' %
1833 (info_dict['id'], render_table(header_line, table)))
1835 def list_thumbnails(self, info_dict):
1836 thumbnails = info_dict.get('thumbnails')
1838 tn_url = info_dict.get('thumbnail')
1840 thumbnails = [{'id': '0', 'url': tn_url}]
1843 '[info] No thumbnails present for %s' % info_dict['id'])
1847 '[info] Thumbnails for %s:' % info_dict['id'])
1848 self.to_screen(render_table(
1849 ['ID', 'width', 'height', 'URL'],
1850 [[t['id'], t.get('width', 'unknown'), t.get('height', 'unknown'), t['url']] for t in thumbnails]))
1852 def list_subtitles(self, video_id, subtitles, name='subtitles'):
1854 self.to_screen('%s has no %s' % (video_id, name))
1857 'Available %s for %s:' % (name, video_id))
1858 self.to_screen(render_table(
1859 ['Language', 'formats'],
1860 [[lang, ', '.join(f['ext'] for f in reversed(formats))]
1861 for lang, formats in subtitles.items()]))
1863 def urlopen(self, req):
1864 """ Start an HTTP download """
1865 return self._opener.open(req, timeout=self._socket_timeout)
1867 def print_debug_header(self):
1868 if not self.params.get('verbose'):
1871 if type('') is not compat_str:
1872 # Python 2.6 on SLES11 SP1 (https://github.com/rg3/youtube-dl/issues/3326)
1873 self.report_warning(
1874 'Your Python is broken! Update to a newer and supported version')
1876 stdout_encoding = getattr(
1877 sys.stdout, 'encoding', 'missing (%s)' % type(sys.stdout).__name__)
1879 '[debug] Encodings: locale %s, fs %s, out %s, pref %s\n' % (
1880 locale.getpreferredencoding(),
1881 sys.getfilesystemencoding(),
1883 self.get_encoding()))
1884 write_string(encoding_str, encoding=None)
1886 self._write_string('[debug] youtube-dl version ' + __version__ + '\n')
1888 sp = subprocess.Popen(
1889 ['git', 'rev-parse', '--short', 'HEAD'],
1890 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
1891 cwd=os.path.dirname(os.path.abspath(__file__)))
1892 out, err = sp.communicate()
1893 out = out.decode().strip()
1894 if re.match('[0-9a-f]+', out):
1895 self._write_string('[debug] Git HEAD: ' + out + '\n')
1901 self._write_string('[debug] Python version %s - %s\n' % (
1902 platform.python_version(), platform_name()))
1904 exe_versions = FFmpegPostProcessor.get_versions(self)
1905 exe_versions['rtmpdump'] = rtmpdump_version()
1906 exe_str = ', '.join(
1908 for exe, v in sorted(exe_versions.items())
1913 self._write_string('[debug] exe versions: %s\n' % exe_str)
1916 for handler in self._opener.handlers:
1917 if hasattr(handler, 'proxies'):
1918 proxy_map.update(handler.proxies)
1919 self._write_string('[debug] Proxy map: ' + compat_str(proxy_map) + '\n')
1921 if self.params.get('call_home', False):
1922 ipaddr = self.urlopen('https://yt-dl.org/ip').read().decode('utf-8')
1923 self._write_string('[debug] Public IP address: %s\n' % ipaddr)
1924 latest_version = self.urlopen(
1925 'https://yt-dl.org/latest/version').read().decode('utf-8')
1926 if version_tuple(latest_version) > version_tuple(__version__):
1927 self.report_warning(
1928 'You are using an outdated version (newest version: %s)! '
1929 'See https://yt-dl.org/update if you need help updating.' %
1932 def _setup_opener(self):
1933 timeout_val = self.params.get('socket_timeout')
1934 self._socket_timeout = 600 if timeout_val is None else float(timeout_val)
1936 opts_cookiefile = self.params.get('cookiefile')
1937 opts_proxy = self.params.get('proxy')
1939 if opts_cookiefile is None:
1940 self.cookiejar = compat_cookiejar.CookieJar()
1942 self.cookiejar = compat_cookiejar.MozillaCookieJar(
1944 if os.access(opts_cookiefile, os.R_OK):
1945 self.cookiejar.load()
1947 cookie_processor = YoutubeDLCookieProcessor(self.cookiejar)
1948 if opts_proxy is not None:
1949 if opts_proxy == '':
1952 proxies = {'http': opts_proxy, 'https': opts_proxy}
1954 proxies = compat_urllib_request.getproxies()
1955 # Set HTTPS proxy to HTTP one if given (https://github.com/rg3/youtube-dl/issues/805)
1956 if 'http' in proxies and 'https' not in proxies:
1957 proxies['https'] = proxies['http']
1958 proxy_handler = PerRequestProxyHandler(proxies)
1960 debuglevel = 1 if self.params.get('debug_printtraffic') else 0
1961 https_handler = make_HTTPS_handler(self.params, debuglevel=debuglevel)
1962 ydlh = YoutubeDLHandler(self.params, debuglevel=debuglevel)
1963 opener = compat_urllib_request.build_opener(
1964 proxy_handler, https_handler, cookie_processor, ydlh)
1966 # Delete the default user-agent header, which would otherwise apply in
1967 # cases where our custom HTTP handler doesn't come into play
1968 # (See https://github.com/rg3/youtube-dl/issues/1309 for details)
1969 opener.addheaders = []
1970 self._opener = opener
1972 def encode(self, s):
1973 if isinstance(s, bytes):
1974 return s # Already encoded
1977 return s.encode(self.get_encoding())
1978 except UnicodeEncodeError as err:
1979 err.reason = err.reason + '. Check your system encoding configuration or use the --encoding option.'
1982 def get_encoding(self):
1983 encoding = self.params.get('encoding')
1984 if encoding is None:
1985 encoding = preferredencoding()
1988 def _write_thumbnails(self, info_dict, filename):
1989 if self.params.get('writethumbnail', False):
1990 thumbnails = info_dict.get('thumbnails')
1992 thumbnails = [thumbnails[-1]]
1993 elif self.params.get('write_all_thumbnails', False):
1994 thumbnails = info_dict.get('thumbnails')
1999 # No thumbnails present, so return immediately
2002 for t in thumbnails:
2003 thumb_ext = determine_ext(t['url'], 'jpg')
2004 suffix = '_%s' % t['id'] if len(thumbnails) > 1 else ''
2005 thumb_display_id = '%s ' % t['id'] if len(thumbnails) > 1 else ''
2006 t['filename'] = thumb_filename = os.path.splitext(filename)[0] + suffix + '.' + thumb_ext
2008 if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)):
2009 self.to_screen('[%s] %s: Thumbnail %sis already present' %
2010 (info_dict['extractor'], info_dict['id'], thumb_display_id))
2012 self.to_screen('[%s] %s: Downloading thumbnail %s...' %
2013 (info_dict['extractor'], info_dict['id'], thumb_display_id))
2015 uf = self.urlopen(t['url'])
2016 with open(encodeFilename(thumb_filename), 'wb') as thumbf:
2017 shutil.copyfileobj(uf, thumbf)
2018 self.to_screen('[%s] %s: Writing thumbnail %sto: %s' %
2019 (info_dict['extractor'], info_dict['id'], thumb_display_id, thumb_filename))
2020 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2021 self.report_warning('Unable to download thumbnail "%s": %s' %
2022 (t['url'], compat_str(err)))