1 from __future__ import unicode_literals
13 import xml.etree.ElementTree
15 from ..compat import (
19 compat_urllib_parse_urlparse,
33 _NO_DEFAULT = object()
36 class InfoExtractor(object):
37 """Information Extractor class.
39 Information extractors are the classes that, given a URL, extract
40 information about the video (or videos) the URL refers to. This
41 information includes the real video URL, the video title, author and
42 others. The information is stored in a dictionary which is then
43 passed to the FileDownloader. The FileDownloader processes this
44 information possibly downloading the video to the file system, among
45 other possible outcomes.
47 The type field determines the the type of the result.
48 By far the most common value (and the default if _type is missing) is
49 "video", which indicates a single video.
51 For a video, the dictionaries must include the following fields:
54 title: Video title, unescaped.
56 Additionally, it must contain either a formats entry or a url one:
58 formats: A list of dictionaries for each format available, ordered
59 from worst to best quality.
62 * url Mandatory. The URL of the video file
63 * ext Will be calculated from url if missing
64 * format A human-readable description of the format
65 ("mp4 container with h264/opus").
66 Calculated from the format_id, width, height.
67 and format_note fields if missing.
68 * format_id A short description of the format
69 ("mp4_h264_opus" or "19").
70 Technically optional, but strongly recommended.
71 * format_note Additional info about the format
72 ("3D" or "DASH video")
73 * width Width of the video, if known
74 * height Height of the video, if known
75 * resolution Textual description of width and height
76 * tbr Average bitrate of audio and video in KBit/s
77 * abr Average audio bitrate in KBit/s
78 * acodec Name of the audio codec in use
79 * asr Audio sampling rate in Hertz
80 * vbr Average video bitrate in KBit/s
82 * vcodec Name of the video codec in use
83 * container Name of the container format
84 * filesize The number of bytes, if known in advance
85 * filesize_approx An estimate for the number of bytes
86 * player_url SWF Player URL (used for rtmpdump).
87 * protocol The protocol that will be used for the actual
89 "http", "https", "rtsp", "rtmp", "m3u8" or so.
90 * preference Order number of this format. If this field is
91 present and not None, the formats get sorted
92 by this field, regardless of all other values.
93 -1 for default (order by other properties),
94 -2 or smaller for less than default.
95 * language_preference Is this in the correct requested
97 10 if it's what the URL is about,
98 -1 for default (don't know),
99 -10 otherwise, other values reserved for now.
100 * quality Order number of the video quality of this
101 format, irrespective of the file format.
102 -1 for default (order by other properties),
103 -2 or smaller for less than default.
104 * source_preference Order number for this video source
105 (quality takes higher priority)
106 -1 for default (order by other properties),
107 -2 or smaller for less than default.
108 * http_referer HTTP Referer header value to set.
109 * http_method HTTP method to use for the download.
110 * http_headers A dictionary of additional HTTP headers
111 to add to the request.
112 * http_post_data Additional data to send with a POST
114 url: Final video URL.
115 ext: Video filename extension.
116 format: The video format, defaults to ext (used for --get-format)
117 player_url: SWF Player URL (used for rtmpdump).
119 The following fields are optional:
121 alt_title: A secondary title of the video.
122 display_id An alternative identifier for the video, not necessarily
123 unique, but available before title. Typically, id is
124 something like "4234987", title "Dancing naked mole rats",
125 and display_id "dancing-naked-mole-rats"
126 thumbnails: A list of dictionaries, with the following entries:
128 * "width" (optional, int)
129 * "height" (optional, int)
130 * "resolution" (optional, string "{width}x{height"},
132 thumbnail: Full URL to a video thumbnail image.
133 description: Full video description.
134 uploader: Full name of the video uploader.
135 timestamp: UNIX timestamp of the moment the video became available.
136 upload_date: Video upload date (YYYYMMDD).
137 If not explicitly set, calculated from timestamp.
138 uploader_id: Nickname or id of the video uploader.
139 location: Physical location where the video was filmed.
140 subtitles: The subtitle file contents as a dictionary in the format
141 {language: subtitles}.
142 duration: Length of the video in seconds, as an integer.
143 view_count: How many users have watched the video on the platform.
144 like_count: Number of positive ratings of the video
145 dislike_count: Number of negative ratings of the video
146 comment_count: Number of comments on the video
147 age_limit: Age restriction for the video, as an integer (years)
148 webpage_url: The url to the video webpage, if given to youtube-dl it
149 should allow to get the same result again. (It will be set
150 by YoutubeDL if it's missing)
151 categories: A list of categories that the video falls in, for example
153 is_live: True, False, or None (=unknown). Whether this video is a
154 live stream that goes on instead of a fixed-length video.
156 Unless mentioned otherwise, the fields should be Unicode strings.
158 Unless mentioned otherwise, None is equivalent to absence of information.
161 _type "playlist" indicates multiple videos.
162 There must be a key "entries", which is a list, an iterable, or a PagedList
163 object, each element of which is a valid dictionary by this specification.
165 Additionally, playlists can have "title" and "id" attributes with the same
166 semantics as videos (see above).
169 _type "multi_video" indicates that there are multiple videos that
170 form a single show, for examples multiple acts of an opera or TV episode.
171 It must have an entries key like a playlist and contain all the keys
172 required for a video at the same time.
175 _type "url" indicates that the video must be extracted from another
176 location, possibly by a different extractor. Its only required key is:
177 "url" - the next URL to extract.
178 The key "ie_key" can be set to the class name (minus the trailing "IE",
179 e.g. "Youtube") if the extractor class is known in advance.
180 Additionally, the dictionary may have any properties of the resolved entity
181 known in advance, for example "title" if the title of the referred video is
185 _type "url_transparent" entities have the same specification as "url", but
186 indicate that the given additional information is more precise than the one
187 associated with the resolved URL.
188 This is useful when a site employs a video service that hosts the video and
189 its technical metadata, but that video service does not embed a useful
190 title, description etc.
193 Subclasses of this one should re-define the _real_initialize() and
194 _real_extract() methods and define a _VALID_URL regexp.
195 Probably, they should also be added to the list of extractors.
197 Finally, the _WORKING attribute should be set to False for broken IEs
198 in order to warn the users and skip the tests.
205 def __init__(self, downloader=None):
206 """Constructor. Receives an optional downloader."""
208 self.set_downloader(downloader)
211 def suitable(cls, url):
212 """Receives a URL and returns True if suitable for this IE."""
214 # This does not use has/getattr intentionally - we want to know whether
215 # we have cached the regexp for *this* class, whereas getattr would also
216 # match the superclass
217 if '_VALID_URL_RE' not in cls.__dict__:
218 cls._VALID_URL_RE = re.compile(cls._VALID_URL)
219 return cls._VALID_URL_RE.match(url) is not None
222 def _match_id(cls, url):
223 if '_VALID_URL_RE' not in cls.__dict__:
224 cls._VALID_URL_RE = re.compile(cls._VALID_URL)
225 m = cls._VALID_URL_RE.match(url)
231 """Getter method for _WORKING."""
234 def initialize(self):
235 """Initializes an instance (authentication, etc)."""
237 self._real_initialize()
240 def extract(self, url):
241 """Extracts URL information and returns it in list of dicts."""
243 return self._real_extract(url)
245 def set_downloader(self, downloader):
246 """Sets the downloader for this IE."""
247 self._downloader = downloader
249 def _real_initialize(self):
250 """Real initialization process. Redefine in subclasses."""
253 def _real_extract(self, url):
254 """Real extraction process. Redefine in subclasses."""
259 """A string for getting the InfoExtractor with get_info_extractor"""
260 return cls.__name__[:-2]
264 return type(self).__name__[:-2]
266 def _request_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True):
267 """ Returns the response handle """
269 self.report_download_webpage(video_id)
270 elif note is not False:
272 self.to_screen('%s' % (note,))
274 self.to_screen('%s: %s' % (video_id, note))
276 return self._downloader.urlopen(url_or_request)
277 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
281 errnote = 'Unable to download webpage'
282 errmsg = '%s: %s' % (errnote, compat_str(err))
284 raise ExtractorError(errmsg, sys.exc_info()[2], cause=err)
286 self._downloader.report_warning(errmsg)
289 def _download_webpage_handle(self, url_or_request, video_id, note=None, errnote=None, fatal=True):
290 """ Returns a tuple (page content as string, URL handle) """
291 # Strip hashes from the URL (#1038)
292 if isinstance(url_or_request, (compat_str, str)):
293 url_or_request = url_or_request.partition('#')[0]
295 urlh = self._request_webpage(url_or_request, video_id, note, errnote, fatal)
299 content = self._webpage_read_content(urlh, url_or_request, video_id, note, errnote, fatal)
300 return (content, urlh)
302 def _webpage_read_content(self, urlh, url_or_request, video_id, note=None, errnote=None, fatal=True, prefix=None):
303 content_type = urlh.headers.get('Content-Type', '')
304 webpage_bytes = urlh.read()
305 if prefix is not None:
306 webpage_bytes = prefix + webpage_bytes
307 m = re.match(r'[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+\s*;\s*charset=(.+)', content_type)
309 encoding = m.group(1)
311 m = re.search(br'<meta[^>]+charset=[\'"]?([^\'")]+)[ /\'">]',
312 webpage_bytes[:1024])
314 encoding = m.group(1).decode('ascii')
315 elif webpage_bytes.startswith(b'\xff\xfe'):
319 if self._downloader.params.get('dump_intermediate_pages', False):
321 url = url_or_request.get_full_url()
322 except AttributeError:
324 self.to_screen('Dumping request to ' + url)
325 dump = base64.b64encode(webpage_bytes).decode('ascii')
326 self._downloader.to_screen(dump)
327 if self._downloader.params.get('write_pages', False):
329 url = url_or_request.get_full_url()
330 except AttributeError:
332 basen = '%s_%s' % (video_id, url)
334 h = '___' + hashlib.md5(basen.encode('utf-8')).hexdigest()
335 basen = basen[:240 - len(h)] + h
336 raw_filename = basen + '.dump'
337 filename = sanitize_filename(raw_filename, restricted=True)
338 self.to_screen('Saving request to ' + filename)
339 # Working around MAX_PATH limitation on Windows (see
340 # http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx)
342 absfilepath = os.path.abspath(filename)
343 if len(absfilepath) > 259:
344 filename = '\\\\?\\' + absfilepath
345 with open(filename, 'wb') as outf:
346 outf.write(webpage_bytes)
349 content = webpage_bytes.decode(encoding, 'replace')
351 content = webpage_bytes.decode('utf-8', 'replace')
353 if ('<title>Access to this site is blocked</title>' in content and
354 'Websense' in content[:512]):
355 msg = 'Access to this webpage has been blocked by Websense filtering software in your network.'
356 blocked_iframe = self._html_search_regex(
357 r'<iframe src="([^"]+)"', content,
358 'Websense information URL', default=None)
360 msg += ' Visit %s for more details' % blocked_iframe
361 raise ExtractorError(msg, expected=True)
365 def _download_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True):
366 """ Returns the data of the page as a string """
367 res = self._download_webpage_handle(url_or_request, video_id, note, errnote, fatal)
374 def _download_xml(self, url_or_request, video_id,
375 note='Downloading XML', errnote='Unable to download XML',
376 transform_source=None, fatal=True):
377 """Return the xml as an xml.etree.ElementTree.Element"""
378 xml_string = self._download_webpage(
379 url_or_request, video_id, note, errnote, fatal=fatal)
380 if xml_string is False:
383 xml_string = transform_source(xml_string)
384 return xml.etree.ElementTree.fromstring(xml_string.encode('utf-8'))
386 def _download_json(self, url_or_request, video_id,
387 note='Downloading JSON metadata',
388 errnote='Unable to download JSON metadata',
389 transform_source=None,
391 json_string = self._download_webpage(
392 url_or_request, video_id, note, errnote, fatal=fatal)
393 if (not fatal) and json_string is False:
395 return self._parse_json(
396 json_string, video_id, transform_source=transform_source, fatal=fatal)
398 def _parse_json(self, json_string, video_id, transform_source=None, fatal=True):
400 json_string = transform_source(json_string)
402 return json.loads(json_string)
403 except ValueError as ve:
404 errmsg = '%s: Failed to parse JSON ' % video_id
406 raise ExtractorError(errmsg, cause=ve)
408 self.report_warning(errmsg + str(ve))
410 def report_warning(self, msg, video_id=None):
411 idstr = '' if video_id is None else '%s: ' % video_id
412 self._downloader.report_warning(
413 '[%s] %s%s' % (self.IE_NAME, idstr, msg))
415 def to_screen(self, msg):
416 """Print msg to screen, prefixing it with '[ie_name]'"""
417 self._downloader.to_screen('[%s] %s' % (self.IE_NAME, msg))
419 def report_extraction(self, id_or_name):
420 """Report information extraction."""
421 self.to_screen('%s: Extracting information' % id_or_name)
423 def report_download_webpage(self, video_id):
424 """Report webpage download."""
425 self.to_screen('%s: Downloading webpage' % video_id)
427 def report_age_confirmation(self):
428 """Report attempt to confirm age."""
429 self.to_screen('Confirming age')
431 def report_login(self):
432 """Report attempt to log in."""
433 self.to_screen('Logging in')
435 # Methods for following #608
437 def url_result(url, ie=None, video_id=None):
438 """Returns a url that points to a page that should be processed"""
439 # TODO: ie should be the class used for getting the info
440 video_info = {'_type': 'url',
443 if video_id is not None:
444 video_info['id'] = video_id
448 def playlist_result(entries, playlist_id=None, playlist_title=None, playlist_description=None):
449 """Returns a playlist"""
450 video_info = {'_type': 'playlist',
453 video_info['id'] = playlist_id
455 video_info['title'] = playlist_title
456 if playlist_description:
457 video_info['description'] = playlist_description
460 def _search_regex(self, pattern, string, name, default=_NO_DEFAULT, fatal=True, flags=0, group=None):
462 Perform a regex search on the given string, using a single or a list of
463 patterns returning the first matching group.
464 In case of failure return a default value or raise a WARNING or a
465 RegexNotFoundError, depending on fatal, specifying the field name.
467 if isinstance(pattern, (str, compat_str, compiled_regex_type)):
468 mobj = re.search(pattern, string, flags)
471 mobj = re.search(p, string, flags)
475 if os.name != 'nt' and sys.stderr.isatty():
476 _name = '\033[0;34m%s\033[0m' % name
482 # return the first matching group
483 return next(g for g in mobj.groups() if g is not None)
485 return mobj.group(group)
486 elif default is not _NO_DEFAULT:
489 raise RegexNotFoundError('Unable to extract %s' % _name)
491 self._downloader.report_warning('unable to extract %s; '
492 'please report this issue on http://yt-dl.org/bug' % _name)
495 def _html_search_regex(self, pattern, string, name, default=_NO_DEFAULT, fatal=True, flags=0, group=None):
497 Like _search_regex, but strips HTML tags and unescapes entities.
499 res = self._search_regex(pattern, string, name, default, fatal, flags, group)
501 return clean_html(res).strip()
505 def _get_login_info(self):
507 Get the the login info as (username, password)
508 It will look in the netrc file using the _NETRC_MACHINE value
509 If there's no info available, return (None, None)
511 if self._downloader is None:
516 downloader_params = self._downloader.params
518 # Attempt to use provided username and password or .netrc data
519 if downloader_params.get('username', None) is not None:
520 username = downloader_params['username']
521 password = downloader_params['password']
522 elif downloader_params.get('usenetrc', False):
524 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
529 raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
530 except (IOError, netrc.NetrcParseError) as err:
531 self._downloader.report_warning('parsing .netrc: %s' % compat_str(err))
533 return (username, password)
535 def _get_tfa_info(self):
537 Get the two-factor authentication info
538 TODO - asking the user will be required for sms/phone verify
539 currently just uses the command line option
540 If there's no info available, return None
542 if self._downloader is None:
544 downloader_params = self._downloader.params
546 if downloader_params.get('twofactor', None) is not None:
547 return downloader_params['twofactor']
551 # Helper functions for extracting OpenGraph info
553 def _og_regexes(prop):
554 content_re = r'content=(?:"([^>]+?)"|\'([^>]+?)\')'
555 property_re = r'(?:name|property)=[\'"]og:%s[\'"]' % re.escape(prop)
556 template = r'<meta[^>]+?%s[^>]+?%s'
558 template % (property_re, content_re),
559 template % (content_re, property_re),
562 def _og_search_property(self, prop, html, name=None, **kargs):
564 name = 'OpenGraph %s' % prop
565 escaped = self._search_regex(self._og_regexes(prop), html, name, flags=re.DOTALL, **kargs)
568 return unescapeHTML(escaped)
570 def _og_search_thumbnail(self, html, **kargs):
571 return self._og_search_property('image', html, 'thumbnail url', fatal=False, **kargs)
573 def _og_search_description(self, html, **kargs):
574 return self._og_search_property('description', html, fatal=False, **kargs)
576 def _og_search_title(self, html, **kargs):
577 return self._og_search_property('title', html, **kargs)
579 def _og_search_video_url(self, html, name='video url', secure=True, **kargs):
580 regexes = self._og_regexes('video') + self._og_regexes('video:url')
582 regexes = self._og_regexes('video:secure_url') + regexes
583 return self._html_search_regex(regexes, html, name, **kargs)
585 def _og_search_url(self, html, **kargs):
586 return self._og_search_property('url', html, **kargs)
588 def _html_search_meta(self, name, html, display_name=None, fatal=False, **kwargs):
589 if display_name is None:
591 return self._html_search_regex(
593 (?=[^>]+(?:itemprop|name|property)=(["\']?)%s\1)
594 [^>]+content=(["\'])(?P<content>.*?)\1''' % re.escape(name),
595 html, display_name, fatal=fatal, group='content', **kwargs)
597 def _dc_search_uploader(self, html):
598 return self._html_search_meta('dc.creator', html, 'uploader')
600 def _rta_search(self, html):
601 # See http://www.rtalabel.org/index.php?content=howtofaq#single
602 if re.search(r'(?ix)<meta\s+name="rating"\s+'
603 r' content="RTA-5042-1996-1400-1577-RTA"',
608 def _media_rating_search(self, html):
609 # See http://www.tjg-designs.com/WP/metadata-code-examples-adding-metadata-to-your-web-pages/
610 rating = self._html_search_meta('rating', html)
622 return RATING_TABLE.get(rating.lower(), None)
624 def _twitter_search_player(self, html):
625 return self._html_search_meta('twitter:player', html,
626 'twitter card player')
628 def _sort_formats(self, formats):
630 raise ExtractorError('No video formats found')
633 # TODO remove the following workaround
634 from ..utils import determine_ext
635 if not f.get('ext') and 'url' in f:
636 f['ext'] = determine_ext(f['url'])
638 preference = f.get('preference')
639 if preference is None:
640 proto = f.get('protocol')
642 proto = compat_urllib_parse_urlparse(f.get('url', '')).scheme
644 preference = 0 if proto in ['http', 'https'] else -0.1
645 if f.get('ext') in ['f4f', 'f4m']: # Not yet supported
648 if f.get('vcodec') == 'none': # audio only
649 if self._downloader.params.get('prefer_free_formats'):
650 ORDER = ['aac', 'mp3', 'm4a', 'webm', 'ogg', 'opus']
652 ORDER = ['webm', 'opus', 'ogg', 'mp3', 'aac', 'm4a']
655 audio_ext_preference = ORDER.index(f['ext'])
657 audio_ext_preference = -1
659 if self._downloader.params.get('prefer_free_formats'):
660 ORDER = ['flv', 'mp4', 'webm']
662 ORDER = ['webm', 'flv', 'mp4']
664 ext_preference = ORDER.index(f['ext'])
667 audio_ext_preference = 0
671 f.get('language_preference') if f.get('language_preference') is not None else -1,
672 f.get('quality') if f.get('quality') is not None else -1,
673 f.get('height') if f.get('height') is not None else -1,
674 f.get('width') if f.get('width') is not None else -1,
676 f.get('tbr') if f.get('tbr') is not None else -1,
677 f.get('vbr') if f.get('vbr') is not None else -1,
678 f.get('abr') if f.get('abr') is not None else -1,
679 audio_ext_preference,
680 f.get('fps') if f.get('fps') is not None else -1,
681 f.get('filesize') if f.get('filesize') is not None else -1,
682 f.get('filesize_approx') if f.get('filesize_approx') is not None else -1,
683 f.get('source_preference') if f.get('source_preference') is not None else -1,
686 formats.sort(key=_formats_key)
688 def http_scheme(self):
689 """ Either "http:" or "https:", depending on the user's preferences """
692 if self._downloader.params.get('prefer_insecure', False)
695 def _proto_relative_url(self, url, scheme=None):
698 if url.startswith('//'):
700 scheme = self.http_scheme()
705 def _sleep(self, timeout, video_id, msg_template=None):
706 if msg_template is None:
707 msg_template = '%(video_id)s: Waiting for %(timeout)s seconds'
708 msg = msg_template % {'video_id': video_id, 'timeout': timeout}
712 def _extract_f4m_formats(self, manifest_url, video_id):
713 manifest = self._download_xml(
714 manifest_url, video_id, 'Downloading f4m manifest',
715 'Unable to download f4m manifest')
718 media_nodes = manifest.findall('{http://ns.adobe.com/f4m/1.0}media')
719 for i, media_el in enumerate(media_nodes):
720 tbr = int_or_none(media_el.attrib.get('bitrate'))
721 format_id = 'f4m-%d' % (i if tbr is None else tbr)
723 'format_id': format_id,
727 'width': int_or_none(media_el.attrib.get('width')),
728 'height': int_or_none(media_el.attrib.get('height')),
730 self._sort_formats(formats)
734 def _extract_m3u8_formats(self, m3u8_url, video_id, ext=None,
735 entry_protocol='m3u8', preference=None):
738 'format_id': 'm3u8-meta',
743 'resolution': 'multiple',
744 'format_note': 'Quality selection URL',
747 format_url = lambda u: (
749 if re.match(r'^https?://', u)
750 else compat_urlparse.urljoin(m3u8_url, u))
752 m3u8_doc = self._download_webpage(
754 note='Downloading m3u8 information',
755 errnote='Failed to download m3u8 information')
758 r'(?P<key>[a-zA-Z_-]+)=(?P<val>"[^"]+"|[^",]+)(?:,|$)')
759 for line in m3u8_doc.splitlines():
760 if line.startswith('#EXT-X-STREAM-INF:'):
762 for m in kv_rex.finditer(line):
764 if v.startswith('"'):
766 last_info[m.group('key')] = v
767 elif line.startswith('#') or not line.strip():
770 if last_info is None:
771 formats.append({'url': format_url(line)})
773 tbr = int_or_none(last_info.get('BANDWIDTH'), scale=1000)
776 'format_id': 'm3u8-%d' % (tbr if tbr else len(formats)),
777 'url': format_url(line.strip()),
780 'protocol': entry_protocol,
781 'preference': preference,
783 codecs = last_info.get('CODECS')
785 # TODO: looks like video codec is not always necessarily goes first
786 va_codecs = codecs.split(',')
788 f['vcodec'] = va_codecs[0].partition('.')[0]
789 if len(va_codecs) > 1 and va_codecs[1]:
790 f['acodec'] = va_codecs[1].partition('.')[0]
791 resolution = last_info.get('RESOLUTION')
793 width_str, height_str = resolution.split('x')
794 f['width'] = int(width_str)
795 f['height'] = int(height_str)
798 self._sort_formats(formats)
801 # TODO: improve extraction
802 def _extract_smil_formats(self, smil_url, video_id):
803 smil = self._download_xml(
804 smil_url, video_id, 'Downloading SMIL file',
805 'Unable to download SMIL file')
807 base = smil.find('./head/meta').get('base')
811 for video in smil.findall('./body/switch/video'):
812 src = video.get('src')
815 bitrate = int_or_none(video.get('system-bitrate') or video.get('systemBitrate'), 1000)
816 width = int_or_none(video.get('width'))
817 height = int_or_none(video.get('height'))
818 proto = video.get('proto')
821 if base.startswith('rtmp'):
823 elif base.startswith('http'):
825 ext = video.get('ext')
827 formats.extend(self._extract_m3u8_formats(src, video_id, ext))
828 elif proto == 'rtmp':
830 streamer = video.get('streamer') or base
835 'format_id': 'rtmp-%d' % (rtmp_count if bitrate is None else bitrate),
840 self._sort_formats(formats)
844 def _live_title(self, name):
845 """ Generate the title for a live video """
846 now = datetime.datetime.now()
847 now_str = now.strftime("%Y-%m-%d %H:%M")
848 return name + ' ' + now_str
850 def _int(self, v, name, fatal=False, **kwargs):
851 res = int_or_none(v, **kwargs)
852 if 'get_attr' in kwargs:
853 print(getattr(v, kwargs['get_attr']))
855 msg = 'Failed to extract %s: Could not parse value %r' % (name, v)
857 raise ExtractorError(msg)
859 self._downloader.report_warning(msg)
862 def _float(self, v, name, fatal=False, **kwargs):
863 res = float_or_none(v, **kwargs)
865 msg = 'Failed to extract %s: Could not parse value %r' % (name, v)
867 raise ExtractorError(msg)
869 self._downloader.report_warning(msg)
872 def _set_cookie(self, domain, name, value, expire_time=None):
873 cookie = compat_cookiejar.Cookie(
874 0, name, value, None, None, domain, None,
875 None, '/', True, False, expire_time, '', None, None, None)
876 self._downloader.cookiejar.set_cookie(cookie)
879 class SearchInfoExtractor(InfoExtractor):
881 Base class for paged search queries extractors.
882 They accept urls in the format _SEARCH_KEY(|all|[0-9]):{query}
883 Instances should define _SEARCH_KEY and _MAX_RESULTS.
887 def _make_valid_url(cls):
888 return r'%s(?P<prefix>|[1-9][0-9]*|all):(?P<query>[\s\S]+)' % cls._SEARCH_KEY
891 def suitable(cls, url):
892 return re.match(cls._make_valid_url(), url) is not None
894 def _real_extract(self, query):
895 mobj = re.match(self._make_valid_url(), query)
897 raise ExtractorError('Invalid search query "%s"' % query)
899 prefix = mobj.group('prefix')
900 query = mobj.group('query')
902 return self._get_n_results(query, 1)
903 elif prefix == 'all':
904 return self._get_n_results(query, self._MAX_RESULTS)
908 raise ExtractorError('invalid download number %s for query "%s"' % (n, query))
909 elif n > self._MAX_RESULTS:
910 self._downloader.report_warning('%s returns max %i results (you requested %i)' % (self._SEARCH_KEY, self._MAX_RESULTS, n))
911 n = self._MAX_RESULTS
912 return self._get_n_results(query, n)
914 def _get_n_results(self, query, n):
915 """Get a specified number of results for a query"""
916 raise NotImplementedError("This method must be implemented by subclasses")
919 def SEARCH_KEY(self):
920 return self._SEARCH_KEY