youtube_dl/extractor/common.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 import base64
   5 import datetime
   6 import hashlib
   7 import json
   8 import netrc
   9 import os
  10 import random
  11 import re
  12 import socket
  13 import sys
  14 import time
  15 import math
  16
  17 from ..compat import (
  18     compat_cookiejar,
  19     compat_cookies,
  20     compat_etree_Element,
  21     compat_etree_fromstring,
  22     compat_getpass,
  23     compat_integer_types,
  24     compat_http_client,
  25     compat_os_name,
  26     compat_str,
  27     compat_urllib_error,
  28     compat_urllib_parse_unquote,
  29     compat_urllib_parse_urlencode,
  30     compat_urllib_request,
  31     compat_urlparse,
  32     compat_xml_parse_error,
  33 )
  34 from ..downloader.f4m import (
  35     get_base_url,
  36     remove_encrypted_media,
  37 )
  38 from ..utils import (
  39     NO_DEFAULT,
  40     age_restricted,
  41     base_url,
  42     bug_reports_message,
  43     clean_html,
  44     compiled_regex_type,
  45     determine_ext,
  46     determine_protocol,
  47     dict_get,
  48     error_to_compat_str,
  49     ExtractorError,
  50     extract_attributes,
  51     fix_xml_ampersands,
  52     float_or_none,
  53     GeoRestrictedError,
  54     GeoUtils,
  55     int_or_none,
  56     js_to_json,
  57     JSON_LD_RE,
  58     mimetype2ext,
  59     orderedSet,
  60     parse_bitrate,
  61     parse_codecs,
  62     parse_duration,
  63     parse_iso8601,
  64     parse_m3u8_attributes,
  65     parse_resolution,
  66     RegexNotFoundError,
  67     sanitized_Request,
  68     sanitize_filename,
  69     str_or_none,
  70     unescapeHTML,
  71     unified_strdate,
  72     unified_timestamp,
  73     update_Request,
  74     update_url_query,
  75     urljoin,
  76     url_basename,
  77     url_or_none,
  78     xpath_element,
  79     xpath_text,
  80     xpath_with_ns,
  81 )
  82
  83
  84 class InfoExtractor(object):
  85     """Information Extractor class.
  86
  87     Information extractors are the classes that, given a URL, extract
  88     information about the video (or videos) the URL refers to. This
  89     information includes the real video URL, the video title, author and
  90     others. The information is stored in a dictionary which is then
  91     passed to the YoutubeDL. The YoutubeDL processes this
  92     information possibly downloading the video to the file system, among
  93     other possible outcomes.
  94
  95     The type field determines the type of the result.
  96     By far the most common value (and the default if _type is missing) is
  97     "video", which indicates a single video.
  98
  99     For a video, the dictionaries must include the following fields:
 100
 101     id:             Video identifier.
 102     title:          Video title, unescaped.
 103
 104     Additionally, it must contain either a formats entry or a url one:
 105
 106     formats:        A list of dictionaries for each format available, ordered
 107                     from worst to best quality.
 108
 109                     Potential fields:
 110                     * url        The mandatory URL representing the media:
 111                                    for plain file media - HTTP URL of this file,
 112                                    for RTMP - RTMP URL,
 113                                    for HLS - URL of the M3U8 media playlist,
 114                                    for HDS - URL of the F4M manifest,
 115                                    for DASH
 116                                      - HTTP URL to plain file media (in case of
 117                                        unfragmented media)
 118                                      - URL of the MPD manifest or base URL
 119                                        representing the media if MPD manifest
 120                                        is parsed froma string (in case of
 121                                        fragmented media)
 122                                    for MSS - URL of the ISM manifest.
 123                     * manifest_url
 124                                  The URL of the manifest file in case of
 125                                  fragmented media:
 126                                    for HLS - URL of the M3U8 master playlist,
 127                                    for HDS - URL of the F4M manifest,
 128                                    for DASH - URL of the MPD manifest,
 129                                    for MSS - URL of the ISM manifest.
 130                     * ext        Will be calculated from URL if missing
 131                     * format     A human-readable description of the format
 132                                  ("mp4 container with h264/opus").
 133                                  Calculated from the format_id, width, height.
 134                                  and format_note fields if missing.
 135                     * format_id  A short description of the format
 136                                  ("mp4_h264_opus" or "19").
 137                                 Technically optional, but strongly recommended.
 138                     * format_note Additional info about the format
 139                                  ("3D" or "DASH video")
 140                     * width      Width of the video, if known
 141                     * height     Height of the video, if known
 142                     * resolution Textual description of width and height
 143                     * tbr        Average bitrate of audio and video in KBit/s
 144                     * abr        Average audio bitrate in KBit/s
 145                     * acodec     Name of the audio codec in use
 146                     * asr        Audio sampling rate in Hertz
 147                     * vbr        Average video bitrate in KBit/s
 148                     * fps        Frame rate
 149                     * vcodec     Name of the video codec in use
 150                     * container  Name of the container format
 151                     * filesize   The number of bytes, if known in advance
 152                     * filesize_approx  An estimate for the number of bytes
 153                     * player_url SWF Player URL (used for rtmpdump).
 154                     * protocol   The protocol that will be used for the actual
 155                                  download, lower-case.
 156                                  "http", "https", "rtsp", "rtmp", "rtmpe",
 157                                  "m3u8", "m3u8_native" or "http_dash_segments".
 158                     * fragment_base_url
 159                                  Base URL for fragments. Each fragment's path
 160                                  value (if present) will be relative to
 161                                  this URL.
 162                     * fragments  A list of fragments of a fragmented media.
 163                                  Each fragment entry must contain either an url
 164                                  or a path. If an url is present it should be
 165                                  considered by a client. Otherwise both path and
 166                                  fragment_base_url must be present. Here is
 167                                  the list of all potential fields:
 168                                  * "url" - fragment's URL
 169                                  * "path" - fragment's path relative to
 170                                             fragment_base_url
 171                                  * "duration" (optional, int or float)
 172                                  * "filesize" (optional, int)
 173                     * preference Order number of this format. If this field is
 174                                  present and not None, the formats get sorted
 175                                  by this field, regardless of all other values.
 176                                  -1 for default (order by other properties),
 177                                  -2 or smaller for less than default.
 178                                  < -1000 to hide the format (if there is
 179                                     another one which is strictly better)
 180                     * language   Language code, e.g. "de" or "en-US".
 181                     * language_preference  Is this in the language mentioned in
 182                                  the URL?
 183                                  10 if it's what the URL is about,
 184                                  -1 for default (don't know),
 185                                  -10 otherwise, other values reserved for now.
 186                     * quality    Order number of the video quality of this
 187                                  format, irrespective of the file format.
 188                                  -1 for default (order by other properties),
 189                                  -2 or smaller for less than default.
 190                     * source_preference  Order number for this video source
 191                                   (quality takes higher priority)
 192                                  -1 for default (order by other properties),
 193                                  -2 or smaller for less than default.
 194                     * http_headers  A dictionary of additional HTTP headers
 195                                  to add to the request.
 196                     * stretched_ratio  If given and not 1, indicates that the
 197                                  video's pixels are not square.
 198                                  width : height ratio as float.
 199                     * no_resume  The server does not support resuming the
 200                                  (HTTP or RTMP) download. Boolean.
 201                     * downloader_options  A dictionary of downloader options as
 202                                  described in FileDownloader
 203
 204     url:            Final video URL.
 205     ext:            Video filename extension.
 206     format:         The video format, defaults to ext (used for --get-format)
 207     player_url:     SWF Player URL (used for rtmpdump).
 208
 209     The following fields are optional:
 210
 211     alt_title:      A secondary title of the video.
 212     display_id      An alternative identifier for the video, not necessarily
 213                     unique, but available before title. Typically, id is
 214                     something like "4234987", title "Dancing naked mole rats",
 215                     and display_id "dancing-naked-mole-rats"
 216     thumbnails:     A list of dictionaries, with the following entries:
 217                         * "id" (optional, string) - Thumbnail format ID
 218                         * "url"
 219                         * "preference" (optional, int) - quality of the image
 220                         * "width" (optional, int)
 221                         * "height" (optional, int)
 222                         * "resolution" (optional, string "{width}x{height"},
 223                                         deprecated)
 224                         * "filesize" (optional, int)
 225     thumbnail:      Full URL to a video thumbnail image.
 226     description:    Full video description.
 227     uploader:       Full name of the video uploader.
 228     license:        License name the video is licensed under.
 229     creator:        The creator of the video.
 230     release_date:   The date (YYYYMMDD) when the video was released.
 231     timestamp:      UNIX timestamp of the moment the video became available.
 232     upload_date:    Video upload date (YYYYMMDD).
 233                     If not explicitly set, calculated from timestamp.
 234     uploader_id:    Nickname or id of the video uploader.
 235     uploader_url:   Full URL to a personal webpage of the video uploader.
 236     channel:        Full name of the channel the video is uploaded on.
 237                     Note that channel fields may or may not repeat uploader
 238                     fields. This depends on a particular extractor.
 239     channel_id:     Id of the channel.
 240     channel_url:    Full URL to a channel webpage.
 241     location:       Physical location where the video was filmed.
 242     subtitles:      The available subtitles as a dictionary in the format
 243                     {tag: subformats}. "tag" is usually a language code, and
 244                     "subformats" is a list sorted from lower to higher
 245                     preference, each element is a dictionary with the "ext"
 246                     entry and one of:
 247                         * "data": The subtitles file contents
 248                         * "url": A URL pointing to the subtitles file
 249                     "ext" will be calculated from URL if missing
 250     automatic_captions: Like 'subtitles', used by the YoutubeIE for
 251                     automatically generated captions
 252     duration:       Length of the video in seconds, as an integer or float.
 253     view_count:     How many users have watched the video on the platform.
 254     like_count:     Number of positive ratings of the video
 255     dislike_count:  Number of negative ratings of the video
 256     repost_count:   Number of reposts of the video
 257     average_rating: Average rating give by users, the scale used depends on the webpage
 258     comment_count:  Number of comments on the video
 259     comments:       A list of comments, each with one or more of the following
 260                     properties (all but one of text or html optional):
 261                         * "author" - human-readable name of the comment author
 262                         * "author_id" - user ID of the comment author
 263                         * "id" - Comment ID
 264                         * "html" - Comment as HTML
 265                         * "text" - Plain text of the comment
 266                         * "timestamp" - UNIX timestamp of comment
 267                         * "parent" - ID of the comment this one is replying to.
 268                                      Set to "root" to indicate that this is a
 269                                      comment to the original video.
 270     age_limit:      Age restriction for the video, as an integer (years)
 271     webpage_url:    The URL to the video webpage, if given to youtube-dl it
 272                     should allow to get the same result again. (It will be set
 273                     by YoutubeDL if it's missing)
 274     categories:     A list of categories that the video falls in, for example
 275                     ["Sports", "Berlin"]
 276     tags:           A list of tags assigned to the video, e.g. ["sweden", "pop music"]
 277     is_live:        True, False, or None (=unknown). Whether this video is a
 278                     live stream that goes on instead of a fixed-length video.
 279     start_time:     Time in seconds where the reproduction should start, as
 280                     specified in the URL.
 281     end_time:       Time in seconds where the reproduction should end, as
 282                     specified in the URL.
 283     chapters:       A list of dictionaries, with the following entries:
 284                         * "start_time" - The start time of the chapter in seconds
 285                         * "end_time" - The end time of the chapter in seconds
 286                         * "title" (optional, string)
 287
 288     The following fields should only be used when the video belongs to some logical
 289     chapter or section:
 290
 291     chapter:        Name or title of the chapter the video belongs to.
 292     chapter_number: Number of the chapter the video belongs to, as an integer.
 293     chapter_id:     Id of the chapter the video belongs to, as a unicode string.
 294
 295     The following fields should only be used when the video is an episode of some
 296     series, programme or podcast:
 297
 298     series:         Title of the series or programme the video episode belongs to.
 299     season:         Title of the season the video episode belongs to.
 300     season_number:  Number of the season the video episode belongs to, as an integer.
 301     season_id:      Id of the season the video episode belongs to, as a unicode string.
 302     episode:        Title of the video episode. Unlike mandatory video title field,
 303                     this field should denote the exact title of the video episode
 304                     without any kind of decoration.
 305     episode_number: Number of the video episode within a season, as an integer.
 306     episode_id:     Id of the video episode, as a unicode string.
 307
 308     The following fields should only be used when the media is a track or a part of
 309     a music album:
 310
 311     track:          Title of the track.
 312     track_number:   Number of the track within an album or a disc, as an integer.
 313     track_id:       Id of the track (useful in case of custom indexing, e.g. 6.iii),
 314                     as a unicode string.
 315     artist:         Artist(s) of the track.
 316     genre:          Genre(s) of the track.
 317     album:          Title of the album the track belongs to.
 318     album_type:     Type of the album (e.g. "Demo", "Full-length", "Split", "Compilation", etc).
 319     album_artist:   List of all artists appeared on the album (e.g.
 320                     "Ash Borer / Fell Voices" or "Various Artists", useful for splits
 321                     and compilations).
 322     disc_number:    Number of the disc or other physical medium the track belongs to,
 323                     as an integer.
 324     release_year:   Year (YYYY) when the album was released.
 325
 326     Unless mentioned otherwise, the fields should be Unicode strings.
 327
 328     Unless mentioned otherwise, None is equivalent to absence of information.
 329
 330
 331     _type "playlist" indicates multiple videos.
 332     There must be a key "entries", which is a list, an iterable, or a PagedList
 333     object, each element of which is a valid dictionary by this specification.
 334
 335     Additionally, playlists can have "id", "title", "description", "uploader",
 336     "uploader_id", "uploader_url" attributes with the same semantics as videos
 337     (see above).
 338
 339
 340     _type "multi_video" indicates that there are multiple videos that
 341     form a single show, for examples multiple acts of an opera or TV episode.
 342     It must have an entries key like a playlist and contain all the keys
 343     required for a video at the same time.
 344
 345
 346     _type "url" indicates that the video must be extracted from another
 347     location, possibly by a different extractor. Its only required key is:
 348     "url" - the next URL to extract.
 349     The key "ie_key" can be set to the class name (minus the trailing "IE",
 350     e.g. "Youtube") if the extractor class is known in advance.
 351     Additionally, the dictionary may have any properties of the resolved entity
 352     known in advance, for example "title" if the title of the referred video is
 353     known ahead of time.
 354
 355
 356     _type "url_transparent" entities have the same specification as "url", but
 357     indicate that the given additional information is more precise than the one
 358     associated with the resolved URL.
 359     This is useful when a site employs a video service that hosts the video and
 360     its technical metadata, but that video service does not embed a useful
 361     title, description etc.
 362
 363
 364     Subclasses of this one should re-define the _real_initialize() and
 365     _real_extract() methods and define a _VALID_URL regexp.
 366     Probably, they should also be added to the list of extractors.
 367
 368     _GEO_BYPASS attribute may be set to False in order to disable
 369     geo restriction bypass mechanisms for a particular extractor.
 370     Though it won't disable explicit geo restriction bypass based on
 371     country code provided with geo_bypass_country.
 372
 373     _GEO_COUNTRIES attribute may contain a list of presumably geo unrestricted
 374     countries for this extractor. One of these countries will be used by
 375     geo restriction bypass mechanism right away in order to bypass
 376     geo restriction, of course, if the mechanism is not disabled.
 377
 378     _GEO_IP_BLOCKS attribute may contain a list of presumably geo unrestricted
 379     IP blocks in CIDR notation for this extractor. One of these IP blocks
 380     will be used by geo restriction bypass mechanism similarly
 381     to _GEO_COUNTRIES.
 382
 383     Finally, the _WORKING attribute should be set to False for broken IEs
 384     in order to warn the users and skip the tests.
 385     """
 386
 387     _ready = False
 388     _downloader = None
 389     _x_forwarded_for_ip = None
 390     _GEO_BYPASS = True
 391     _GEO_COUNTRIES = None
 392     _GEO_IP_BLOCKS = None
 393     _WORKING = True
 394
 395     def __init__(self, downloader=None):
 396         """Constructor. Receives an optional downloader."""
 397         self._ready = False
 398         self._x_forwarded_for_ip = None
 399         self.set_downloader(downloader)
 400
 401     @classmethod
 402     def suitable(cls, url):
 403         """Receives a URL and returns True if suitable for this IE."""
 404
 405         # This does not use has/getattr intentionally - we want to know whether
 406         # we have cached the regexp for *this* class, whereas getattr would also
 407         # match the superclass
 408         if '_VALID_URL_RE' not in cls.__dict__:
 409             cls._VALID_URL_RE = re.compile(cls._VALID_URL)
 410         return cls._VALID_URL_RE.match(url) is not None
 411
 412     @classmethod
 413     def _match_id(cls, url):
 414         if '_VALID_URL_RE' not in cls.__dict__:
 415             cls._VALID_URL_RE = re.compile(cls._VALID_URL)
 416         m = cls._VALID_URL_RE.match(url)
 417         assert m
 418         return compat_str(m.group('id'))
 419
 420     @classmethod
 421     def working(cls):
 422         """Getter method for _WORKING."""
 423         return cls._WORKING
 424
 425     def initialize(self):
 426         """Initializes an instance (authentication, etc)."""
 427         self._initialize_geo_bypass({
 428             'countries': self._GEO_COUNTRIES,
 429             'ip_blocks': self._GEO_IP_BLOCKS,
 430         })
 431         if not self._ready:
 432             self._real_initialize()
 433             self._ready = True
 434
 435     def _initialize_geo_bypass(self, geo_bypass_context):
 436         """
 437         Initialize geo restriction bypass mechanism.
 438
 439         This method is used to initialize geo bypass mechanism based on faking
 440         X-Forwarded-For HTTP header. A random country from provided country list
 441         is selected and a random IP belonging to this country is generated. This
 442         IP will be passed as X-Forwarded-For HTTP header in all subsequent
 443         HTTP requests.
 444
 445         This method will be used for initial geo bypass mechanism initialization
 446         during the instance initialization with _GEO_COUNTRIES and
 447         _GEO_IP_BLOCKS.
 448
 449         You may also manually call it from extractor's code if geo bypass
 450         information is not available beforehand (e.g. obtained during
 451         extraction) or due to some other reason. In this case you should pass
 452         this information in geo bypass context passed as first argument. It may
 453         contain following fields:
 454
 455         countries:  List of geo unrestricted countries (similar
 456                     to _GEO_COUNTRIES)
 457         ip_blocks:  List of geo unrestricted IP blocks in CIDR notation
 458                     (similar to _GEO_IP_BLOCKS)
 459
 460         """
 461         if not self._x_forwarded_for_ip:
 462
 463             # Geo bypass mechanism is explicitly disabled by user
 464             if not self._downloader.params.get('geo_bypass', True):
 465                 return
 466
 467             if not geo_bypass_context:
 468                 geo_bypass_context = {}
 469
 470             # Backward compatibility: previously _initialize_geo_bypass
 471             # expected a list of countries, some 3rd party code may still use
 472             # it this way
 473             if isinstance(geo_bypass_context, (list, tuple)):
 474                 geo_bypass_context = {
 475                     'countries': geo_bypass_context,
 476                 }
 477
 478             # The whole point of geo bypass mechanism is to fake IP
 479             # as X-Forwarded-For HTTP header based on some IP block or
 480             # country code.
 481
 482             # Path 1: bypassing based on IP block in CIDR notation
 483
 484             # Explicit IP block specified by user, use it right away
 485             # regardless of whether extractor is geo bypassable or not
 486             ip_block = self._downloader.params.get('geo_bypass_ip_block', None)
 487
 488             # Otherwise use random IP block from geo bypass context but only
 489             # if extractor is known as geo bypassable
 490             if not ip_block:
 491                 ip_blocks = geo_bypass_context.get('ip_blocks')
 492                 if self._GEO_BYPASS and ip_blocks:
 493                     ip_block = random.choice(ip_blocks)
 494
 495             if ip_block:
 496                 self._x_forwarded_for_ip = GeoUtils.random_ipv4(ip_block)
 497                 if self._downloader.params.get('verbose', False):
 498                     self._downloader.to_screen(
 499                         '[debug] Using fake IP %s as X-Forwarded-For.'
 500                         % self._x_forwarded_for_ip)
 501                 return
 502
 503             # Path 2: bypassing based on country code
 504
 505             # Explicit country code specified by user, use it right away
 506             # regardless of whether extractor is geo bypassable or not
 507             country = self._downloader.params.get('geo_bypass_country', None)
 508
 509             # Otherwise use random country code from geo bypass context but
 510             # only if extractor is known as geo bypassable
 511             if not country:
 512                 countries = geo_bypass_context.get('countries')
 513                 if self._GEO_BYPASS and countries:
 514                     country = random.choice(countries)
 515
 516             if country:
 517                 self._x_forwarded_for_ip = GeoUtils.random_ipv4(country)
 518                 if self._downloader.params.get('verbose', False):
 519                     self._downloader.to_screen(
 520                         '[debug] Using fake IP %s (%s) as X-Forwarded-For.'
 521                         % (self._x_forwarded_for_ip, country.upper()))
 522
 523     def extract(self, url):
 524         """Extracts URL information and returns it in list of dicts."""
 525         try:
 526             for _ in range(2):
 527                 try:
 528                     self.initialize()
 529                     ie_result = self._real_extract(url)
 530                     if self._x_forwarded_for_ip:
 531                         ie_result['__x_forwarded_for_ip'] = self._x_forwarded_for_ip
 532                     return ie_result
 533                 except GeoRestrictedError as e:
 534                     if self.__maybe_fake_ip_and_retry(e.countries):
 535                         continue
 536                     raise
 537         except ExtractorError:
 538             raise
 539         except compat_http_client.IncompleteRead as e:
 540             raise ExtractorError('A network error has occurred.', cause=e, expected=True)
 541         except (KeyError, StopIteration) as e:
 542             raise ExtractorError('An extractor error has occurred.', cause=e)
 543
 544     def __maybe_fake_ip_and_retry(self, countries):
 545         if (not self._downloader.params.get('geo_bypass_country', None) and
 546                 self._GEO_BYPASS and
 547                 self._downloader.params.get('geo_bypass', True) and
 548                 not self._x_forwarded_for_ip and
 549                 countries):
 550             country_code = random.choice(countries)
 551             self._x_forwarded_for_ip = GeoUtils.random_ipv4(country_code)
 552             if self._x_forwarded_for_ip:
 553                 self.report_warning(
 554                     'Video is geo restricted. Retrying extraction with fake IP %s (%s) as X-Forwarded-For.'
 555                     % (self._x_forwarded_for_ip, country_code.upper()))
 556                 return True
 557         return False
 558
 559     def set_downloader(self, downloader):
 560         """Sets the downloader for this IE."""
 561         self._downloader = downloader
 562
 563     def _real_initialize(self):
 564         """Real initialization process. Redefine in subclasses."""
 565         pass
 566
 567     def _real_extract(self, url):
 568         """Real extraction process. Redefine in subclasses."""
 569         pass
 570
 571     @classmethod
 572     def ie_key(cls):
 573         """A string for getting the InfoExtractor with get_info_extractor"""
 574         return compat_str(cls.__name__[:-2])
 575
 576     @property
 577     def IE_NAME(self):
 578         return compat_str(type(self).__name__[:-2])
 579
 580     @staticmethod
 581     def __can_accept_status_code(err, expected_status):
 582         assert isinstance(err, compat_urllib_error.HTTPError)
 583         if expected_status is None:
 584             return False
 585         if isinstance(expected_status, compat_integer_types):
 586             return err.code == expected_status
 587         elif isinstance(expected_status, (list, tuple)):
 588             return err.code in expected_status
 589         elif callable(expected_status):
 590             return expected_status(err.code) is True
 591         else:
 592             assert False
 593
 594     def _request_webpage(self, url_or_request, video_id, note=None, errnote=None, fatal=True, data=None, headers={}, query={}, expected_status=None):
 595         """
 596         Return the response handle.
 597
 598         See _download_webpage docstring for arguments specification.
 599         """
 600         if note is None:
 601             self.report_download_webpage(video_id)
 602         elif note is not False:
 603             if video_id is None:
 604                 self.to_screen('%s' % (note,))
 605             else:
 606                 self.to_screen('%s: %s' % (video_id, note))
 607
 608         # Some sites check X-Forwarded-For HTTP header in order to figure out
 609         # the origin of the client behind proxy. This allows bypassing geo
 610         # restriction by faking this header's value to IP that belongs to some
 611         # geo unrestricted country. We will do so once we encounter any
 612         # geo restriction error.
 613         if self._x_forwarded_for_ip:
 614             if 'X-Forwarded-For' not in headers:
 615                 headers['X-Forwarded-For'] = self._x_forwarded_for_ip
 616
 617         if isinstance(url_or_request, compat_urllib_request.Request):
 618             url_or_request = update_Request(
 619                 url_or_request, data=data, headers=headers, query=query)
 620         else:
 621             if query:
 622                 url_or_request = update_url_query(url_or_request, query)
 623             if data is not None or headers:
 624                 url_or_request = sanitized_Request(url_or_request, data, headers)
 625         try:
 626             return self._downloader.urlopen(url_or_request)
 627         except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
 628             if isinstance(err, compat_urllib_error.HTTPError):
 629                 if self.__can_accept_status_code(err, expected_status):
 630                     # Retain reference to error to prevent file object from
 631                     # being closed before it can be read. Works around the
 632                     # effects of <https://bugs.python.org/issue15002>
 633                     # introduced in Python 3.4.1.
 634                     err.fp._error = err
 635                     return err.fp
 636
 637             if errnote is False:
 638                 return False
 639             if errnote is None:
 640                 errnote = 'Unable to download webpage'
 641
 642             errmsg = '%s: %s' % (errnote, error_to_compat_str(err))
 643             if fatal:
 644                 raise ExtractorError(errmsg, sys.exc_info()[2], cause=err)
 645             else:
 646                 self._downloader.report_warning(errmsg)
 647                 return False
 648
 649     def _download_webpage_handle(self, url_or_request, video_id, note=None, errnote=None, fatal=True, encoding=None, data=None, headers={}, query={}, expected_status=None):
 650         """
 651         Return a tuple (page content as string, URL handle).
 652
 653         See _download_webpage docstring for arguments specification.
 654         """
 655         # Strip hashes from the URL (#1038)
 656         if isinstance(url_or_request, (compat_str, str)):
 657             url_or_request = url_or_request.partition('#')[0]
 658
 659         urlh = self._request_webpage(url_or_request, video_id, note, errnote, fatal, data=data, headers=headers, query=query, expected_status=expected_status)
 660         if urlh is False:
 661             assert not fatal
 662             return False
 663         content = self._webpage_read_content(urlh, url_or_request, video_id, note, errnote, fatal, encoding=encoding)
 664         return (content, urlh)
 665
 666     @staticmethod
 667     def _guess_encoding_from_content(content_type, webpage_bytes):
 668         m = re.match(r'[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+\s*;\s*charset=(.+)', content_type)
 669         if m:
 670             encoding = m.group(1)
 671         else:
 672             m = re.search(br'<meta[^>]+charset=[\'"]?([^\'")]+)[ /\'">]',
 673                           webpage_bytes[:1024])
 674             if m:
 675                 encoding = m.group(1).decode('ascii')
 676             elif webpage_bytes.startswith(b'\xff\xfe'):
 677                 encoding = 'utf-16'
 678             else:
 679                 encoding = 'utf-8'
 680
 681         return encoding
 682
 683     def __check_blocked(self, content):
 684         first_block = content[:512]
 685         if ('<title>Access to this site is blocked</title>' in content and
 686                 'Websense' in first_block):
 687             msg = 'Access to this webpage has been blocked by Websense filtering software in your network.'
 688             blocked_iframe = self._html_search_regex(
 689                 r'<iframe src="([^"]+)"', content,
 690                 'Websense information URL', default=None)
 691             if blocked_iframe:
 692                 msg += ' Visit %s for more details' % blocked_iframe
 693             raise ExtractorError(msg, expected=True)
 694         if '<title>The URL you requested has been blocked</title>' in first_block:
 695             msg = (
 696                 'Access to this webpage has been blocked by Indian censorship. '
 697                 'Use a VPN or proxy server (with --proxy) to route around it.')
 698             block_msg = self._html_search_regex(
 699                 r'</h1><p>(.*?)</p>',
 700                 content, 'block message', default=None)
 701             if block_msg:
 702                 msg += ' (Message: "%s")' % block_msg.replace('\n', ' ')
 703             raise ExtractorError(msg, expected=True)
 704         if ('<title>TTK :: Доступ к ресурсу ограничен</title>' in content and
 705                 'blocklist.rkn.gov.ru' in content):
 706             raise ExtractorError(
 707                 'Access to this webpage has been blocked by decision of the Russian government. '
 708                 'Visit http://blocklist.rkn.gov.ru/ for a block reason.',
 709                 expected=True)
 710
 711     def _webpage_read_content(self, urlh, url_or_request, video_id, note=None, errnote=None, fatal=True, prefix=None, encoding=None):
 712         content_type = urlh.headers.get('Content-Type', '')
 713         webpage_bytes = urlh.read()
 714         if prefix is not None:
 715             webpage_bytes = prefix + webpage_bytes
 716         if not encoding:
 717             encoding = self._guess_encoding_from_content(content_type, webpage_bytes)
 718         if self._downloader.params.get('dump_intermediate_pages', False):
 719             self.to_screen('Dumping request to ' + urlh.geturl())
 720             dump = base64.b64encode(webpage_bytes).decode('ascii')
 721             self._downloader.to_screen(dump)
 722         if self._downloader.params.get('write_pages', False):
 723             basen = '%s_%s' % (video_id, urlh.geturl())
 724             if len(basen) > 240:
 725                 h = '___' + hashlib.md5(basen.encode('utf-8')).hexdigest()
 726                 basen = basen[:240 - len(h)] + h
 727             raw_filename = basen + '.dump'
 728             filename = sanitize_filename(raw_filename, restricted=True)
 729             self.to_screen('Saving request to ' + filename)
 730             # Working around MAX_PATH limitation on Windows (see
 731             # http://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx)
 732             if compat_os_name == 'nt':
 733                 absfilepath = os.path.abspath(filename)
 734                 if len(absfilepath) > 259:
 735                     filename = '\\\\?\\' + absfilepath
 736             with open(filename, 'wb') as outf:
 737                 outf.write(webpage_bytes)
 738
 739         try:
 740             content = webpage_bytes.decode(encoding, 'replace')
 741         except LookupError:
 742             content = webpage_bytes.decode('utf-8', 'replace')
 743
 744         self.__check_blocked(content)
 745
 746         return content
 747
 748     def _download_webpage(
 749             self, url_or_request, video_id, note=None, errnote=None,
 750             fatal=True, tries=1, timeout=5, encoding=None, data=None,
 751             headers={}, query={}, expected_status=None):
 752         """
 753         Return the data of the page as a string.
 754
 755         Arguments:
 756         url_or_request -- plain text URL as a string or
 757             a compat_urllib_request.Requestobject
 758         video_id -- Video/playlist/item identifier (string)
 759
 760         Keyword arguments:
 761         note -- note printed before downloading (string)
 762         errnote -- note printed in case of an error (string)
 763         fatal -- flag denoting whether error should be considered fatal,
 764             i.e. whether it should cause ExtractionError to be raised,
 765             otherwise a warning will be reported and extraction continued
 766         tries -- number of tries
 767         timeout -- sleep interval between tries
 768         encoding -- encoding for a page content decoding, guessed automatically
 769             when not explicitly specified
 770         data -- POST data (bytes)
 771         headers -- HTTP headers (dict)
 772         query -- URL query (dict)
 773         expected_status -- allows to accept failed HTTP requests (non 2xx
 774             status code) by explicitly specifying a set of accepted status
 775             codes. Can be any of the following entities:
 776                 - an integer type specifying an exact failed status code to
 777                   accept
 778                 - a list or a tuple of integer types specifying a list of
 779                   failed status codes to accept
 780                 - a callable accepting an actual failed status code and
 781                   returning True if it should be accepted
 782             Note that this argument does not affect success status codes (2xx)
 783             which are always accepted.
 784         """
 785
 786         success = False
 787         try_count = 0
 788         while success is False:
 789             try:
 790                 res = self._download_webpage_handle(
 791                     url_or_request, video_id, note, errnote, fatal,
 792                     encoding=encoding, data=data, headers=headers, query=query,
 793                     expected_status=expected_status)
 794                 success = True
 795             except compat_http_client.IncompleteRead as e:
 796                 try_count += 1
 797                 if try_count >= tries:
 798                     raise e
 799                 self._sleep(timeout, video_id)
 800         if res is False:
 801             return res
 802         else:
 803             content, _ = res
 804             return content
 805
 806     def _download_xml_handle(
 807             self, url_or_request, video_id, note='Downloading XML',
 808             errnote='Unable to download XML', transform_source=None,
 809             fatal=True, encoding=None, data=None, headers={}, query={},
 810             expected_status=None):
 811         """
 812         Return a tuple (xml as an compat_etree_Element, URL handle).
 813
 814         See _download_webpage docstring for arguments specification.
 815         """
 816         res = self._download_webpage_handle(
 817             url_or_request, video_id, note, errnote, fatal=fatal,
 818             encoding=encoding, data=data, headers=headers, query=query,
 819             expected_status=expected_status)
 820         if res is False:
 821             return res
 822         xml_string, urlh = res
 823         return self._parse_xml(
 824             xml_string, video_id, transform_source=transform_source,
 825             fatal=fatal), urlh
 826
 827     def _download_xml(
 828             self, url_or_request, video_id,
 829             note='Downloading XML', errnote='Unable to download XML',
 830             transform_source=None, fatal=True, encoding=None,
 831             data=None, headers={}, query={}, expected_status=None):
 832         """
 833         Return the xml as an compat_etree_Element.
 834
 835         See _download_webpage docstring for arguments specification.
 836         """
 837         res = self._download_xml_handle(
 838             url_or_request, video_id, note=note, errnote=errnote,
 839             transform_source=transform_source, fatal=fatal, encoding=encoding,
 840             data=data, headers=headers, query=query,
 841             expected_status=expected_status)
 842         return res if res is False else res[0]
 843
 844     def _parse_xml(self, xml_string, video_id, transform_source=None, fatal=True):
 845         if transform_source:
 846             xml_string = transform_source(xml_string)
 847         try:
 848             return compat_etree_fromstring(xml_string.encode('utf-8'))
 849         except compat_xml_parse_error as ve:
 850             errmsg = '%s: Failed to parse XML ' % video_id
 851             if fatal:
 852                 raise ExtractorError(errmsg, cause=ve)
 853             else:
 854                 self.report_warning(errmsg + str(ve))
 855
 856     def _download_json_handle(
 857             self, url_or_request, video_id, note='Downloading JSON metadata',
 858             errnote='Unable to download JSON metadata', transform_source=None,
 859             fatal=True, encoding=None, data=None, headers={}, query={},
 860             expected_status=None):
 861         """
 862         Return a tuple (JSON object, URL handle).
 863
 864         See _download_webpage docstring for arguments specification.
 865         """
 866         res = self._download_webpage_handle(
 867             url_or_request, video_id, note, errnote, fatal=fatal,
 868             encoding=encoding, data=data, headers=headers, query=query,
 869             expected_status=expected_status)
 870         if res is False:
 871             return res
 872         json_string, urlh = res
 873         return self._parse_json(
 874             json_string, video_id, transform_source=transform_source,
 875             fatal=fatal), urlh
 876
 877     def _download_json(
 878             self, url_or_request, video_id, note='Downloading JSON metadata',
 879             errnote='Unable to download JSON metadata', transform_source=None,
 880             fatal=True, encoding=None, data=None, headers={}, query={},
 881             expected_status=None):
 882         """
 883         Return the JSON object as a dict.
 884
 885         See _download_webpage docstring for arguments specification.
 886         """
 887         res = self._download_json_handle(
 888             url_or_request, video_id, note=note, errnote=errnote,
 889             transform_source=transform_source, fatal=fatal, encoding=encoding,
 890             data=data, headers=headers, query=query,
 891             expected_status=expected_status)
 892         return res if res is False else res[0]
 893
 894     def _parse_json(self, json_string, video_id, transform_source=None, fatal=True):
 895         if transform_source:
 896             json_string = transform_source(json_string)
 897         try:
 898             return json.loads(json_string)
 899         except ValueError as ve:
 900             errmsg = '%s: Failed to parse JSON ' % video_id
 901             if fatal:
 902                 raise ExtractorError(errmsg, cause=ve)
 903             else:
 904                 self.report_warning(errmsg + str(ve))
 905
 906     def report_warning(self, msg, video_id=None):
 907         idstr = '' if video_id is None else '%s: ' % video_id
 908         self._downloader.report_warning(
 909             '[%s] %s%s' % (self.IE_NAME, idstr, msg))
 910
 911     def to_screen(self, msg):
 912         """Print msg to screen, prefixing it with '[ie_name]'"""
 913         self._downloader.to_screen('[%s] %s' % (self.IE_NAME, msg))
 914
 915     def report_extraction(self, id_or_name):
 916         """Report information extraction."""
 917         self.to_screen('%s: Extracting information' % id_or_name)
 918
 919     def report_download_webpage(self, video_id):
 920         """Report webpage download."""
 921         self.to_screen('%s: Downloading webpage' % video_id)
 922
 923     def report_age_confirmation(self):
 924         """Report attempt to confirm age."""
 925         self.to_screen('Confirming age')
 926
 927     def report_login(self):
 928         """Report attempt to log in."""
 929         self.to_screen('Logging in')
 930
 931     @staticmethod
 932     def raise_login_required(msg='This video is only available for registered users'):
 933         raise ExtractorError(
 934             '%s. Use --username and --password or --netrc to provide account credentials.' % msg,
 935             expected=True)
 936
 937     @staticmethod
 938     def raise_geo_restricted(msg='This video is not available from your location due to geo restriction', countries=None):
 939         raise GeoRestrictedError(msg, countries=countries)
 940
 941     # Methods for following #608
 942     @staticmethod
 943     def url_result(url, ie=None, video_id=None, video_title=None):
 944         """Returns a URL that points to a page that should be processed"""
 945         # TODO: ie should be the class used for getting the info
 946         video_info = {'_type': 'url',
 947                       'url': url,
 948                       'ie_key': ie}
 949         if video_id is not None:
 950             video_info['id'] = video_id
 951         if video_title is not None:
 952             video_info['title'] = video_title
 953         return video_info
 954
 955     def playlist_from_matches(self, matches, playlist_id=None, playlist_title=None, getter=None, ie=None):
 956         urls = orderedSet(
 957             self.url_result(self._proto_relative_url(getter(m) if getter else m), ie)
 958             for m in matches)
 959         return self.playlist_result(
 960             urls, playlist_id=playlist_id, playlist_title=playlist_title)
 961
 962     @staticmethod
 963     def playlist_result(entries, playlist_id=None, playlist_title=None, playlist_description=None):
 964         """Returns a playlist"""
 965         video_info = {'_type': 'playlist',
 966                       'entries': entries}
 967         if playlist_id:
 968             video_info['id'] = playlist_id
 969         if playlist_title:
 970             video_info['title'] = playlist_title
 971         if playlist_description:
 972             video_info['description'] = playlist_description
 973         return video_info
 974
 975     def _search_regex(self, pattern, string, name, default=NO_DEFAULT, fatal=True, flags=0, group=None):
 976         """
 977         Perform a regex search on the given string, using a single or a list of
 978         patterns returning the first matching group.
 979         In case of failure return a default value or raise a WARNING or a
 980         RegexNotFoundError, depending on fatal, specifying the field name.
 981         """
 982         if isinstance(pattern, (str, compat_str, compiled_regex_type)):
 983             mobj = re.search(pattern, string, flags)
 984         else:
 985             for p in pattern:
 986                 mobj = re.search(p, string, flags)
 987                 if mobj:
 988                     break
 989
 990         if not self._downloader.params.get('no_color') and compat_os_name != 'nt' and sys.stderr.isatty():
 991             _name = '\033[0;34m%s\033[0m' % name
 992         else:
 993             _name = name
 994
 995         if mobj:
 996             if group is None:
 997                 # return the first matching group
 998                 return next(g for g in mobj.groups() if g is not None)
 999             else:
1000                 return mobj.group(group)
1001         elif default is not NO_DEFAULT:
1002             return default
1003         elif fatal:
1004             raise RegexNotFoundError('Unable to extract %s' % _name)
1005         else:
1006             self._downloader.report_warning('unable to extract %s' % _name + bug_reports_message())
1007             return None
1008
1009     def _html_search_regex(self, pattern, string, name, default=NO_DEFAULT, fatal=True, flags=0, group=None):
1010         """
1011         Like _search_regex, but strips HTML tags and unescapes entities.
1012         """
1013         res = self._search_regex(pattern, string, name, default, fatal, flags, group)
1014         if res:
1015             return clean_html(res).strip()
1016         else:
1017             return res
1018
1019     def _get_netrc_login_info(self, netrc_machine=None):
1020         username = None
1021         password = None
1022         netrc_machine = netrc_machine or self._NETRC_MACHINE
1023
1024         if self._downloader.params.get('usenetrc', False):
1025             try:
1026                 info = netrc.netrc().authenticators(netrc_machine)
1027                 if info is not None:
1028                     username = info[0]
1029                     password = info[2]
1030                 else:
1031                     raise netrc.NetrcParseError(
1032                         'No authenticators for %s' % netrc_machine)
1033             except (IOError, netrc.NetrcParseError) as err:
1034                 self._downloader.report_warning(
1035                     'parsing .netrc: %s' % error_to_compat_str(err))
1036
1037         return username, password
1038
1039     def _get_login_info(self, username_option='username', password_option='password', netrc_machine=None):
1040         """
1041         Get the login info as (username, password)
1042         First look for the manually specified credentials using username_option
1043         and password_option as keys in params dictionary. If no such credentials
1044         available look in the netrc file using the netrc_machine or _NETRC_MACHINE
1045         value.
1046         If there's no info available, return (None, None)
1047         """
1048         if self._downloader is None:
1049             return (None, None)
1050
1051         downloader_params = self._downloader.params
1052
1053         # Attempt to use provided username and password or .netrc data
1054         if downloader_params.get(username_option) is not None:
1055             username = downloader_params[username_option]
1056             password = downloader_params[password_option]
1057         else:
1058             username, password = self._get_netrc_login_info(netrc_machine)
1059
1060         return username, password
1061
1062     def _get_tfa_info(self, note='two-factor verification code'):
1063         """
1064         Get the two-factor authentication info
1065         TODO - asking the user will be required for sms/phone verify
1066         currently just uses the command line option
1067         If there's no info available, return None
1068         """
1069         if self._downloader is None:
1070             return None
1071         downloader_params = self._downloader.params
1072
1073         if downloader_params.get('twofactor') is not None:
1074             return downloader_params['twofactor']
1075
1076         return compat_getpass('Type %s and press [Return]: ' % note)
1077
1078     # Helper functions for extracting OpenGraph info
1079     @staticmethod
1080     def _og_regexes(prop):
1081         content_re = r'content=(?:"([^"]+?)"|\'([^\']+?)\'|\s*([^\s"\'=<>`]+?))'
1082         property_re = (r'(?:name|property)=(?:\'og[:-]%(prop)s\'|"og[:-]%(prop)s"|\s*og[:-]%(prop)s\b)'
1083                        % {'prop': re.escape(prop)})
1084         template = r'<meta[^>]+?%s[^>]+?%s'
1085         return [
1086             template % (property_re, content_re),
1087             template % (content_re, property_re),
1088         ]
1089
1090     @staticmethod
1091     def _meta_regex(prop):
1092         return r'''(?isx)<meta
1093                     (?=[^>]+(?:itemprop|name|property|id|http-equiv)=(["\']?)%s\1)
1094                     [^>]+?content=(["\'])(?P<content>.*?)\2''' % re.escape(prop)
1095
1096     def _og_search_property(self, prop, html, name=None, **kargs):
1097         if not isinstance(prop, (list, tuple)):
1098             prop = [prop]
1099         if name is None:
1100             name = 'OpenGraph %s' % prop[0]
1101         og_regexes = []
1102         for p in prop:
1103             og_regexes.extend(self._og_regexes(p))
1104         escaped = self._search_regex(og_regexes, html, name, flags=re.DOTALL, **kargs)
1105         if escaped is None:
1106             return None
1107         return unescapeHTML(escaped)
1108
1109     def _og_search_thumbnail(self, html, **kargs):
1110         return self._og_search_property('image', html, 'thumbnail URL', fatal=False, **kargs)
1111
1112     def _og_search_description(self, html, **kargs):
1113         return self._og_search_property('description', html, fatal=False, **kargs)
1114
1115     def _og_search_title(self, html, **kargs):
1116         return self._og_search_property('title', html, **kargs)
1117
1118     def _og_search_video_url(self, html, name='video url', secure=True, **kargs):
1119         regexes = self._og_regexes('video') + self._og_regexes('video:url')
1120         if secure:
1121             regexes = self._og_regexes('video:secure_url') + regexes
1122         return self._html_search_regex(regexes, html, name, **kargs)
1123
1124     def _og_search_url(self, html, **kargs):
1125         return self._og_search_property('url', html, **kargs)
1126
1127     def _html_search_meta(self, name, html, display_name=None, fatal=False, **kwargs):
1128         if not isinstance(name, (list, tuple)):
1129             name = [name]
1130         if display_name is None:
1131             display_name = name[0]
1132         return self._html_search_regex(
1133             [self._meta_regex(n) for n in name],
1134             html, display_name, fatal=fatal, group='content', **kwargs)
1135
1136     def _dc_search_uploader(self, html):
1137         return self._html_search_meta('dc.creator', html, 'uploader')
1138
1139     def _rta_search(self, html):
1140         # See http://www.rtalabel.org/index.php?content=howtofaq#single
1141         if re.search(r'(?ix)<meta\s+name="rating"\s+'
1142                      r'     content="RTA-5042-1996-1400-1577-RTA"',
1143                      html):
1144             return 18
1145         return 0
1146
1147     def _media_rating_search(self, html):
1148         # See http://www.tjg-designs.com/WP/metadata-code-examples-adding-metadata-to-your-web-pages/
1149         rating = self._html_search_meta('rating', html)
1150
1151         if not rating:
1152             return None
1153
1154         RATING_TABLE = {
1155             'safe for kids': 0,
1156             'general': 8,
1157             '14 years': 14,
1158             'mature': 17,
1159             'restricted': 19,
1160         }
1161         return RATING_TABLE.get(rating.lower())
1162
1163     def _family_friendly_search(self, html):
1164         # See http://schema.org/VideoObject
1165         family_friendly = self._html_search_meta(
1166             'isFamilyFriendly', html, default=None)
1167
1168         if not family_friendly:
1169             return None
1170
1171         RATING_TABLE = {
1172             '1': 0,
1173             'true': 0,
1174             '0': 18,
1175             'false': 18,
1176         }
1177         return RATING_TABLE.get(family_friendly.lower())
1178
1179     def _twitter_search_player(self, html):
1180         return self._html_search_meta('twitter:player', html,
1181                                       'twitter card player')
1182
1183     def _search_json_ld(self, html, video_id, expected_type=None, **kwargs):
1184         json_ld = self._search_regex(
1185             JSON_LD_RE, html, 'JSON-LD', group='json_ld', **kwargs)
1186         default = kwargs.get('default', NO_DEFAULT)
1187         if not json_ld:
1188             return default if default is not NO_DEFAULT else {}
1189         # JSON-LD may be malformed and thus `fatal` should be respected.
1190         # At the same time `default` may be passed that assumes `fatal=False`
1191         # for _search_regex. Let's simulate the same behavior here as well.
1192         fatal = kwargs.get('fatal', True) if default == NO_DEFAULT else False
1193         return self._json_ld(json_ld, video_id, fatal=fatal, expected_type=expected_type)
1194
1195     def _json_ld(self, json_ld, video_id, fatal=True, expected_type=None):
1196         if isinstance(json_ld, compat_str):
1197             json_ld = self._parse_json(json_ld, video_id, fatal=fatal)
1198         if not json_ld:
1199             return {}
1200         info = {}
1201         if not isinstance(json_ld, (list, tuple, dict)):
1202             return info
1203         if isinstance(json_ld, dict):
1204             json_ld = [json_ld]
1205
1206         INTERACTION_TYPE_MAP = {
1207             'CommentAction': 'comment',
1208             'AgreeAction': 'like',
1209             'DisagreeAction': 'dislike',
1210             'LikeAction': 'like',
1211             'DislikeAction': 'dislike',
1212             'ListenAction': 'view',
1213             'WatchAction': 'view',
1214             'ViewAction': 'view',
1215         }
1216
1217         def extract_interaction_statistic(e):
1218             interaction_statistic = e.get('interactionStatistic')
1219             if not isinstance(interaction_statistic, list):
1220                 return
1221             for is_e in interaction_statistic:
1222                 if not isinstance(is_e, dict):
1223                     continue
1224                 if is_e.get('@type') != 'InteractionCounter':
1225                     continue
1226                 interaction_type = is_e.get('interactionType')
1227                 if not isinstance(interaction_type, compat_str):
1228                     continue
1229                 interaction_count = int_or_none(is_e.get('userInteractionCount'))
1230                 if interaction_count is None:
1231                     continue
1232                 count_kind = INTERACTION_TYPE_MAP.get(interaction_type.split('/')[-1])
1233                 if not count_kind:
1234                     continue
1235                 count_key = '%s_count' % count_kind
1236                 if info.get(count_key) is not None:
1237                     continue
1238                 info[count_key] = interaction_count
1239
1240         def extract_video_object(e):
1241             assert e['@type'] == 'VideoObject'
1242             info.update({
1243                 'url': url_or_none(e.get('contentUrl')),
1244                 'title': unescapeHTML(e.get('name')),
1245                 'description': unescapeHTML(e.get('description')),
1246                 'thumbnail': url_or_none(e.get('thumbnailUrl') or e.get('thumbnailURL')),
1247                 'duration': parse_duration(e.get('duration')),
1248                 'timestamp': unified_timestamp(e.get('uploadDate')),
1249                 'filesize': float_or_none(e.get('contentSize')),
1250                 'tbr': int_or_none(e.get('bitrate')),
1251                 'width': int_or_none(e.get('width')),
1252                 'height': int_or_none(e.get('height')),
1253                 'view_count': int_or_none(e.get('interactionCount')),
1254             })
1255             extract_interaction_statistic(e)
1256
1257         for e in json_ld:
1258             if isinstance(e.get('@context'), compat_str) and re.match(r'^https?://schema.org/?$', e.get('@context')):
1259                 item_type = e.get('@type')
1260                 if expected_type is not None and expected_type != item_type:
1261                     return info
1262                 if item_type in ('TVEpisode', 'Episode'):
1263                     episode_name = unescapeHTML(e.get('name'))
1264                     info.update({
1265                         'episode': episode_name,
1266                         'episode_number': int_or_none(e.get('episodeNumber')),
1267                         'description': unescapeHTML(e.get('description')),
1268                     })
1269                     if not info.get('title') and episode_name:
1270                         info['title'] = episode_name
1271                     part_of_season = e.get('partOfSeason')
1272                     if isinstance(part_of_season, dict) and part_of_season.get('@type') in ('TVSeason', 'Season', 'CreativeWorkSeason'):
1273                         info.update({
1274                             'season': unescapeHTML(part_of_season.get('name')),
1275                             'season_number': int_or_none(part_of_season.get('seasonNumber')),
1276                         })
1277                     part_of_series = e.get('partOfSeries') or e.get('partOfTVSeries')
1278                     if isinstance(part_of_series, dict) and part_of_series.get('@type') in ('TVSeries', 'Series', 'CreativeWorkSeries'):
1279                         info['series'] = unescapeHTML(part_of_series.get('name'))
1280                 elif item_type == 'Movie':
1281                     info.update({
1282                         'title': unescapeHTML(e.get('name')),
1283                         'description': unescapeHTML(e.get('description')),
1284                         'duration': parse_duration(e.get('duration')),
1285                         'timestamp': unified_timestamp(e.get('dateCreated')),
1286                     })
1287                 elif item_type in ('Article', 'NewsArticle'):
1288                     info.update({
1289                         'timestamp': parse_iso8601(e.get('datePublished')),
1290                         'title': unescapeHTML(e.get('headline')),
1291                         'description': unescapeHTML(e.get('articleBody')),
1292                     })
1293                 elif item_type == 'VideoObject':
1294                     extract_video_object(e)
1295                     continue
1296                 video = e.get('video')
1297                 if isinstance(video, dict) and video.get('@type') == 'VideoObject':
1298                     extract_video_object(video)
1299                 break
1300         return dict((k, v) for k, v in info.items() if v is not None)
1301
1302     @staticmethod
1303     def _hidden_inputs(html):
1304         html = re.sub(r'<!--(?:(?!<!--).)*-->', '', html)
1305         hidden_inputs = {}
1306         for input in re.findall(r'(?i)(<input[^>]+>)', html):
1307             attrs = extract_attributes(input)
1308             if not input:
1309                 continue
1310             if attrs.get('type') not in ('hidden', 'submit'):
1311                 continue
1312             name = attrs.get('name') or attrs.get('id')
1313             value = attrs.get('value')
1314             if name and value is not None:
1315                 hidden_inputs[name] = value
1316         return hidden_inputs
1317
1318     def _form_hidden_inputs(self, form_id, html):
1319         form = self._search_regex(
1320             r'(?is)<form[^>]+?id=(["\'])%s\1[^>]*>(?P<form>.+?)</form>' % form_id,
1321             html, '%s form' % form_id, group='form')
1322         return self._hidden_inputs(form)
1323
1324     def _sort_formats(self, formats, field_preference=None):
1325         if not formats:
1326             raise ExtractorError('No video formats found')
1327
1328         for f in formats:
1329             # Automatically determine tbr when missing based on abr and vbr (improves
1330             # formats sorting in some cases)
1331             if 'tbr' not in f and f.get('abr') is not None and f.get('vbr') is not None:
1332                 f['tbr'] = f['abr'] + f['vbr']
1333
1334         def _formats_key(f):
1335             # TODO remove the following workaround
1336             from ..utils import determine_ext
1337             if not f.get('ext') and 'url' in f:
1338                 f['ext'] = determine_ext(f['url'])
1339
1340             if isinstance(field_preference, (list, tuple)):
1341                 return tuple(
1342                     f.get(field)
1343                     if f.get(field) is not None
1344                     else ('' if field == 'format_id' else -1)
1345                     for field in field_preference)
1346
1347             preference = f.get('preference')
1348             if preference is None:
1349                 preference = 0
1350                 if f.get('ext') in ['f4f', 'f4m']:  # Not yet supported
1351                     preference -= 0.5
1352
1353             protocol = f.get('protocol') or determine_protocol(f)
1354             proto_preference = 0 if protocol in ['http', 'https'] else (-0.5 if protocol == 'rtsp' else -0.1)
1355
1356             if f.get('vcodec') == 'none':  # audio only
1357                 preference -= 50
1358                 if self._downloader.params.get('prefer_free_formats'):
1359                     ORDER = ['aac', 'mp3', 'm4a', 'webm', 'ogg', 'opus']
1360                 else:
1361                     ORDER = ['webm', 'opus', 'ogg', 'mp3', 'aac', 'm4a']
1362                 ext_preference = 0
1363                 try:
1364                     audio_ext_preference = ORDER.index(f['ext'])
1365                 except ValueError:
1366                     audio_ext_preference = -1
1367             else:
1368                 if f.get('acodec') == 'none':  # video only
1369                     preference -= 40
1370                 if self._downloader.params.get('prefer_free_formats'):
1371                     ORDER = ['flv', 'mp4', 'webm']
1372                 else:
1373                     ORDER = ['webm', 'flv', 'mp4']
1374                 try:
1375                     ext_preference = ORDER.index(f['ext'])
1376                 except ValueError:
1377                     ext_preference = -1
1378                 audio_ext_preference = 0
1379
1380             return (
1381                 preference,
1382                 f.get('language_preference') if f.get('language_preference') is not None else -1,
1383                 f.get('quality') if f.get('quality') is not None else -1,
1384                 f.get('tbr') if f.get('tbr') is not None else -1,
1385                 f.get('filesize') if f.get('filesize') is not None else -1,
1386                 f.get('vbr') if f.get('vbr') is not None else -1,
1387                 f.get('height') if f.get('height') is not None else -1,
1388                 f.get('width') if f.get('width') is not None else -1,
1389                 proto_preference,
1390                 ext_preference,
1391                 f.get('abr') if f.get('abr') is not None else -1,
1392                 audio_ext_preference,
1393                 f.get('fps') if f.get('fps') is not None else -1,
1394                 f.get('filesize_approx') if f.get('filesize_approx') is not None else -1,
1395                 f.get('source_preference') if f.get('source_preference') is not None else -1,
1396                 f.get('format_id') if f.get('format_id') is not None else '',
1397             )
1398         formats.sort(key=_formats_key)
1399
1400     def _check_formats(self, formats, video_id):
1401         if formats:
1402             formats[:] = filter(
1403                 lambda f: self._is_valid_url(
1404                     f['url'], video_id,
1405                     item='%s video format' % f.get('format_id') if f.get('format_id') else 'video'),
1406                 formats)
1407
1408     @staticmethod
1409     def _remove_duplicate_formats(formats):
1410         format_urls = set()
1411         unique_formats = []
1412         for f in formats:
1413             if f['url'] not in format_urls:
1414                 format_urls.add(f['url'])
1415                 unique_formats.append(f)
1416         formats[:] = unique_formats
1417
1418     def _is_valid_url(self, url, video_id, item='video', headers={}):
1419         url = self._proto_relative_url(url, scheme='http:')
1420         # For now assume non HTTP(S) URLs always valid
1421         if not (url.startswith('http://') or url.startswith('https://')):
1422             return True
1423         try:
1424             self._request_webpage(url, video_id, 'Checking %s URL' % item, headers=headers)
1425             return True
1426         except ExtractorError as e:
1427             if isinstance(e.cause, compat_urllib_error.URLError):
1428                 self.to_screen(
1429                     '%s: %s URL is invalid, skipping' % (video_id, item))
1430                 return False
1431             raise
1432
1433     def http_scheme(self):
1434         """ Either "http:" or "https:", depending on the user's preferences """
1435         return (
1436             'http:'
1437             if self._downloader.params.get('prefer_insecure', False)
1438             else 'https:')
1439
1440     def _proto_relative_url(self, url, scheme=None):
1441         if url is None:
1442             return url
1443         if url.startswith('//'):
1444             if scheme is None:
1445                 scheme = self.http_scheme()
1446             return scheme + url
1447         else:
1448             return url
1449
1450     def _sleep(self, timeout, video_id, msg_template=None):
1451         if msg_template is None:
1452             msg_template = '%(video_id)s: Waiting for %(timeout)s seconds'
1453         msg = msg_template % {'video_id': video_id, 'timeout': timeout}
1454         self.to_screen(msg)
1455         time.sleep(timeout)
1456
1457     def _extract_f4m_formats(self, manifest_url, video_id, preference=None, f4m_id=None,
1458                              transform_source=lambda s: fix_xml_ampersands(s).strip(),
1459                              fatal=True, m3u8_id=None):
1460         manifest = self._download_xml(
1461             manifest_url, video_id, 'Downloading f4m manifest',
1462             'Unable to download f4m manifest',
1463             # Some manifests may be malformed, e.g. prosiebensat1 generated manifests
1464             # (see https://github.com/ytdl-org/youtube-dl/issues/6215#issuecomment-121704244)
1465             transform_source=transform_source,
1466             fatal=fatal)
1467
1468         if manifest is False:
1469             return []
1470
1471         return self._parse_f4m_formats(
1472             manifest, manifest_url, video_id, preference=preference, f4m_id=f4m_id,
1473             transform_source=transform_source, fatal=fatal, m3u8_id=m3u8_id)
1474
1475     def _parse_f4m_formats(self, manifest, manifest_url, video_id, preference=None, f4m_id=None,
1476                            transform_source=lambda s: fix_xml_ampersands(s).strip(),
1477                            fatal=True, m3u8_id=None):
1478         if not isinstance(manifest, compat_etree_Element) and not fatal:
1479             return []
1480
1481         # currently youtube-dl cannot decode the playerVerificationChallenge as Akamai uses Adobe Alchemy
1482         akamai_pv = manifest.find('{http://ns.adobe.com/f4m/1.0}pv-2.0')
1483         if akamai_pv is not None and ';' in akamai_pv.text:
1484             playerVerificationChallenge = akamai_pv.text.split(';')[0]
1485             if playerVerificationChallenge.strip() != '':
1486                 return []
1487
1488         formats = []
1489         manifest_version = '1.0'
1490         media_nodes = manifest.findall('{http://ns.adobe.com/f4m/1.0}media')
1491         if not media_nodes:
1492             manifest_version = '2.0'
1493             media_nodes = manifest.findall('{http://ns.adobe.com/f4m/2.0}media')
1494         # Remove unsupported DRM protected media from final formats
1495         # rendition (see https://github.com/ytdl-org/youtube-dl/issues/8573).
1496         media_nodes = remove_encrypted_media(media_nodes)
1497         if not media_nodes:
1498             return formats
1499
1500         manifest_base_url = get_base_url(manifest)
1501
1502         bootstrap_info = xpath_element(
1503             manifest, ['{http://ns.adobe.com/f4m/1.0}bootstrapInfo', '{http://ns.adobe.com/f4m/2.0}bootstrapInfo'],
1504             'bootstrap info', default=None)
1505
1506         vcodec = None
1507         mime_type = xpath_text(
1508             manifest, ['{http://ns.adobe.com/f4m/1.0}mimeType', '{http://ns.adobe.com/f4m/2.0}mimeType'],
1509             'base URL', default=None)
1510         if mime_type and mime_type.startswith('audio/'):
1511             vcodec = 'none'
1512
1513         for i, media_el in enumerate(media_nodes):
1514             tbr = int_or_none(media_el.attrib.get('bitrate'))
1515             width = int_or_none(media_el.attrib.get('width'))
1516             height = int_or_none(media_el.attrib.get('height'))
1517             format_id = '-'.join(filter(None, [f4m_id, compat_str(i if tbr is None else tbr)]))
1518             # If <bootstrapInfo> is present, the specified f4m is a
1519             # stream-level manifest, and only set-level manifests may refer to
1520             # external resources.  See section 11.4 and section 4 of F4M spec
1521             if bootstrap_info is None:
1522                 media_url = None
1523                 # @href is introduced in 2.0, see section 11.6 of F4M spec
1524                 if manifest_version == '2.0':
1525                     media_url = media_el.attrib.get('href')
1526                 if media_url is None:
1527                     media_url = media_el.attrib.get('url')
1528                 if not media_url:
1529                     continue
1530                 manifest_url = (
1531                     media_url if media_url.startswith('http://') or media_url.startswith('https://')
1532                     else ((manifest_base_url or '/'.join(manifest_url.split('/')[:-1])) + '/' + media_url))
1533                 # If media_url is itself a f4m manifest do the recursive extraction
1534                 # since bitrates in parent manifest (this one) and media_url manifest
1535                 # may differ leading to inability to resolve the format by requested
1536                 # bitrate in f4m downloader
1537                 ext = determine_ext(manifest_url)
1538                 if ext == 'f4m':
1539                     f4m_formats = self._extract_f4m_formats(
1540                         manifest_url, video_id, preference=preference, f4m_id=f4m_id,
1541                         transform_source=transform_source, fatal=fatal)
1542                     # Sometimes stream-level manifest contains single media entry that
1543                     # does not contain any quality metadata (e.g. http://matchtv.ru/#live-player).
1544                     # At the same time parent's media entry in set-level manifest may
1545                     # contain it. We will copy it from parent in such cases.
1546                     if len(f4m_formats) == 1:
1547                         f = f4m_formats[0]
1548                         f.update({
1549                             'tbr': f.get('tbr') or tbr,
1550                             'width': f.get('width') or width,
1551                             'height': f.get('height') or height,
1552                             'format_id': f.get('format_id') if not tbr else format_id,
1553                             'vcodec': vcodec,
1554                         })
1555                     formats.extend(f4m_formats)
1556                     continue
1557                 elif ext == 'm3u8':
1558                     formats.extend(self._extract_m3u8_formats(
1559                         manifest_url, video_id, 'mp4', preference=preference,
1560                         m3u8_id=m3u8_id, fatal=fatal))
1561                     continue
1562             formats.append({
1563                 'format_id': format_id,
1564                 'url': manifest_url,
1565                 'manifest_url': manifest_url,
1566                 'ext': 'flv' if bootstrap_info is not None else None,
1567                 'protocol': 'f4m',
1568                 'tbr': tbr,
1569                 'width': width,
1570                 'height': height,
1571                 'vcodec': vcodec,
1572                 'preference': preference,
1573             })
1574         return formats
1575
1576     def _m3u8_meta_format(self, m3u8_url, ext=None, preference=None, m3u8_id=None):
1577         return {
1578             'format_id': '-'.join(filter(None, [m3u8_id, 'meta'])),
1579             'url': m3u8_url,
1580             'ext': ext,
1581             'protocol': 'm3u8',
1582             'preference': preference - 100 if preference else -100,
1583             'resolution': 'multiple',
1584             'format_note': 'Quality selection URL',
1585         }
1586
1587     def _extract_m3u8_formats(self, m3u8_url, video_id, ext=None,
1588                               entry_protocol='m3u8', preference=None,
1589                               m3u8_id=None, note=None, errnote=None,
1590                               fatal=True, live=False):
1591         res = self._download_webpage_handle(
1592             m3u8_url, video_id,
1593             note=note or 'Downloading m3u8 information',
1594             errnote=errnote or 'Failed to download m3u8 information',
1595             fatal=fatal)
1596
1597         if res is False:
1598             return []
1599
1600         m3u8_doc, urlh = res
1601         m3u8_url = urlh.geturl()
1602
1603         return self._parse_m3u8_formats(
1604             m3u8_doc, m3u8_url, ext=ext, entry_protocol=entry_protocol,
1605             preference=preference, m3u8_id=m3u8_id, live=live)
1606
1607     def _parse_m3u8_formats(self, m3u8_doc, m3u8_url, ext=None,
1608                             entry_protocol='m3u8', preference=None,
1609                             m3u8_id=None, live=False):
1610         if '#EXT-X-FAXS-CM:' in m3u8_doc:  # Adobe Flash Access
1611             return []
1612
1613         if re.search(r'#EXT-X-SESSION-KEY:.*?URI="skd://', m3u8_doc):  # Apple FairPlay
1614             return []
1615
1616         formats = []
1617
1618         format_url = lambda u: (
1619             u
1620             if re.match(r'^https?://', u)
1621             else compat_urlparse.urljoin(m3u8_url, u))
1622
1623         # References:
1624         # 1. https://tools.ietf.org/html/draft-pantos-http-live-streaming-21
1625         # 2. https://github.com/ytdl-org/youtube-dl/issues/12211
1626         # 3. https://github.com/ytdl-org/youtube-dl/issues/18923
1627
1628         # We should try extracting formats only from master playlists [1, 4.3.4],
1629         # i.e. playlists that describe available qualities. On the other hand
1630         # media playlists [1, 4.3.3] should be returned as is since they contain
1631         # just the media without qualities renditions.
1632         # Fortunately, master playlist can be easily distinguished from media
1633         # playlist based on particular tags availability. As of [1, 4.3.3, 4.3.4]
1634         # master playlist tags MUST NOT appear in a media playist and vice versa.
1635         # As of [1, 4.3.3.1] #EXT-X-TARGETDURATION tag is REQUIRED for every
1636         # media playlist and MUST NOT appear in master playlist thus we can
1637         # clearly detect media playlist with this criterion.
1638
1639         if '#EXT-X-TARGETDURATION' in m3u8_doc:  # media playlist, return as is
1640             return [{
1641                 'url': m3u8_url,
1642                 'format_id': m3u8_id,
1643                 'ext': ext,
1644                 'protocol': entry_protocol,
1645                 'preference': preference,
1646             }]
1647
1648         groups = {}
1649         last_stream_inf = {}
1650
1651         def extract_media(x_media_line):
1652             media = parse_m3u8_attributes(x_media_line)
1653             # As per [1, 4.3.4.1] TYPE, GROUP-ID and NAME are REQUIRED
1654             media_type, group_id, name = media.get('TYPE'), media.get('GROUP-ID'), media.get('NAME')
1655             if not (media_type and group_id and name):
1656                 return
1657             groups.setdefault(group_id, []).append(media)
1658             if media_type not in ('VIDEO', 'AUDIO'):
1659                 return
1660             media_url = media.get('URI')
1661             if media_url:
1662                 format_id = []
1663                 for v in (m3u8_id, group_id, name):
1664                     if v:
1665                         format_id.append(v)
1666                 f = {
1667                     'format_id': '-'.join(format_id),
1668                     'url': format_url(media_url),
1669                     'manifest_url': m3u8_url,
1670                     'language': media.get('LANGUAGE'),
1671                     'ext': ext,
1672                     'protocol': entry_protocol,
1673                     'preference': preference,
1674                 }
1675                 if media_type == 'AUDIO':
1676                     f['vcodec'] = 'none'
1677                 formats.append(f)
1678
1679         def build_stream_name():
1680             # Despite specification does not mention NAME attribute for
1681             # EXT-X-STREAM-INF tag it still sometimes may be present (see [1]
1682             # or vidio test in TestInfoExtractor.test_parse_m3u8_formats)
1683             # 1. http://www.vidio.com/watch/165683-dj_ambred-booyah-live-2015
1684             stream_name = last_stream_inf.get('NAME')
1685             if stream_name:
1686                 return stream_name
1687             # If there is no NAME in EXT-X-STREAM-INF it will be obtained
1688             # from corresponding rendition group
1689             stream_group_id = last_stream_inf.get('VIDEO')
1690             if not stream_group_id:
1691                 return
1692             stream_group = groups.get(stream_group_id)
1693             if not stream_group:
1694                 return stream_group_id
1695             rendition = stream_group[0]
1696             return rendition.get('NAME') or stream_group_id
1697
1698         # parse EXT-X-MEDIA tags before EXT-X-STREAM-INF in order to have the
1699         # chance to detect video only formats when EXT-X-STREAM-INF tags
1700         # precede EXT-X-MEDIA tags in HLS manifest such as [3].
1701         for line in m3u8_doc.splitlines():
1702             if line.startswith('#EXT-X-MEDIA:'):
1703                 extract_media(line)
1704
1705         for line in m3u8_doc.splitlines():
1706             if line.startswith('#EXT-X-STREAM-INF:'):
1707                 last_stream_inf = parse_m3u8_attributes(line)
1708             elif line.startswith('#') or not line.strip():
1709                 continue
1710             else:
1711                 tbr = float_or_none(
1712                     last_stream_inf.get('AVERAGE-BANDWIDTH') or
1713                     last_stream_inf.get('BANDWIDTH'), scale=1000)
1714                 format_id = []
1715                 if m3u8_id:
1716                     format_id.append(m3u8_id)
1717                 stream_name = build_stream_name()
1718                 # Bandwidth of live streams may differ over time thus making
1719                 # format_id unpredictable. So it's better to keep provided
1720                 # format_id intact.
1721                 if not live:
1722                     format_id.append(stream_name if stream_name else '%d' % (tbr if tbr else len(formats)))
1723                 manifest_url = format_url(line.strip())
1724                 f = {
1725                     'format_id': '-'.join(format_id),
1726                     'url': manifest_url,
1727                     'manifest_url': m3u8_url,
1728                     'tbr': tbr,
1729                     'ext': ext,
1730                     'fps': float_or_none(last_stream_inf.get('FRAME-RATE')),
1731                     'protocol': entry_protocol,
1732                     'preference': preference,
1733                 }
1734                 resolution = last_stream_inf.get('RESOLUTION')
1735                 if resolution:
1736                     mobj = re.search(r'(?P<width>\d+)[xX](?P<height>\d+)', resolution)
1737                     if mobj:
1738                         f['width'] = int(mobj.group('width'))
1739                         f['height'] = int(mobj.group('height'))
1740                 # Unified Streaming Platform
1741                 mobj = re.search(
1742                     r'audio.*?(?:%3D|=)(\d+)(?:-video.*?(?:%3D|=)(\d+))?', f['url'])
1743                 if mobj:
1744                     abr, vbr = mobj.groups()
1745                     abr, vbr = float_or_none(abr, 1000), float_or_none(vbr, 1000)
1746                     f.update({
1747                         'vbr': vbr,
1748                         'abr': abr,
1749                     })
1750                 codecs = parse_codecs(last_stream_inf.get('CODECS'))
1751                 f.update(codecs)
1752                 audio_group_id = last_stream_inf.get('AUDIO')
1753                 # As per [1, 4.3.4.1.1] any EXT-X-STREAM-INF tag which
1754                 # references a rendition group MUST have a CODECS attribute.
1755                 # However, this is not always respected, for example, [2]
1756                 # contains EXT-X-STREAM-INF tag which references AUDIO
1757                 # rendition group but does not have CODECS and despite
1758                 # referencing an audio group it represents a complete
1759                 # (with audio and video) format. So, for such cases we will
1760                 # ignore references to rendition groups and treat them
1761                 # as complete formats.
1762                 if audio_group_id and codecs and f.get('vcodec') != 'none':
1763                     audio_group = groups.get(audio_group_id)
1764                     if audio_group and audio_group[0].get('URI'):
1765                         # TODO: update acodec for audio only formats with
1766                         # the same GROUP-ID
1767                         f['acodec'] = 'none'
1768                 formats.append(f)
1769                 last_stream_inf = {}
1770         return formats
1771
1772     @staticmethod
1773     def _xpath_ns(path, namespace=None):
1774         if not namespace:
1775             return path
1776         out = []
1777         for c in path.split('/'):
1778             if not c or c == '.':
1779                 out.append(c)
1780             else:
1781                 out.append('{%s}%s' % (namespace, c))
1782         return '/'.join(out)
1783
1784     def _extract_smil_formats(self, smil_url, video_id, fatal=True, f4m_params=None, transform_source=None):
1785         smil = self._download_smil(smil_url, video_id, fatal=fatal, transform_source=transform_source)
1786
1787         if smil is False:
1788             assert not fatal
1789             return []
1790
1791         namespace = self._parse_smil_namespace(smil)
1792
1793         return self._parse_smil_formats(
1794             smil, smil_url, video_id, namespace=namespace, f4m_params=f4m_params)
1795
1796     def _extract_smil_info(self, smil_url, video_id, fatal=True, f4m_params=None):
1797         smil = self._download_smil(smil_url, video_id, fatal=fatal)
1798         if smil is False:
1799             return {}
1800         return self._parse_smil(smil, smil_url, video_id, f4m_params=f4m_params)
1801
1802     def _download_smil(self, smil_url, video_id, fatal=True, transform_source=None):
1803         return self._download_xml(
1804             smil_url, video_id, 'Downloading SMIL file',
1805             'Unable to download SMIL file', fatal=fatal, transform_source=transform_source)
1806
1807     def _parse_smil(self, smil, smil_url, video_id, f4m_params=None):
1808         namespace = self._parse_smil_namespace(smil)
1809
1810         formats = self._parse_smil_formats(
1811             smil, smil_url, video_id, namespace=namespace, f4m_params=f4m_params)
1812         subtitles = self._parse_smil_subtitles(smil, namespace=namespace)
1813
1814         video_id = os.path.splitext(url_basename(smil_url))[0]
1815         title = None
1816         description = None
1817         upload_date = None
1818         for meta in smil.findall(self._xpath_ns('./head/meta', namespace)):
1819             name = meta.attrib.get('name')
1820             content = meta.attrib.get('content')
1821             if not name or not content:
1822                 continue
1823             if not title and name == 'title':
1824                 title = content
1825             elif not description and name in ('description', 'abstract'):
1826                 description = content
1827             elif not upload_date and name == 'date':
1828                 upload_date = unified_strdate(content)
1829
1830         thumbnails = [{
1831             'id': image.get('type'),
1832             'url': image.get('src'),
1833             'width': int_or_none(image.get('width')),
1834             'height': int_or_none(image.get('height')),
1835         } for image in smil.findall(self._xpath_ns('.//image', namespace)) if image.get('src')]
1836
1837         return {
1838             'id': video_id,
1839             'title': title or video_id,
1840             'description': description,
1841             'upload_date': upload_date,
1842             'thumbnails': thumbnails,
1843             'formats': formats,
1844             'subtitles': subtitles,
1845         }
1846
1847     def _parse_smil_namespace(self, smil):
1848         return self._search_regex(
1849             r'(?i)^{([^}]+)?}smil$', smil.tag, 'namespace', default=None)
1850
1851     def _parse_smil_formats(self, smil, smil_url, video_id, namespace=None, f4m_params=None, transform_rtmp_url=None):
1852         base = smil_url
1853         for meta in smil.findall(self._xpath_ns('./head/meta', namespace)):
1854             b = meta.get('base') or meta.get('httpBase')
1855             if b:
1856                 base = b
1857                 break
1858
1859         formats = []
1860         rtmp_count = 0
1861         http_count = 0
1862         m3u8_count = 0
1863
1864         srcs = []
1865         media = smil.findall(self._xpath_ns('.//video', namespace)) + smil.findall(self._xpath_ns('.//audio', namespace))
1866         for medium in media:
1867             src = medium.get('src')
1868             if not src or src in srcs:
1869                 continue
1870             srcs.append(src)
1871
1872             bitrate = float_or_none(medium.get('system-bitrate') or medium.get('systemBitrate'), 1000)
1873             filesize = int_or_none(medium.get('size') or medium.get('fileSize'))
1874             width = int_or_none(medium.get('width'))
1875             height = int_or_none(medium.get('height'))
1876             proto = medium.get('proto')
1877             ext = medium.get('ext')
1878             src_ext = determine_ext(src)
1879             streamer = medium.get('streamer') or base
1880
1881             if proto == 'rtmp' or streamer.startswith('rtmp'):
1882                 rtmp_count += 1
1883                 formats.append({
1884                     'url': streamer,
1885                     'play_path': src,
1886                     'ext': 'flv',
1887                     'format_id': 'rtmp-%d' % (rtmp_count if bitrate is None else bitrate),
1888                     'tbr': bitrate,
1889                     'filesize': filesize,
1890                     'width': width,
1891                     'height': height,
1892                 })
1893                 if transform_rtmp_url:
1894                     streamer, src = transform_rtmp_url(streamer, src)
1895                     formats[-1].update({
1896                         'url': streamer,
1897                         'play_path': src,
1898                     })
1899                 continue
1900
1901             src_url = src if src.startswith('http') else compat_urlparse.urljoin(base, src)
1902             src_url = src_url.strip()
1903
1904             if proto == 'm3u8' or src_ext == 'm3u8':
1905                 m3u8_formats = self._extract_m3u8_formats(
1906                     src_url, video_id, ext or 'mp4', m3u8_id='hls', fatal=False)
1907                 if len(m3u8_formats) == 1:
1908                     m3u8_count += 1
1909                     m3u8_formats[0].update({
1910                         'format_id': 'hls-%d' % (m3u8_count if bitrate is None else bitrate),
1911                         'tbr': bitrate,
1912                         'width': width,
1913                         'height': height,
1914                     })
1915                 formats.extend(m3u8_formats)
1916             elif src_ext == 'f4m':
1917                 f4m_url = src_url
1918                 if not f4m_params:
1919                     f4m_params = {
1920                         'hdcore': '3.2.0',
1921                         'plugin': 'flowplayer-3.2.0.1',
1922                     }
1923                 f4m_url += '&' if '?' in f4m_url else '?'
1924                 f4m_url += compat_urllib_parse_urlencode(f4m_params)
1925                 formats.extend(self._extract_f4m_formats(f4m_url, video_id, f4m_id='hds', fatal=False))
1926             elif src_ext == 'mpd':
1927                 formats.extend(self._extract_mpd_formats(
1928                     src_url, video_id, mpd_id='dash', fatal=False))
1929             elif re.search(r'\.ism/[Mm]anifest', src_url):
1930                 formats.extend(self._extract_ism_formats(
1931                     src_url, video_id, ism_id='mss', fatal=False))
1932             elif src_url.startswith('http') and self._is_valid_url(src, video_id):
1933                 http_count += 1
1934                 formats.append({
1935                     'url': src_url,
1936                     'ext': ext or src_ext or 'flv',
1937                     'format_id': 'http-%d' % (bitrate or http_count),
1938                     'tbr': bitrate,
1939                     'filesize': filesize,
1940                     'width': width,
1941                     'height': height,
1942                 })
1943
1944         return formats
1945
1946     def _parse_smil_subtitles(self, smil, namespace=None, subtitles_lang='en'):
1947         urls = []
1948         subtitles = {}
1949         for num, textstream in enumerate(smil.findall(self._xpath_ns('.//textstream', namespace))):
1950             src = textstream.get('src')
1951             if not src or src in urls:
1952                 continue
1953             urls.append(src)
1954             ext = textstream.get('ext') or mimetype2ext(textstream.get('type')) or determine_ext(src)
1955             lang = textstream.get('systemLanguage') or textstream.get('systemLanguageName') or textstream.get('lang') or subtitles_lang
1956             subtitles.setdefault(lang, []).append({
1957                 'url': src,
1958                 'ext': ext,
1959             })
1960         return subtitles
1961
1962     def _extract_xspf_playlist(self, xspf_url, playlist_id, fatal=True):
1963         xspf = self._download_xml(
1964             xspf_url, playlist_id, 'Downloading xpsf playlist',
1965             'Unable to download xspf manifest', fatal=fatal)
1966         if xspf is False:
1967             return []
1968         return self._parse_xspf(
1969             xspf, playlist_id, xspf_url=xspf_url,
1970             xspf_base_url=base_url(xspf_url))
1971
1972     def _parse_xspf(self, xspf_doc, playlist_id, xspf_url=None, xspf_base_url=None):
1973         NS_MAP = {
1974             'xspf': 'http://xspf.org/ns/0/',
1975             's1': 'http://static.streamone.nl/player/ns/0',
1976         }
1977
1978         entries = []
1979         for track in xspf_doc.findall(xpath_with_ns('./xspf:trackList/xspf:track', NS_MAP)):
1980             title = xpath_text(
1981                 track, xpath_with_ns('./xspf:title', NS_MAP), 'title', default=playlist_id)
1982             description = xpath_text(
1983                 track, xpath_with_ns('./xspf:annotation', NS_MAP), 'description')
1984             thumbnail = xpath_text(
1985                 track, xpath_with_ns('./xspf:image', NS_MAP), 'thumbnail')
1986             duration = float_or_none(
1987                 xpath_text(track, xpath_with_ns('./xspf:duration', NS_MAP), 'duration'), 1000)
1988
1989             formats = []
1990             for location in track.findall(xpath_with_ns('./xspf:location', NS_MAP)):
1991                 format_url = urljoin(xspf_base_url, location.text)
1992                 if not format_url:
1993                     continue
1994                 formats.append({
1995                     'url': format_url,
1996                     'manifest_url': xspf_url,
1997                     'format_id': location.get(xpath_with_ns('s1:label', NS_MAP)),
1998                     'width': int_or_none(location.get(xpath_with_ns('s1:width', NS_MAP))),
1999                     'height': int_or_none(location.get(xpath_with_ns('s1:height', NS_MAP))),
2000                 })
2001             self._sort_formats(formats)
2002
2003             entries.append({
2004                 'id': playlist_id,
2005                 'title': title,
2006                 'description': description,
2007                 'thumbnail': thumbnail,
2008                 'duration': duration,
2009                 'formats': formats,
2010             })
2011         return entries
2012
2013     def _extract_mpd_formats(self, mpd_url, video_id, mpd_id=None, note=None, errnote=None, fatal=True, formats_dict={}):
2014         res = self._download_xml_handle(
2015             mpd_url, video_id,
2016             note=note or 'Downloading MPD manifest',
2017             errnote=errnote or 'Failed to download MPD manifest',
2018             fatal=fatal)
2019         if res is False:
2020             return []
2021         mpd_doc, urlh = res
2022         mpd_base_url = base_url(urlh.geturl())
2023
2024         return self._parse_mpd_formats(
2025             mpd_doc, mpd_id=mpd_id, mpd_base_url=mpd_base_url,
2026             formats_dict=formats_dict, mpd_url=mpd_url)
2027
2028     def _parse_mpd_formats(self, mpd_doc, mpd_id=None, mpd_base_url='', formats_dict={}, mpd_url=None):
2029         """
2030         Parse formats from MPD manifest.
2031         References:
2032          1. MPEG-DASH Standard, ISO/IEC 23009-1:2014(E),
2033             http://standards.iso.org/ittf/PubliclyAvailableStandards/c065274_ISO_IEC_23009-1_2014.zip
2034          2. https://en.wikipedia.org/wiki/Dynamic_Adaptive_Streaming_over_HTTP
2035         """
2036         if mpd_doc.get('type') == 'dynamic':
2037             return []
2038
2039         namespace = self._search_regex(r'(?i)^{([^}]+)?}MPD$', mpd_doc.tag, 'namespace', default=None)
2040
2041         def _add_ns(path):
2042             return self._xpath_ns(path, namespace)
2043
2044         def is_drm_protected(element):
2045             return element.find(_add_ns('ContentProtection')) is not None
2046
2047         def extract_multisegment_info(element, ms_parent_info):
2048             ms_info = ms_parent_info.copy()
2049
2050             # As per [1, 5.3.9.2.2] SegmentList and SegmentTemplate share some
2051             # common attributes and elements.  We will only extract relevant
2052             # for us.
2053             def extract_common(source):
2054                 segment_timeline = source.find(_add_ns('SegmentTimeline'))
2055                 if segment_timeline is not None:
2056                     s_e = segment_timeline.findall(_add_ns('S'))
2057                     if s_e:
2058                         ms_info['total_number'] = 0
2059                         ms_info['s'] = []
2060                         for s in s_e:
2061                             r = int(s.get('r', 0))
2062                             ms_info['total_number'] += 1 + r
2063                             ms_info['s'].append({
2064                                 't': int(s.get('t', 0)),
2065                                 # @d is mandatory (see [1, 5.3.9.6.2, Table 17, page 60])
2066                                 'd': int(s.attrib['d']),
2067                                 'r': r,
2068                             })
2069                 start_number = source.get('startNumber')
2070                 if start_number:
2071                     ms_info['start_number'] = int(start_number)
2072                 timescale = source.get('timescale')
2073                 if timescale:
2074                     ms_info['timescale'] = int(timescale)
2075                 segment_duration = source.get('duration')
2076                 if segment_duration:
2077                     ms_info['segment_duration'] = float(segment_duration)
2078
2079             def extract_Initialization(source):
2080                 initialization = source.find(_add_ns('Initialization'))
2081                 if initialization is not None:
2082                     ms_info['initialization_url'] = initialization.attrib['sourceURL']
2083
2084             segment_list = element.find(_add_ns('SegmentList'))
2085             if segment_list is not None:
2086                 extract_common(segment_list)
2087                 extract_Initialization(segment_list)
2088                 segment_urls_e = segment_list.findall(_add_ns('SegmentURL'))
2089                 if segment_urls_e:
2090                     ms_info['segment_urls'] = [segment.attrib['media'] for segment in segment_urls_e]
2091             else:
2092                 segment_template = element.find(_add_ns('SegmentTemplate'))
2093                 if segment_template is not None:
2094                     extract_common(segment_template)
2095                     media = segment_template.get('media')
2096                     if media:
2097                         ms_info['media'] = media
2098                     initialization = segment_template.get('initialization')
2099                     if initialization:
2100                         ms_info['initialization'] = initialization
2101                     else:
2102                         extract_Initialization(segment_template)
2103             return ms_info
2104
2105         mpd_duration = parse_duration(mpd_doc.get('mediaPresentationDuration'))
2106         formats = []
2107         for period in mpd_doc.findall(_add_ns('Period')):
2108             period_duration = parse_duration(period.get('duration')) or mpd_duration
2109             period_ms_info = extract_multisegment_info(period, {
2110                 'start_number': 1,
2111                 'timescale': 1,
2112             })
2113             for adaptation_set in period.findall(_add_ns('AdaptationSet')):
2114                 if is_drm_protected(adaptation_set):
2115                     continue
2116                 adaption_set_ms_info = extract_multisegment_info(adaptation_set, period_ms_info)
2117                 for representation in adaptation_set.findall(_add_ns('Representation')):
2118                     if is_drm_protected(representation):
2119                         continue
2120                     representation_attrib = adaptation_set.attrib.copy()
2121                     representation_attrib.update(representation.attrib)
2122                     # According to [1, 5.3.7.2, Table 9, page 41], @mimeType is mandatory
2123                     mime_type = representation_attrib['mimeType']
2124                     content_type = mime_type.split('/')[0]
2125                     if content_type == 'text':
2126                         # TODO implement WebVTT downloading
2127                         pass
2128                     elif content_type in ('video', 'audio'):
2129                         base_url = ''
2130                         for element in (representation, adaptation_set, period, mpd_doc):
2131                             base_url_e = element.find(_add_ns('BaseURL'))
2132                             if base_url_e is not None:
2133                                 base_url = base_url_e.text + base_url
2134                                 if re.match(r'^https?://', base_url):
2135                                     break
2136                         if mpd_base_url and not re.match(r'^https?://', base_url):
2137                             if not mpd_base_url.endswith('/') and not base_url.startswith('/'):
2138                                 mpd_base_url += '/'
2139                             base_url = mpd_base_url + base_url
2140                         representation_id = representation_attrib.get('id')
2141                         lang = representation_attrib.get('lang')
2142                         url_el = representation.find(_add_ns('BaseURL'))
2143                         filesize = int_or_none(url_el.attrib.get('{http://youtube.com/yt/2012/10/10}contentLength') if url_el is not None else None)
2144                         bandwidth = int_or_none(representation_attrib.get('bandwidth'))
2145                         f = {
2146                             'format_id': '%s-%s' % (mpd_id, representation_id) if mpd_id else representation_id,
2147                             'manifest_url': mpd_url,
2148                             'ext': mimetype2ext(mime_type),
2149                             'width': int_or_none(representation_attrib.get('width')),
2150                             'height': int_or_none(representation_attrib.get('height')),
2151                             'tbr': float_or_none(bandwidth, 1000),
2152                             'asr': int_or_none(representation_attrib.get('audioSamplingRate')),
2153                             'fps': int_or_none(representation_attrib.get('frameRate')),
2154                             'language': lang if lang not in ('mul', 'und', 'zxx', 'mis') else None,
2155                             'format_note': 'DASH %s' % content_type,
2156                             'filesize': filesize,
2157                             'container': mimetype2ext(mime_type) + '_dash',
2158                         }
2159                         f.update(parse_codecs(representation_attrib.get('codecs')))
2160                         representation_ms_info = extract_multisegment_info(representation, adaption_set_ms_info)
2161
2162                         def prepare_template(template_name, identifiers):
2163                             tmpl = representation_ms_info[template_name]
2164                             # First of, % characters outside $...$ templates
2165                             # must be escaped by doubling for proper processing
2166                             # by % operator string formatting used further (see
2167                             # https://github.com/ytdl-org/youtube-dl/issues/16867).
2168                             t = ''
2169                             in_template = False
2170                             for c in tmpl:
2171                                 t += c
2172                                 if c == '$':
2173                                     in_template = not in_template
2174                                 elif c == '%' and not in_template:
2175                                     t += c
2176                             # Next, $...$ templates are translated to their
2177                             # %(...) counterparts to be used with % operator
2178                             t = t.replace('$RepresentationID$', representation_id)
2179                             t = re.sub(r'\$(%s)\$' % '|'.join(identifiers), r'%(\1)d', t)
2180                             t = re.sub(r'\$(%s)%%([^$]+)\$' % '|'.join(identifiers), r'%(\1)\2', t)
2181                             t.replace('$$', '$')
2182                             return t
2183
2184                         # @initialization is a regular template like @media one
2185                         # so it should be handled just the same way (see
2186                         # https://github.com/ytdl-org/youtube-dl/issues/11605)
2187                         if 'initialization' in representation_ms_info:
2188                             initialization_template = prepare_template(
2189                                 'initialization',
2190                                 # As per [1, 5.3.9.4.2, Table 15, page 54] $Number$ and
2191                                 # $Time$ shall not be included for @initialization thus
2192                                 # only $Bandwidth$ remains
2193                                 ('Bandwidth', ))
2194                             representation_ms_info['initialization_url'] = initialization_template % {
2195                                 'Bandwidth': bandwidth,
2196                             }
2197
2198                         def location_key(location):
2199                             return 'url' if re.match(r'^https?://', location) else 'path'
2200
2201                         if 'segment_urls' not in representation_ms_info and 'media' in representation_ms_info:
2202
2203                             media_template = prepare_template('media', ('Number', 'Bandwidth', 'Time'))
2204                             media_location_key = location_key(media_template)
2205
2206                             # As per [1, 5.3.9.4.4, Table 16, page 55] $Number$ and $Time$
2207                             # can't be used at the same time
2208                             if '%(Number' in media_template and 's' not in representation_ms_info:
2209                                 segment_duration = None
2210                                 if 'total_number' not in representation_ms_info and 'segment_duration' in representation_ms_info:
2211                                     segment_duration = float_or_none(representation_ms_info['segment_duration'], representation_ms_info['timescale'])
2212                                     representation_ms_info['total_number'] = int(math.ceil(float(period_duration) / segment_duration))
2213                                 representation_ms_info['fragments'] = [{
2214                                     media_location_key: media_template % {
2215                                         'Number': segment_number,
2216                                         'Bandwidth': bandwidth,
2217                                     },
2218                                     'duration': segment_duration,
2219                                 } for segment_number in range(
2220                                     representation_ms_info['start_number'],
2221                                     representation_ms_info['total_number'] + representation_ms_info['start_number'])]
2222                             else:
2223                                 # $Number*$ or $Time$ in media template with S list available
2224                                 # Example $Number*$: http://www.svtplay.se/klipp/9023742/stopptid-om-bjorn-borg
2225                                 # Example $Time$: https://play.arkena.com/embed/avp/v2/player/media/b41dda37-d8e7-4d3f-b1b5-9a9db578bdfe/1/129411
2226                                 representation_ms_info['fragments'] = []
2227                                 segment_time = 0
2228                                 segment_d = None
2229                                 segment_number = representation_ms_info['start_number']
2230
2231                                 def add_segment_url():
2232                                     segment_url = media_template % {
2233                                         'Time': segment_time,
2234                                         'Bandwidth': bandwidth,
2235                                         'Number': segment_number,
2236                                     }
2237                                     representation_ms_info['fragments'].append({
2238                                         media_location_key: segment_url,
2239                                         'duration': float_or_none(segment_d, representation_ms_info['timescale']),
2240                                     })
2241
2242                                 for num, s in enumerate(representation_ms_info['s']):
2243                                     segment_time = s.get('t') or segment_time
2244                                     segment_d = s['d']
2245                                     add_segment_url()
2246                                     segment_number += 1
2247                                     for r in range(s.get('r', 0)):
2248                                         segment_time += segment_d
2249                                         add_segment_url()
2250                                         segment_number += 1
2251                                     segment_time += segment_d
2252                         elif 'segment_urls' in representation_ms_info and 's' in representation_ms_info:
2253                             # No media template
2254                             # Example: https://www.youtube.com/watch?v=iXZV5uAYMJI
2255                             # or any YouTube dashsegments video
2256                             fragments = []
2257                             segment_index = 0
2258                             timescale = representation_ms_info['timescale']
2259                             for s in representation_ms_info['s']:
2260                                 duration = float_or_none(s['d'], timescale)
2261                                 for r in range(s.get('r', 0) + 1):
2262                                     segment_uri = representation_ms_info['segment_urls'][segment_index]
2263                                     fragments.append({
2264                                         location_key(segment_uri): segment_uri,
2265                                         'duration': duration,
2266                                     })
2267                                     segment_index += 1
2268                             representation_ms_info['fragments'] = fragments
2269                         elif 'segment_urls' in representation_ms_info:
2270                             # Segment URLs with no SegmentTimeline
2271                             # Example: https://www.seznam.cz/zpravy/clanek/cesko-zasahne-vitr-o-sile-vichrice-muze-byt-i-zivotu-nebezpecny-39091
2272                             # https://github.com/ytdl-org/youtube-dl/pull/14844
2273                             fragments = []
2274                             segment_duration = float_or_none(
2275                                 representation_ms_info['segment_duration'],
2276                                 representation_ms_info['timescale']) if 'segment_duration' in representation_ms_info else None
2277                             for segment_url in representation_ms_info['segment_urls']:
2278                                 fragment = {
2279                                     location_key(segment_url): segment_url,
2280                                 }
2281                                 if segment_duration:
2282                                     fragment['duration'] = segment_duration
2283                                 fragments.append(fragment)
2284                             representation_ms_info['fragments'] = fragments
2285                         # If there is a fragments key available then we correctly recognized fragmented media.
2286                         # Otherwise we will assume unfragmented media with direct access. Technically, such
2287                         # assumption is not necessarily correct since we may simply have no support for
2288                         # some forms of fragmented media renditions yet, but for now we'll use this fallback.
2289                         if 'fragments' in representation_ms_info:
2290                             f.update({
2291                                 # NB: mpd_url may be empty when MPD manifest is parsed from a string
2292                                 'url': mpd_url or base_url,
2293                                 'fragment_base_url': base_url,
2294                                 'fragments': [],
2295                                 'protocol': 'http_dash_segments',
2296                             })
2297                             if 'initialization_url' in representation_ms_info:
2298                                 initialization_url = representation_ms_info['initialization_url']
2299                                 if not f.get('url'):
2300                                     f['url'] = initialization_url
2301                                 f['fragments'].append({location_key(initialization_url): initialization_url})
2302                             f['fragments'].extend(representation_ms_info['fragments'])
2303                         else:
2304                             # Assuming direct URL to unfragmented media.
2305                             f['url'] = base_url
2306
2307                         # According to [1, 5.3.5.2, Table 7, page 35] @id of Representation
2308                         # is not necessarily unique within a Period thus formats with
2309                         # the same `format_id` are quite possible. There are numerous examples
2310                         # of such manifests (see https://github.com/ytdl-org/youtube-dl/issues/15111,
2311                         # https://github.com/ytdl-org/youtube-dl/issues/13919)
2312                         full_info = formats_dict.get(representation_id, {}).copy()
2313                         full_info.update(f)
2314                         formats.append(full_info)
2315                     else:
2316                         self.report_warning('Unknown MIME type %s in DASH manifest' % mime_type)
2317         return formats
2318
2319     def _extract_ism_formats(self, ism_url, video_id, ism_id=None, note=None, errnote=None, fatal=True):
2320         res = self._download_xml_handle(
2321             ism_url, video_id,
2322             note=note or 'Downloading ISM manifest',
2323             errnote=errnote or 'Failed to download ISM manifest',
2324             fatal=fatal)
2325         if res is False:
2326             return []
2327         ism_doc, urlh = res
2328
2329         return self._parse_ism_formats(ism_doc, urlh.geturl(), ism_id)
2330
2331     def _parse_ism_formats(self, ism_doc, ism_url, ism_id=None):
2332         """
2333         Parse formats from ISM manifest.
2334         References:
2335          1. [MS-SSTR]: Smooth Streaming Protocol,
2336             https://msdn.microsoft.com/en-us/library/ff469518.aspx
2337         """
2338         if ism_doc.get('IsLive') == 'TRUE' or ism_doc.find('Protection') is not None:
2339             return []
2340
2341         duration = int(ism_doc.attrib['Duration'])
2342         timescale = int_or_none(ism_doc.get('TimeScale')) or 10000000
2343
2344         formats = []
2345         for stream in ism_doc.findall('StreamIndex'):
2346             stream_type = stream.get('Type')
2347             if stream_type not in ('video', 'audio'):
2348                 continue
2349             url_pattern = stream.attrib['Url']
2350             stream_timescale = int_or_none(stream.get('TimeScale')) or timescale
2351             stream_name = stream.get('Name')
2352             for track in stream.findall('QualityLevel'):
2353                 fourcc = track.get('FourCC', 'AACL' if track.get('AudioTag') == '255' else None)
2354                 # TODO: add support for WVC1 and WMAP
2355                 if fourcc not in ('H264', 'AVC1', 'AACL'):
2356                     self.report_warning('%s is not a supported codec' % fourcc)
2357                     continue
2358                 tbr = int(track.attrib['Bitrate']) // 1000
2359                 # [1] does not mention Width and Height attributes. However,
2360                 # they're often present while MaxWidth and MaxHeight are
2361                 # missing, so should be used as fallbacks
2362                 width = int_or_none(track.get('MaxWidth') or track.get('Width'))
2363                 height = int_or_none(track.get('MaxHeight') or track.get('Height'))
2364                 sampling_rate = int_or_none(track.get('SamplingRate'))
2365
2366                 track_url_pattern = re.sub(r'{[Bb]itrate}', track.attrib['Bitrate'], url_pattern)
2367                 track_url_pattern = compat_urlparse.urljoin(ism_url, track_url_pattern)
2368
2369                 fragments = []
2370                 fragment_ctx = {
2371                     'time': 0,
2372                 }
2373                 stream_fragments = stream.findall('c')
2374                 for stream_fragment_index, stream_fragment in enumerate(stream_fragments):
2375                     fragment_ctx['time'] = int_or_none(stream_fragment.get('t')) or fragment_ctx['time']
2376                     fragment_repeat = int_or_none(stream_fragment.get('r')) or 1
2377                     fragment_ctx['duration'] = int_or_none(stream_fragment.get('d'))
2378                     if not fragment_ctx['duration']:
2379                         try:
2380                             next_fragment_time = int(stream_fragment[stream_fragment_index + 1].attrib['t'])
2381                         except IndexError:
2382                             next_fragment_time = duration
2383                         fragment_ctx['duration'] = (next_fragment_time - fragment_ctx['time']) / fragment_repeat
2384                     for _ in range(fragment_repeat):
2385                         fragments.append({
2386                             'url': re.sub(r'{start[ _]time}', compat_str(fragment_ctx['time']), track_url_pattern),
2387                             'duration': fragment_ctx['duration'] / stream_timescale,
2388                         })
2389                         fragment_ctx['time'] += fragment_ctx['duration']
2390
2391                 format_id = []
2392                 if ism_id:
2393                     format_id.append(ism_id)
2394                 if stream_name:
2395                     format_id.append(stream_name)
2396                 format_id.append(compat_str(tbr))
2397
2398                 formats.append({
2399                     'format_id': '-'.join(format_id),
2400                     'url': ism_url,
2401                     'manifest_url': ism_url,
2402                     'ext': 'ismv' if stream_type == 'video' else 'isma',
2403                     'width': width,
2404                     'height': height,
2405                     'tbr': tbr,
2406                     'asr': sampling_rate,
2407                     'vcodec': 'none' if stream_type == 'audio' else fourcc,
2408                     'acodec': 'none' if stream_type == 'video' else fourcc,
2409                     'protocol': 'ism',
2410                     'fragments': fragments,
2411                     '_download_params': {
2412                         'duration': duration,
2413                         'timescale': stream_timescale,
2414                         'width': width or 0,
2415                         'height': height or 0,
2416                         'fourcc': fourcc,
2417                         'codec_private_data': track.get('CodecPrivateData'),
2418                         'sampling_rate': sampling_rate,
2419                         'channels': int_or_none(track.get('Channels', 2)),
2420                         'bits_per_sample': int_or_none(track.get('BitsPerSample', 16)),
2421                         'nal_unit_length_field': int_or_none(track.get('NALUnitLengthField', 4)),
2422                     },
2423                 })
2424         return formats
2425
2426     def _parse_html5_media_entries(self, base_url, webpage, video_id, m3u8_id=None, m3u8_entry_protocol='m3u8', mpd_id=None, preference=None):
2427         def absolute_url(item_url):
2428             return urljoin(base_url, item_url)
2429
2430         def parse_content_type(content_type):
2431             if not content_type:
2432                 return {}
2433             ctr = re.search(r'(?P<mimetype>[^/]+/[^;]+)(?:;\s*codecs="?(?P<codecs>[^"]+))?', content_type)
2434             if ctr:
2435                 mimetype, codecs = ctr.groups()
2436                 f = parse_codecs(codecs)
2437                 f['ext'] = mimetype2ext(mimetype)
2438                 return f
2439             return {}
2440
2441         def _media_formats(src, cur_media_type, type_info={}):
2442             full_url = absolute_url(src)
2443             ext = type_info.get('ext') or determine_ext(full_url)
2444             if ext == 'm3u8':
2445                 is_plain_url = False
2446                 formats = self._extract_m3u8_formats(
2447                     full_url, video_id, ext='mp4',
2448                     entry_protocol=m3u8_entry_protocol, m3u8_id=m3u8_id,
2449                     preference=preference, fatal=False)
2450             elif ext == 'mpd':
2451                 is_plain_url = False
2452                 formats = self._extract_mpd_formats(
2453                     full_url, video_id, mpd_id=mpd_id, fatal=False)
2454             else:
2455                 is_plain_url = True
2456                 formats = [{
2457                     'url': full_url,
2458                     'vcodec': 'none' if cur_media_type == 'audio' else None,
2459                 }]
2460             return is_plain_url, formats
2461
2462         entries = []
2463         # amp-video and amp-audio are very similar to their HTML5 counterparts
2464         # so we wll include them right here (see
2465         # https://www.ampproject.org/docs/reference/components/amp-video)
2466         media_tags = [(media_tag, media_type, '')
2467                       for media_tag, media_type
2468                       in re.findall(r'(?s)(<(?:amp-)?(video|audio)[^>]*/>)', webpage)]
2469         media_tags.extend(re.findall(
2470             # We only allow video|audio followed by a whitespace or '>'.
2471             # Allowing more characters may end up in significant slow down (see
2472             # https://github.com/ytdl-org/youtube-dl/issues/11979, example URL:
2473             # http://www.porntrex.com/maps/videositemap.xml).
2474             r'(?s)(<(?P<tag>(?:amp-)?(?:video|audio))(?:\s+[^>]*)?>)(.*?)</(?P=tag)>', webpage))
2475         for media_tag, media_type, media_content in media_tags:
2476             media_info = {
2477                 'formats': [],
2478                 'subtitles': {},
2479             }
2480             media_attributes = extract_attributes(media_tag)
2481             src = media_attributes.get('src')
2482             if src:
2483                 _, formats = _media_formats(src, media_type)
2484                 media_info['formats'].extend(formats)
2485             media_info['thumbnail'] = absolute_url(media_attributes.get('poster'))
2486             if media_content:
2487                 for source_tag in re.findall(r'<source[^>]+>', media_content):
2488                     s_attr = extract_attributes(source_tag)
2489                     # data-video-src and data-src are non standard but seen
2490                     # several times in the wild
2491                     src = dict_get(s_attr, ('src', 'data-video-src', 'data-src'))
2492                     if not src:
2493                         continue
2494                     f = parse_content_type(s_attr.get('type'))
2495                     is_plain_url, formats = _media_formats(src, media_type, f)
2496                     if is_plain_url:
2497                         # width, height, res, label and title attributes are
2498                         # all not standard but seen several times in the wild
2499                         labels = [
2500                             s_attr.get(lbl)
2501                             for lbl in ('label', 'title')
2502                             if str_or_none(s_attr.get(lbl))
2503                         ]
2504                         width = int_or_none(s_attr.get('width'))
2505                         height = (int_or_none(s_attr.get('height')) or
2506                                   int_or_none(s_attr.get('res')))
2507                         if not width or not height:
2508                             for lbl in labels:
2509                                 resolution = parse_resolution(lbl)
2510                                 if not resolution:
2511                                     continue
2512                                 width = width or resolution.get('width')
2513                                 height = height or resolution.get('height')
2514                         for lbl in labels:
2515                             tbr = parse_bitrate(lbl)
2516                             if tbr:
2517                                 break
2518                         else:
2519                             tbr = None
2520                         f.update({
2521                             'width': width,
2522                             'height': height,
2523                             'tbr': tbr,
2524                             'format_id': s_attr.get('label') or s_attr.get('title'),
2525                         })
2526                         f.update(formats[0])
2527                         media_info['formats'].append(f)
2528                     else:
2529                         media_info['formats'].extend(formats)
2530                 for track_tag in re.findall(r'<track[^>]+>', media_content):
2531                     track_attributes = extract_attributes(track_tag)
2532                     kind = track_attributes.get('kind')
2533                     if not kind or kind in ('subtitles', 'captions'):
2534                         src = track_attributes.get('src')
2535                         if not src:
2536                             continue
2537                         lang = track_attributes.get('srclang') or track_attributes.get('lang') or track_attributes.get('label')
2538                         media_info['subtitles'].setdefault(lang, []).append({
2539                             'url': absolute_url(src),
2540                         })
2541             for f in media_info['formats']:
2542                 f.setdefault('http_headers', {})['Referer'] = base_url
2543             if media_info['formats'] or media_info['subtitles']:
2544                 entries.append(media_info)
2545         return entries
2546
2547     def _extract_akamai_formats(self, manifest_url, video_id, hosts={}):
2548         formats = []
2549         hdcore_sign = 'hdcore=3.7.0'
2550         f4m_url = re.sub(r'(https?://[^/]+)/i/', r'\1/z/', manifest_url).replace('/master.m3u8', '/manifest.f4m')
2551         hds_host = hosts.get('hds')
2552         if hds_host:
2553             f4m_url = re.sub(r'(https?://)[^/]+', r'\1' + hds_host, f4m_url)
2554         if 'hdcore=' not in f4m_url:
2555             f4m_url += ('&' if '?' in f4m_url else '?') + hdcore_sign
2556         f4m_formats = self._extract_f4m_formats(
2557             f4m_url, video_id, f4m_id='hds', fatal=False)
2558         for entry in f4m_formats:
2559             entry.update({'extra_param_to_segment_url': hdcore_sign})
2560         formats.extend(f4m_formats)
2561         m3u8_url = re.sub(r'(https?://[^/]+)/z/', r'\1/i/', manifest_url).replace('/manifest.f4m', '/master.m3u8')
2562         hls_host = hosts.get('hls')
2563         if hls_host:
2564             m3u8_url = re.sub(r'(https?://)[^/]+', r'\1' + hls_host, m3u8_url)
2565         formats.extend(self._extract_m3u8_formats(
2566             m3u8_url, video_id, 'mp4', 'm3u8_native',
2567             m3u8_id='hls', fatal=False))
2568         return formats
2569
2570     def _extract_wowza_formats(self, url, video_id, m3u8_entry_protocol='m3u8_native', skip_protocols=[]):
2571         query = compat_urlparse.urlparse(url).query
2572         url = re.sub(r'/(?:manifest|playlist|jwplayer)\.(?:m3u8|f4m|mpd|smil)', '', url)
2573         mobj = re.search(
2574             r'(?:(?:http|rtmp|rtsp)(?P<s>s)?:)?(?P<url>//[^?]+)', url)
2575         url_base = mobj.group('url')
2576         http_base_url = '%s%s:%s' % ('http', mobj.group('s') or '', url_base)
2577         formats = []
2578
2579         def manifest_url(manifest):
2580             m_url = '%s/%s' % (http_base_url, manifest)
2581             if query:
2582                 m_url += '?%s' % query
2583             return m_url
2584
2585         if 'm3u8' not in skip_protocols:
2586             formats.extend(self._extract_m3u8_formats(
2587                 manifest_url('playlist.m3u8'), video_id, 'mp4',
2588                 m3u8_entry_protocol, m3u8_id='hls', fatal=False))
2589         if 'f4m' not in skip_protocols:
2590             formats.extend(self._extract_f4m_formats(
2591                 manifest_url('manifest.f4m'),
2592                 video_id, f4m_id='hds', fatal=False))
2593         if 'dash' not in skip_protocols:
2594             formats.extend(self._extract_mpd_formats(
2595                 manifest_url('manifest.mpd'),
2596                 video_id, mpd_id='dash', fatal=False))
2597         if re.search(r'(?:/smil:|\.smil)', url_base):
2598             if 'smil' not in skip_protocols:
2599                 rtmp_formats = self._extract_smil_formats(
2600                     manifest_url('jwplayer.smil'),
2601                     video_id, fatal=False)
2602                 for rtmp_format in rtmp_formats:
2603                     rtsp_format = rtmp_format.copy()
2604                     rtsp_format['url'] = '%s/%s' % (rtmp_format['url'], rtmp_format['play_path'])
2605                     del rtsp_format['play_path']
2606                     del rtsp_format['ext']
2607                     rtsp_format.update({
2608                         'url': rtsp_format['url'].replace('rtmp://', 'rtsp://'),
2609                         'format_id': rtmp_format['format_id'].replace('rtmp', 'rtsp'),
2610                         'protocol': 'rtsp',
2611                     })
2612                     formats.extend([rtmp_format, rtsp_format])
2613         else:
2614             for protocol in ('rtmp', 'rtsp'):
2615                 if protocol not in skip_protocols:
2616                     formats.append({
2617                         'url': '%s:%s' % (protocol, url_base),
2618                         'format_id': protocol,
2619                         'protocol': protocol,
2620                     })
2621         return formats
2622
2623     def _find_jwplayer_data(self, webpage, video_id=None, transform_source=js_to_json):
2624         mobj = re.search(
2625             r'(?s)jwplayer\((?P<quote>[\'"])[^\'" ]+(?P=quote)\)(?!</script>).*?\.setup\s*\((?P<options>[^)]+)\)',
2626             webpage)
2627         if mobj:
2628             try:
2629                 jwplayer_data = self._parse_json(mobj.group('options'),
2630                                                  video_id=video_id,
2631                                                  transform_source=transform_source)
2632             except ExtractorError:
2633                 pass
2634             else:
2635                 if isinstance(jwplayer_data, dict):
2636                     return jwplayer_data
2637
2638     def _extract_jwplayer_data(self, webpage, video_id, *args, **kwargs):
2639         jwplayer_data = self._find_jwplayer_data(
2640             webpage, video_id, transform_source=js_to_json)
2641         return self._parse_jwplayer_data(
2642             jwplayer_data, video_id, *args, **kwargs)
2643
2644     def _parse_jwplayer_data(self, jwplayer_data, video_id=None, require_title=True,
2645                              m3u8_id=None, mpd_id=None, rtmp_params=None, base_url=None):
2646         # JWPlayer backward compatibility: flattened playlists
2647         # https://github.com/jwplayer/jwplayer/blob/v7.4.3/src/js/api/config.js#L81-L96
2648         if 'playlist' not in jwplayer_data:
2649             jwplayer_data = {'playlist': [jwplayer_data]}
2650
2651         entries = []
2652
2653         # JWPlayer backward compatibility: single playlist item
2654         # https://github.com/jwplayer/jwplayer/blob/v7.7.0/src/js/playlist/playlist.js#L10
2655         if not isinstance(jwplayer_data['playlist'], list):
2656             jwplayer_data['playlist'] = [jwplayer_data['playlist']]
2657
2658         for video_data in jwplayer_data['playlist']:
2659             # JWPlayer backward compatibility: flattened sources
2660             # https://github.com/jwplayer/jwplayer/blob/v7.4.3/src/js/playlist/item.js#L29-L35
2661             if 'sources' not in video_data:
2662                 video_data['sources'] = [video_data]
2663
2664             this_video_id = video_id or video_data['mediaid']
2665
2666             formats = self._parse_jwplayer_formats(
2667                 video_data['sources'], video_id=this_video_id, m3u8_id=m3u8_id,
2668                 mpd_id=mpd_id, rtmp_params=rtmp_params, base_url=base_url)
2669
2670             subtitles = {}
2671             tracks = video_data.get('tracks')
2672             if tracks and isinstance(tracks, list):
2673                 for track in tracks:
2674                     if not isinstance(track, dict):
2675                         continue
2676                     track_kind = track.get('kind')
2677                     if not track_kind or not isinstance(track_kind, compat_str):
2678                         continue
2679                     if track_kind.lower() not in ('captions', 'subtitles'):
2680                         continue
2681                     track_url = urljoin(base_url, track.get('file'))
2682                     if not track_url:
2683                         continue
2684                     subtitles.setdefault(track.get('label') or 'en', []).append({
2685                         'url': self._proto_relative_url(track_url)
2686                     })
2687
2688             entry = {
2689                 'id': this_video_id,
2690                 'title': unescapeHTML(video_data['title'] if require_title else video_data.get('title')),
2691                 'description': video_data.get('description'),
2692                 'thumbnail': urljoin(base_url, self._proto_relative_url(video_data.get('image'))),
2693                 'timestamp': int_or_none(video_data.get('pubdate')),
2694                 'duration': float_or_none(jwplayer_data.get('duration') or video_data.get('duration')),
2695                 'subtitles': subtitles,
2696             }
2697             # https://github.com/jwplayer/jwplayer/blob/master/src/js/utils/validator.js#L32
2698             if len(formats) == 1 and re.search(r'^(?:http|//).*(?:youtube\.com|youtu\.be)/.+', formats[0]['url']):
2699                 entry.update({
2700                     '_type': 'url_transparent',
2701                     'url': formats[0]['url'],
2702                 })
2703             else:
2704                 self._sort_formats(formats)
2705                 entry['formats'] = formats
2706             entries.append(entry)
2707         if len(entries) == 1:
2708             return entries[0]
2709         else:
2710             return self.playlist_result(entries)
2711
2712     def _parse_jwplayer_formats(self, jwplayer_sources_data, video_id=None,
2713                                 m3u8_id=None, mpd_id=None, rtmp_params=None, base_url=None):
2714         urls = []
2715         formats = []
2716         for source in jwplayer_sources_data:
2717             if not isinstance(source, dict):
2718                 continue
2719             source_url = urljoin(
2720                 base_url, self._proto_relative_url(source.get('file')))
2721             if not source_url or source_url in urls:
2722                 continue
2723             urls.append(source_url)
2724             source_type = source.get('type') or ''
2725             ext = mimetype2ext(source_type) or determine_ext(source_url)
2726             if source_type == 'hls' or ext == 'm3u8':
2727                 formats.extend(self._extract_m3u8_formats(
2728                     source_url, video_id, 'mp4', entry_protocol='m3u8_native',
2729                     m3u8_id=m3u8_id, fatal=False))
2730             elif source_type == 'dash' or ext == 'mpd':
2731                 formats.extend(self._extract_mpd_formats(
2732                     source_url, video_id, mpd_id=mpd_id, fatal=False))
2733             elif ext == 'smil':
2734                 formats.extend(self._extract_smil_formats(
2735                     source_url, video_id, fatal=False))
2736             # https://github.com/jwplayer/jwplayer/blob/master/src/js/providers/default.js#L67
2737             elif source_type.startswith('audio') or ext in (
2738                     'oga', 'aac', 'mp3', 'mpeg', 'vorbis'):
2739                 formats.append({
2740                     'url': source_url,
2741                     'vcodec': 'none',
2742                     'ext': ext,
2743                 })
2744             else:
2745                 height = int_or_none(source.get('height'))
2746                 if height is None:
2747                     # Often no height is provided but there is a label in
2748                     # format like "1080p", "720p SD", or 1080.
2749                     height = int_or_none(self._search_regex(
2750                         r'^(\d{3,4})[pP]?(?:\b|$)', compat_str(source.get('label') or ''),
2751                         'height', default=None))
2752                 a_format = {
2753                     'url': source_url,
2754                     'width': int_or_none(source.get('width')),
2755                     'height': height,
2756                     'tbr': int_or_none(source.get('bitrate')),
2757                     'ext': ext,
2758                 }
2759                 if source_url.startswith('rtmp'):
2760                     a_format['ext'] = 'flv'
2761                     # See com/longtailvideo/jwplayer/media/RTMPMediaProvider.as
2762                     # of jwplayer.flash.swf
2763                     rtmp_url_parts = re.split(
2764                         r'((?:mp4|mp3|flv):)', source_url, 1)
2765                     if len(rtmp_url_parts) == 3:
2766                         rtmp_url, prefix, play_path = rtmp_url_parts
2767                         a_format.update({
2768                             'url': rtmp_url,
2769                             'play_path': prefix + play_path,
2770                         })
2771                     if rtmp_params:
2772                         a_format.update(rtmp_params)
2773                 formats.append(a_format)
2774         return formats
2775
2776     def _live_title(self, name):
2777         """ Generate the title for a live video """
2778         now = datetime.datetime.now()
2779         now_str = now.strftime('%Y-%m-%d %H:%M')
2780         return name + ' ' + now_str
2781
2782     def _int(self, v, name, fatal=False, **kwargs):
2783         res = int_or_none(v, **kwargs)
2784         if 'get_attr' in kwargs:
2785             print(getattr(v, kwargs['get_attr']))
2786         if res is None:
2787             msg = 'Failed to extract %s: Could not parse value %r' % (name, v)
2788             if fatal:
2789                 raise ExtractorError(msg)
2790             else:
2791                 self._downloader.report_warning(msg)
2792         return res
2793
2794     def _float(self, v, name, fatal=False, **kwargs):
2795         res = float_or_none(v, **kwargs)
2796         if res is None:
2797             msg = 'Failed to extract %s: Could not parse value %r' % (name, v)
2798             if fatal:
2799                 raise ExtractorError(msg)
2800             else:
2801                 self._downloader.report_warning(msg)
2802         return res
2803
2804     def _set_cookie(self, domain, name, value, expire_time=None, port=None,
2805                     path='/', secure=False, discard=False, rest={}, **kwargs):
2806         cookie = compat_cookiejar.Cookie(
2807             0, name, value, port, port is not None, domain, True,
2808             domain.startswith('.'), path, True, secure, expire_time,
2809             discard, None, None, rest)
2810         self._downloader.cookiejar.set_cookie(cookie)
2811
2812     def _get_cookies(self, url):
2813         """ Return a compat_cookies.SimpleCookie with the cookies for the url """
2814         req = sanitized_Request(url)
2815         self._downloader.cookiejar.add_cookie_header(req)
2816         return compat_cookies.SimpleCookie(req.get_header('Cookie'))
2817
2818     def get_testcases(self, include_onlymatching=False):
2819         t = getattr(self, '_TEST', None)
2820         if t:
2821             assert not hasattr(self, '_TESTS'), \
2822                 '%s has _TEST and _TESTS' % type(self).__name__
2823             tests = [t]
2824         else:
2825             tests = getattr(self, '_TESTS', [])
2826         for t in tests:
2827             if not include_onlymatching and t.get('only_matching', False):
2828                 continue
2829             t['name'] = type(self).__name__[:-len('IE')]
2830             yield t
2831
2832     def is_suitable(self, age_limit):
2833         """ Test whether the extractor is generally suitable for the given
2834         age limit (i.e. pornographic sites are not, all others usually are) """
2835
2836         any_restricted = False
2837         for tc in self.get_testcases(include_onlymatching=False):
2838             if tc.get('playlist', []):
2839                 tc = tc['playlist'][0]
2840             is_restricted = age_restricted(
2841                 tc.get('info_dict', {}).get('age_limit'), age_limit)
2842             if not is_restricted:
2843                 return True
2844             any_restricted = any_restricted or is_restricted
2845         return not any_restricted
2846
2847     def extract_subtitles(self, *args, **kwargs):
2848         if (self._downloader.params.get('writesubtitles', False) or
2849                 self._downloader.params.get('listsubtitles')):
2850             return self._get_subtitles(*args, **kwargs)
2851         return {}
2852
2853     def _get_subtitles(self, *args, **kwargs):
2854         raise NotImplementedError('This method must be implemented by subclasses')
2855
2856     @staticmethod
2857     def _merge_subtitle_items(subtitle_list1, subtitle_list2):
2858         """ Merge subtitle items for one language. Items with duplicated URLs
2859         will be dropped. """
2860         list1_urls = set([item['url'] for item in subtitle_list1])
2861         ret = list(subtitle_list1)
2862         ret.extend([item for item in subtitle_list2 if item['url'] not in list1_urls])
2863         return ret
2864
2865     @classmethod
2866     def _merge_subtitles(cls, subtitle_dict1, subtitle_dict2):
2867         """ Merge two subtitle dictionaries, language by language. """
2868         ret = dict(subtitle_dict1)
2869         for lang in subtitle_dict2:
2870             ret[lang] = cls._merge_subtitle_items(subtitle_dict1.get(lang, []), subtitle_dict2[lang])
2871         return ret
2872
2873     def extract_automatic_captions(self, *args, **kwargs):
2874         if (self._downloader.params.get('writeautomaticsub', False) or
2875                 self._downloader.params.get('listsubtitles')):
2876             return self._get_automatic_captions(*args, **kwargs)
2877         return {}
2878
2879     def _get_automatic_captions(self, *args, **kwargs):
2880         raise NotImplementedError('This method must be implemented by subclasses')
2881
2882     def mark_watched(self, *args, **kwargs):
2883         if (self._downloader.params.get('mark_watched', False) and
2884                 (self._get_login_info()[0] is not None or
2885                     self._downloader.params.get('cookiefile') is not None)):
2886             self._mark_watched(*args, **kwargs)
2887
2888     def _mark_watched(self, *args, **kwargs):
2889         raise NotImplementedError('This method must be implemented by subclasses')
2890
2891     def geo_verification_headers(self):
2892         headers = {}
2893         geo_verification_proxy = self._downloader.params.get('geo_verification_proxy')
2894         if geo_verification_proxy:
2895             headers['Ytdl-request-proxy'] = geo_verification_proxy
2896         return headers
2897
2898     def _generic_id(self, url):
2899         return compat_urllib_parse_unquote(os.path.splitext(url.rstrip('/').split('/')[-1])[0])
2900
2901     def _generic_title(self, url):
2902         return compat_urllib_parse_unquote(os.path.splitext(url_basename(url))[0])
2903
2904
2905 class SearchInfoExtractor(InfoExtractor):
2906     """
2907     Base class for paged search queries extractors.
2908     They accept URLs in the format _SEARCH_KEY(|all|[0-9]):{query}
2909     Instances should define _SEARCH_KEY and _MAX_RESULTS.
2910     """
2911
2912     @classmethod
2913     def _make_valid_url(cls):
2914         return r'%s(?P<prefix>|[1-9][0-9]*|all):(?P<query>[\s\S]+)' % cls._SEARCH_KEY
2915
2916     @classmethod
2917     def suitable(cls, url):
2918         return re.match(cls._make_valid_url(), url) is not None
2919
2920     def _real_extract(self, query):
2921         mobj = re.match(self._make_valid_url(), query)
2922         if mobj is None:
2923             raise ExtractorError('Invalid search query "%s"' % query)
2924
2925         prefix = mobj.group('prefix')
2926         query = mobj.group('query')
2927         if prefix == '':
2928             return self._get_n_results(query, 1)
2929         elif prefix == 'all':
2930             return self._get_n_results(query, self._MAX_RESULTS)
2931         else:
2932             n = int(prefix)
2933             if n <= 0:
2934                 raise ExtractorError('invalid download number %s for query "%s"' % (n, query))
2935             elif n > self._MAX_RESULTS:
2936                 self._downloader.report_warning('%s returns max %i results (you requested %i)' % (self._SEARCH_KEY, self._MAX_RESULTS, n))
2937                 n = self._MAX_RESULTS
2938             return self._get_n_results(query, n)
2939
2940     def _get_n_results(self, query, n):
2941         """Get a specified number of results for a query"""
2942         raise NotImplementedError('This method must be implemented by subclasses')
2943
2944     @property
2945     def SEARCH_KEY(self):
2946         return self._SEARCH_KEY