]> gitweb @ CieloNegro.org - youtube-dl.git/blob - youtube_dl/extractor/youtube.py
9424d5e2669a72e791a0ba0a0120de0bfec27fc8
[youtube-dl.git] / youtube_dl / extractor / youtube.py
1 # coding: utf-8
2
3 import collections
4 import errno
5 import io
6 import itertools
7 import json
8 import os.path
9 import re
10 import string
11 import struct
12 import traceback
13 import zlib
14
15 from .common import InfoExtractor, SearchInfoExtractor
16 from .subtitles import SubtitlesInfoExtractor
17 from ..utils import (
18     compat_chr,
19     compat_parse_qs,
20     compat_urllib_parse,
21     compat_urllib_request,
22     compat_urlparse,
23     compat_str,
24
25     clean_html,
26     get_cachedir,
27     get_element_by_id,
28     get_element_by_attribute,
29     ExtractorError,
30     unescapeHTML,
31     unified_strdate,
32     orderedSet,
33     write_json_file,
34 )
35
36 class YoutubeBaseInfoExtractor(InfoExtractor):
37     """Provide base functions for Youtube extractors"""
38     _LOGIN_URL = 'https://accounts.google.com/ServiceLogin'
39     _LANG_URL = r'https://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
40     _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
41     _NETRC_MACHINE = 'youtube'
42     # If True it will raise an error if no login info is provided
43     _LOGIN_REQUIRED = False
44
45     def _set_language(self):
46         return bool(self._download_webpage(
47             self._LANG_URL, None,
48             note=u'Setting language', errnote='unable to set language',
49             fatal=False))
50
51     def _login(self):
52         (username, password) = self._get_login_info()
53         # No authentication to be performed
54         if username is None:
55             if self._LOGIN_REQUIRED:
56                 raise ExtractorError(u'No login info available, needed for using %s.' % self.IE_NAME, expected=True)
57             return False
58
59         login_page = self._download_webpage(
60             self._LOGIN_URL, None,
61             note=u'Downloading login page',
62             errnote=u'unable to fetch login page', fatal=False)
63         if login_page is False:
64             return
65
66         galx = self._search_regex(r'(?s)<input.+?name="GALX".+?value="(.+?)"',
67                                   login_page, u'Login GALX parameter')
68
69         # Log in
70         login_form_strs = {
71                 u'continue': u'https://www.youtube.com/signin?action_handle_signin=true&feature=sign_in_button&hl=en_US&nomobiletemp=1',
72                 u'Email': username,
73                 u'GALX': galx,
74                 u'Passwd': password,
75                 u'PersistentCookie': u'yes',
76                 u'_utf8': u'霱',
77                 u'bgresponse': u'js_disabled',
78                 u'checkConnection': u'',
79                 u'checkedDomains': u'youtube',
80                 u'dnConn': u'',
81                 u'pstMsg': u'0',
82                 u'rmShown': u'1',
83                 u'secTok': u'',
84                 u'signIn': u'Sign in',
85                 u'timeStmp': u'',
86                 u'service': u'youtube',
87                 u'uilel': u'3',
88                 u'hl': u'en_US',
89         }
90         # Convert to UTF-8 *before* urlencode because Python 2.x's urlencode
91         # chokes on unicode
92         login_form = dict((k.encode('utf-8'), v.encode('utf-8')) for k,v in login_form_strs.items())
93         login_data = compat_urllib_parse.urlencode(login_form).encode('ascii')
94
95         req = compat_urllib_request.Request(self._LOGIN_URL, login_data)
96         login_results = self._download_webpage(
97             req, None,
98             note=u'Logging in', errnote=u'unable to log in', fatal=False)
99         if login_results is False:
100             return False
101         if re.search(r'(?i)<form[^>]* id="gaia_loginform"', login_results) is not None:
102             self._downloader.report_warning(u'unable to log in: bad username or password')
103             return False
104         return True
105
106     def _confirm_age(self):
107         age_form = {
108             'next_url': '/',
109             'action_confirm': 'Confirm',
110         }
111         req = compat_urllib_request.Request(self._AGE_URL, compat_urllib_parse.urlencode(age_form))
112
113         self._download_webpage(
114             req, None,
115             note=u'Confirming age', errnote=u'Unable to confirm age')
116         return True
117
118     def _real_initialize(self):
119         if self._downloader is None:
120             return
121         if not self._set_language():
122             return
123         if not self._login():
124             return
125         self._confirm_age()
126
127
128 class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
129     IE_DESC = u'YouTube.com'
130     _VALID_URL = r"""(?x)^
131                      (
132                          (?:https?://|//)?                                    # http(s):// or protocol-independent URL (optional)
133                          (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/|
134                             tube\.majestyc\.net/|
135                             youtube\.googleapis\.com/)                        # the various hostnames, with wildcard subdomains
136                          (?:.*?\#/)?                                          # handle anchor (#/) redirect urls
137                          (?:                                                  # the various things that can precede the ID:
138                              (?:(?:v|embed|e)/)                               # v/ or embed/ or e/
139                              |(?:                                             # or the v= param in all its forms
140                                  (?:(?:watch|movie)(?:_popup)?(?:\.php)?)?    # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
141                                  (?:\?|\#!?)                                  # the params delimiter ? or # or #!
142                                  (?:.*?&)?                                    # any other preceding param (like /?s=tuff&v=xxxx)
143                                  v=
144                              )
145                          ))
146                          |youtu\.be/                                          # just youtu.be/xxxx
147                          )
148                      )?                                                       # all until now is optional -> you can pass the naked ID
149                      ([0-9A-Za-z_-]{11})                                      # here is it! the YouTube video ID
150                      (?(1).+)?                                                # if we found the ID, everything can follow
151                      $"""
152     _NEXT_URL_RE = r'[\?&]next_url=([^&]+)'
153     _formats = {
154         '5': {'ext': 'flv', 'width': 400, 'height': 240},
155         '6': {'ext': 'flv', 'width': 450, 'height': 270},
156         '13': {'ext': '3gp'},
157         '17': {'ext': '3gp', 'width': 176, 'height': 144},
158         '18': {'ext': 'mp4', 'width': 640, 'height': 360},
159         '22': {'ext': 'mp4', 'width': 1280, 'height': 720},
160         '34': {'ext': 'flv', 'width': 640, 'height': 360},
161         '35': {'ext': 'flv', 'width': 854, 'height': 480},
162         '36': {'ext': '3gp', 'width': 320, 'height': 240},
163         '37': {'ext': 'mp4', 'width': 1920, 'height': 1080},
164         '38': {'ext': 'mp4', 'width': 4096, 'height': 3072},
165         '43': {'ext': 'webm', 'width': 640, 'height': 360},
166         '44': {'ext': 'webm', 'width': 854, 'height': 480},
167         '45': {'ext': 'webm', 'width': 1280, 'height': 720},
168         '46': {'ext': 'webm', 'width': 1920, 'height': 1080},
169
170
171         # 3d videos
172         '82': {'ext': 'mp4', 'height': 360, 'resolution': '360p', 'format_note': '3D', 'preference': -20},
173         '83': {'ext': 'mp4', 'height': 480, 'resolution': '480p', 'format_note': '3D', 'preference': -20},
174         '84': {'ext': 'mp4', 'height': 720, 'resolution': '720p', 'format_note': '3D', 'preference': -20},
175         '85': {'ext': 'mp4', 'height': 1080, 'resolution': '1080p', 'format_note': '3D', 'preference': -20},
176         '100': {'ext': 'webm', 'height': 360, 'resolution': '360p', 'format_note': '3D', 'preference': -20},
177         '101': {'ext': 'webm', 'height': 480, 'resolution': '480p', 'format_note': '3D', 'preference': -20},
178         '102': {'ext': 'webm', 'height': 720, 'resolution': '720p', 'format_note': '3D', 'preference': -20},
179
180         # Apple HTTP Live Streaming
181         '92': {'ext': 'mp4', 'height': 240, 'resolution': '240p', 'format_note': 'HLS', 'preference': -10},
182         '93': {'ext': 'mp4', 'height': 360, 'resolution': '360p', 'format_note': 'HLS', 'preference': -10},
183         '94': {'ext': 'mp4', 'height': 480, 'resolution': '480p', 'format_note': 'HLS', 'preference': -10},
184         '95': {'ext': 'mp4', 'height': 720, 'resolution': '720p', 'format_note': 'HLS', 'preference': -10},
185         '96': {'ext': 'mp4', 'height': 1080, 'resolution': '1080p', 'format_note': 'HLS', 'preference': -10},
186         '132': {'ext': 'mp4', 'height': 240, 'resolution': '240p', 'format_note': 'HLS', 'preference': -10},
187         '151': {'ext': 'mp4', 'height': 72, 'resolution': '72p', 'format_note': 'HLS', 'preference': -10},
188
189         # DASH mp4 video
190         '133': {'ext': 'mp4', 'height': 240, 'resolution': '240p', 'format_note': 'DASH video', 'preference': -40},
191         '134': {'ext': 'mp4', 'height': 360, 'resolution': '360p', 'format_note': 'DASH video', 'preference': -40},
192         '135': {'ext': 'mp4', 'height': 480, 'resolution': '480p', 'format_note': 'DASH video', 'preference': -40},
193         '136': {'ext': 'mp4', 'height': 720, 'resolution': '720p', 'format_note': 'DASH video', 'preference': -40},
194         '137': {'ext': 'mp4', 'height': 1080, 'resolution': '1080p', 'format_note': 'DASH video', 'preference': -40},
195         '138': {'ext': 'mp4', 'height': 1081, 'resolution': '>1080p', 'format_note': 'DASH video', 'preference': -40},
196         '160': {'ext': 'mp4', 'height': 192, 'resolution': '192p', 'format_note': 'DASH video', 'preference': -40},
197         '264': {'ext': 'mp4', 'height': 1080, 'resolution': '1080p', 'format_note': 'DASH video', 'preference': -40},
198
199         # Dash mp4 audio
200         '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'vcodec': 'none', 'abr': 48, 'preference': -50},
201         '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'vcodec': 'none', 'abr': 128, 'preference': -50},
202         '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'vcodec': 'none', 'abr': 256, 'preference': -50},
203
204         # Dash webm
205         '242': {'ext': 'webm', 'height': 240, 'resolution': '240p', 'format_note': 'DASH webm', 'preference': -40},
206         '243': {'ext': 'webm', 'height': 360, 'resolution': '360p', 'format_note': 'DASH webm', 'preference': -40},
207         '244': {'ext': 'webm', 'height': 480, 'resolution': '480p', 'format_note': 'DASH webm', 'preference': -40},
208         '245': {'ext': 'webm', 'height': 480, 'resolution': '480p', 'format_note': 'DASH webm', 'preference': -40},
209         '246': {'ext': 'webm', 'height': 480, 'resolution': '480p', 'format_note': 'DASH webm', 'preference': -40},
210         '247': {'ext': 'webm', 'height': 720, 'resolution': '720p', 'format_note': 'DASH webm', 'preference': -40},
211         '248': {'ext': 'webm', 'height': 1080, 'resolution': '1080p', 'format_note': 'DASH webm', 'preference': -40},
212
213         # Dash webm audio
214         '171': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH webm audio', 'abr': 48, 'preference': -50},
215         '172': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH webm audio', 'abr': 256, 'preference': -50},
216     }
217
218     IE_NAME = u'youtube'
219     _TESTS = [
220         {
221             u"url":  u"http://www.youtube.com/watch?v=BaW_jenozKc",
222             u"file":  u"BaW_jenozKc.mp4",
223             u"info_dict": {
224                 u"title": u"youtube-dl test video \"'/\\ä↭𝕐",
225                 u"uploader": u"Philipp Hagemeister",
226                 u"uploader_id": u"phihag",
227                 u"upload_date": u"20121002",
228                 u"description": u"test chars:  \"'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de ."
229             }
230         },
231         {
232             u"url":  u"http://www.youtube.com/watch?v=UxxajLWwzqY",
233             u"file":  u"UxxajLWwzqY.mp4",
234             u"note": u"Test generic use_cipher_signature video (#897)",
235             u"info_dict": {
236                 u"upload_date": u"20120506",
237                 u"title": u"Icona Pop - I Love It (feat. Charli XCX) [OFFICIAL VIDEO]",
238                 u"description": u"md5:5b292926389560516e384ac437c0ec07",
239                 u"uploader": u"Icona Pop",
240                 u"uploader_id": u"IconaPop"
241             }
242         },
243         {
244             u"url":  u"https://www.youtube.com/watch?v=07FYdnEawAQ",
245             u"file":  u"07FYdnEawAQ.mp4",
246             u"note": u"Test VEVO video with age protection (#956)",
247             u"info_dict": {
248                 u"upload_date": u"20130703",
249                 u"title": u"Justin Timberlake - Tunnel Vision (Explicit)",
250                 u"description": u"md5:64249768eec3bc4276236606ea996373",
251                 u"uploader": u"justintimberlakeVEVO",
252                 u"uploader_id": u"justintimberlakeVEVO"
253             }
254         },
255         {
256             u"url":  u"//www.YouTube.com/watch?v=yZIXLfi8CZQ",
257             u"file":  u"yZIXLfi8CZQ.mp4",
258             u"note": u"Embed-only video (#1746)",
259             u"info_dict": {
260                 u"upload_date": u"20120608",
261                 u"title": u"Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012",
262                 u"description": u"md5:09b78bd971f1e3e289601dfba15ca4f7",
263                 u"uploader": u"SET India",
264                 u"uploader_id": u"setindia"
265             }
266         },
267     ]
268
269
270     @classmethod
271     def suitable(cls, url):
272         """Receives a URL and returns True if suitable for this IE."""
273         if YoutubePlaylistIE.suitable(url): return False
274         return re.match(cls._VALID_URL, url) is not None
275
276     def __init__(self, *args, **kwargs):
277         super(YoutubeIE, self).__init__(*args, **kwargs)
278         self._player_cache = {}
279
280     def report_video_info_webpage_download(self, video_id):
281         """Report attempt to download video info webpage."""
282         self.to_screen(u'%s: Downloading video info webpage' % video_id)
283
284     def report_information_extraction(self, video_id):
285         """Report attempt to extract video information."""
286         self.to_screen(u'%s: Extracting video information' % video_id)
287
288     def report_unavailable_format(self, video_id, format):
289         """Report extracted video URL."""
290         self.to_screen(u'%s: Format %s not available' % (video_id, format))
291
292     def report_rtmp_download(self):
293         """Indicate the download will use the RTMP protocol."""
294         self.to_screen(u'RTMP download detected')
295
296     def _extract_signature_function(self, video_id, player_url, slen):
297         id_m = re.match(r'.*-(?P<id>[a-zA-Z0-9_-]+)\.(?P<ext>[a-z]+)$',
298                         player_url)
299         player_type = id_m.group('ext')
300         player_id = id_m.group('id')
301
302         # Read from filesystem cache
303         func_id = '%s_%s_%d' % (player_type, player_id, slen)
304         assert os.path.basename(func_id) == func_id
305         cache_dir = get_cachedir(self._downloader.params)
306
307         cache_enabled = cache_dir is not None
308         if cache_enabled:
309             cache_fn = os.path.join(os.path.expanduser(cache_dir),
310                                     u'youtube-sigfuncs',
311                                     func_id + '.json')
312             try:
313                 with io.open(cache_fn, 'r', encoding='utf-8') as cachef:
314                     cache_spec = json.load(cachef)
315                 return lambda s: u''.join(s[i] for i in cache_spec)
316             except IOError:
317                 pass  # No cache available
318
319         if player_type == 'js':
320             code = self._download_webpage(
321                 player_url, video_id,
322                 note=u'Downloading %s player %s' % (player_type, player_id),
323                 errnote=u'Download of %s failed' % player_url)
324             res = self._parse_sig_js(code)
325         elif player_type == 'swf':
326             urlh = self._request_webpage(
327                 player_url, video_id,
328                 note=u'Downloading %s player %s' % (player_type, player_id),
329                 errnote=u'Download of %s failed' % player_url)
330             code = urlh.read()
331             res = self._parse_sig_swf(code)
332         else:
333             assert False, 'Invalid player type %r' % player_type
334
335         if cache_enabled:
336             try:
337                 test_string = u''.join(map(compat_chr, range(slen)))
338                 cache_res = res(test_string)
339                 cache_spec = [ord(c) for c in cache_res]
340                 try:
341                     os.makedirs(os.path.dirname(cache_fn))
342                 except OSError as ose:
343                     if ose.errno != errno.EEXIST:
344                         raise
345                 write_json_file(cache_spec, cache_fn)
346             except Exception:
347                 tb = traceback.format_exc()
348                 self._downloader.report_warning(
349                     u'Writing cache to %r failed: %s' % (cache_fn, tb))
350
351         return res
352
353     def _print_sig_code(self, func, slen):
354         def gen_sig_code(idxs):
355             def _genslice(start, end, step):
356                 starts = u'' if start == 0 else str(start)
357                 ends = (u':%d' % (end+step)) if end + step >= 0 else u':'
358                 steps = u'' if step == 1 else (u':%d' % step)
359                 return u's[%s%s%s]' % (starts, ends, steps)
360
361             step = None
362             start = '(Never used)'  # Quelch pyflakes warnings - start will be
363                                     # set as soon as step is set
364             for i, prev in zip(idxs[1:], idxs[:-1]):
365                 if step is not None:
366                     if i - prev == step:
367                         continue
368                     yield _genslice(start, prev, step)
369                     step = None
370                     continue
371                 if i - prev in [-1, 1]:
372                     step = i - prev
373                     start = prev
374                     continue
375                 else:
376                     yield u's[%d]' % prev
377             if step is None:
378                 yield u's[%d]' % i
379             else:
380                 yield _genslice(start, i, step)
381
382         test_string = u''.join(map(compat_chr, range(slen)))
383         cache_res = func(test_string)
384         cache_spec = [ord(c) for c in cache_res]
385         expr_code = u' + '.join(gen_sig_code(cache_spec))
386         code = u'if len(s) == %d:\n    return %s\n' % (slen, expr_code)
387         self.to_screen(u'Extracted signature function:\n' + code)
388
389     def _parse_sig_js(self, jscode):
390         funcname = self._search_regex(
391             r'signature=([a-zA-Z]+)', jscode,
392             u'Initial JS player signature function name')
393
394         functions = {}
395
396         def argidx(varname):
397             return string.lowercase.index(varname)
398
399         def interpret_statement(stmt, local_vars, allow_recursion=20):
400             if allow_recursion < 0:
401                 raise ExtractorError(u'Recursion limit reached')
402
403             if stmt.startswith(u'var '):
404                 stmt = stmt[len(u'var '):]
405             ass_m = re.match(r'^(?P<out>[a-z]+)(?:\[(?P<index>[^\]]+)\])?' +
406                              r'=(?P<expr>.*)$', stmt)
407             if ass_m:
408                 if ass_m.groupdict().get('index'):
409                     def assign(val):
410                         lvar = local_vars[ass_m.group('out')]
411                         idx = interpret_expression(ass_m.group('index'),
412                                                    local_vars, allow_recursion)
413                         assert isinstance(idx, int)
414                         lvar[idx] = val
415                         return val
416                     expr = ass_m.group('expr')
417                 else:
418                     def assign(val):
419                         local_vars[ass_m.group('out')] = val
420                         return val
421                     expr = ass_m.group('expr')
422             elif stmt.startswith(u'return '):
423                 assign = lambda v: v
424                 expr = stmt[len(u'return '):]
425             else:
426                 raise ExtractorError(
427                     u'Cannot determine left side of statement in %r' % stmt)
428
429             v = interpret_expression(expr, local_vars, allow_recursion)
430             return assign(v)
431
432         def interpret_expression(expr, local_vars, allow_recursion):
433             if expr.isdigit():
434                 return int(expr)
435
436             if expr.isalpha():
437                 return local_vars[expr]
438
439             m = re.match(r'^(?P<in>[a-z]+)\.(?P<member>.*)$', expr)
440             if m:
441                 member = m.group('member')
442                 val = local_vars[m.group('in')]
443                 if member == 'split("")':
444                     return list(val)
445                 if member == 'join("")':
446                     return u''.join(val)
447                 if member == 'length':
448                     return len(val)
449                 if member == 'reverse()':
450                     return val[::-1]
451                 slice_m = re.match(r'slice\((?P<idx>.*)\)', member)
452                 if slice_m:
453                     idx = interpret_expression(
454                         slice_m.group('idx'), local_vars, allow_recursion-1)
455                     return val[idx:]
456
457             m = re.match(
458                 r'^(?P<in>[a-z]+)\[(?P<idx>.+)\]$', expr)
459             if m:
460                 val = local_vars[m.group('in')]
461                 idx = interpret_expression(m.group('idx'), local_vars,
462                                            allow_recursion-1)
463                 return val[idx]
464
465             m = re.match(r'^(?P<a>.+?)(?P<op>[%])(?P<b>.+?)$', expr)
466             if m:
467                 a = interpret_expression(m.group('a'),
468                                          local_vars, allow_recursion)
469                 b = interpret_expression(m.group('b'),
470                                          local_vars, allow_recursion)
471                 return a % b
472
473             m = re.match(
474                 r'^(?P<func>[a-zA-Z]+)\((?P<args>[a-z0-9,]+)\)$', expr)
475             if m:
476                 fname = m.group('func')
477                 if fname not in functions:
478                     functions[fname] = extract_function(fname)
479                 argvals = [int(v) if v.isdigit() else local_vars[v]
480                            for v in m.group('args').split(',')]
481                 return functions[fname](argvals)
482             raise ExtractorError(u'Unsupported JS expression %r' % expr)
483
484         def extract_function(funcname):
485             func_m = re.search(
486                 r'function ' + re.escape(funcname) +
487                 r'\((?P<args>[a-z,]+)\){(?P<code>[^}]+)}',
488                 jscode)
489             argnames = func_m.group('args').split(',')
490
491             def resf(args):
492                 local_vars = dict(zip(argnames, args))
493                 for stmt in func_m.group('code').split(';'):
494                     res = interpret_statement(stmt, local_vars)
495                 return res
496             return resf
497
498         initial_function = extract_function(funcname)
499         return lambda s: initial_function([s])
500
501     def _parse_sig_swf(self, file_contents):
502         if file_contents[1:3] != b'WS':
503             raise ExtractorError(
504                 u'Not an SWF file; header is %r' % file_contents[:3])
505         if file_contents[:1] == b'C':
506             content = zlib.decompress(file_contents[8:])
507         else:
508             raise NotImplementedError(u'Unsupported compression format %r' %
509                                       file_contents[:1])
510
511         def extract_tags(content):
512             pos = 0
513             while pos < len(content):
514                 header16 = struct.unpack('<H', content[pos:pos+2])[0]
515                 pos += 2
516                 tag_code = header16 >> 6
517                 tag_len = header16 & 0x3f
518                 if tag_len == 0x3f:
519                     tag_len = struct.unpack('<I', content[pos:pos+4])[0]
520                     pos += 4
521                 assert pos+tag_len <= len(content)
522                 yield (tag_code, content[pos:pos+tag_len])
523                 pos += tag_len
524
525         code_tag = next(tag
526                         for tag_code, tag in extract_tags(content)
527                         if tag_code == 82)
528         p = code_tag.index(b'\0', 4) + 1
529         code_reader = io.BytesIO(code_tag[p:])
530
531         # Parse ABC (AVM2 ByteCode)
532         def read_int(reader=None):
533             if reader is None:
534                 reader = code_reader
535             res = 0
536             shift = 0
537             for _ in range(5):
538                 buf = reader.read(1)
539                 assert len(buf) == 1
540                 b = struct.unpack('<B', buf)[0]
541                 res = res | ((b & 0x7f) << shift)
542                 if b & 0x80 == 0:
543                     break
544                 shift += 7
545             return res
546
547         def u30(reader=None):
548             res = read_int(reader)
549             assert res & 0xf0000000 == 0
550             return res
551         u32 = read_int
552
553         def s32(reader=None):
554             v = read_int(reader)
555             if v & 0x80000000 != 0:
556                 v = - ((v ^ 0xffffffff) + 1)
557             return v
558
559         def read_string(reader=None):
560             if reader is None:
561                 reader = code_reader
562             slen = u30(reader)
563             resb = reader.read(slen)
564             assert len(resb) == slen
565             return resb.decode('utf-8')
566
567         def read_bytes(count, reader=None):
568             if reader is None:
569                 reader = code_reader
570             resb = reader.read(count)
571             assert len(resb) == count
572             return resb
573
574         def read_byte(reader=None):
575             resb = read_bytes(1, reader=reader)
576             res = struct.unpack('<B', resb)[0]
577             return res
578
579         # minor_version + major_version
580         read_bytes(2 + 2)
581
582         # Constant pool
583         int_count = u30()
584         for _c in range(1, int_count):
585             s32()
586         uint_count = u30()
587         for _c in range(1, uint_count):
588             u32()
589         double_count = u30()
590         read_bytes((double_count-1) * 8)
591         string_count = u30()
592         constant_strings = [u'']
593         for _c in range(1, string_count):
594             s = read_string()
595             constant_strings.append(s)
596         namespace_count = u30()
597         for _c in range(1, namespace_count):
598             read_bytes(1)  # kind
599             u30()  # name
600         ns_set_count = u30()
601         for _c in range(1, ns_set_count):
602             count = u30()
603             for _c2 in range(count):
604                 u30()
605         multiname_count = u30()
606         MULTINAME_SIZES = {
607             0x07: 2,  # QName
608             0x0d: 2,  # QNameA
609             0x0f: 1,  # RTQName
610             0x10: 1,  # RTQNameA
611             0x11: 0,  # RTQNameL
612             0x12: 0,  # RTQNameLA
613             0x09: 2,  # Multiname
614             0x0e: 2,  # MultinameA
615             0x1b: 1,  # MultinameL
616             0x1c: 1,  # MultinameLA
617         }
618         multinames = [u'']
619         for _c in range(1, multiname_count):
620             kind = u30()
621             assert kind in MULTINAME_SIZES, u'Invalid multiname kind %r' % kind
622             if kind == 0x07:
623                 u30()  # namespace_idx
624                 name_idx = u30()
625                 multinames.append(constant_strings[name_idx])
626             else:
627                 multinames.append('[MULTINAME kind: %d]' % kind)
628                 for _c2 in range(MULTINAME_SIZES[kind]):
629                     u30()
630
631         # Methods
632         method_count = u30()
633         MethodInfo = collections.namedtuple(
634             'MethodInfo',
635             ['NEED_ARGUMENTS', 'NEED_REST'])
636         method_infos = []
637         for method_id in range(method_count):
638             param_count = u30()
639             u30()  # return type
640             for _ in range(param_count):
641                 u30()  # param type
642             u30()  # name index (always 0 for youtube)
643             flags = read_byte()
644             if flags & 0x08 != 0:
645                 # Options present
646                 option_count = u30()
647                 for c in range(option_count):
648                     u30()  # val
649                     read_bytes(1)  # kind
650             if flags & 0x80 != 0:
651                 # Param names present
652                 for _ in range(param_count):
653                     u30()  # param name
654             mi = MethodInfo(flags & 0x01 != 0, flags & 0x04 != 0)
655             method_infos.append(mi)
656
657         # Metadata
658         metadata_count = u30()
659         for _c in range(metadata_count):
660             u30()  # name
661             item_count = u30()
662             for _c2 in range(item_count):
663                 u30()  # key
664                 u30()  # value
665
666         def parse_traits_info():
667             trait_name_idx = u30()
668             kind_full = read_byte()
669             kind = kind_full & 0x0f
670             attrs = kind_full >> 4
671             methods = {}
672             if kind in [0x00, 0x06]:  # Slot or Const
673                 u30()  # Slot id
674                 u30()  # type_name_idx
675                 vindex = u30()
676                 if vindex != 0:
677                     read_byte()  # vkind
678             elif kind in [0x01, 0x02, 0x03]:  # Method / Getter / Setter
679                 u30()  # disp_id
680                 method_idx = u30()
681                 methods[multinames[trait_name_idx]] = method_idx
682             elif kind == 0x04:  # Class
683                 u30()  # slot_id
684                 u30()  # classi
685             elif kind == 0x05:  # Function
686                 u30()  # slot_id
687                 function_idx = u30()
688                 methods[function_idx] = multinames[trait_name_idx]
689             else:
690                 raise ExtractorError(u'Unsupported trait kind %d' % kind)
691
692             if attrs & 0x4 != 0:  # Metadata present
693                 metadata_count = u30()
694                 for _c3 in range(metadata_count):
695                     u30()  # metadata index
696
697             return methods
698
699         # Classes
700         TARGET_CLASSNAME = u'SignatureDecipher'
701         searched_idx = multinames.index(TARGET_CLASSNAME)
702         searched_class_id = None
703         class_count = u30()
704         for class_id in range(class_count):
705             name_idx = u30()
706             if name_idx == searched_idx:
707                 # We found the class we're looking for!
708                 searched_class_id = class_id
709             u30()  # super_name idx
710             flags = read_byte()
711             if flags & 0x08 != 0:  # Protected namespace is present
712                 u30()  # protected_ns_idx
713             intrf_count = u30()
714             for _c2 in range(intrf_count):
715                 u30()
716             u30()  # iinit
717             trait_count = u30()
718             for _c2 in range(trait_count):
719                 parse_traits_info()
720
721         if searched_class_id is None:
722             raise ExtractorError(u'Target class %r not found' %
723                                  TARGET_CLASSNAME)
724
725         method_names = {}
726         method_idxs = {}
727         for class_id in range(class_count):
728             u30()  # cinit
729             trait_count = u30()
730             for _c2 in range(trait_count):
731                 trait_methods = parse_traits_info()
732                 if class_id == searched_class_id:
733                     method_names.update(trait_methods.items())
734                     method_idxs.update(dict(
735                         (idx, name)
736                         for name, idx in trait_methods.items()))
737
738         # Scripts
739         script_count = u30()
740         for _c in range(script_count):
741             u30()  # init
742             trait_count = u30()
743             for _c2 in range(trait_count):
744                 parse_traits_info()
745
746         # Method bodies
747         method_body_count = u30()
748         Method = collections.namedtuple('Method', ['code', 'local_count'])
749         methods = {}
750         for _c in range(method_body_count):
751             method_idx = u30()
752             u30()  # max_stack
753             local_count = u30()
754             u30()  # init_scope_depth
755             u30()  # max_scope_depth
756             code_length = u30()
757             code = read_bytes(code_length)
758             if method_idx in method_idxs:
759                 m = Method(code, local_count)
760                 methods[method_idxs[method_idx]] = m
761             exception_count = u30()
762             for _c2 in range(exception_count):
763                 u30()  # from
764                 u30()  # to
765                 u30()  # target
766                 u30()  # exc_type
767                 u30()  # var_name
768             trait_count = u30()
769             for _c2 in range(trait_count):
770                 parse_traits_info()
771
772         assert p + code_reader.tell() == len(code_tag)
773         assert len(methods) == len(method_idxs)
774
775         method_pyfunctions = {}
776
777         def extract_function(func_name):
778             if func_name in method_pyfunctions:
779                 return method_pyfunctions[func_name]
780             if func_name not in methods:
781                 raise ExtractorError(u'Cannot find function %r' % func_name)
782             m = methods[func_name]
783
784             def resfunc(args):
785                 registers = ['(this)'] + list(args) + [None] * m.local_count
786                 stack = []
787                 coder = io.BytesIO(m.code)
788                 while True:
789                     opcode = struct.unpack('!B', coder.read(1))[0]
790                     if opcode == 36:  # pushbyte
791                         v = struct.unpack('!B', coder.read(1))[0]
792                         stack.append(v)
793                     elif opcode == 44:  # pushstring
794                         idx = u30(coder)
795                         stack.append(constant_strings[idx])
796                     elif opcode == 48:  # pushscope
797                         # We don't implement the scope register, so we'll just
798                         # ignore the popped value
799                         stack.pop()
800                     elif opcode == 70:  # callproperty
801                         index = u30(coder)
802                         mname = multinames[index]
803                         arg_count = u30(coder)
804                         args = list(reversed(
805                             [stack.pop() for _ in range(arg_count)]))
806                         obj = stack.pop()
807                         if mname == u'split':
808                             assert len(args) == 1
809                             assert isinstance(args[0], compat_str)
810                             assert isinstance(obj, compat_str)
811                             if args[0] == u'':
812                                 res = list(obj)
813                             else:
814                                 res = obj.split(args[0])
815                             stack.append(res)
816                         elif mname == u'slice':
817                             assert len(args) == 1
818                             assert isinstance(args[0], int)
819                             assert isinstance(obj, list)
820                             res = obj[args[0]:]
821                             stack.append(res)
822                         elif mname == u'join':
823                             assert len(args) == 1
824                             assert isinstance(args[0], compat_str)
825                             assert isinstance(obj, list)
826                             res = args[0].join(obj)
827                             stack.append(res)
828                         elif mname in method_pyfunctions:
829                             stack.append(method_pyfunctions[mname](args))
830                         else:
831                             raise NotImplementedError(
832                                 u'Unsupported property %r on %r'
833                                 % (mname, obj))
834                     elif opcode == 72:  # returnvalue
835                         res = stack.pop()
836                         return res
837                     elif opcode == 79:  # callpropvoid
838                         index = u30(coder)
839                         mname = multinames[index]
840                         arg_count = u30(coder)
841                         args = list(reversed(
842                             [stack.pop() for _ in range(arg_count)]))
843                         obj = stack.pop()
844                         if mname == u'reverse':
845                             assert isinstance(obj, list)
846                             obj.reverse()
847                         else:
848                             raise NotImplementedError(
849                                 u'Unsupported (void) property %r on %r'
850                                 % (mname, obj))
851                     elif opcode == 93:  # findpropstrict
852                         index = u30(coder)
853                         mname = multinames[index]
854                         res = extract_function(mname)
855                         stack.append(res)
856                     elif opcode == 97:  # setproperty
857                         index = u30(coder)
858                         value = stack.pop()
859                         idx = stack.pop()
860                         obj = stack.pop()
861                         assert isinstance(obj, list)
862                         assert isinstance(idx, int)
863                         obj[idx] = value
864                     elif opcode == 98:  # getlocal
865                         index = u30(coder)
866                         stack.append(registers[index])
867                     elif opcode == 99:  # setlocal
868                         index = u30(coder)
869                         value = stack.pop()
870                         registers[index] = value
871                     elif opcode == 102:  # getproperty
872                         index = u30(coder)
873                         pname = multinames[index]
874                         if pname == u'length':
875                             obj = stack.pop()
876                             assert isinstance(obj, list)
877                             stack.append(len(obj))
878                         else:  # Assume attribute access
879                             idx = stack.pop()
880                             assert isinstance(idx, int)
881                             obj = stack.pop()
882                             assert isinstance(obj, list)
883                             stack.append(obj[idx])
884                     elif opcode == 128:  # coerce
885                         u30(coder)
886                     elif opcode == 133:  # coerce_s
887                         assert isinstance(stack[-1], (type(None), compat_str))
888                     elif opcode == 164:  # modulo
889                         value2 = stack.pop()
890                         value1 = stack.pop()
891                         res = value1 % value2
892                         stack.append(res)
893                     elif opcode == 208:  # getlocal_0
894                         stack.append(registers[0])
895                     elif opcode == 209:  # getlocal_1
896                         stack.append(registers[1])
897                     elif opcode == 210:  # getlocal_2
898                         stack.append(registers[2])
899                     elif opcode == 211:  # getlocal_3
900                         stack.append(registers[3])
901                     elif opcode == 214:  # setlocal_2
902                         registers[2] = stack.pop()
903                     elif opcode == 215:  # setlocal_3
904                         registers[3] = stack.pop()
905                     else:
906                         raise NotImplementedError(
907                             u'Unsupported opcode %d' % opcode)
908
909             method_pyfunctions[func_name] = resfunc
910             return resfunc
911
912         initial_function = extract_function(u'decipher')
913         return lambda s: initial_function([s])
914
915     def _decrypt_signature(self, s, video_id, player_url, age_gate=False):
916         """Turn the encrypted s field into a working signature"""
917
918         if player_url is not None:
919             if player_url.startswith(u'//'):
920                 player_url = u'https:' + player_url
921             try:
922                 player_id = (player_url, len(s))
923                 if player_id not in self._player_cache:
924                     func = self._extract_signature_function(
925                         video_id, player_url, len(s)
926                     )
927                     self._player_cache[player_id] = func
928                 func = self._player_cache[player_id]
929                 if self._downloader.params.get('youtube_print_sig_code'):
930                     self._print_sig_code(func, len(s))
931                 return func(s)
932             except Exception:
933                 tb = traceback.format_exc()
934                 self._downloader.report_warning(
935                     u'Automatic signature extraction failed: ' + tb)
936
937             self._downloader.report_warning(
938                 u'Warning: Falling back to static signature algorithm')
939
940         return self._static_decrypt_signature(
941             s, video_id, player_url, age_gate)
942
943     def _static_decrypt_signature(self, s, video_id, player_url, age_gate):
944         if age_gate:
945             # The videos with age protection use another player, so the
946             # algorithms can be different.
947             if len(s) == 86:
948                 return s[2:63] + s[82] + s[64:82] + s[63]
949
950         if len(s) == 93:
951             return s[86:29:-1] + s[88] + s[28:5:-1]
952         elif len(s) == 92:
953             return s[25] + s[3:25] + s[0] + s[26:42] + s[79] + s[43:79] + s[91] + s[80:83]
954         elif len(s) == 91:
955             return s[84:27:-1] + s[86] + s[26:5:-1]
956         elif len(s) == 90:
957             return s[25] + s[3:25] + s[2] + s[26:40] + s[77] + s[41:77] + s[89] + s[78:81]
958         elif len(s) == 89:
959             return s[84:78:-1] + s[87] + s[77:60:-1] + s[0] + s[59:3:-1]
960         elif len(s) == 88:
961             return s[7:28] + s[87] + s[29:45] + s[55] + s[46:55] + s[2] + s[56:87] + s[28]
962         elif len(s) == 87:
963             return s[6:27] + s[4] + s[28:39] + s[27] + s[40:59] + s[2] + s[60:]
964         elif len(s) == 86:
965             return s[80:72:-1] + s[16] + s[71:39:-1] + s[72] + s[38:16:-1] + s[82] + s[15::-1]
966         elif len(s) == 85:
967             return s[3:11] + s[0] + s[12:55] + s[84] + s[56:84]
968         elif len(s) == 84:
969             return s[78:70:-1] + s[14] + s[69:37:-1] + s[70] + s[36:14:-1] + s[80] + s[:14][::-1]
970         elif len(s) == 83:
971             return s[80:63:-1] + s[0] + s[62:0:-1] + s[63]
972         elif len(s) == 82:
973             return s[80:37:-1] + s[7] + s[36:7:-1] + s[0] + s[6:0:-1] + s[37]
974         elif len(s) == 81:
975             return s[56] + s[79:56:-1] + s[41] + s[55:41:-1] + s[80] + s[40:34:-1] + s[0] + s[33:29:-1] + s[34] + s[28:9:-1] + s[29] + s[8:0:-1] + s[9]
976         elif len(s) == 80:
977             return s[1:19] + s[0] + s[20:68] + s[19] + s[69:80]
978         elif len(s) == 79:
979             return s[54] + s[77:54:-1] + s[39] + s[53:39:-1] + s[78] + s[38:34:-1] + s[0] + s[33:29:-1] + s[34] + s[28:9:-1] + s[29] + s[8:0:-1] + s[9]
980
981         else:
982             raise ExtractorError(u'Unable to decrypt signature, key length %d not supported; retrying might work' % (len(s)))
983
984     def _get_available_subtitles(self, video_id, webpage):
985         try:
986             sub_list = self._download_webpage(
987                 'http://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id,
988                 video_id, note=False)
989         except ExtractorError as err:
990             self._downloader.report_warning(u'unable to download video subtitles: %s' % compat_str(err))
991             return {}
992         lang_list = re.findall(r'name="([^"]*)"[^>]+lang_code="([\w\-]+)"', sub_list)
993
994         sub_lang_list = {}
995         for l in lang_list:
996             lang = l[1]
997             params = compat_urllib_parse.urlencode({
998                 'lang': lang,
999                 'v': video_id,
1000                 'fmt': self._downloader.params.get('subtitlesformat', 'srt'),
1001                 'name': l[0].encode('utf-8'),
1002             })
1003             url = u'http://www.youtube.com/api/timedtext?' + params
1004             sub_lang_list[lang] = url
1005         if not sub_lang_list:
1006             self._downloader.report_warning(u'video doesn\'t have subtitles')
1007             return {}
1008         return sub_lang_list
1009
1010     def _get_available_automatic_caption(self, video_id, webpage):
1011         """We need the webpage for getting the captions url, pass it as an
1012            argument to speed up the process."""
1013         sub_format = self._downloader.params.get('subtitlesformat', 'srt')
1014         self.to_screen(u'%s: Looking for automatic captions' % video_id)
1015         mobj = re.search(r';ytplayer.config = ({.*?});', webpage)
1016         err_msg = u'Couldn\'t find automatic captions for %s' % video_id
1017         if mobj is None:
1018             self._downloader.report_warning(err_msg)
1019             return {}
1020         player_config = json.loads(mobj.group(1))
1021         try:
1022             args = player_config[u'args']
1023             caption_url = args[u'ttsurl']
1024             timestamp = args[u'timestamp']
1025             # We get the available subtitles
1026             list_params = compat_urllib_parse.urlencode({
1027                 'type': 'list',
1028                 'tlangs': 1,
1029                 'asrs': 1,
1030             })
1031             list_url = caption_url + '&' + list_params
1032             caption_list = self._download_xml(list_url, video_id)
1033             original_lang_node = caption_list.find('track')
1034             if original_lang_node is None or original_lang_node.attrib.get('kind') != 'asr' :
1035                 self._downloader.report_warning(u'Video doesn\'t have automatic captions')
1036                 return {}
1037             original_lang = original_lang_node.attrib['lang_code']
1038
1039             sub_lang_list = {}
1040             for lang_node in caption_list.findall('target'):
1041                 sub_lang = lang_node.attrib['lang_code']
1042                 params = compat_urllib_parse.urlencode({
1043                     'lang': original_lang,
1044                     'tlang': sub_lang,
1045                     'fmt': sub_format,
1046                     'ts': timestamp,
1047                     'kind': 'asr',
1048                 })
1049                 sub_lang_list[sub_lang] = caption_url + '&' + params
1050             return sub_lang_list
1051         # An extractor error can be raise by the download process if there are
1052         # no automatic captions but there are subtitles
1053         except (KeyError, ExtractorError):
1054             self._downloader.report_warning(err_msg)
1055             return {}
1056
1057     def _extract_id(self, url):
1058         mobj = re.match(self._VALID_URL, url, re.VERBOSE)
1059         if mobj is None:
1060             raise ExtractorError(u'Invalid URL: %s' % url)
1061         video_id = mobj.group(2)
1062         return video_id
1063
1064     def _get_video_url_list(self, url_map):
1065         """
1066         Transform a dictionary in the format {itag:url} to a list of (itag, url)
1067         with the requested formats.
1068         """
1069         existing_formats = [x for x in self._formats if x in url_map]
1070         if len(existing_formats) == 0:
1071             raise ExtractorError(u'no known formats available for video')
1072         video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats
1073         video_url_list.reverse() # order worst to best
1074         return video_url_list
1075
1076     def _extract_from_m3u8(self, manifest_url, video_id):
1077         url_map = {}
1078         def _get_urls(_manifest):
1079             lines = _manifest.split('\n')
1080             urls = filter(lambda l: l and not l.startswith('#'),
1081                             lines)
1082             return urls
1083         manifest = self._download_webpage(manifest_url, video_id, u'Downloading formats manifest')
1084         formats_urls = _get_urls(manifest)
1085         for format_url in formats_urls:
1086             itag = self._search_regex(r'itag/(\d+?)/', format_url, 'itag')
1087             url_map[itag] = format_url
1088         return url_map
1089
1090     def _extract_annotations(self, video_id):
1091         url = 'https://www.youtube.com/annotations_invideo?features=1&legacy=1&video_id=%s' % video_id
1092         return self._download_webpage(url, video_id, note=u'Searching for annotations.', errnote=u'Unable to download video annotations.')
1093
1094     def _real_extract(self, url):
1095         # Extract original video URL from URL with redirection, like age verification, using next_url parameter
1096         mobj = re.search(self._NEXT_URL_RE, url)
1097         if mobj:
1098             url = 'https://www.youtube.com/' + compat_urllib_parse.unquote(mobj.group(1)).lstrip('/')
1099         video_id = self._extract_id(url)
1100
1101         # Get video webpage
1102         url = 'https://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1' % video_id
1103         video_webpage = self._download_webpage(url, video_id)
1104
1105         # Attempt to extract SWF player URL
1106         mobj = re.search(r'swfConfig.*?"(https?:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)
1107         if mobj is not None:
1108             player_url = re.sub(r'\\(.)', r'\1', mobj.group(1))
1109         else:
1110             player_url = None
1111
1112         # Get video info
1113         self.report_video_info_webpage_download(video_id)
1114         if re.search(r'player-age-gate-content">', video_webpage) is not None:
1115             self.report_age_confirmation()
1116             age_gate = True
1117             # We simulate the access to the video from www.youtube.com/v/{video_id}
1118             # this can be viewed without login into Youtube
1119             data = compat_urllib_parse.urlencode({'video_id': video_id,
1120                                                   'el': 'player_embedded',
1121                                                   'gl': 'US',
1122                                                   'hl': 'en',
1123                                                   'eurl': 'https://youtube.googleapis.com/v/' + video_id,
1124                                                   'asv': 3,
1125                                                   'sts':'1588',
1126                                                   })
1127             video_info_url = 'https://www.youtube.com/get_video_info?' + data
1128             video_info_webpage = self._download_webpage(video_info_url, video_id,
1129                                     note=False,
1130                                     errnote='unable to download video info webpage')
1131             video_info = compat_parse_qs(video_info_webpage)
1132         else:
1133             age_gate = False
1134             for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']:
1135                 video_info_url = ('https://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
1136                         % (video_id, el_type))
1137                 video_info_webpage = self._download_webpage(video_info_url, video_id,
1138                                         note=False,
1139                                         errnote='unable to download video info webpage')
1140                 video_info = compat_parse_qs(video_info_webpage)
1141                 if 'token' in video_info:
1142                     break
1143         if 'token' not in video_info:
1144             if 'reason' in video_info:
1145                 raise ExtractorError(u'YouTube said: %s' % video_info['reason'][0], expected=True)
1146             else:
1147                 raise ExtractorError(u'"token" parameter not in video info for unknown reason')
1148
1149         if 'view_count' in video_info:
1150             view_count = int(video_info['view_count'][0])
1151         else:
1152             view_count = None
1153
1154         # Check for "rental" videos
1155         if 'ypc_video_rental_bar_text' in video_info and 'author' not in video_info:
1156             raise ExtractorError(u'"rental" videos not supported')
1157
1158         # Start extracting information
1159         self.report_information_extraction(video_id)
1160
1161         # uploader
1162         if 'author' not in video_info:
1163             raise ExtractorError(u'Unable to extract uploader name')
1164         video_uploader = compat_urllib_parse.unquote_plus(video_info['author'][0])
1165
1166         # uploader_id
1167         video_uploader_id = None
1168         mobj = re.search(r'<link itemprop="url" href="http://www.youtube.com/(?:user|channel)/([^"]+)">', video_webpage)
1169         if mobj is not None:
1170             video_uploader_id = mobj.group(1)
1171         else:
1172             self._downloader.report_warning(u'unable to extract uploader nickname')
1173
1174         # title
1175         if 'title' in video_info:
1176             video_title = compat_urllib_parse.unquote_plus(video_info['title'][0])
1177         else:
1178             self._downloader.report_warning(u'Unable to extract video title')
1179             video_title = u'_'
1180
1181         # thumbnail image
1182         # We try first to get a high quality image:
1183         m_thumb = re.search(r'<span itemprop="thumbnail".*?href="(.*?)">',
1184                             video_webpage, re.DOTALL)
1185         if m_thumb is not None:
1186             video_thumbnail = m_thumb.group(1)
1187         elif 'thumbnail_url' not in video_info:
1188             self._downloader.report_warning(u'unable to extract video thumbnail')
1189             video_thumbnail = None
1190         else:   # don't panic if we can't find it
1191             video_thumbnail = compat_urllib_parse.unquote_plus(video_info['thumbnail_url'][0])
1192
1193         # upload date
1194         upload_date = None
1195         mobj = re.search(r'id="eow-date.*?>(.*?)</span>', video_webpage, re.DOTALL)
1196         if mobj is not None:
1197             upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())
1198             upload_date = unified_strdate(upload_date)
1199
1200         # description
1201         video_description = get_element_by_id("eow-description", video_webpage)
1202         if video_description:
1203             video_description = re.sub(r'''(?x)
1204                 <a\s+
1205                     (?:[a-zA-Z-]+="[^"]+"\s+)*?
1206                     title="([^"]+)"\s+
1207                     (?:[a-zA-Z-]+="[^"]+"\s+)*?
1208                     class="yt-uix-redirect-link"\s*>
1209                 [^<]+
1210                 </a>
1211             ''', r'\1', video_description)
1212             video_description = clean_html(video_description)
1213         else:
1214             fd_mobj = re.search(r'<meta name="description" content="([^"]+)"', video_webpage)
1215             if fd_mobj:
1216                 video_description = unescapeHTML(fd_mobj.group(1))
1217             else:
1218                 video_description = u''
1219
1220         def _extract_count(klass):
1221             count = self._search_regex(
1222                 r'class="%s">([\d,]+)</span>' % re.escape(klass),
1223                 video_webpage, klass, default=None)
1224             if count is not None:
1225                 return int(count.replace(',', ''))
1226             return None
1227         like_count = _extract_count(u'likes-count')
1228         dislike_count = _extract_count(u'dislikes-count')
1229
1230         # subtitles
1231         video_subtitles = self.extract_subtitles(video_id, video_webpage)
1232
1233         if self._downloader.params.get('listsubtitles', False):
1234             self._list_available_subtitles(video_id, video_webpage)
1235             return
1236
1237         if 'length_seconds' not in video_info:
1238             self._downloader.report_warning(u'unable to extract video duration')
1239             video_duration = None
1240         else:
1241             video_duration = int(compat_urllib_parse.unquote_plus(video_info['length_seconds'][0]))
1242
1243         # annotations
1244         video_annotations = None
1245         if self._downloader.params.get('writeannotations', False):
1246                 video_annotations = self._extract_annotations(video_id)
1247
1248         # Decide which formats to download
1249
1250         try:
1251             mobj = re.search(r';ytplayer.config = ({.*?});', video_webpage)
1252             if not mobj:
1253                 raise ValueError('Could not find vevo ID')
1254             info = json.loads(mobj.group(1))
1255             args = info['args']
1256             # Easy way to know if the 's' value is in url_encoded_fmt_stream_map
1257             # this signatures are encrypted
1258             if 'url_encoded_fmt_stream_map' not in args:
1259                 raise ValueError(u'No stream_map present')  # caught below
1260             re_signature = re.compile(r'[&,]s=')
1261             m_s = re_signature.search(args['url_encoded_fmt_stream_map'])
1262             if m_s is not None:
1263                 self.to_screen(u'%s: Encrypted signatures detected.' % video_id)
1264                 video_info['url_encoded_fmt_stream_map'] = [args['url_encoded_fmt_stream_map']]
1265             m_s = re_signature.search(args.get('adaptive_fmts', u''))
1266             if m_s is not None:
1267                 if 'adaptive_fmts' in video_info:
1268                     video_info['adaptive_fmts'][0] += ',' + args['adaptive_fmts']
1269                 else:
1270                     video_info['adaptive_fmts'] = [args['adaptive_fmts']]
1271         except ValueError:
1272             pass
1273
1274         if 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
1275             self.report_rtmp_download()
1276             video_url_list = [(None, video_info['conn'][0])]
1277         elif len(video_info.get('url_encoded_fmt_stream_map', [])) >= 1 or len(video_info.get('adaptive_fmts', [])) >= 1:
1278             encoded_url_map = video_info.get('url_encoded_fmt_stream_map', [''])[0] + ',' + video_info.get('adaptive_fmts',[''])[0]
1279             if 'rtmpe%3Dyes' in encoded_url_map:
1280                 raise ExtractorError('rtmpe downloads are not supported, see https://github.com/rg3/youtube-dl/issues/343 for more information.', expected=True)
1281             url_map = {}
1282             for url_data_str in encoded_url_map.split(','):
1283                 url_data = compat_parse_qs(url_data_str)
1284                 if 'itag' in url_data and 'url' in url_data:
1285                     url = url_data['url'][0]
1286                     if 'sig' in url_data:
1287                         url += '&signature=' + url_data['sig'][0]
1288                     elif 's' in url_data:
1289                         encrypted_sig = url_data['s'][0]
1290                         if self._downloader.params.get('verbose'):
1291                             if age_gate:
1292                                 if player_url is None:
1293                                     player_version = 'unknown'
1294                                 else:
1295                                     player_version = self._search_regex(
1296                                         r'-(.+)\.swf$', player_url,
1297                                         u'flash player', fatal=False)
1298                                 player_desc = 'flash player %s' % player_version
1299                             else:
1300                                 player_version = self._search_regex(
1301                                     r'html5player-(.+?)\.js', video_webpage,
1302                                     'html5 player', fatal=False)
1303                                 player_desc = u'html5 player %s' % player_version
1304
1305                             parts_sizes = u'.'.join(compat_str(len(part)) for part in encrypted_sig.split('.'))
1306                             self.to_screen(u'encrypted signature length %d (%s), itag %s, %s' %
1307                                 (len(encrypted_sig), parts_sizes, url_data['itag'][0], player_desc))
1308
1309                         if not age_gate:
1310                             jsplayer_url_json = self._search_regex(
1311                                 r'"assets":.+?"js":\s*("[^"]+")',
1312                                 video_webpage, u'JS player URL')
1313                             player_url = json.loads(jsplayer_url_json)
1314
1315                         signature = self._decrypt_signature(
1316                             encrypted_sig, video_id, player_url, age_gate)
1317                         url += '&signature=' + signature
1318                     if 'ratebypass' not in url:
1319                         url += '&ratebypass=yes'
1320                     url_map[url_data['itag'][0]] = url
1321             video_url_list = self._get_video_url_list(url_map)
1322         elif video_info.get('hlsvp'):
1323             manifest_url = video_info['hlsvp'][0]
1324             url_map = self._extract_from_m3u8(manifest_url, video_id)
1325             video_url_list = self._get_video_url_list(url_map)
1326         else:
1327             raise ExtractorError(u'no conn, hlsvp or url_encoded_fmt_stream_map information found in video info')
1328
1329         formats = []
1330         for itag, video_real_url in video_url_list:
1331             dct = {
1332                 'format_id': itag,
1333                 'url': video_real_url,
1334                 'player_url': player_url,
1335             }
1336             dct.update(self._formats[itag])
1337             formats.append(dct)
1338
1339         self._sort_formats(formats)
1340
1341         return {
1342             'id':           video_id,
1343             'uploader':     video_uploader,
1344             'uploader_id':  video_uploader_id,
1345             'upload_date':  upload_date,
1346             'title':        video_title,
1347             'thumbnail':    video_thumbnail,
1348             'description':  video_description,
1349             'subtitles':    video_subtitles,
1350             'duration':     video_duration,
1351             'age_limit':    18 if age_gate else 0,
1352             'annotations':  video_annotations,
1353             'webpage_url': 'https://www.youtube.com/watch?v=%s' % video_id,
1354             'view_count':   view_count,
1355             'like_count': like_count,
1356             'dislike_count': dislike_count,
1357             'formats':      formats,
1358         }
1359
1360 class YoutubePlaylistIE(YoutubeBaseInfoExtractor):
1361     IE_DESC = u'YouTube.com playlists'
1362     _VALID_URL = r"""(?:
1363                         (?:https?://)?
1364                         (?:\w+\.)?
1365                         youtube\.com/
1366                         (?:
1367                            (?:course|view_play_list|my_playlists|artist|playlist|watch)
1368                            \? (?:.*?&)*? (?:p|a|list)=
1369                         |  p/
1370                         )
1371                         ((?:PL|EC|UU|FL|RD)?[0-9A-Za-z-_]{10,})
1372                         .*
1373                      |
1374                         ((?:PL|EC|UU|FL|RD)[0-9A-Za-z-_]{10,})
1375                      )"""
1376     _TEMPLATE_URL = 'https://www.youtube.com/playlist?list=%s&page=%s'
1377     _MORE_PAGES_INDICATOR = r'data-link-type="next"'
1378     _VIDEO_RE = r'href="/watch\?v=(?P<id>[0-9A-Za-z_-]{11})&amp;[^"]*?index=(?P<index>\d+)'
1379     IE_NAME = u'youtube:playlist'
1380
1381     @classmethod
1382     def suitable(cls, url):
1383         """Receives a URL and returns True if suitable for this IE."""
1384         return re.match(cls._VALID_URL, url, re.VERBOSE) is not None
1385
1386     def _real_initialize(self):
1387         self._login()
1388
1389     def _ids_to_results(self, ids):
1390         return [self.url_result(vid_id, 'Youtube', video_id=vid_id)
1391                        for vid_id in ids]
1392
1393     def _extract_mix(self, playlist_id):
1394         # The mixes are generated from a a single video
1395         # the id of the playlist is just 'RD' + video_id
1396         url = 'https://youtube.com/watch?v=%s&list=%s' % (playlist_id[-11:], playlist_id)
1397         webpage = self._download_webpage(url, playlist_id, u'Downloading Youtube mix')
1398         title_span = (get_element_by_attribute('class', 'title long-title', webpage) or
1399             get_element_by_attribute('class', 'title ', webpage))
1400         title = clean_html(title_span)
1401         video_re = r'data-index="\d+".*?href="/watch\?v=([0-9A-Za-z_-]{11})&amp;[^"]*?list=%s' % re.escape(playlist_id)
1402         ids = orderedSet(re.findall(video_re, webpage))
1403         url_results = self._ids_to_results(ids)
1404
1405         return self.playlist_result(url_results, playlist_id, title)
1406
1407     def _real_extract(self, url):
1408         # Extract playlist id
1409         mobj = re.match(self._VALID_URL, url, re.VERBOSE)
1410         if mobj is None:
1411             raise ExtractorError(u'Invalid URL: %s' % url)
1412         playlist_id = mobj.group(1) or mobj.group(2)
1413
1414         # Check if it's a video-specific URL
1415         query_dict = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
1416         if 'v' in query_dict:
1417             video_id = query_dict['v'][0]
1418             if self._downloader.params.get('noplaylist'):
1419                 self.to_screen(u'Downloading just video %s because of --no-playlist' % video_id)
1420                 return self.url_result(video_id, 'Youtube', video_id=video_id)
1421             else:
1422                 self.to_screen(u'Downloading playlist PL%s - add --no-playlist to just download video %s' % (playlist_id, video_id))
1423
1424         if playlist_id.startswith('RD'):
1425             # Mixes require a custom extraction process
1426             return self._extract_mix(playlist_id)
1427         if playlist_id.startswith('TL'):
1428             raise ExtractorError(u'For downloading YouTube.com top lists, use '
1429                 u'the "yttoplist" keyword, for example "youtube-dl \'yttoplist:music:Top Tracks\'"', expected=True)
1430
1431         # Extract the video ids from the playlist pages
1432         ids = []
1433
1434         for page_num in itertools.count(1):
1435             url = self._TEMPLATE_URL % (playlist_id, page_num)
1436             page = self._download_webpage(url, playlist_id, u'Downloading page #%s' % page_num)
1437             matches = re.finditer(self._VIDEO_RE, page)
1438             # We remove the duplicates and the link with index 0
1439             # (it's not the first video of the playlist)
1440             new_ids = orderedSet(m.group('id') for m in matches if m.group('index') != '0')
1441             ids.extend(new_ids)
1442
1443             if re.search(self._MORE_PAGES_INDICATOR, page) is None:
1444                 break
1445
1446         playlist_title = self._og_search_title(page)
1447
1448         url_results = self._ids_to_results(ids)
1449         return self.playlist_result(url_results, playlist_id, playlist_title)
1450
1451
1452 class YoutubeTopListIE(YoutubePlaylistIE):
1453     IE_NAME = u'youtube:toplist'
1454     IE_DESC = (u'YouTube.com top lists, "yttoplist:{channel}:{list title}"'
1455         u' (Example: "yttoplist:music:Top Tracks")')
1456     _VALID_URL = r'yttoplist:(?P<chann>.*?):(?P<title>.*?)$'
1457
1458     def _real_extract(self, url):
1459         mobj = re.match(self._VALID_URL, url)
1460         channel = mobj.group('chann')
1461         title = mobj.group('title')
1462         query = compat_urllib_parse.urlencode({'title': title})
1463         playlist_re = 'href="([^"]+?%s[^"]+?)"' % re.escape(query)
1464         channel_page = self._download_webpage('https://www.youtube.com/%s' % channel, title)
1465         link = self._html_search_regex(playlist_re, channel_page, u'list')
1466         url = compat_urlparse.urljoin('https://www.youtube.com/', link)
1467         
1468         video_re = r'data-index="\d+".*?data-video-id="([0-9A-Za-z_-]{11})"'
1469         ids = []
1470         # sometimes the webpage doesn't contain the videos
1471         # retry until we get them
1472         for i in itertools.count(0):
1473             msg = u'Downloading Youtube mix'
1474             if i > 0:
1475                 msg += ', retry #%d' % i
1476             webpage = self._download_webpage(url, title, msg)
1477             ids = orderedSet(re.findall(video_re, webpage))
1478             if ids:
1479                 break
1480         url_results = self._ids_to_results(ids)
1481         return self.playlist_result(url_results, playlist_title=title)
1482
1483
1484 class YoutubeChannelIE(InfoExtractor):
1485     IE_DESC = u'YouTube.com channels'
1486     _VALID_URL = r"^(?:https?://)?(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/([0-9A-Za-z_-]+)"
1487     _MORE_PAGES_INDICATOR = 'yt-uix-load-more'
1488     _MORE_PAGES_URL = 'http://www.youtube.com/c4_browse_ajax?action_load_more_videos=1&flow=list&paging=%s&view=0&sort=da&channel_id=%s'
1489     IE_NAME = u'youtube:channel'
1490
1491     def extract_videos_from_page(self, page):
1492         ids_in_page = []
1493         for mobj in re.finditer(r'href="/watch\?v=([0-9A-Za-z_-]+)&?', page):
1494             if mobj.group(1) not in ids_in_page:
1495                 ids_in_page.append(mobj.group(1))
1496         return ids_in_page
1497
1498     def _real_extract(self, url):
1499         # Extract channel id
1500         mobj = re.match(self._VALID_URL, url)
1501         if mobj is None:
1502             raise ExtractorError(u'Invalid URL: %s' % url)
1503
1504         # Download channel page
1505         channel_id = mobj.group(1)
1506         video_ids = []
1507         url = 'https://www.youtube.com/channel/%s/videos' % channel_id
1508         channel_page = self._download_webpage(url, channel_id)
1509         autogenerated = re.search(r'''(?x)
1510                 class="[^"]*?(?:
1511                     channel-header-autogenerated-label|
1512                     yt-channel-title-autogenerated
1513                 )[^"]*"''', channel_page) is not None
1514
1515         if autogenerated:
1516             # The videos are contained in a single page
1517             # the ajax pages can't be used, they are empty
1518             video_ids = self.extract_videos_from_page(channel_page)
1519         else:
1520             # Download all channel pages using the json-based channel_ajax query
1521             for pagenum in itertools.count(1):
1522                 url = self._MORE_PAGES_URL % (pagenum, channel_id)
1523                 page = self._download_webpage(url, channel_id,
1524                                               u'Downloading page #%s' % pagenum)
1525     
1526                 page = json.loads(page)
1527     
1528                 ids_in_page = self.extract_videos_from_page(page['content_html'])
1529                 video_ids.extend(ids_in_page)
1530     
1531                 if self._MORE_PAGES_INDICATOR not in page['load_more_widget_html']:
1532                     break
1533
1534         self._downloader.to_screen(u'[youtube] Channel %s: Found %i videos' % (channel_id, len(video_ids)))
1535
1536         url_entries = [self.url_result(video_id, 'Youtube', video_id=video_id)
1537                        for video_id in video_ids]
1538         return self.playlist_result(url_entries, channel_id)
1539
1540
1541 class YoutubeUserIE(InfoExtractor):
1542     IE_DESC = u'YouTube.com user videos (URL or "ytuser" keyword)'
1543     _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:user/)?(?!(?:attribution_link|watch)(?:$|[^a-z_A-Z0-9-])))|ytuser:)(?!feed/)([A-Za-z0-9_-]+)'
1544     _TEMPLATE_URL = 'http://gdata.youtube.com/feeds/api/users/%s'
1545     _GDATA_PAGE_SIZE = 50
1546     _GDATA_URL = 'http://gdata.youtube.com/feeds/api/users/%s/uploads?max-results=%d&start-index=%d&alt=json'
1547     IE_NAME = u'youtube:user'
1548
1549     @classmethod
1550     def suitable(cls, url):
1551         # Don't return True if the url can be extracted with other youtube
1552         # extractor, the regex would is too permissive and it would match.
1553         other_ies = iter(klass for (name, klass) in globals().items() if name.endswith('IE') and klass is not cls)
1554         if any(ie.suitable(url) for ie in other_ies): return False
1555         else: return super(YoutubeUserIE, cls).suitable(url)
1556
1557     def _real_extract(self, url):
1558         # Extract username
1559         mobj = re.match(self._VALID_URL, url)
1560         if mobj is None:
1561             raise ExtractorError(u'Invalid URL: %s' % url)
1562
1563         username = mobj.group(1)
1564
1565         # Download video ids using YouTube Data API. Result size per
1566         # query is limited (currently to 50 videos) so we need to query
1567         # page by page until there are no video ids - it means we got
1568         # all of them.
1569
1570         url_results = []
1571
1572         for pagenum in itertools.count(0):
1573             start_index = pagenum * self._GDATA_PAGE_SIZE + 1
1574
1575             gdata_url = self._GDATA_URL % (username, self._GDATA_PAGE_SIZE, start_index)
1576             page = self._download_webpage(gdata_url, username,
1577                                           u'Downloading video ids from %d to %d' % (start_index, start_index + self._GDATA_PAGE_SIZE))
1578
1579             try:
1580                 response = json.loads(page)
1581             except ValueError as err:
1582                 raise ExtractorError(u'Invalid JSON in API response: ' + compat_str(err))
1583             if 'entry' not in response['feed']:
1584                 # Number of videos is a multiple of self._MAX_RESULTS
1585                 break
1586
1587             # Extract video identifiers
1588             entries = response['feed']['entry']
1589             for entry in entries:
1590                 title = entry['title']['$t']
1591                 video_id = entry['id']['$t'].split('/')[-1]
1592                 url_results.append({
1593                     '_type': 'url',
1594                     'url': video_id,
1595                     'ie_key': 'Youtube',
1596                     'id': 'video_id',
1597                     'title': title,
1598                 })
1599
1600             # A little optimization - if current page is not
1601             # "full", ie. does not contain PAGE_SIZE video ids then
1602             # we can assume that this page is the last one - there
1603             # are no more ids on further pages - no need to query
1604             # again.
1605
1606             if len(entries) < self._GDATA_PAGE_SIZE:
1607                 break
1608
1609         return self.playlist_result(url_results, playlist_title=username)
1610
1611
1612 class YoutubeSearchIE(SearchInfoExtractor):
1613     IE_DESC = u'YouTube.com searches'
1614     _API_URL = 'https://gdata.youtube.com/feeds/api/videos?q=%s&start-index=%i&max-results=50&v=2&alt=jsonc'
1615     _MAX_RESULTS = 1000
1616     IE_NAME = u'youtube:search'
1617     _SEARCH_KEY = 'ytsearch'
1618
1619     def _get_n_results(self, query, n):
1620         """Get a specified number of results for a query"""
1621
1622         video_ids = []
1623         pagenum = 0
1624         limit = n
1625
1626         while (50 * pagenum) < limit:
1627             result_url = self._API_URL % (compat_urllib_parse.quote_plus(query), (50*pagenum)+1)
1628             data_json = self._download_webpage(
1629                 result_url, video_id=u'query "%s"' % query,
1630                 note=u'Downloading page %s' % (pagenum + 1),
1631                 errnote=u'Unable to download API page')
1632             data = json.loads(data_json)
1633             api_response = data['data']
1634
1635             if 'items' not in api_response:
1636                 raise ExtractorError(u'[youtube] No video results')
1637
1638             new_ids = list(video['id'] for video in api_response['items'])
1639             video_ids += new_ids
1640
1641             limit = min(n, api_response['totalItems'])
1642             pagenum += 1
1643
1644         if len(video_ids) > n:
1645             video_ids = video_ids[:n]
1646         videos = [self.url_result(video_id, 'Youtube', video_id=video_id)
1647                   for video_id in video_ids]
1648         return self.playlist_result(videos, query)
1649
1650 class YoutubeSearchDateIE(YoutubeSearchIE):
1651     IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
1652     _API_URL = 'https://gdata.youtube.com/feeds/api/videos?q=%s&start-index=%i&max-results=50&v=2&alt=jsonc&orderby=published'
1653     _SEARCH_KEY = 'ytsearchdate'
1654     IE_DESC = u'YouTube.com searches, newest videos first'
1655
1656 class YoutubeShowIE(InfoExtractor):
1657     IE_DESC = u'YouTube.com (multi-season) shows'
1658     _VALID_URL = r'https?://www\.youtube\.com/show/(.*)'
1659     IE_NAME = u'youtube:show'
1660
1661     def _real_extract(self, url):
1662         mobj = re.match(self._VALID_URL, url)
1663         show_name = mobj.group(1)
1664         webpage = self._download_webpage(url, show_name, u'Downloading show webpage')
1665         # There's one playlist for each season of the show
1666         m_seasons = list(re.finditer(r'href="(/playlist\?list=.*?)"', webpage))
1667         self.to_screen(u'%s: Found %s seasons' % (show_name, len(m_seasons)))
1668         return [self.url_result('https://www.youtube.com' + season.group(1), 'YoutubePlaylist') for season in m_seasons]
1669
1670
1671 class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor):
1672     """
1673     Base class for extractors that fetch info from
1674     http://www.youtube.com/feed_ajax
1675     Subclasses must define the _FEED_NAME and _PLAYLIST_TITLE properties.
1676     """
1677     _LOGIN_REQUIRED = True
1678     # use action_load_personal_feed instead of action_load_system_feed
1679     _PERSONAL_FEED = False
1680
1681     @property
1682     def _FEED_TEMPLATE(self):
1683         action = 'action_load_system_feed'
1684         if self._PERSONAL_FEED:
1685             action = 'action_load_personal_feed'
1686         return 'http://www.youtube.com/feed_ajax?%s=1&feed_name=%s&paging=%%s' % (action, self._FEED_NAME)
1687
1688     @property
1689     def IE_NAME(self):
1690         return u'youtube:%s' % self._FEED_NAME
1691
1692     def _real_initialize(self):
1693         self._login()
1694
1695     def _real_extract(self, url):
1696         feed_entries = []
1697         paging = 0
1698         for i in itertools.count(1):
1699             info = self._download_webpage(self._FEED_TEMPLATE % paging,
1700                                           u'%s feed' % self._FEED_NAME,
1701                                           u'Downloading page %s' % i)
1702             info = json.loads(info)
1703             feed_html = info['feed_html']
1704             m_ids = re.finditer(r'"/watch\?v=(.*?)["&]', feed_html)
1705             ids = orderedSet(m.group(1) for m in m_ids)
1706             feed_entries.extend(
1707                 self.url_result(video_id, 'Youtube', video_id=video_id)
1708                 for video_id in ids)
1709             if info['paging'] is None:
1710                 break
1711             paging = info['paging']
1712         return self.playlist_result(feed_entries, playlist_title=self._PLAYLIST_TITLE)
1713
1714 class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
1715     IE_DESC = u'YouTube.com subscriptions feed, "ytsubs" keyword(requires authentication)'
1716     _VALID_URL = r'https?://www\.youtube\.com/feed/subscriptions|:ytsubs(?:criptions)?'
1717     _FEED_NAME = 'subscriptions'
1718     _PLAYLIST_TITLE = u'Youtube Subscriptions'
1719
1720 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
1721     IE_DESC = u'YouTube.com recommended videos, "ytrec" keyword (requires authentication)'
1722     _VALID_URL = r'https?://www\.youtube\.com/feed/recommended|:ytrec(?:ommended)?'
1723     _FEED_NAME = 'recommended'
1724     _PLAYLIST_TITLE = u'Youtube Recommended videos'
1725
1726 class YoutubeWatchLaterIE(YoutubeFeedsInfoExtractor):
1727     IE_DESC = u'Youtube watch later list, "ytwatchlater" keyword (requires authentication)'
1728     _VALID_URL = r'https?://www\.youtube\.com/feed/watch_later|:ytwatchlater'
1729     _FEED_NAME = 'watch_later'
1730     _PLAYLIST_TITLE = u'Youtube Watch Later'
1731     _PERSONAL_FEED = True
1732
1733 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
1734     IE_DESC = u'Youtube watch history, "ythistory" keyword (requires authentication)'
1735     _VALID_URL = u'https?://www\.youtube\.com/feed/history|:ythistory'
1736     _FEED_NAME = 'history'
1737     _PERSONAL_FEED = True
1738     _PLAYLIST_TITLE = u'Youtube Watch History'
1739
1740 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
1741     IE_NAME = u'youtube:favorites'
1742     IE_DESC = u'YouTube.com favourite videos, "ytfav" keyword (requires authentication)'
1743     _VALID_URL = r'https?://www\.youtube\.com/my_favorites|:ytfav(?:ou?rites)?'
1744     _LOGIN_REQUIRED = True
1745
1746     def _real_extract(self, url):
1747         webpage = self._download_webpage('https://www.youtube.com/my_favorites', 'Youtube Favourites videos')
1748         playlist_id = self._search_regex(r'list=(.+?)["&]', webpage, u'favourites playlist id')
1749         return self.url_result(playlist_id, 'YoutubePlaylist')
1750
1751
1752 class YoutubeTruncatedURLIE(InfoExtractor):
1753     IE_NAME = 'youtube:truncated_url'
1754     IE_DESC = False  # Do not list
1755     _VALID_URL = r'(?:https?://)?[^/]+/watch\?feature=[a-z_]+$'
1756
1757     def _real_extract(self, url):
1758         raise ExtractorError(
1759             u'Did you forget to quote the URL? Remember that & is a meta '
1760             u'character in most shells, so you want to put the URL in quotes, '
1761             u'like  youtube-dl '
1762             u'\'http://www.youtube.com/watch?feature=foo&v=BaW_jenozKc\''
1763             u' (or simply  youtube-dl BaW_jenozKc  ).',
1764             expected=True)