3 from __future__ import unicode_literals
13 from .common import InfoExtractor, SearchInfoExtractor
14 from ..jsinterp import JSInterpreter
15 from ..swfinterp import SWFInterpreter
16 from ..compat import (
20 compat_urllib_parse_unquote,
21 compat_urllib_parse_unquote_plus,
22 compat_urllib_parse_urlparse,
32 get_element_by_attribute,
50 class YoutubeBaseInfoExtractor(InfoExtractor):
51 """Provide base functions for Youtube extractors"""
52 _LOGIN_URL = 'https://accounts.google.com/ServiceLogin'
53 _TWOFACTOR_URL = 'https://accounts.google.com/signin/challenge'
54 _NETRC_MACHINE = 'youtube'
55 # If True it will raise an error if no login info is provided
56 _LOGIN_REQUIRED = False
58 def _set_language(self):
60 '.youtube.com', 'PREF', 'f1=50000000&hl=en',
61 # YouTube sets the expire time to about two months
62 expire_time=time.time() + 2 * 30 * 24 * 3600)
64 def _ids_to_results(self, ids):
66 self.url_result(vid_id, 'Youtube', video_id=vid_id)
71 Attempt to log in to YouTube.
72 True is returned if successful or skipped.
73 False is returned if login failed.
75 If _LOGIN_REQUIRED is set and no authentication was provided, an error is raised.
77 (username, password) = self._get_login_info()
78 # No authentication to be performed
80 if self._LOGIN_REQUIRED:
81 raise ExtractorError('No login info available, needed for using %s.' % self.IE_NAME, expected=True)
84 login_page = self._download_webpage(
85 self._LOGIN_URL, None,
86 note='Downloading login page',
87 errnote='unable to fetch login page', fatal=False)
88 if login_page is False:
91 galx = self._search_regex(r'(?s)<input.+?name="GALX".+?value="(.+?)"',
92 login_page, 'Login GALX parameter')
96 'continue': 'https://www.youtube.com/signin?action_handle_signin=true&feature=sign_in_button&hl=en_US&nomobiletemp=1',
101 'PersistentCookie': 'yes',
103 'bgresponse': 'js_disabled',
104 'checkConnection': '',
105 'checkedDomains': 'youtube',
112 'service': 'youtube',
117 login_data = compat_urllib_parse.urlencode(encode_dict(login_form_strs)).encode('ascii')
119 req = sanitized_Request(self._LOGIN_URL, login_data)
120 login_results = self._download_webpage(
122 note='Logging in', errnote='unable to log in', fatal=False)
123 if login_results is False:
126 if re.search(r'id="errormsg_0_Passwd"', login_results) is not None:
127 raise ExtractorError('Please use your account password and a two-factor code instead of an application-specific password.', expected=True)
130 # TODO add SMS and phone call support - these require making a request and then prompting the user
132 if re.search(r'(?i)<form[^>]* id="challenge"', login_results) is not None:
133 tfa_code = self._get_tfa_info('2-step verification code')
136 self._downloader.report_warning(
137 'Two-factor authentication required. Provide it either interactively or with --twofactor <code>'
138 '(Note that only TOTP (Google Authenticator App) codes work at this time.)')
141 tfa_code = remove_start(tfa_code, 'G-')
143 tfa_form_strs = self._form_hidden_inputs('challenge', login_results)
145 tfa_form_strs.update({
150 tfa_data = compat_urllib_parse.urlencode(encode_dict(tfa_form_strs)).encode('ascii')
152 tfa_req = sanitized_Request(self._TWOFACTOR_URL, tfa_data)
153 tfa_results = self._download_webpage(
155 note='Submitting TFA code', errnote='unable to submit tfa', fatal=False)
157 if tfa_results is False:
160 if re.search(r'(?i)<form[^>]* id="challenge"', tfa_results) is not None:
161 self._downloader.report_warning('Two-factor code expired or invalid. Please try again, or use a one-use backup code instead.')
163 if re.search(r'(?i)<form[^>]* id="gaia_loginform"', tfa_results) is not None:
164 self._downloader.report_warning('unable to log in - did the page structure change?')
166 if re.search(r'smsauth-interstitial-reviewsettings', tfa_results) is not None:
167 self._downloader.report_warning('Your Google account has a security notice. Please log in on your web browser, resolve the notice, and try again.')
170 if re.search(r'(?i)<form[^>]* id="gaia_loginform"', login_results) is not None:
171 self._downloader.report_warning('unable to log in: bad username or password')
175 def _real_initialize(self):
176 if self._downloader is None:
179 if not self._login():
183 class YoutubeEntryListBaseInfoExtractor(InfoExtractor):
184 # Extract entries from page with "Load more" button
185 def _entries(self, page, playlist_id):
186 more_widget_html = content_html = page
187 for page_num in itertools.count(1):
188 for entry in self._process_page(content_html):
191 mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)
195 more = self._download_json(
196 'https://youtube.com/%s' % mobj.group('more'), playlist_id,
197 'Downloading page #%s' % page_num,
198 transform_source=uppercase_escape)
199 content_html = more['content_html']
200 if not content_html.strip():
201 # Some webpages show a "Load more" button but they don't
204 more_widget_html = more['load_more_widget_html']
207 class YoutubePlaylistBaseInfoExtractor(YoutubeEntryListBaseInfoExtractor):
208 def _process_page(self, content):
209 for video_id, video_title in self.extract_videos_from_page(content):
210 yield self.url_result(video_id, 'Youtube', video_id, video_title)
212 def extract_videos_from_page(self, page):
215 for mobj in re.finditer(self._VIDEO_RE, page):
216 # The link with index 0 is not the first video of the playlist (not sure if still actual)
217 if 'index' in mobj.groupdict() and mobj.group('id') == '0':
219 video_id = mobj.group('id')
220 video_title = unescapeHTML(mobj.group('title'))
222 video_title = video_title.strip()
224 idx = ids_in_page.index(video_id)
225 if video_title and not titles_in_page[idx]:
226 titles_in_page[idx] = video_title
228 ids_in_page.append(video_id)
229 titles_in_page.append(video_title)
230 return zip(ids_in_page, titles_in_page)
233 class YoutubePlaylistsBaseInfoExtractor(YoutubeEntryListBaseInfoExtractor):
234 def _process_page(self, content):
235 for playlist_id in re.findall(r'href="/?playlist\?list=(.+?)"', content):
236 yield self.url_result(
237 'https://www.youtube.com/playlist?list=%s' % playlist_id, 'YoutubePlaylist')
239 def _real_extract(self, url):
240 playlist_id = self._match_id(url)
241 webpage = self._download_webpage(url, playlist_id)
242 title = self._og_search_title(webpage, fatal=False)
243 return self.playlist_result(self._entries(webpage, playlist_id), playlist_id, title)
246 class YoutubeIE(YoutubeBaseInfoExtractor):
247 IE_DESC = 'YouTube.com'
248 _VALID_URL = r"""(?x)^
250 (?:https?://|//) # http(s):// or protocol-independent URL
251 (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/|
252 (?:www\.)?deturl\.com/www\.youtube\.com/|
253 (?:www\.)?pwnyoutube\.com/|
254 (?:www\.)?yourepeat\.com/|
255 tube\.majestyc\.net/|
256 youtube\.googleapis\.com/) # the various hostnames, with wildcard subdomains
257 (?:.*?\#/)? # handle anchor (#/) redirect urls
258 (?: # the various things that can precede the ID:
259 (?:(?:v|embed|e)/(?!videoseries)) # v/ or embed/ or e/
260 |(?: # or the v= param in all its forms
261 (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)? # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
262 (?:\?|\#!?) # the params delimiter ? or # or #!
263 (?:.*?[&;])?? # any other preceding param (like /?s=tuff&v=xxxx or ?s=tuff&v=V36LpHqtcDY)
268 youtu\.be| # just youtu.be/xxxx
269 vid\.plus # or vid.plus/xxxx
271 |(?:www\.)?cleanvideosearch\.com/media/action/yt/watch\?videoId=
273 )? # all until now is optional -> you can pass the naked ID
274 ([0-9A-Za-z_-]{11}) # here is it! the YouTube video ID
275 (?!.*?&list=) # combined list/video URLs are handled by the playlist IE
276 (?(1).+)? # if we found the ID, everything can follow
278 _NEXT_URL_RE = r'[\?&]next_url=([^&]+)'
280 '5': {'ext': 'flv', 'width': 400, 'height': 240},
281 '6': {'ext': 'flv', 'width': 450, 'height': 270},
282 '13': {'ext': '3gp'},
283 '17': {'ext': '3gp', 'width': 176, 'height': 144},
284 '18': {'ext': 'mp4', 'width': 640, 'height': 360},
285 '22': {'ext': 'mp4', 'width': 1280, 'height': 720},
286 '34': {'ext': 'flv', 'width': 640, 'height': 360},
287 '35': {'ext': 'flv', 'width': 854, 'height': 480},
288 '36': {'ext': '3gp', 'width': 320, 'height': 240},
289 '37': {'ext': 'mp4', 'width': 1920, 'height': 1080},
290 '38': {'ext': 'mp4', 'width': 4096, 'height': 3072},
291 '43': {'ext': 'webm', 'width': 640, 'height': 360},
292 '44': {'ext': 'webm', 'width': 854, 'height': 480},
293 '45': {'ext': 'webm', 'width': 1280, 'height': 720},
294 '46': {'ext': 'webm', 'width': 1920, 'height': 1080},
295 '59': {'ext': 'mp4', 'width': 854, 'height': 480},
296 '78': {'ext': 'mp4', 'width': 854, 'height': 480},
300 '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'preference': -20},
301 '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'preference': -20},
302 '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'preference': -20},
303 '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'preference': -20},
304 '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'preference': -20},
305 '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'preference': -20},
306 '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'preference': -20},
308 # Apple HTTP Live Streaming
309 '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'preference': -10},
310 '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'preference': -10},
311 '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'preference': -10},
312 '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'preference': -10},
313 '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'preference': -10},
314 '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'preference': -10},
315 '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'preference': -10},
318 '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
319 '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
320 '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
321 '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
322 '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
323 '138': {'ext': 'mp4', 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40}, # Height can vary (https://github.com/rg3/youtube-dl/issues/4559)
324 '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
325 '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
326 '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'h264'},
327 '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'h264'},
328 '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'vcodec': 'h264'},
331 '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'vcodec': 'none', 'abr': 48, 'preference': -50, 'container': 'm4a_dash'},
332 '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'vcodec': 'none', 'abr': 128, 'preference': -50, 'container': 'm4a_dash'},
333 '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'vcodec': 'none', 'abr': 256, 'preference': -50, 'container': 'm4a_dash'},
336 '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
337 '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
338 '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
339 '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
340 '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
341 '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'acodec': 'none', 'container': 'webm', 'vcodec': 'vp8', 'preference': -40},
342 '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'container': 'webm', 'vcodec': 'vp9'},
343 '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
344 '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
345 '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
346 '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
347 '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
348 '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
349 '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
350 '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
351 # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
352 '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40},
353 '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'vp9'},
354 '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'vp9'},
355 '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'vp9'},
356 '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'vcodec': 'vp9'},
357 '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'acodec': 'none', 'preference': -40, 'fps': 60, 'vcodec': 'vp9'},
360 '171': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'abr': 128, 'preference': -50},
361 '172': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'abr': 256, 'preference': -50},
363 # Dash webm audio with opus inside
364 '249': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50, 'preference': -50},
365 '250': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70, 'preference': -50},
366 '251': {'ext': 'webm', 'vcodec': 'none', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160, 'preference': -50},
369 '_rtmp': {'protocol': 'rtmp'},
375 'url': 'http://www.youtube.com/watch?v=BaW_jenozKcj&t=1s&end=9',
379 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
380 'uploader': 'Philipp Hagemeister',
381 'uploader_id': 'phihag',
382 'upload_date': '20121002',
383 'description': 'test chars: "\'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
384 'categories': ['Science & Technology'],
385 'tags': ['youtube-dl'],
387 'dislike_count': int,
393 'url': 'http://www.youtube.com/watch?v=UxxajLWwzqY',
394 'note': 'Test generic use_cipher_signature video (#897)',
398 'upload_date': '20120506',
399 'title': 'Icona Pop - I Love It (feat. Charli XCX) [OFFICIAL VIDEO]',
400 'alt_title': 'I Love It (feat. Charli XCX)',
401 'description': 'md5:782e8651347686cba06e58f71ab51773',
402 'tags': ['Icona Pop i love it', 'sweden', 'pop music', 'big beat records', 'big beat', 'charli',
403 'xcx', 'charli xcx', 'girls', 'hbo', 'i love it', "i don't care", 'icona', 'pop',
404 'iconic ep', 'iconic', 'love', 'it'],
405 'uploader': 'Icona Pop',
406 'uploader_id': 'IconaPop',
407 'creator': 'Icona Pop',
411 'url': 'https://www.youtube.com/watch?v=07FYdnEawAQ',
412 'note': 'Test VEVO video with age protection (#956)',
416 'upload_date': '20130703',
417 'title': 'Justin Timberlake - Tunnel Vision (Explicit)',
418 'alt_title': 'Tunnel Vision',
419 'description': 'md5:64249768eec3bc4276236606ea996373',
420 'uploader': 'justintimberlakeVEVO',
421 'uploader_id': 'justintimberlakeVEVO',
422 'creator': 'Justin Timberlake',
427 'url': '//www.YouTube.com/watch?v=yZIXLfi8CZQ',
428 'note': 'Embed-only video (#1746)',
432 'upload_date': '20120608',
433 'title': 'Principal Sexually Assaults A Teacher - Episode 117 - 8th June 2012',
434 'description': 'md5:09b78bd971f1e3e289601dfba15ca4f7',
435 'uploader': 'SET India',
436 'uploader_id': 'setindia',
441 'url': 'http://www.youtube.com/watch?v=BaW_jenozKcj&v=UxxajLWwzqY',
442 'note': 'Use the first video ID in the URL',
446 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
447 'uploader': 'Philipp Hagemeister',
448 'uploader_id': 'phihag',
449 'upload_date': '20121002',
450 'description': 'test chars: "\'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
451 'categories': ['Science & Technology'],
452 'tags': ['youtube-dl'],
454 'dislike_count': int,
457 'skip_download': True,
461 'url': 'http://www.youtube.com/watch?v=a9LDPn-MO4I',
462 'note': '256k DASH audio (format 141) via DASH manifest',
466 'upload_date': '20121002',
467 'uploader_id': '8KVIDEO',
469 'uploader': '8KVIDEO',
470 'title': 'UHDTV TEST 8K VIDEO.mp4'
473 'youtube_include_dash_manifest': True,
477 # DASH manifest with encrypted signature
479 'url': 'https://www.youtube.com/watch?v=IB3lcPjvWLA',
483 'title': 'Afrojack, Spree Wilson - The Spark ft. Spree Wilson',
484 'description': 'md5:12e7067fa6735a77bdcbb58cb1187d2d',
485 'uploader': 'AfrojackVEVO',
486 'uploader_id': 'AfrojackVEVO',
487 'upload_date': '20131011',
490 'youtube_include_dash_manifest': True,
494 # JS player signature function name containing $
496 'url': 'https://www.youtube.com/watch?v=nfWlot6h_JM',
500 'title': 'Taylor Swift - Shake It Off',
501 'alt_title': 'Shake It Off',
502 'description': 'md5:95f66187cd7c8b2c13eb78e1223b63c3',
503 'uploader': 'TaylorSwiftVEVO',
504 'uploader_id': 'TaylorSwiftVEVO',
505 'upload_date': '20140818',
506 'creator': 'Taylor Swift',
509 'youtube_include_dash_manifest': True,
515 'url': 'https://www.youtube.com/watch?v=T4XJQO3qol8',
519 'upload_date': '20100909',
520 'uploader': 'The Amazing Atheist',
521 'uploader_id': 'TheAmazingAtheist',
522 'title': 'Burning Everyone\'s Koran',
523 'description': 'SUBSCRIBE: http://www.youtube.com/saturninefilms\n\nEven Obama has taken a stand against freedom on this issue: http://www.huffingtonpost.com/2010/09/09/obama-gma-interview-quran_n_710282.html',
526 # Normal age-gate video (No vevo, embed allowed)
528 'url': 'http://youtube.com/watch?v=HtVdAasjOgU',
532 'title': 'The Witcher 3: Wild Hunt - The Sword Of Destiny Trailer',
533 'description': 're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
534 'uploader': 'The Witcher',
535 'uploader_id': 'WitcherGame',
536 'upload_date': '20140605',
540 # Age-gate video with encrypted signature
542 'url': 'http://www.youtube.com/watch?v=6kLq3WMV1nU',
546 'title': 'Dedication To My Ex (Miss That) (Lyric Video)',
547 'description': 'md5:33765bb339e1b47e7e72b5490139bb41',
548 'uploader': 'LloydVEVO',
549 'uploader_id': 'LloydVEVO',
550 'upload_date': '20110629',
554 # video_info is None (https://github.com/rg3/youtube-dl/issues/4421)
556 'url': '__2ABJjxzNo',
560 'upload_date': '20100430',
561 'uploader_id': 'deadmau5',
562 'creator': 'deadmau5',
563 'description': 'md5:12c56784b8032162bb936a5f76d55360',
564 'uploader': 'deadmau5',
565 'title': 'Deadmau5 - Some Chords (HD)',
566 'alt_title': 'Some Chords',
568 'expected_warnings': [
569 'DASH manifest missing',
572 # Olympics (https://github.com/rg3/youtube-dl/issues/4431)
574 'url': 'lqQg6PlCWgI',
578 'upload_date': '20150827',
579 'uploader_id': 'olympic',
580 'description': 'HO09 - Women - GER-AUS - Hockey - 31 July 2012 - London 2012 Olympic Games',
581 'uploader': 'Olympics',
582 'title': 'Hockey - Women - GER-AUS - London 2012 Olympic Games',
585 'skip_download': 'requires avconv',
590 'url': 'https://www.youtube.com/watch?v=_b-2C3KPAM0',
594 'stretched_ratio': 16 / 9.,
595 'upload_date': '20110310',
596 'uploader_id': 'AllenMeow',
597 'description': 'made by Wacom from Korea | 字幕&加油添醋 by TY\'s Allen | 感謝heylisa00cavey1001同學熱情提供梗及翻譯',
599 'title': '[A-made] 變態妍字幕版 太妍 我就是這樣的人',
602 # url_encoded_fmt_stream_map is empty string
604 'url': 'qEJwOuvDf7I',
608 'title': 'Обсуждение судебной практики по выборам 14 сентября 2014 года в Санкт-Петербурге',
610 'upload_date': '20150404',
611 'uploader_id': 'spbelect',
612 'uploader': 'Наблюдатели Петербурга',
615 'skip_download': 'requires avconv',
618 # Extraction from multiple DASH manifests (https://github.com/rg3/youtube-dl/pull/6097)
620 'url': 'https://www.youtube.com/watch?v=FIl7x6_3R5Y',
624 'title': 'md5:7b81415841e02ecd4313668cde88737a',
625 'description': 'md5:116377fd2963b81ec4ce64b542173306',
626 'upload_date': '20150625',
627 'uploader_id': 'dorappi2000',
628 'uploader': 'dorappi2000',
629 'formats': 'mincount:33',
632 # DASH manifest with segment_list
634 'url': 'https://www.youtube.com/embed/CsmdDsKjzN8',
635 'md5': '8ce563a1d667b599d21064e982ab9e31',
639 'upload_date': '20150501', # According to '<meta itemprop="datePublished"', but in other places it's 20150510
640 'uploader': 'Airtek',
641 'description': 'Retransmisión en directo de la XVIII media maratón de Zaragoza.',
642 'uploader_id': 'UCzTzUmjXxxacNnL8I3m4LnQ',
643 'title': 'Retransmisión XVIII Media maratón Zaragoza 2015',
646 'youtube_include_dash_manifest': True,
647 'format': '135', # bestvideo
651 # Multifeed videos (multiple cameras), URL is for Main Camera
652 'url': 'https://www.youtube.com/watch?v=jqWvoWXjCVs',
655 'title': 'teamPGP: Rocket League Noob Stream',
656 'description': 'md5:dc7872fb300e143831327f1bae3af010',
662 'title': 'teamPGP: Rocket League Noob Stream (Main Camera)',
663 'description': 'md5:dc7872fb300e143831327f1bae3af010',
664 'upload_date': '20150721',
665 'uploader': 'Beer Games Beer',
666 'uploader_id': 'beergamesbeer',
672 'title': 'teamPGP: Rocket League Noob Stream (kreestuh)',
673 'description': 'md5:dc7872fb300e143831327f1bae3af010',
674 'upload_date': '20150721',
675 'uploader': 'Beer Games Beer',
676 'uploader_id': 'beergamesbeer',
682 'title': 'teamPGP: Rocket League Noob Stream (grizzle)',
683 'description': 'md5:dc7872fb300e143831327f1bae3af010',
684 'upload_date': '20150721',
685 'uploader': 'Beer Games Beer',
686 'uploader_id': 'beergamesbeer',
692 'title': 'teamPGP: Rocket League Noob Stream (zim)',
693 'description': 'md5:dc7872fb300e143831327f1bae3af010',
694 'upload_date': '20150721',
695 'uploader': 'Beer Games Beer',
696 'uploader_id': 'beergamesbeer',
700 'skip_download': True,
704 'url': 'http://vid.plus/FlRa-iH7PGw',
705 'only_matching': True,
708 # Title with JS-like syntax "};" (see https://github.com/rg3/youtube-dl/issues/7468)
709 # Also tests cut-off URL expansion in video description (see
710 # https://github.com/rg3/youtube-dl/issues/1892,
711 # https://github.com/rg3/youtube-dl/issues/8164)
712 'url': 'https://www.youtube.com/watch?v=lsguqyKfVQg',
716 'title': '{dark walk}; Loki/AC/Dishonored; collab w/Elflover21',
717 'alt_title': 'Dark Walk',
718 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
719 'upload_date': '20151119',
720 'uploader_id': 'IronSoulElf',
721 'uploader': 'IronSoulElf',
722 'creator': 'Todd Haberman, Daniel Law Heath & Aaron Kaplan',
725 'skip_download': True,
729 # Tags with '};' (see https://github.com/rg3/youtube-dl/issues/7468)
730 'url': 'https://www.youtube.com/watch?v=Ms7iBXnlUO8',
731 'only_matching': True,
734 # Video with yt:stretch=17:0
735 'url': 'https://www.youtube.com/watch?v=Q39EVAstoRM',
739 'title': 'Clash Of Clans#14 Dicas De Ataque Para CV 4',
740 'description': 'md5:ee18a25c350637c8faff806845bddee9',
741 'upload_date': '20151107',
742 'uploader_id': 'UCCr7TALkRbo3EtFzETQF1LA',
743 'uploader': 'CH GAMER DROID',
746 'skip_download': True,
750 'url': 'https://www.youtube.com/watch?feature=player_embedded&amp;v=V36LpHqtcDY',
751 'only_matching': True,
755 def __init__(self, *args, **kwargs):
756 super(YoutubeIE, self).__init__(*args, **kwargs)
757 self._player_cache = {}
759 def report_video_info_webpage_download(self, video_id):
760 """Report attempt to download video info webpage."""
761 self.to_screen('%s: Downloading video info webpage' % video_id)
763 def report_information_extraction(self, video_id):
764 """Report attempt to extract video information."""
765 self.to_screen('%s: Extracting video information' % video_id)
767 def report_unavailable_format(self, video_id, format):
768 """Report extracted video URL."""
769 self.to_screen('%s: Format %s not available' % (video_id, format))
771 def report_rtmp_download(self):
772 """Indicate the download will use the RTMP protocol."""
773 self.to_screen('RTMP download detected')
775 def _signature_cache_id(self, example_sig):
776 """ Return a string representation of a signature """
777 return '.'.join(compat_str(len(part)) for part in example_sig.split('.'))
779 def _extract_signature_function(self, video_id, player_url, example_sig):
781 r'.*?-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player(?:-new)?|/base)?\.(?P<ext>[a-z]+)$',
784 raise ExtractorError('Cannot identify player %r' % player_url)
785 player_type = id_m.group('ext')
786 player_id = id_m.group('id')
788 # Read from filesystem cache
789 func_id = '%s_%s_%s' % (
790 player_type, player_id, self._signature_cache_id(example_sig))
791 assert os.path.basename(func_id) == func_id
793 cache_spec = self._downloader.cache.load('youtube-sigfuncs', func_id)
794 if cache_spec is not None:
795 return lambda s: ''.join(s[i] for i in cache_spec)
798 'Downloading player %s' % player_url
799 if self._downloader.params.get('verbose') else
800 'Downloading %s player %s' % (player_type, player_id)
802 if player_type == 'js':
803 code = self._download_webpage(
804 player_url, video_id,
806 errnote='Download of %s failed' % player_url)
807 res = self._parse_sig_js(code)
808 elif player_type == 'swf':
809 urlh = self._request_webpage(
810 player_url, video_id,
812 errnote='Download of %s failed' % player_url)
814 res = self._parse_sig_swf(code)
816 assert False, 'Invalid player type %r' % player_type
818 test_string = ''.join(map(compat_chr, range(len(example_sig))))
819 cache_res = res(test_string)
820 cache_spec = [ord(c) for c in cache_res]
822 self._downloader.cache.store('youtube-sigfuncs', func_id, cache_spec)
825 def _print_sig_code(self, func, example_sig):
826 def gen_sig_code(idxs):
827 def _genslice(start, end, step):
828 starts = '' if start == 0 else str(start)
829 ends = (':%d' % (end + step)) if end + step >= 0 else ':'
830 steps = '' if step == 1 else (':%d' % step)
831 return 's[%s%s%s]' % (starts, ends, steps)
834 # Quelch pyflakes warnings - start will be set when step is set
835 start = '(Never used)'
836 for i, prev in zip(idxs[1:], idxs[:-1]):
840 yield _genslice(start, prev, step)
843 if i - prev in [-1, 1]:
852 yield _genslice(start, i, step)
854 test_string = ''.join(map(compat_chr, range(len(example_sig))))
855 cache_res = func(test_string)
856 cache_spec = [ord(c) for c in cache_res]
857 expr_code = ' + '.join(gen_sig_code(cache_spec))
858 signature_id_tuple = '(%s)' % (
859 ', '.join(compat_str(len(p)) for p in example_sig.split('.')))
860 code = ('if tuple(len(p) for p in s.split(\'.\')) == %s:\n'
861 ' return %s\n') % (signature_id_tuple, expr_code)
862 self.to_screen('Extracted signature function:\n' + code)
864 def _parse_sig_js(self, jscode):
865 funcname = self._search_regex(
866 r'\.sig\|\|([a-zA-Z0-9$]+)\(', jscode,
867 'Initial JS player signature function name')
869 jsi = JSInterpreter(jscode)
870 initial_function = jsi.extract_function(funcname)
871 return lambda s: initial_function([s])
873 def _parse_sig_swf(self, file_contents):
874 swfi = SWFInterpreter(file_contents)
875 TARGET_CLASSNAME = 'SignatureDecipher'
876 searched_class = swfi.extract_class(TARGET_CLASSNAME)
877 initial_function = swfi.extract_function(searched_class, 'decipher')
878 return lambda s: initial_function([s])
880 def _decrypt_signature(self, s, video_id, player_url, age_gate=False):
881 """Turn the encrypted s field into a working signature"""
883 if player_url is None:
884 raise ExtractorError('Cannot decrypt signature without player_url')
886 if player_url.startswith('//'):
887 player_url = 'https:' + player_url
889 player_id = (player_url, self._signature_cache_id(s))
890 if player_id not in self._player_cache:
891 func = self._extract_signature_function(
892 video_id, player_url, s
894 self._player_cache[player_id] = func
895 func = self._player_cache[player_id]
896 if self._downloader.params.get('youtube_print_sig_code'):
897 self._print_sig_code(func, s)
899 except Exception as e:
900 tb = traceback.format_exc()
901 raise ExtractorError(
902 'Signature extraction failed: ' + tb, cause=e)
904 def _get_subtitles(self, video_id, webpage):
906 subs_doc = self._download_xml(
907 'https://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id,
908 video_id, note=False)
909 except ExtractorError as err:
910 self._downloader.report_warning('unable to download video subtitles: %s' % error_to_compat_str(err))
914 for track in subs_doc.findall('track'):
915 lang = track.attrib['lang_code']
916 if lang in sub_lang_list:
919 for ext in ['sbv', 'vtt', 'srt']:
920 params = compat_urllib_parse.urlencode({
924 'name': track.attrib['name'].encode('utf-8'),
927 'url': 'https://www.youtube.com/api/timedtext?' + params,
930 sub_lang_list[lang] = sub_formats
931 if not sub_lang_list:
932 self._downloader.report_warning('video doesn\'t have subtitles')
936 def _get_ytplayer_config(self, video_id, webpage):
938 # User data may contain arbitrary character sequences that may affect
939 # JSON extraction with regex, e.g. when '};' is contained the second
940 # regex won't capture the whole JSON. Yet working around by trying more
941 # concrete regex first keeping in mind proper quoted string handling
942 # to be implemented in future that will replace this workaround (see
943 # https://github.com/rg3/youtube-dl/issues/7468,
944 # https://github.com/rg3/youtube-dl/pull/7599)
945 r';ytplayer\.config\s*=\s*({.+?});ytplayer',
946 r';ytplayer\.config\s*=\s*({.+?});',
948 config = self._search_regex(
949 patterns, webpage, 'ytplayer.config', default=None)
951 return self._parse_json(
952 uppercase_escape(config), video_id, fatal=False)
954 def _get_automatic_captions(self, video_id, webpage):
955 """We need the webpage for getting the captions url, pass it as an
956 argument to speed up the process."""
957 self.to_screen('%s: Looking for automatic captions' % video_id)
958 player_config = self._get_ytplayer_config(video_id, webpage)
959 err_msg = 'Couldn\'t find automatic captions for %s' % video_id
960 if not player_config:
961 self._downloader.report_warning(err_msg)
964 args = player_config['args']
965 caption_url = args['ttsurl']
966 timestamp = args['timestamp']
967 # We get the available subtitles
968 list_params = compat_urllib_parse.urlencode({
973 list_url = caption_url + '&' + list_params
974 caption_list = self._download_xml(list_url, video_id)
975 original_lang_node = caption_list.find('track')
976 if original_lang_node is None:
977 self._downloader.report_warning('Video doesn\'t have automatic captions')
979 original_lang = original_lang_node.attrib['lang_code']
980 caption_kind = original_lang_node.attrib.get('kind', '')
983 for lang_node in caption_list.findall('target'):
984 sub_lang = lang_node.attrib['lang_code']
986 for ext in ['sbv', 'vtt', 'srt']:
987 params = compat_urllib_parse.urlencode({
988 'lang': original_lang,
992 'kind': caption_kind,
995 'url': caption_url + '&' + params,
998 sub_lang_list[sub_lang] = sub_formats
1000 # An extractor error can be raise by the download process if there are
1001 # no automatic captions but there are subtitles
1002 except (KeyError, ExtractorError):
1003 self._downloader.report_warning(err_msg)
1007 def extract_id(cls, url):
1008 mobj = re.match(cls._VALID_URL, url, re.VERBOSE)
1010 raise ExtractorError('Invalid URL: %s' % url)
1011 video_id = mobj.group(2)
1014 def _extract_from_m3u8(self, manifest_url, video_id):
1017 def _get_urls(_manifest):
1018 lines = _manifest.split('\n')
1019 urls = filter(lambda l: l and not l.startswith('#'),
1022 manifest = self._download_webpage(manifest_url, video_id, 'Downloading formats manifest')
1023 formats_urls = _get_urls(manifest)
1024 for format_url in formats_urls:
1025 itag = self._search_regex(r'itag/(\d+?)/', format_url, 'itag')
1026 url_map[itag] = format_url
1029 def _extract_annotations(self, video_id):
1030 url = 'https://www.youtube.com/annotations_invideo?features=1&legacy=1&video_id=%s' % video_id
1031 return self._download_webpage(url, video_id, note='Searching for annotations.', errnote='Unable to download video annotations.')
1033 def _parse_dash_manifest(
1034 self, video_id, dash_manifest_url, player_url, age_gate, fatal=True):
1035 def decrypt_sig(mobj):
1037 dec_s = self._decrypt_signature(s, video_id, player_url, age_gate)
1038 return '/signature/%s' % dec_s
1039 dash_manifest_url = re.sub(r'/s/([a-fA-F0-9\.]+)', decrypt_sig, dash_manifest_url)
1040 dash_doc = self._download_xml(
1041 dash_manifest_url, video_id,
1042 note='Downloading DASH manifest',
1043 errnote='Could not download DASH manifest',
1046 if dash_doc is False:
1050 for a in dash_doc.findall('.//{urn:mpeg:DASH:schema:MPD:2011}AdaptationSet'):
1051 mime_type = a.attrib.get('mimeType')
1052 for r in a.findall('{urn:mpeg:DASH:schema:MPD:2011}Representation'):
1053 url_el = r.find('{urn:mpeg:DASH:schema:MPD:2011}BaseURL')
1056 if mime_type == 'text/vtt':
1057 # TODO implement WebVTT downloading
1059 elif mime_type.startswith('audio/') or mime_type.startswith('video/'):
1060 segment_list = r.find('{urn:mpeg:DASH:schema:MPD:2011}SegmentList')
1061 format_id = r.attrib['id']
1062 video_url = url_el.text
1063 filesize = int_or_none(url_el.attrib.get('{http://youtube.com/yt/2012/10/10}contentLength'))
1065 'format_id': format_id,
1067 'width': int_or_none(r.attrib.get('width')),
1068 'height': int_or_none(r.attrib.get('height')),
1069 'tbr': int_or_none(r.attrib.get('bandwidth'), 1000),
1070 'asr': int_or_none(r.attrib.get('audioSamplingRate')),
1071 'filesize': filesize,
1072 'fps': int_or_none(r.attrib.get('frameRate')),
1074 if segment_list is not None:
1076 'initialization_url': segment_list.find('{urn:mpeg:DASH:schema:MPD:2011}Initialization').attrib['sourceURL'],
1077 'segment_urls': [segment.attrib.get('media') for segment in segment_list.findall('{urn:mpeg:DASH:schema:MPD:2011}SegmentURL')],
1078 'protocol': 'http_dash_segments',
1081 existing_format = next(
1082 fo for fo in formats
1083 if fo['format_id'] == format_id)
1084 except StopIteration:
1085 full_info = self._formats.get(format_id, {}).copy()
1087 codecs = r.attrib.get('codecs')
1089 if full_info.get('acodec') == 'none' and 'vcodec' not in full_info:
1090 full_info['vcodec'] = codecs
1091 elif full_info.get('vcodec') == 'none' and 'acodec' not in full_info:
1092 full_info['acodec'] = codecs
1093 formats.append(full_info)
1095 existing_format.update(f)
1097 self.report_warning('Unknown MIME type %s in DASH manifest' % mime_type)
1100 def _real_extract(self, url):
1101 url, smuggled_data = unsmuggle_url(url, {})
1104 'http' if self._downloader.params.get('prefer_insecure', False)
1109 parsed_url = compat_urllib_parse_urlparse(url)
1110 for component in [parsed_url.fragment, parsed_url.query]:
1111 query = compat_parse_qs(component)
1112 if start_time is None and 't' in query:
1113 start_time = parse_duration(query['t'][0])
1114 if start_time is None and 'start' in query:
1115 start_time = parse_duration(query['start'][0])
1116 if end_time is None and 'end' in query:
1117 end_time = parse_duration(query['end'][0])
1119 # Extract original video URL from URL with redirection, like age verification, using next_url parameter
1120 mobj = re.search(self._NEXT_URL_RE, url)
1122 url = proto + '://www.youtube.com/' + compat_urllib_parse_unquote(mobj.group(1)).lstrip('/')
1123 video_id = self.extract_id(url)
1126 url = proto + '://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1&bpctr=9999999999' % video_id
1127 video_webpage = self._download_webpage(url, video_id)
1129 # Attempt to extract SWF player URL
1130 mobj = re.search(r'swfConfig.*?"(https?:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)
1131 if mobj is not None:
1132 player_url = re.sub(r'\\(.)', r'\1', mobj.group(1))
1138 def add_dash_mpd(video_info):
1139 dash_mpd = video_info.get('dashmpd')
1140 if dash_mpd and dash_mpd[0] not in dash_mpds:
1141 dash_mpds.append(dash_mpd[0])
1144 embed_webpage = None
1146 if re.search(r'player-age-gate-content">', video_webpage) is not None:
1148 # We simulate the access to the video from www.youtube.com/v/{video_id}
1149 # this can be viewed without login into Youtube
1150 url = proto + '://www.youtube.com/embed/%s' % video_id
1151 embed_webpage = self._download_webpage(url, video_id, 'Downloading embed webpage')
1152 data = compat_urllib_parse.urlencode({
1153 'video_id': video_id,
1154 'eurl': 'https://youtube.googleapis.com/v/' + video_id,
1155 'sts': self._search_regex(
1156 r'"sts"\s*:\s*(\d+)', embed_webpage, 'sts', default=''),
1158 video_info_url = proto + '://www.youtube.com/get_video_info?' + data
1159 video_info_webpage = self._download_webpage(
1160 video_info_url, video_id,
1161 note='Refetching age-gated info webpage',
1162 errnote='unable to download video info webpage')
1163 video_info = compat_parse_qs(video_info_webpage)
1164 add_dash_mpd(video_info)
1168 # Try looking directly into the video webpage
1169 ytplayer_config = self._get_ytplayer_config(video_id, video_webpage)
1171 args = ytplayer_config['args']
1172 if args.get('url_encoded_fmt_stream_map'):
1173 # Convert to the same format returned by compat_parse_qs
1174 video_info = dict((k, [v]) for k, v in args.items())
1175 add_dash_mpd(video_info)
1176 if args.get('livestream') == '1' or args.get('live_playback') == 1:
1178 if not video_info or self._downloader.params.get('youtube_include_dash_manifest', True):
1179 # We also try looking in get_video_info since it may contain different dashmpd
1180 # URL that points to a DASH manifest with possibly different itag set (some itags
1181 # are missing from DASH manifest pointed by webpage's dashmpd, some - from DASH
1182 # manifest pointed by get_video_info's dashmpd).
1183 # The general idea is to take a union of itags of both DASH manifests (for example
1184 # video with such 'manifest behavior' see https://github.com/rg3/youtube-dl/issues/6093)
1185 self.report_video_info_webpage_download(video_id)
1186 for el_type in ['&el=info', '&el=embedded', '&el=detailpage', '&el=vevo', '']:
1188 '%s://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
1189 % (proto, video_id, el_type))
1190 video_info_webpage = self._download_webpage(
1192 video_id, note=False,
1193 errnote='unable to download video info webpage')
1194 get_video_info = compat_parse_qs(video_info_webpage)
1195 if get_video_info.get('use_cipher_signature') != ['True']:
1196 add_dash_mpd(get_video_info)
1198 video_info = get_video_info
1199 if 'token' in get_video_info:
1200 # Different get_video_info requests may report different results, e.g.
1201 # some may report video unavailability, but some may serve it without
1202 # any complaint (see https://github.com/rg3/youtube-dl/issues/7362,
1203 # the original webpage as well as el=info and el=embedded get_video_info
1204 # requests report video unavailability due to geo restriction while
1205 # el=detailpage succeeds and returns valid data). This is probably
1206 # due to YouTube measures against IP ranges of hosting providers.
1207 # Working around by preferring the first succeeded video_info containing
1208 # the token if no such video_info yet was found.
1209 if 'token' not in video_info:
1210 video_info = get_video_info
1212 if 'token' not in video_info:
1213 if 'reason' in video_info:
1214 if 'The uploader has not made this video available in your country.' in video_info['reason']:
1215 regions_allowed = self._html_search_meta('regionsAllowed', video_webpage, default=None)
1217 raise ExtractorError('YouTube said: This video is available in %s only' % (
1218 ', '.join(map(ISO3166Utils.short2full, regions_allowed.split(',')))),
1220 raise ExtractorError(
1221 'YouTube said: %s' % video_info['reason'][0],
1222 expected=True, video_id=video_id)
1224 raise ExtractorError(
1225 '"token" parameter not in video info for unknown reason',
1229 if 'title' in video_info:
1230 video_title = video_info['title'][0]
1232 self._downloader.report_warning('Unable to extract video title')
1236 video_description = get_element_by_id("eow-description", video_webpage)
1237 if video_description:
1238 video_description = re.sub(r'''(?x)
1240 (?:[a-zA-Z-]+="[^"]+"\s+)*?
1241 (?:title|href)="([^"]+)"\s+
1242 (?:[a-zA-Z-]+="[^"]+"\s+)*?
1243 class="(?:yt-uix-redirect-link|yt-uix-sessionlink[^"]*)"[^>]*>
1246 ''', r'\1', video_description)
1247 video_description = clean_html(video_description)
1249 fd_mobj = re.search(r'<meta name="description" content="([^"]+)"', video_webpage)
1251 video_description = unescapeHTML(fd_mobj.group(1))
1253 video_description = ''
1255 if 'multifeed_metadata_list' in video_info and not smuggled_data.get('force_singlefeed', False):
1256 if not self._downloader.params.get('noplaylist'):
1259 multifeed_metadata_list = compat_urllib_parse_unquote_plus(video_info['multifeed_metadata_list'][0])
1260 for feed in multifeed_metadata_list.split(','):
1261 feed_data = compat_parse_qs(feed)
1263 '_type': 'url_transparent',
1264 'ie_key': 'Youtube',
1266 '%s://www.youtube.com/watch?v=%s' % (proto, feed_data['id'][0]),
1267 {'force_singlefeed': True}),
1268 'title': '%s (%s)' % (video_title, feed_data['title'][0]),
1270 feed_ids.append(feed_data['id'][0])
1272 'Downloading multifeed video (%s) - add --no-playlist to just download video %s'
1273 % (', '.join(feed_ids), video_id))
1274 return self.playlist_result(entries, video_id, video_title, video_description)
1275 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
1277 if 'view_count' in video_info:
1278 view_count = int(video_info['view_count'][0])
1282 # Check for "rental" videos
1283 if 'ypc_video_rental_bar_text' in video_info and 'author' not in video_info:
1284 raise ExtractorError('"rental" videos not supported')
1286 # Start extracting information
1287 self.report_information_extraction(video_id)
1290 if 'author' not in video_info:
1291 raise ExtractorError('Unable to extract uploader name')
1292 video_uploader = compat_urllib_parse_unquote_plus(video_info['author'][0])
1295 video_uploader_id = None
1296 mobj = re.search(r'<link itemprop="url" href="http://www.youtube.com/(?:user|channel)/([^"]+)">', video_webpage)
1297 if mobj is not None:
1298 video_uploader_id = mobj.group(1)
1300 self._downloader.report_warning('unable to extract uploader nickname')
1303 # We try first to get a high quality image:
1304 m_thumb = re.search(r'<span itemprop="thumbnail".*?href="(.*?)">',
1305 video_webpage, re.DOTALL)
1306 if m_thumb is not None:
1307 video_thumbnail = m_thumb.group(1)
1308 elif 'thumbnail_url' not in video_info:
1309 self._downloader.report_warning('unable to extract video thumbnail')
1310 video_thumbnail = None
1311 else: # don't panic if we can't find it
1312 video_thumbnail = compat_urllib_parse_unquote_plus(video_info['thumbnail_url'][0])
1315 upload_date = self._html_search_meta(
1316 'datePublished', video_webpage, 'upload date', default=None)
1318 upload_date = self._search_regex(
1319 [r'(?s)id="eow-date.*?>(.*?)</span>',
1320 r'id="watch-uploader-info".*?>.*?(?:Published|Uploaded|Streamed live|Started) on (.+?)</strong>'],
1321 video_webpage, 'upload date', default=None)
1323 upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())
1324 upload_date = unified_strdate(upload_date)
1326 m_music = re.search(
1327 r'<h4[^>]+class="title"[^>]*>\s*Music\s*</h4>\s*<ul[^>]*>\s*<li>(?P<title>.+?) by (?P<creator>.+?)(?:\(.+?\))?</li',
1330 video_alt_title = remove_quotes(unescapeHTML(m_music.group('title')))
1331 video_creator = clean_html(m_music.group('creator'))
1333 video_alt_title = video_creator = None
1335 m_cat_container = self._search_regex(
1336 r'(?s)<h4[^>]*>\s*Category\s*</h4>\s*<ul[^>]*>(.*?)</ul>',
1337 video_webpage, 'categories', default=None)
1339 category = self._html_search_regex(
1340 r'(?s)<a[^<]+>(.*?)</a>', m_cat_container, 'category',
1342 video_categories = None if category is None else [category]
1344 video_categories = None
1347 unescapeHTML(m.group('content'))
1348 for m in re.finditer(self._meta_regex('og:video:tag'), video_webpage)]
1350 def _extract_count(count_name):
1351 return str_to_int(self._search_regex(
1352 r'-%s-button[^>]+><span[^>]+class="yt-uix-button-content"[^>]*>([\d,]+)</span>'
1353 % re.escape(count_name),
1354 video_webpage, count_name, default=None))
1356 like_count = _extract_count('like')
1357 dislike_count = _extract_count('dislike')
1360 video_subtitles = self.extract_subtitles(video_id, video_webpage)
1361 automatic_captions = self.extract_automatic_captions(video_id, video_webpage)
1363 if 'length_seconds' not in video_info:
1364 self._downloader.report_warning('unable to extract video duration')
1365 video_duration = None
1367 video_duration = int(compat_urllib_parse_unquote_plus(video_info['length_seconds'][0]))
1370 video_annotations = None
1371 if self._downloader.params.get('writeannotations', False):
1372 video_annotations = self._extract_annotations(video_id)
1374 def _map_to_format_list(urlmap):
1376 for itag, video_real_url in urlmap.items():
1379 'url': video_real_url,
1380 'player_url': player_url,
1382 if itag in self._formats:
1383 dct.update(self._formats[itag])
1387 if 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
1388 self.report_rtmp_download()
1390 'format_id': '_rtmp',
1392 'url': video_info['conn'][0],
1393 'player_url': player_url,
1395 elif len(video_info.get('url_encoded_fmt_stream_map', [''])[0]) >= 1 or len(video_info.get('adaptive_fmts', [''])[0]) >= 1:
1396 encoded_url_map = video_info.get('url_encoded_fmt_stream_map', [''])[0] + ',' + video_info.get('adaptive_fmts', [''])[0]
1397 if 'rtmpe%3Dyes' in encoded_url_map:
1398 raise ExtractorError('rtmpe downloads are not supported, see https://github.com/rg3/youtube-dl/issues/343 for more information.', expected=True)
1400 for url_data_str in encoded_url_map.split(','):
1401 url_data = compat_parse_qs(url_data_str)
1402 if 'itag' not in url_data or 'url' not in url_data:
1404 format_id = url_data['itag'][0]
1405 url = url_data['url'][0]
1407 if 'sig' in url_data:
1408 url += '&signature=' + url_data['sig'][0]
1409 elif 's' in url_data:
1410 encrypted_sig = url_data['s'][0]
1411 ASSETS_RE = r'"assets":.+?"js":\s*("[^"]+")'
1413 jsplayer_url_json = self._search_regex(
1415 embed_webpage if age_gate else video_webpage,
1416 'JS player URL (1)', default=None)
1417 if not jsplayer_url_json and not age_gate:
1418 # We need the embed website after all
1419 if embed_webpage is None:
1420 embed_url = proto + '://www.youtube.com/embed/%s' % video_id
1421 embed_webpage = self._download_webpage(
1422 embed_url, video_id, 'Downloading embed webpage')
1423 jsplayer_url_json = self._search_regex(
1424 ASSETS_RE, embed_webpage, 'JS player URL')
1426 player_url = json.loads(jsplayer_url_json)
1427 if player_url is None:
1428 player_url_json = self._search_regex(
1429 r'ytplayer\.config.*?"url"\s*:\s*("[^"]+")',
1430 video_webpage, 'age gate player URL')
1431 player_url = json.loads(player_url_json)
1433 if self._downloader.params.get('verbose'):
1434 if player_url is None:
1435 player_version = 'unknown'
1436 player_desc = 'unknown'
1438 if player_url.endswith('swf'):
1439 player_version = self._search_regex(
1440 r'-(.+?)(?:/watch_as3)?\.swf$', player_url,
1441 'flash player', fatal=False)
1442 player_desc = 'flash player %s' % player_version
1444 player_version = self._search_regex(
1445 [r'html5player-([^/]+?)(?:/html5player(?:-new)?)?\.js', r'(?:www|player)-([^/]+)/base\.js'],
1447 'html5 player', fatal=False)
1448 player_desc = 'html5 player %s' % player_version
1450 parts_sizes = self._signature_cache_id(encrypted_sig)
1451 self.to_screen('{%s} signature length %s, %s' %
1452 (format_id, parts_sizes, player_desc))
1454 signature = self._decrypt_signature(
1455 encrypted_sig, video_id, player_url, age_gate)
1456 url += '&signature=' + signature
1457 if 'ratebypass' not in url:
1458 url += '&ratebypass=yes'
1460 # Some itags are not included in DASH manifest thus corresponding formats will
1461 # lack metadata (see https://github.com/rg3/youtube-dl/pull/5993).
1462 # Trying to extract metadata from url_encoded_fmt_stream_map entry.
1463 mobj = re.search(r'^(?P<width>\d+)[xX](?P<height>\d+)$', url_data.get('size', [''])[0])
1464 width, height = (int(mobj.group('width')), int(mobj.group('height'))) if mobj else (None, None)
1466 'format_id': format_id,
1468 'player_url': player_url,
1469 'filesize': int_or_none(url_data.get('clen', [None])[0]),
1470 'tbr': float_or_none(url_data.get('bitrate', [None])[0], 1000),
1473 'fps': int_or_none(url_data.get('fps', [None])[0]),
1474 'format_note': url_data.get('quality_label', [None])[0] or url_data.get('quality', [None])[0],
1476 type_ = url_data.get('type', [None])[0]
1478 type_split = type_.split(';')
1479 kind_ext = type_split[0].split('/')
1480 if len(kind_ext) == 2:
1481 kind, ext = kind_ext
1483 if kind in ('audio', 'video'):
1485 for mobj in re.finditer(
1486 r'(?P<key>[a-zA-Z_-]+)=(?P<quote>["\']?)(?P<val>.+?)(?P=quote)(?:;|$)', type_):
1487 if mobj.group('key') == 'codecs':
1488 codecs = mobj.group('val')
1491 codecs = codecs.split(',')
1492 if len(codecs) == 2:
1493 acodec, vcodec = codecs[1], codecs[0]
1495 acodec, vcodec = (codecs[0], 'none') if kind == 'audio' else ('none', codecs[0])
1500 if format_id in self._formats:
1501 dct.update(self._formats[format_id])
1503 elif video_info.get('hlsvp'):
1504 manifest_url = video_info['hlsvp'][0]
1505 url_map = self._extract_from_m3u8(manifest_url, video_id)
1506 formats = _map_to_format_list(url_map)
1507 # Accept-Encoding header causes failures in live streams on Youtube and Youtube Gaming
1508 for a_format in formats:
1509 a_format.setdefault('http_headers', {})['Youtubedl-no-compression'] = 'True'
1511 raise ExtractorError('no conn, hlsvp or url_encoded_fmt_stream_map information found in video info')
1513 # Look for the DASH manifest
1514 if self._downloader.params.get('youtube_include_dash_manifest', True):
1515 dash_mpd_fatal = True
1516 for dash_manifest_url in dash_mpds:
1519 for df in self._parse_dash_manifest(
1520 video_id, dash_manifest_url, player_url, age_gate, dash_mpd_fatal):
1521 # Do not overwrite DASH format found in some previous DASH manifest
1522 if df['format_id'] not in dash_formats:
1523 dash_formats[df['format_id']] = df
1524 # Additional DASH manifests may end up in HTTP Error 403 therefore
1525 # allow them to fail without bug report message if we already have
1526 # some DASH manifest succeeded. This is temporary workaround to reduce
1527 # burst of bug reports until we figure out the reason and whether it
1528 # can be fixed at all.
1529 dash_mpd_fatal = False
1530 except (ExtractorError, KeyError) as e:
1531 self.report_warning(
1532 'Skipping DASH manifest: %r' % e, video_id)
1534 # Remove the formats we found through non-DASH, they
1535 # contain less info and it can be wrong, because we use
1536 # fixed values (for example the resolution). See
1537 # https://github.com/rg3/youtube-dl/issues/5774 for an
1539 formats = [f for f in formats if f['format_id'] not in dash_formats.keys()]
1540 formats.extend(dash_formats.values())
1542 # Check for malformed aspect ratio
1543 stretched_m = re.search(
1544 r'<meta\s+property="og:video:tag".*?content="yt:stretch=(?P<w>[0-9]+):(?P<h>[0-9]+)">',
1547 w = float(stretched_m.group('w'))
1548 h = float(stretched_m.group('h'))
1549 # yt:stretch may hold invalid ratio data (e.g. for Q39EVAstoRM ratio is 17:0).
1550 # We will only process correct ratios.
1554 if f.get('vcodec') != 'none':
1555 f['stretched_ratio'] = ratio
1557 self._sort_formats(formats)
1561 'uploader': video_uploader,
1562 'uploader_id': video_uploader_id,
1563 'upload_date': upload_date,
1564 'creator': video_creator,
1565 'title': video_title,
1566 'alt_title': video_alt_title,
1567 'thumbnail': video_thumbnail,
1568 'description': video_description,
1569 'categories': video_categories,
1571 'subtitles': video_subtitles,
1572 'automatic_captions': automatic_captions,
1573 'duration': video_duration,
1574 'age_limit': 18 if age_gate else 0,
1575 'annotations': video_annotations,
1576 'webpage_url': proto + '://www.youtube.com/watch?v=%s' % video_id,
1577 'view_count': view_count,
1578 'like_count': like_count,
1579 'dislike_count': dislike_count,
1580 'average_rating': float_or_none(video_info.get('avg_rating', [None])[0]),
1583 'start_time': start_time,
1584 'end_time': end_time,
1588 class YoutubePlaylistIE(YoutubeBaseInfoExtractor, YoutubePlaylistBaseInfoExtractor):
1589 IE_DESC = 'YouTube.com playlists'
1590 _VALID_URL = r"""(?x)(?:
1595 (?:course|view_play_list|my_playlists|artist|playlist|watch|embed/videoseries)
1596 \? (?:.*?[&;])*? (?:p|a|list)=
1600 (?:PL|LL|EC|UU|FL|RD|UL)?[0-9A-Za-z-_]{10,}
1601 # Top tracks, they can also include dots
1606 ((?:PL|LL|EC|UU|FL|RD|UL)[0-9A-Za-z-_]{10,})
1608 _TEMPLATE_URL = 'https://www.youtube.com/playlist?list=%s'
1609 _VIDEO_RE = r'href="\s*/watch\?v=(?P<id>[0-9A-Za-z_-]{11})&[^"]*?index=(?P<index>\d+)(?:[^>]+>(?P<title>[^<]+))?'
1610 IE_NAME = 'youtube:playlist'
1612 'url': 'https://www.youtube.com/playlist?list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re',
1614 'title': 'ytdl test PL',
1615 'id': 'PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re',
1617 'playlist_count': 3,
1619 'url': 'https://www.youtube.com/playlist?list=PLtPgu7CB4gbZDA7i_euNxn75ISqxwZPYx',
1621 'id': 'PLtPgu7CB4gbZDA7i_euNxn75ISqxwZPYx',
1622 'title': 'YDL_Empty_List',
1624 'playlist_count': 0,
1626 'note': 'Playlist with deleted videos (#651). As a bonus, the video #51 is also twice in this list.',
1627 'url': 'https://www.youtube.com/playlist?list=PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
1629 'title': '29C3: Not my department',
1630 'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
1632 'playlist_count': 95,
1634 'note': 'issue #673',
1635 'url': 'PLBB231211A4F62143',
1637 'title': '[OLD]Team Fortress 2 (Class-based LP)',
1638 'id': 'PLBB231211A4F62143',
1640 'playlist_mincount': 26,
1642 'note': 'Large playlist',
1643 'url': 'https://www.youtube.com/playlist?list=UUBABnxM4Ar9ten8Mdjj1j0Q',
1645 'title': 'Uploads from Cauchemar',
1646 'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
1648 'playlist_mincount': 799,
1650 'url': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
1652 'title': 'YDL_safe_search',
1653 'id': 'PLtPgu7CB4gbY9oDN3drwC3cMbJggS7dKl',
1655 'playlist_count': 2,
1658 'url': 'http://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
1659 'playlist_count': 4,
1662 'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
1665 'note': 'Embedded SWF player',
1666 'url': 'http://www.youtube.com/p/YN5VISEtHet5D4NEvfTd0zcgFk84NqFZ?hl=en_US&fs=1&rel=0',
1667 'playlist_count': 4,
1670 'id': 'YN5VISEtHet5D4NEvfTd0zcgFk84NqFZ',
1673 'note': 'Buggy playlist: the webpage has a "Load more" button but it doesn\'t have more videos',
1674 'url': 'https://www.youtube.com/playlist?list=UUXw-G3eDE9trcvY2sBMM_aA',
1676 'title': 'Uploads from Interstellar Movie',
1677 'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
1679 'playlist_mincout': 21,
1682 def _real_initialize(self):
1685 def _extract_mix(self, playlist_id):
1686 # The mixes are generated from a single video
1687 # the id of the playlist is just 'RD' + video_id
1688 url = 'https://youtube.com/watch?v=%s&list=%s' % (playlist_id[-11:], playlist_id)
1689 webpage = self._download_webpage(
1690 url, playlist_id, 'Downloading Youtube mix')
1691 search_title = lambda class_name: get_element_by_attribute('class', class_name, webpage)
1693 search_title('playlist-title') or
1694 search_title('title long-title') or
1695 search_title('title'))
1696 title = clean_html(title_span)
1697 ids = orderedSet(re.findall(
1698 r'''(?xs)data-video-username=".*?".*?
1699 href="/watch\?v=([0-9A-Za-z_-]{11})&[^"]*?list=%s''' % re.escape(playlist_id),
1701 url_results = self._ids_to_results(ids)
1703 return self.playlist_result(url_results, playlist_id, title)
1705 def _extract_playlist(self, playlist_id):
1706 url = self._TEMPLATE_URL % playlist_id
1707 page = self._download_webpage(url, playlist_id)
1709 for match in re.findall(r'<div class="yt-alert-message">([^<]+)</div>', page):
1710 match = match.strip()
1711 # Check if the playlist exists or is private
1712 if re.match(r'[^<]*(The|This) playlist (does not exist|is private)[^<]*', match):
1713 raise ExtractorError(
1714 'The playlist doesn\'t exist or is private, use --username or '
1715 '--netrc to access it.',
1717 elif re.match(r'[^<]*Invalid parameters[^<]*', match):
1718 raise ExtractorError(
1719 'Invalid parameters. Maybe URL is incorrect.',
1721 elif re.match(r'[^<]*Choose your language[^<]*', match):
1724 self.report_warning('Youtube gives an alert message: ' + match)
1726 playlist_title = self._html_search_regex(
1727 r'(?s)<h1 class="pl-header-title[^"]*"[^>]*>\s*(.*?)\s*</h1>',
1730 return self.playlist_result(self._entries(page, playlist_id), playlist_id, playlist_title)
1732 def _real_extract(self, url):
1733 # Extract playlist id
1734 mobj = re.match(self._VALID_URL, url)
1736 raise ExtractorError('Invalid URL: %s' % url)
1737 playlist_id = mobj.group(1) or mobj.group(2)
1739 # Check if it's a video-specific URL
1740 query_dict = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
1741 if 'v' in query_dict:
1742 video_id = query_dict['v'][0]
1743 if self._downloader.params.get('noplaylist'):
1744 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
1745 return self.url_result(video_id, 'Youtube', video_id=video_id)
1747 self.to_screen('Downloading playlist %s - add --no-playlist to just download video %s' % (playlist_id, video_id))
1749 if playlist_id.startswith('RD') or playlist_id.startswith('UL'):
1750 # Mixes require a custom extraction process
1751 return self._extract_mix(playlist_id)
1753 return self._extract_playlist(playlist_id)
1756 class YoutubeChannelIE(YoutubePlaylistBaseInfoExtractor):
1757 IE_DESC = 'YouTube.com channels'
1758 _VALID_URL = r'https?://(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/(?P<id>[0-9A-Za-z_-]+)'
1759 _TEMPLATE_URL = 'https://www.youtube.com/channel/%s/videos'
1760 _VIDEO_RE = r'(?:title="(?P<title>[^"]+)"[^>]+)?href="/watch\?v=(?P<id>[0-9A-Za-z_-]+)&?'
1761 IE_NAME = 'youtube:channel'
1763 'note': 'paginated channel',
1764 'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w',
1765 'playlist_mincount': 91,
1767 'id': 'UUKfVa3S1e4PHvxWcwyMMg8w',
1768 'title': 'Uploads from lex will',
1771 'note': 'Age restricted channel',
1772 # from https://www.youtube.com/user/DeusExOfficial
1773 'url': 'https://www.youtube.com/channel/UCs0ifCMCm1icqRbqhUINa0w',
1774 'playlist_mincount': 64,
1776 'id': 'UUs0ifCMCm1icqRbqhUINa0w',
1777 'title': 'Uploads from Deus Ex',
1782 def suitable(cls, url):
1783 return False if YoutubePlaylistsIE.suitable(url) else super(YoutubeChannelIE, cls).suitable(url)
1785 def _real_extract(self, url):
1786 channel_id = self._match_id(url)
1788 url = self._TEMPLATE_URL % channel_id
1790 # Channel by page listing is restricted to 35 pages of 30 items, i.e. 1050 videos total (see #5778)
1791 # Workaround by extracting as a playlist if managed to obtain channel playlist URL
1792 # otherwise fallback on channel by page extraction
1793 channel_page = self._download_webpage(
1794 url + '?view=57', channel_id,
1795 'Downloading channel page', fatal=False)
1796 if channel_page is False:
1797 channel_playlist_id = False
1799 channel_playlist_id = self._html_search_meta(
1800 'channelId', channel_page, 'channel id', default=None)
1801 if not channel_playlist_id:
1802 channel_playlist_id = self._search_regex(
1803 r'data-(?:channel-external-|yt)id="([^"]+)"',
1804 channel_page, 'channel id', default=None)
1805 if channel_playlist_id and channel_playlist_id.startswith('UC'):
1806 playlist_id = 'UU' + channel_playlist_id[2:]
1807 return self.url_result(
1808 compat_urlparse.urljoin(url, '/playlist?list=%s' % playlist_id), 'YoutubePlaylist')
1810 channel_page = self._download_webpage(url, channel_id, 'Downloading page #1')
1811 autogenerated = re.search(r'''(?x)
1813 channel-header-autogenerated-label|
1814 yt-channel-title-autogenerated
1815 )[^"]*"''', channel_page) is not None
1818 # The videos are contained in a single page
1819 # the ajax pages can't be used, they are empty
1822 video_id, 'Youtube', video_id=video_id,
1823 video_title=video_title)
1824 for video_id, video_title in self.extract_videos_from_page(channel_page)]
1825 return self.playlist_result(entries, channel_id)
1827 return self.playlist_result(self._entries(channel_page, channel_id), channel_id)
1830 class YoutubeUserIE(YoutubeChannelIE):
1831 IE_DESC = 'YouTube.com user videos (URL or "ytuser" keyword)'
1832 _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:user/)?(?!(?:attribution_link|watch|results)(?:$|[^a-z_A-Z0-9-])))|ytuser:)(?!feed/)(?P<id>[A-Za-z0-9_-]+)'
1833 _TEMPLATE_URL = 'https://www.youtube.com/user/%s/videos'
1834 IE_NAME = 'youtube:user'
1837 'url': 'https://www.youtube.com/user/TheLinuxFoundation',
1838 'playlist_mincount': 320,
1840 'title': 'TheLinuxFoundation',
1843 'url': 'ytuser:phihag',
1844 'only_matching': True,
1848 def suitable(cls, url):
1849 # Don't return True if the url can be extracted with other youtube
1850 # extractor, the regex would is too permissive and it would match.
1851 other_ies = iter(klass for (name, klass) in globals().items() if name.endswith('IE') and klass is not cls)
1852 if any(ie.suitable(url) for ie in other_ies):
1855 return super(YoutubeUserIE, cls).suitable(url)
1858 class YoutubePlaylistsIE(YoutubePlaylistsBaseInfoExtractor):
1859 IE_DESC = 'YouTube.com user/channel playlists'
1860 _VALID_URL = r'https?://(?:\w+\.)?youtube\.com/(?:user|channel)/(?P<id>[^/]+)/playlists'
1861 IE_NAME = 'youtube:playlists'
1864 'url': 'http://www.youtube.com/user/ThirstForScience/playlists',
1865 'playlist_mincount': 4,
1867 'id': 'ThirstForScience',
1868 'title': 'Thirst for Science',
1871 # with "Load more" button
1872 'url': 'http://www.youtube.com/user/igorkle1/playlists?view=1&sort=dd',
1873 'playlist_mincount': 70,
1876 'title': 'Игорь Клейнер',
1879 'url': 'https://www.youtube.com/channel/UCiU1dHvZObB2iP6xkJ__Icw/playlists',
1880 'playlist_mincount': 17,
1882 'id': 'UCiU1dHvZObB2iP6xkJ__Icw',
1883 'title': 'Chem Player',
1888 class YoutubeSearchIE(SearchInfoExtractor, YoutubePlaylistIE):
1889 IE_DESC = 'YouTube.com searches'
1890 # there doesn't appear to be a real limit, for example if you search for
1891 # 'python' you get more than 8.000.000 results
1892 _MAX_RESULTS = float('inf')
1893 IE_NAME = 'youtube:search'
1894 _SEARCH_KEY = 'ytsearch'
1895 _EXTRA_QUERY_ARGS = {}
1898 def _get_n_results(self, query, n):
1899 """Get a specified number of results for a query"""
1904 for pagenum in itertools.count(1):
1906 'search_query': query.encode('utf-8'),
1910 url_query.update(self._EXTRA_QUERY_ARGS)
1911 result_url = 'https://www.youtube.com/results?' + compat_urllib_parse.urlencode(url_query)
1912 data = self._download_json(
1913 result_url, video_id='query "%s"' % query,
1914 note='Downloading page %s' % pagenum,
1915 errnote='Unable to download API page')
1916 html_content = data[1]['body']['content']
1918 if 'class="search-message' in html_content:
1919 raise ExtractorError(
1920 '[youtube] No video results', expected=True)
1922 new_videos = self._ids_to_results(orderedSet(re.findall(
1923 r'href="/watch\?v=(.{11})', html_content)))
1924 videos += new_videos
1925 if not new_videos or len(videos) > limit:
1930 return self.playlist_result(videos, query)
1933 class YoutubeSearchDateIE(YoutubeSearchIE):
1934 IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
1935 _SEARCH_KEY = 'ytsearchdate'
1936 IE_DESC = 'YouTube.com searches, newest videos first'
1937 _EXTRA_QUERY_ARGS = {'search_sort': 'video_date_uploaded'}
1940 class YoutubeSearchURLIE(InfoExtractor):
1941 IE_DESC = 'YouTube.com search URLs'
1942 IE_NAME = 'youtube:search_url'
1943 _VALID_URL = r'https?://(?:www\.)?youtube\.com/results\?(.*?&)?search_query=(?P<query>[^&]+)(?:[&]|$)'
1945 'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
1946 'playlist_mincount': 5,
1948 'title': 'youtube-dl test video',
1952 def _real_extract(self, url):
1953 mobj = re.match(self._VALID_URL, url)
1954 query = compat_urllib_parse_unquote_plus(mobj.group('query'))
1956 webpage = self._download_webpage(url, query)
1957 result_code = self._search_regex(
1958 r'(?s)<ol[^>]+class="item-section"(.*?)</ol>', webpage, 'result HTML')
1960 part_codes = re.findall(
1961 r'(?s)<h3[^>]+class="[^"]*yt-lockup-title[^"]*"[^>]*>(.*?)</h3>', result_code)
1963 for part_code in part_codes:
1964 part_title = self._html_search_regex(
1965 [r'(?s)title="([^"]+)"', r'>([^<]+)</a>'], part_code, 'item title', fatal=False)
1966 part_url_snippet = self._html_search_regex(
1967 r'(?s)href="([^"]+)"', part_code, 'item URL')
1968 part_url = compat_urlparse.urljoin(
1969 'https://www.youtube.com/', part_url_snippet)
1973 'title': part_title,
1977 '_type': 'playlist',
1983 class YoutubeShowIE(YoutubePlaylistsBaseInfoExtractor):
1984 IE_DESC = 'YouTube.com (multi-season) shows'
1985 _VALID_URL = r'https?://www\.youtube\.com/show/(?P<id>[^?#]*)'
1986 IE_NAME = 'youtube:show'
1988 'url': 'https://www.youtube.com/show/airdisasters',
1989 'playlist_mincount': 5,
1991 'id': 'airdisasters',
1992 'title': 'Air Disasters',
1996 def _real_extract(self, url):
1997 playlist_id = self._match_id(url)
1998 return super(YoutubeShowIE, self)._real_extract(
1999 'https://www.youtube.com/show/%s/playlists' % playlist_id)
2002 class YoutubeFeedsInfoExtractor(YoutubeBaseInfoExtractor):
2004 Base class for feed extractors
2005 Subclasses must define the _FEED_NAME and _PLAYLIST_TITLE properties.
2007 _LOGIN_REQUIRED = True
2011 return 'youtube:%s' % self._FEED_NAME
2013 def _real_initialize(self):
2016 def _real_extract(self, url):
2017 page = self._download_webpage(
2018 'https://www.youtube.com/feed/%s' % self._FEED_NAME, self._PLAYLIST_TITLE)
2020 # The extraction process is the same as for playlists, but the regex
2021 # for the video ids doesn't contain an index
2023 more_widget_html = content_html = page
2024 for page_num in itertools.count(1):
2025 matches = re.findall(r'href="\s*/watch\?v=([0-9A-Za-z_-]{11})', content_html)
2027 # 'recommended' feed has infinite 'load more' and each new portion spins
2028 # the same videos in (sometimes) slightly different order, so we'll check
2029 # for unicity and break when portion has no new videos
2030 new_ids = filter(lambda video_id: video_id not in ids, orderedSet(matches))
2036 mobj = re.search(r'data-uix-load-more-href="/?(?P<more>[^"]+)"', more_widget_html)
2040 more = self._download_json(
2041 'https://youtube.com/%s' % mobj.group('more'), self._PLAYLIST_TITLE,
2042 'Downloading page #%s' % page_num,
2043 transform_source=uppercase_escape)
2044 content_html = more['content_html']
2045 more_widget_html = more['load_more_widget_html']
2047 return self.playlist_result(
2048 self._ids_to_results(ids), playlist_title=self._PLAYLIST_TITLE)
2051 class YoutubeWatchLaterIE(YoutubePlaylistIE):
2052 IE_NAME = 'youtube:watchlater'
2053 IE_DESC = 'Youtube watch later list, ":ytwatchlater" for short (requires authentication)'
2054 _VALID_URL = r'https?://www\.youtube\.com/(?:feed/watch_later|playlist\?list=WL)|:ytwatchlater'
2056 _TESTS = [] # override PlaylistIE tests
2058 def _real_extract(self, url):
2059 return self._extract_playlist('WL')
2062 class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
2063 IE_NAME = 'youtube:favorites'
2064 IE_DESC = 'YouTube.com favourite videos, ":ytfav" for short (requires authentication)'
2065 _VALID_URL = r'https?://www\.youtube\.com/my_favorites|:ytfav(?:ou?rites)?'
2066 _LOGIN_REQUIRED = True
2068 def _real_extract(self, url):
2069 webpage = self._download_webpage('https://www.youtube.com/my_favorites', 'Youtube Favourites videos')
2070 playlist_id = self._search_regex(r'list=(.+?)["&]', webpage, 'favourites playlist id')
2071 return self.url_result(playlist_id, 'YoutubePlaylist')
2074 class YoutubeRecommendedIE(YoutubeFeedsInfoExtractor):
2075 IE_DESC = 'YouTube.com recommended videos, ":ytrec" for short (requires authentication)'
2076 _VALID_URL = r'https?://www\.youtube\.com/feed/recommended|:ytrec(?:ommended)?'
2077 _FEED_NAME = 'recommended'
2078 _PLAYLIST_TITLE = 'Youtube Recommended videos'
2081 class YoutubeSubscriptionsIE(YoutubeFeedsInfoExtractor):
2082 IE_DESC = 'YouTube.com subscriptions feed, "ytsubs" keyword (requires authentication)'
2083 _VALID_URL = r'https?://www\.youtube\.com/feed/subscriptions|:ytsubs(?:criptions)?'
2084 _FEED_NAME = 'subscriptions'
2085 _PLAYLIST_TITLE = 'Youtube Subscriptions'
2088 class YoutubeHistoryIE(YoutubeFeedsInfoExtractor):
2089 IE_DESC = 'Youtube watch history, ":ythistory" for short (requires authentication)'
2090 _VALID_URL = 'https?://www\.youtube\.com/feed/history|:ythistory'
2091 _FEED_NAME = 'history'
2092 _PLAYLIST_TITLE = 'Youtube History'
2095 class YoutubeTruncatedURLIE(InfoExtractor):
2096 IE_NAME = 'youtube:truncated_url'
2097 IE_DESC = False # Do not list
2098 _VALID_URL = r'''(?x)
2100 (?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie)?\.com/
2103 annotation_id=annotation_[^&]+|
2109 attribution_link\?a=[^&]+
2115 'url': 'http://www.youtube.com/watch?annotation_id=annotation_3951667041',
2116 'only_matching': True,
2118 'url': 'http://www.youtube.com/watch?',
2119 'only_matching': True,
2121 'url': 'https://www.youtube.com/watch?x-yt-cl=84503534',
2122 'only_matching': True,
2124 'url': 'https://www.youtube.com/watch?feature=foo',
2125 'only_matching': True,
2127 'url': 'https://www.youtube.com/watch?hl=en-GB',
2128 'only_matching': True,
2130 'url': 'https://www.youtube.com/watch?t=2372',
2131 'only_matching': True,
2134 def _real_extract(self, url):
2135 raise ExtractorError(
2136 'Did you forget to quote the URL? Remember that & is a meta '
2137 'character in most shells, so you want to put the URL in quotes, '
2139 '"http://www.youtube.com/watch?feature=foo&v=BaW_jenozKc" '
2140 ' or simply youtube-dl BaW_jenozKc .',
2144 class YoutubeTruncatedIDIE(InfoExtractor):
2145 IE_NAME = 'youtube:truncated_id'
2146 IE_DESC = False # Do not list
2147 _VALID_URL = r'https?://(?:www\.)?youtube\.com/watch\?v=(?P<id>[0-9A-Za-z_-]{1,10})$'
2150 'url': 'https://www.youtube.com/watch?v=N_708QY7Ob',
2151 'only_matching': True,
2154 def _real_extract(self, url):
2155 video_id = self._match_id(url)
2156 raise ExtractorError(
2157 'Incomplete YouTube ID %s. URL %s looks truncated.' % (video_id, url),