youtube_dl/extractor/ted.py

   1 from __future__ import unicode_literals
   2
   3 import json
   4 import re
   5
   6 from .common import InfoExtractor
   7
   8 from ..compat import compat_str
   9 from ..utils import int_or_none
  10
  11
  12 class TEDIE(InfoExtractor):
  13     IE_NAME = 'ted'
  14     _VALID_URL = r'''(?x)
  15         (?P<proto>https?://)
  16         (?P<type>www|embed(?:-ssl)?)(?P<urlmain>\.ted\.com/
  17         (
  18             (?P<type_playlist>playlists(?:/\d+)?) # We have a playlist
  19             |
  20             ((?P<type_talk>talks)) # We have a simple talk
  21             |
  22             (?P<type_watch>watch)/[^/]+/[^/]+
  23         )
  24         (/lang/(.*?))? # The url may contain the language
  25         /(?P<name>[\w-]+) # Here goes the name and then ".html"
  26         .*)$
  27         '''
  28     _TESTS = [{
  29         'url': 'http://www.ted.com/talks/dan_dennett_on_our_consciousness.html',
  30         'md5': '0de43ac406aa3e4ea74b66c9c7789b13',
  31         'info_dict': {
  32             'id': '102',
  33             'ext': 'mp4',
  34             'title': 'The illusion of consciousness',
  35             'description': ('Philosopher Dan Dennett makes a compelling '
  36                             'argument that not only don\'t we understand our own '
  37                             'consciousness, but that half the time our brains are '
  38                             'actively fooling us.'),
  39             'uploader': 'Dan Dennett',
  40             'width': 853,
  41             'duration': 1308,
  42         }
  43     }, {
  44         'url': 'http://www.ted.com/watch/ted-institute/ted-bcg/vishal-sikka-the-beauty-and-power-of-algorithms',
  45         'md5': 'b899ac15e345fb39534d913f7606082b',
  46         'info_dict': {
  47             'id': 'tSVI8ta_P4w',
  48             'ext': 'mp4',
  49             'title': 'Vishal Sikka: The beauty and power of algorithms',
  50             'thumbnail': 're:^https?://.+\.jpg',
  51             'description': 'md5:6261fdfe3e02f4f579cbbfc00aff73f4',
  52             'upload_date': '20140122',
  53             'uploader_id': 'TEDInstitute',
  54             'uploader': 'TED Institute',
  55         },
  56         'add_ie': ['Youtube'],
  57     }, {
  58         'url': 'http://www.ted.com/talks/gabby_giffords_and_mark_kelly_be_passionate_be_courageous_be_your_best',
  59         'md5': '71b3ab2f4233012dce09d515c9c39ce2',
  60         'info_dict': {
  61             'id': '1972',
  62             'ext': 'mp4',
  63             'title': 'Be passionate. Be courageous. Be your best.',
  64             'uploader': 'Gabby Giffords and Mark Kelly',
  65             'description': 'md5:5174aed4d0f16021b704120360f72b92',
  66             'duration': 1128,
  67         },
  68     }, {
  69         'url': 'http://www.ted.com/playlists/who_are_the_hackers',
  70         'info_dict': {
  71             'id': '10',
  72             'title': 'Who are the hackers?',
  73         },
  74         'playlist_mincount': 6,
  75     }, {
  76         # contains a youtube video
  77         'url': 'https://www.ted.com/talks/douglas_adams_parrots_the_universe_and_everything',
  78         'add_ie': ['Youtube'],
  79         'info_dict': {
  80             'id': '_ZG8HBuDjgc',
  81             'ext': 'webm',
  82             'title': 'Douglas Adams: Parrots the Universe and Everything',
  83             'description': 'md5:01ad1e199c49ac640cb1196c0e9016af',
  84             'uploader': 'University of California Television (UCTV)',
  85             'uploader_id': 'UCtelevision',
  86             'upload_date': '20080522',
  87         },
  88         'params': {
  89             'skip_download': True,
  90         },
  91     }, {
  92         # YouTube video
  93         'url': 'http://www.ted.com/talks/jeffrey_kluger_the_sibling_bond',
  94         'add_ie': ['Youtube'],
  95         'info_dict': {
  96             'id': 'aFBIPO-P7LM',
  97             'ext': 'mp4',
  98             'title': 'The hidden power of siblings: Jeff Kluger at TEDxAsheville',
  99             'description': 'md5:3d7a4f50d95ca5dd67104e2a20f43fe1',
 100             'uploader': 'TEDx Talks',
 101             'uploader_id': 'TEDxTalks',
 102             'upload_date': '20111216',
 103         },
 104         'params': {
 105             'skip_download': True,
 106         },
 107     }]
 108
 109     _NATIVE_FORMATS = {
 110         'low': {'width': 320, 'height': 180},
 111         'medium': {'width': 512, 'height': 288},
 112         'high': {'width': 854, 'height': 480},
 113     }
 114
 115     def _extract_info(self, webpage):
 116         info_json = self._search_regex(r'q\("\w+.init",({.+})\)</script>',
 117                                        webpage, 'info json')
 118         return json.loads(info_json)
 119
 120     def _real_extract(self, url):
 121         m = re.match(self._VALID_URL, url, re.VERBOSE)
 122         if m.group('type').startswith('embed'):
 123             desktop_url = m.group('proto') + 'www' + m.group('urlmain')
 124             return self.url_result(desktop_url, 'TED')
 125         name = m.group('name')
 126         if m.group('type_talk'):
 127             return self._talk_info(url, name)
 128         elif m.group('type_watch'):
 129             return self._watch_info(url, name)
 130         else:
 131             return self._playlist_videos_info(url, name)
 132
 133     def _playlist_videos_info(self, url, name):
 134         '''Returns the videos of the playlist'''
 135
 136         webpage = self._download_webpage(url, name,
 137                                          'Downloading playlist webpage')
 138         info = self._extract_info(webpage)
 139         playlist_info = info['playlist']
 140
 141         playlist_entries = [
 142             self.url_result('http://www.ted.com/talks/' + talk['slug'], self.ie_key())
 143             for talk in info['talks']
 144         ]
 145         return self.playlist_result(
 146             playlist_entries,
 147             playlist_id=compat_str(playlist_info['id']),
 148             playlist_title=playlist_info['title'])
 149
 150     def _talk_info(self, url, video_name):
 151         webpage = self._download_webpage(url, video_name)
 152         self.report_extraction(video_name)
 153
 154         talk_info = self._extract_info(webpage)['talks'][0]
 155
 156         external = talk_info.get('external')
 157         if external:
 158             service = external['service']
 159             self.to_screen('Found video from %s' % service)
 160             ext_url = None
 161             if service.lower() == 'youtube':
 162                 ext_url = external.get('code')
 163             return {
 164                 '_type': 'url',
 165                 'url': ext_url or external['uri'],
 166             }
 167
 168         formats = [{
 169             'url': format_url,
 170             'format_id': format_id,
 171             'format': format_id,
 172         } for (format_id, format_url) in talk_info['nativeDownloads'].items() if format_url is not None]
 173         if formats:
 174             for f in formats:
 175                 finfo = self._NATIVE_FORMATS.get(f['format_id'])
 176                 if finfo:
 177                     f.update(finfo)
 178
 179         http_url = None
 180         for format_id, resources in talk_info['resources'].items():
 181             if format_id == 'h264':
 182                 for resource in resources:
 183                     h264_url = resource.get('file')
 184                     if not h264_url:
 185                         continue
 186                     bitrate = int_or_none(resource.get('bitrate'))
 187                     formats.append({
 188                         'url': h264_url,
 189                         'format_id': '%s-%sk' % (format_id, bitrate),
 190                         'tbr': bitrate,
 191                     })
 192                     if re.search('\d+k', h264_url):
 193                         http_url = h264_url
 194             elif format_id == 'rtmp':
 195                 streamer = talk_info.get('streamer')
 196                 if not streamer:
 197                     continue
 198                 for resource in resources:
 199                     formats.append({
 200                         'format_id': '%s-%s' % (format_id, resource.get('name')),
 201                         'url': streamer,
 202                         'play_path': resource['file'],
 203                         'ext': 'flv',
 204                         'width': int_or_none(resource.get('width')),
 205                         'height': int_or_none(resource.get('height')),
 206                         'tbr': int_or_none(resource.get('bitrate')),
 207                     })
 208             elif format_id == 'hls':
 209                 formats.extend(self._extract_m3u8_formats(
 210                     resources.get('stream'), video_name, 'mp4', m3u8_id=format_id, fatal=False))
 211
 212         m3u8_formats = list(filter(
 213             lambda f: f.get('protocol') == 'm3u8' and f.get('vcodec') != 'none' and f.get('resolution') != 'multiple',
 214             formats))
 215         if http_url:
 216             for m3u8_format in m3u8_formats:
 217                 bitrate = self._search_regex(r'(\d+k)', m3u8_format['url'], 'bitrate', default=None)
 218                 if not bitrate:
 219                     continue
 220                 f = m3u8_format.copy()
 221                 f.update({
 222                     'url': re.sub(r'\d+k', bitrate, http_url),
 223                     'format_id': m3u8_format['format_id'].replace('hls', 'http'),
 224                     'protocol': 'http',
 225                 })
 226                 formats.append(f)
 227
 228         audio_download = talk_info.get('audioDownload')
 229         if audio_download:
 230             formats.append({
 231                 'url': audio_download,
 232                 'format_id': 'audio',
 233                 'vcodec': 'none',
 234             })
 235
 236         self._sort_formats(formats)
 237
 238         video_id = compat_str(talk_info['id'])
 239
 240         thumbnail = talk_info['thumb']
 241         if not thumbnail.startswith('http'):
 242             thumbnail = 'http://' + thumbnail
 243         return {
 244             'id': video_id,
 245             'title': talk_info['title'].strip(),
 246             'uploader': talk_info['speaker'],
 247             'thumbnail': thumbnail,
 248             'description': self._og_search_description(webpage),
 249             'subtitles': self._get_subtitles(video_id, talk_info),
 250             'formats': formats,
 251             'duration': talk_info.get('duration'),
 252         }
 253
 254     def _get_subtitles(self, video_id, talk_info):
 255         languages = [lang['languageCode'] for lang in talk_info.get('languages', [])]
 256         if languages:
 257             sub_lang_list = {}
 258             for l in languages:
 259                 sub_lang_list[l] = [
 260                     {
 261                         'url': 'http://www.ted.com/talks/subtitles/id/%s/lang/%s/format/%s' % (video_id, l, ext),
 262                         'ext': ext,
 263                     }
 264                     for ext in ['ted', 'srt']
 265                 ]
 266             return sub_lang_list
 267         else:
 268             return {}
 269
 270     def _watch_info(self, url, name):
 271         webpage = self._download_webpage(url, name)
 272
 273         config_json = self._html_search_regex(
 274             r'"pages\.jwplayer"\s*,\s*({.+?})\s*\)\s*</script>',
 275             webpage, 'config', default=None)
 276         if not config_json:
 277             embed_url = self._search_regex(
 278                 r"<iframe[^>]+class='pages-video-embed__video__object'[^>]+src='([^']+)'", webpage, 'embed url')
 279             return self.url_result(self._proto_relative_url(embed_url))
 280         config = json.loads(config_json)['config']
 281         video_url = config['video']['url']
 282         thumbnail = config.get('image', {}).get('url')
 283
 284         title = self._html_search_regex(
 285             r"(?s)<h1(?:\s+class='[^']+')?>(.+?)</h1>", webpage, 'title')
 286         description = self._html_search_regex(
 287             [
 288                 r'(?s)<h4 class="[^"]+" id="h3--about-this-talk">.*?</h4>(.*?)</div>',
 289                 r'(?s)<p><strong>About this talk:</strong>\s+(.*?)</p>',
 290             ],
 291             webpage, 'description', fatal=False)
 292
 293         return {
 294             'id': name,
 295             'url': video_url,
 296             'title': title,
 297             'thumbnail': thumbnail,
 298             'description': description,
 299         }