1 # -*- coding: utf-8 -*-
2 from __future__ import unicode_literals
12 from .once import OnceIE
13 from ..compat import (
15 compat_urllib_parse_urlparse,
33 default_ns = 'http://www.w3.org/2005/SMIL21/Language'
34 _x = lambda p: xpath_with_ns(p, {'smil': default_ns})
37 class ThePlatformBaseIE(OnceIE):
38 def _extract_theplatform_smil(self, smil_url, video_id, note='Downloading SMIL data'):
39 meta = self._download_xml(smil_url, video_id, note=note, query={'format': 'SMIL'})
40 error_element = find_xpath_attr(meta, _x('.//smil:ref'), 'src')
41 if error_element is not None and error_element.attrib['src'].startswith(
42 'http://link.theplatform.com/s/errorFiles/Unavailable.'):
43 raise ExtractorError(error_element.attrib['abstract'], expected=True)
45 smil_formats = self._parse_smil_formats(
46 meta, smil_url, video_id, namespace=default_ns,
47 # the parameters are from syfy.com, other sites may use others,
48 # they also work for nbc.com
49 f4m_params={'g': 'UXWGVKRWHFSP', 'hdcore': '3.0.3'},
50 transform_rtmp_url=lambda streamer, src: (streamer, 'mp4:' + src))
53 for _format in smil_formats:
54 if OnceIE.suitable(_format['url']):
55 formats.extend(self._extract_once_formats(_format['url']))
57 media_url = _format['url']
58 if determine_ext(media_url) == 'm3u8':
59 hdnea2 = self._get_cookies(media_url).get('hdnea2')
61 _format['url'] = update_url_query(media_url, {'hdnea3': hdnea2.value})
63 formats.append(_format)
65 subtitles = self._parse_smil_subtitles(meta, default_ns)
67 return formats, subtitles
69 def _download_theplatform_metadata(self, path, video_id):
70 info_url = 'http://link.theplatform.com/s/%s?format=preview' % path
71 return self._download_json(info_url, video_id)
73 def _parse_theplatform_metadata(self, info):
75 captions = info.get('captions')
76 if isinstance(captions, list):
77 for caption in captions:
78 lang, src, mime = caption.get('lang', 'en'), caption.get('src'), caption.get('type')
80 'ext': mimetype2ext(mime),
85 'title': info['title'],
86 'subtitles': subtitles,
87 'description': info['description'],
88 'thumbnail': info['defaultThumbnailUrl'],
89 'duration': int_or_none(info.get('duration'), 1000),
90 'timestamp': int_or_none(info.get('pubDate'), 1000) or None,
91 'uploader': info.get('billingCode'),
94 def _extract_theplatform_metadata(self, path, video_id):
95 info = self._download_theplatform_metadata(path, video_id)
96 return self._parse_theplatform_metadata(info)
99 class ThePlatformIE(ThePlatformBaseIE):
100 _VALID_URL = r'''(?x)
101 (?:https?://(?:link|player)\.theplatform\.com/[sp]/(?P<provider_id>[^/]+)/
102 (?:(?:(?:[^/]+/)+select/)?(?P<media>media/(?:guid/\d+/)?)|(?P<config>(?:[^/\?]+/(?:swf|config)|onsite)/select/))?
103 |theplatform:)(?P<id>[^/\?&]+)'''
106 # from http://www.metacafe.com/watch/cb-e9I_cZgTgIPd/blackberrys_big_bold_z30/
107 'url': 'http://link.theplatform.com/s/dJ5BDC/e9I_cZgTgIPd/meta.smil?format=smil&Tracking=true&mbr=true',
109 'id': 'e9I_cZgTgIPd',
111 'title': 'Blackberry\'s big, bold Z30',
112 'description': 'The Z30 is Blackberry\'s biggest, baddest mobile messaging device yet.',
114 'timestamp': 1383239700,
115 'upload_date': '20131031',
116 'uploader': 'CBSI-NEW',
120 'skip_download': True,
123 # from http://www.cnet.com/videos/tesla-model-s-a-second-step-towards-a-cleaner-motoring-future/
124 'url': 'http://link.theplatform.com/s/kYEXFC/22d_qsQ6MIRT',
126 'id': '22d_qsQ6MIRT',
128 'description': 'md5:ac330c9258c04f9d7512cf26b9595409',
129 'title': 'Tesla Model S: A second step towards a cleaner motoring future',
130 'timestamp': 1426176191,
131 'upload_date': '20150312',
132 'uploader': 'CBSI-NEW',
136 'skip_download': True,
139 'url': 'https://player.theplatform.com/p/D6x-PC/pulse_preview/embed/select/media/yMBg9E8KFxZD',
141 'id': 'yMBg9E8KFxZD',
143 'description': 'md5:644ad9188d655b742f942bf2e06b002d',
144 'title': 'HIGHLIGHTS: USA bag first ever series Cup win',
148 'url': 'http://player.theplatform.com/p/NnzsPC/widget/select/media/4Y0TlYUr_ZT7',
149 'only_matching': True,
151 'url': 'http://player.theplatform.com/p/2E2eJC/nbcNewsOffsite?guid=tdy_or_siri_150701',
152 'md5': 'fb96bb3d85118930a5b055783a3bd992',
154 'id': 'tdy_or_siri_150701',
156 'title': 'iPhone Siri’s sassy response to a math question has people talking',
157 'description': 'md5:a565d1deadd5086f3331d57298ec6333',
159 'thumbnail': 're:^https?://.*\.jpg$',
160 'timestamp': 1435752600,
161 'upload_date': '20150701',
162 'uploader': 'NBCU-NEWS',
165 # From http://www.nbc.com/the-blacklist/video/sir-crispin-crandall/2928790?onid=137781#vc137781=1
166 # geo-restricted (US), HLS encrypted with AES-128
167 'url': 'http://player.theplatform.com/p/NnzsPC/onsite_universal/select/media/guid/2410887629/2928790?fwsitesection=nbc_the_blacklist_video_library&autoPlay=true&carouselID=137781',
168 'only_matching': True,
170 _SERVICE_PROVIDER_TEMPLATE = 'https://sp.auth.adobe.com/adobe-services/%s'
173 def _extract_urls(cls, webpage):
177 property=(["'])(?:og:video(?::(?:secure_)?url)?|twitter:player)\1\s+
178 content=(["'])(?P<url>https?://player\.theplatform\.com/p/.+?)\2
181 return [m.group('url')]
183 matches = re.findall(
184 r'<(?:iframe|script)[^>]+src=(["\'])((?:https?:)?//player\.theplatform\.com/p/.+?)\1', webpage)
186 return list(zip(*matches))[1]
189 def _sign_url(url, sig_key, sig_secret, life=600, include_qs=False):
190 flags = '10' if include_qs else '00'
191 expiration_date = '%x' % (int(time.time()) + life)
194 return binascii.b2a_hex(str.encode('ascii')).decode('ascii')
196 def hex_to_bytes(hex):
197 return binascii.a2b_hex(hex.encode('ascii'))
199 relative_path = re.match(r'https?://link.theplatform.com/s/([^?]+)', url).group(1)
200 clear_text = hex_to_bytes(flags + expiration_date + str_to_hex(relative_path))
201 checksum = hmac.new(sig_key.encode('ascii'), clear_text, hashlib.sha1).hexdigest()
202 sig = flags + expiration_date + checksum + str_to_hex(sig_secret)
203 return '%s&sig=%s' % (url, sig)
205 def _extract_mvpd_auth(self, url, video_id, requestor_id, resource):
206 def xml_text(xml_str, tag):
207 return self._search_regex(
208 '<%s>(.+?)</%s>' % (tag, tag), xml_str, tag)
211 'ap_42': 'anonymous',
212 'ap_11': 'Linux i686',
213 'ap_z': 'Mozilla/5.0 (X11; Linux i686; rv:47.0) Gecko/20100101 Firefox/47.0',
214 'User-Agent': 'Mozilla/5.0 (X11; Linux i686; rv:47.0) Gecko/20100101 Firefox/47.0',
217 guid = xml_text(resource, 'guid')
218 requestor_info = self._downloader.cache.load('mvpd', requestor_id) or {}
219 authn_token = requestor_info.get('authn_token')
221 token_expires = unified_timestamp(xml_text(authn_token, 'simpleTokenExpires').replace('_GMT', ''))
222 if token_expires and token_expires >= time.time():
225 # TODO add support for other TV Providers
227 login_info = netrc.netrc().authenticators(mso_id)
231 def post_form(form_page, note, data={}):
232 post_url = self._html_search_regex(r'<form[^>]+action=(["\'])(?P<url>.+?)\1', form_page, 'post url', group='url')
233 return self._download_webpage(
234 post_url, video_id, note, data=urlencode_postdata(data or self._hidden_inputs(form_page)), headers={
235 'Content-Type': 'application/x-www-form-urlencoded',
238 provider_redirect_page = self._download_webpage(
239 self._SERVICE_PROVIDER_TEMPLATE % 'authenticate/saml', video_id,
240 'Downloading Provider Redirect Page', query={
243 'requestor_id': requestor_id,
244 'no_iframe': 'false',
245 'domain_name': 'adobe.com',
248 provider_login_page = post_form(
249 provider_redirect_page, 'Downloading Provider Login Page')
250 mvpd_confirm_page = post_form(provider_login_page, 'Logging in', {
251 'username': login_info[0],
252 'password': login_info[2],
254 post_form(mvpd_confirm_page, 'Confirming Login')
256 session = self._download_webpage(
257 self._SERVICE_PROVIDER_TEMPLATE % 'session', video_id,
258 'Retrieving Session', data=urlencode_postdata({
260 'requestor_id': requestor_id,
261 }), headers=mvpd_headers)
262 authn_token = unescapeHTML(xml_text(session, 'authnToken'))
263 requestor_info['authn_token'] = authn_token
264 self._downloader.cache.store('mvpd', requestor_id, requestor_info)
266 authz_token = requestor_info.get(guid)
268 authorize = self._download_webpage(
269 self._SERVICE_PROVIDER_TEMPLATE % 'authorize', video_id,
270 'Retrieving Authorization Token', data=urlencode_postdata({
271 'resource_id': resource,
272 'requestor_id': requestor_id,
273 'authentication_token': authn_token,
274 'mso_id': xml_text(authn_token, 'simpleTokenMsoID'),
276 }), headers=mvpd_headers)
277 authz_token = unescapeHTML(xml_text(authorize, 'authzToken'))
278 requestor_info[guid] = authz_token
279 self._downloader.cache.store('mvpd', requestor_id, requestor_info)
281 mvpd_headers.update({
282 'ap_19': xml_text(authn_token, 'simpleSamlNameID'),
283 'ap_23': xml_text(authn_token, 'simpleSamlSessionIndex'),
286 return self._download_webpage(
287 self._SERVICE_PROVIDER_TEMPLATE % 'shortAuthorize',
288 video_id, 'Retrieving Media Token', data=urlencode_postdata({
289 'authz_token': authz_token,
290 'requestor_id': requestor_id,
291 'session_guid': xml_text(authn_token, 'simpleTokenAuthenticationGuid'),
292 'hashed_guid': 'false',
293 }), headers=mvpd_headers)
295 def _real_extract(self, url):
296 url, smuggled_data = unsmuggle_url(url, {})
298 mobj = re.match(self._VALID_URL, url)
299 provider_id = mobj.group('provider_id')
300 video_id = mobj.group('id')
303 provider_id = 'dJ5BDC'
305 path = provider_id + '/'
306 if mobj.group('media'):
307 path += mobj.group('media')
310 qs_dict = compat_parse_qs(compat_urllib_parse_urlparse(url).query)
311 if 'guid' in qs_dict:
312 webpage = self._download_webpage(url, video_id)
313 scripts = re.findall(r'<script[^>]+src="([^"]+)"', webpage)
315 # feed id usually locates in the last script.
316 # Seems there's no pattern for the interested script filename, so
318 for script in reversed(scripts):
319 feed_script = self._download_webpage(
320 self._proto_relative_url(script, 'http:'),
321 video_id, 'Downloading feed script')
322 feed_id = self._search_regex(
323 r'defaultFeedId\s*:\s*"([^"]+)"', feed_script,
324 'default feed id', default=None)
325 if feed_id is not None:
328 raise ExtractorError('Unable to find feed id')
329 return self.url_result('http://feed.theplatform.com/f/%s/%s?byGuid=%s' % (
330 provider_id, feed_id, qs_dict['guid'][0]))
332 if smuggled_data.get('force_smil_url', False):
334 # Explicitly specified SMIL (see https://github.com/rg3/youtube-dl/issues/7385)
335 elif '/guid/' in url:
337 source_url = smuggled_data.get('source_url')
339 headers['Referer'] = source_url
340 request = sanitized_Request(url, headers=headers)
341 webpage = self._download_webpage(request, video_id)
342 smil_url = self._search_regex(
343 r'<link[^>]+href=(["\'])(?P<url>.+?)\1[^>]+type=["\']application/smil\+xml',
344 webpage, 'smil url', group='url')
345 path = self._search_regex(
346 r'link\.theplatform\.com/s/((?:[^/?#&]+/)+[^/?#&]+)', smil_url, 'path')
347 smil_url += '?' if '?' not in smil_url else '&' + 'formats=m3u,mpeg4'
348 elif mobj.group('config'):
349 config_url = url + '&form=json'
350 config_url = config_url.replace('swf/', 'config/')
351 config_url = config_url.replace('onsite/', 'onsite/config/')
352 config = self._download_json(config_url, video_id, 'Downloading config')
353 if 'releaseUrl' in config:
354 release_url = config['releaseUrl']
356 release_url = 'http://link.theplatform.com/s/%s?mbr=true' % path
357 smil_url = release_url + '&formats=MPEG4&manifest=f4m'
359 smil_url = 'http://link.theplatform.com/s/%s?mbr=true' % path
361 sig = smuggled_data.get('sig')
363 smil_url = self._sign_url(smil_url, sig['key'], sig['secret'])
365 formats, subtitles = self._extract_theplatform_smil(smil_url, video_id)
366 self._sort_formats(formats)
368 ret = self._extract_theplatform_metadata(path, video_id)
369 combined_subtitles = self._merge_subtitles(ret.get('subtitles', {}), subtitles)
373 'subtitles': combined_subtitles,
379 class ThePlatformFeedIE(ThePlatformBaseIE):
380 _URL_TEMPLATE = '%s//feed.theplatform.com/f/%s/%s?form=json&%s'
381 _VALID_URL = r'https?://feed\.theplatform\.com/f/(?P<provider_id>[^/]+)/(?P<feed_id>[^?/]+)\?(?:[^&]+&)*(?P<filter>by(?:Gui|I)d=(?P<id>[\w-]+))'
383 # From http://player.theplatform.com/p/7wvmTC/MSNBCEmbeddedOffSite?guid=n_hardball_5biden_140207
384 'url': 'http://feed.theplatform.com/f/7wvmTC/msnbc_video-p-test?form=json&pretty=true&range=-40&byGuid=n_hardball_5biden_140207',
385 'md5': '6e32495b5073ab414471b615c5ded394',
387 'id': 'n_hardball_5biden_140207',
389 'title': 'The Biden factor: will Joe run in 2016?',
390 'description': 'Could Vice President Joe Biden be preparing a 2016 campaign? Mark Halperin and Sam Stein weigh in.',
391 'thumbnail': 're:^https?://.*\.jpg$',
392 'upload_date': '20140208',
393 'timestamp': 1391824260,
395 'categories': ['MSNBC/Issues/Democrats', 'MSNBC/Issues/Elections/Election 2016'],
396 'uploader': 'NBCU-NEWS',
400 def _extract_feed_info(self, provider_id, feed_id, filter_query, video_id, custom_fields=None, asset_types_query={}):
401 real_url = self._URL_TEMPLATE % (self.http_scheme(), provider_id, feed_id, filter_query)
402 entry = self._download_json(real_url, video_id)['entries'][0]
406 first_video_id = None
409 for item in entry['media$content']:
410 smil_url = item['plfile$url']
411 cur_video_id = ThePlatformIE._match_id(smil_url)
412 if first_video_id is None:
413 first_video_id = cur_video_id
414 duration = float_or_none(item.get('plfile$duration'))
415 for asset_type in item['plfile$assetTypes']:
416 if asset_type in asset_types:
418 asset_types.append(asset_type)
421 'formats': item['plfile$format'],
422 'assetTypes': asset_type,
424 if asset_type in asset_types_query:
425 query.update(asset_types_query[asset_type])
426 cur_formats, cur_subtitles = self._extract_theplatform_smil(update_url_query(
427 smil_url, query), video_id, 'Downloading SMIL data for %s' % asset_type)
428 formats.extend(cur_formats)
429 subtitles = self._merge_subtitles(subtitles, cur_subtitles)
431 self._sort_formats(formats)
434 'url': thumbnail['plfile$url'],
435 'width': int_or_none(thumbnail.get('plfile$width')),
436 'height': int_or_none(thumbnail.get('plfile$height')),
437 } for thumbnail in entry.get('media$thumbnails', [])]
439 timestamp = int_or_none(entry.get('media$availableDate'), scale=1000)
440 categories = [item['media$name'] for item in entry.get('media$categories', [])]
442 ret = self._extract_theplatform_metadata('%s/%s' % (provider_id, first_video_id), video_id)
443 subtitles = self._merge_subtitles(subtitles, ret['subtitles'])
447 'subtitles': subtitles,
448 'thumbnails': thumbnails,
449 'duration': duration,
450 'timestamp': timestamp,
451 'categories': categories,
454 ret.update(custom_fields(entry))
458 def _real_extract(self, url):
459 mobj = re.match(self._VALID_URL, url)
461 video_id = mobj.group('id')
462 provider_id = mobj.group('provider_id')
463 feed_id = mobj.group('feed_id')
464 filter_query = mobj.group('filter')
466 return self._extract_feed_info(provider_id, feed_id, filter_query, video_id)