]> gitweb @ CieloNegro.org - youtube-dl.git/blob - youtube_dl/extractor/zdf.py
[hidive] add support for authentication(closes #16534)
[youtube-dl.git] / youtube_dl / extractor / zdf.py
1 # coding: utf-8
2 from __future__ import unicode_literals
3
4 import re
5
6 from .common import InfoExtractor
7 from ..compat import compat_str
8 from ..utils import (
9     determine_ext,
10     int_or_none,
11     NO_DEFAULT,
12     orderedSet,
13     parse_codecs,
14     qualities,
15     try_get,
16     unified_timestamp,
17     update_url_query,
18     urljoin,
19 )
20
21
22 class ZDFBaseIE(InfoExtractor):
23     def _call_api(self, url, player, referrer, video_id, item):
24         return self._download_json(
25             url, video_id, 'Downloading JSON %s' % item,
26             headers={
27                 'Referer': referrer,
28                 'Api-Auth': 'Bearer %s' % player['apiToken'],
29             })
30
31     def _extract_player(self, webpage, video_id, fatal=True):
32         return self._parse_json(
33             self._search_regex(
34                 r'(?s)data-zdfplayer-jsb=(["\'])(?P<json>{.+?})\1', webpage,
35                 'player JSON', default='{}' if not fatal else NO_DEFAULT,
36                 group='json'),
37             video_id)
38
39
40 class ZDFIE(ZDFBaseIE):
41     _VALID_URL = r'https?://www\.zdf\.de/(?:[^/]+/)*(?P<id>[^/?]+)\.html'
42     _QUALITIES = ('auto', 'low', 'med', 'high', 'veryhigh')
43
44     _TESTS = [{
45         'url': 'https://www.zdf.de/dokumentation/terra-x/die-magie-der-farben-von-koenigspurpur-und-jeansblau-100.html',
46         'info_dict': {
47             'id': 'die-magie-der-farben-von-koenigspurpur-und-jeansblau-100',
48             'ext': 'mp4',
49             'title': 'Die Magie der Farben (2/2)',
50             'description': 'md5:a89da10c928c6235401066b60a6d5c1a',
51             'duration': 2615,
52             'timestamp': 1465021200,
53             'upload_date': '20160604',
54         },
55     }, {
56         'url': 'https://www.zdf.de/service-und-hilfe/die-neue-zdf-mediathek/zdfmediathek-trailer-100.html',
57         'only_matching': True,
58     }, {
59         'url': 'https://www.zdf.de/filme/taunuskrimi/die-lebenden-und-die-toten-1---ein-taunuskrimi-100.html',
60         'only_matching': True,
61     }, {
62         'url': 'https://www.zdf.de/dokumentation/planet-e/planet-e-uebersichtsseite-weitere-dokumentationen-von-planet-e-100.html',
63         'only_matching': True,
64     }]
65
66     @staticmethod
67     def _extract_subtitles(src):
68         subtitles = {}
69         for caption in try_get(src, lambda x: x['captions'], list) or []:
70             subtitle_url = caption.get('uri')
71             if subtitle_url and isinstance(subtitle_url, compat_str):
72                 lang = caption.get('language', 'deu')
73                 subtitles.setdefault(lang, []).append({
74                     'url': subtitle_url,
75                 })
76         return subtitles
77
78     def _extract_format(self, video_id, formats, format_urls, meta):
79         format_url = meta.get('url')
80         if not format_url or not isinstance(format_url, compat_str):
81             return
82         if format_url in format_urls:
83             return
84         format_urls.add(format_url)
85         mime_type = meta.get('mimeType')
86         ext = determine_ext(format_url)
87         if mime_type == 'application/x-mpegURL' or ext == 'm3u8':
88             formats.extend(self._extract_m3u8_formats(
89                 format_url, video_id, 'mp4', m3u8_id='hls',
90                 entry_protocol='m3u8_native', fatal=False))
91         elif mime_type == 'application/f4m+xml' or ext == 'f4m':
92             formats.extend(self._extract_f4m_formats(
93                 update_url_query(format_url, {'hdcore': '3.7.0'}), video_id, f4m_id='hds', fatal=False))
94         else:
95             f = parse_codecs(meta.get('mimeCodec'))
96             format_id = ['http']
97             for p in (meta.get('type'), meta.get('quality')):
98                 if p and isinstance(p, compat_str):
99                     format_id.append(p)
100             f.update({
101                 'url': format_url,
102                 'format_id': '-'.join(format_id),
103                 'format_note': meta.get('quality'),
104                 'language': meta.get('language'),
105                 'quality': qualities(self._QUALITIES)(meta.get('quality')),
106                 'preference': -10,
107             })
108             formats.append(f)
109
110     def _extract_entry(self, url, player, content, video_id):
111         title = content.get('title') or content['teaserHeadline']
112
113         t = content['mainVideoContent']['http://zdf.de/rels/target']
114
115         ptmd_path = t.get('http://zdf.de/rels/streams/ptmd')
116
117         if not ptmd_path:
118             ptmd_path = t[
119                 'http://zdf.de/rels/streams/ptmd-template'].replace(
120                 '{playerId}', 'portal')
121
122         ptmd = self._call_api(
123             urljoin(url, ptmd_path), player, url, video_id, 'metadata')
124
125         formats = []
126         track_uris = set()
127         for p in ptmd['priorityList']:
128             formitaeten = p.get('formitaeten')
129             if not isinstance(formitaeten, list):
130                 continue
131             for f in formitaeten:
132                 f_qualities = f.get('qualities')
133                 if not isinstance(f_qualities, list):
134                     continue
135                 for quality in f_qualities:
136                     tracks = try_get(quality, lambda x: x['audio']['tracks'], list)
137                     if not tracks:
138                         continue
139                     for track in tracks:
140                         self._extract_format(
141                             video_id, formats, track_uris, {
142                                 'url': track.get('uri'),
143                                 'type': f.get('type'),
144                                 'mimeType': f.get('mimeType'),
145                                 'quality': quality.get('quality'),
146                                 'language': track.get('language'),
147                             })
148         self._sort_formats(formats)
149
150         thumbnails = []
151         layouts = try_get(
152             content, lambda x: x['teaserImageRef']['layouts'], dict)
153         if layouts:
154             for layout_key, layout_url in layouts.items():
155                 if not isinstance(layout_url, compat_str):
156                     continue
157                 thumbnail = {
158                     'url': layout_url,
159                     'format_id': layout_key,
160                 }
161                 mobj = re.search(r'(?P<width>\d+)x(?P<height>\d+)', layout_key)
162                 if mobj:
163                     thumbnail.update({
164                         'width': int(mobj.group('width')),
165                         'height': int(mobj.group('height')),
166                     })
167                 thumbnails.append(thumbnail)
168
169         return {
170             'id': video_id,
171             'title': title,
172             'description': content.get('leadParagraph') or content.get('teasertext'),
173             'duration': int_or_none(t.get('duration')),
174             'timestamp': unified_timestamp(content.get('editorialDate')),
175             'thumbnails': thumbnails,
176             'subtitles': self._extract_subtitles(ptmd),
177             'formats': formats,
178         }
179
180     def _extract_regular(self, url, player, video_id):
181         content = self._call_api(
182             player['content'], player, url, video_id, 'content')
183         return self._extract_entry(player['content'], player, content, video_id)
184
185     def _extract_mobile(self, video_id):
186         document = self._download_json(
187             'https://zdf-cdn.live.cellular.de/mediathekV2/document/%s' % video_id,
188             video_id)['document']
189
190         title = document['titel']
191
192         formats = []
193         format_urls = set()
194         for f in document['formitaeten']:
195             self._extract_format(video_id, formats, format_urls, f)
196         self._sort_formats(formats)
197
198         thumbnails = []
199         teaser_bild = document.get('teaserBild')
200         if isinstance(teaser_bild, dict):
201             for thumbnail_key, thumbnail in teaser_bild.items():
202                 thumbnail_url = try_get(
203                     thumbnail, lambda x: x['url'], compat_str)
204                 if thumbnail_url:
205                     thumbnails.append({
206                         'url': thumbnail_url,
207                         'id': thumbnail_key,
208                         'width': int_or_none(thumbnail.get('width')),
209                         'height': int_or_none(thumbnail.get('height')),
210                     })
211
212         return {
213             'id': video_id,
214             'title': title,
215             'description': document.get('beschreibung'),
216             'duration': int_or_none(document.get('length')),
217             'timestamp': unified_timestamp(try_get(
218                 document, lambda x: x['meta']['editorialDate'], compat_str)),
219             'thumbnails': thumbnails,
220             'subtitles': self._extract_subtitles(document),
221             'formats': formats,
222         }
223
224     def _real_extract(self, url):
225         video_id = self._match_id(url)
226
227         webpage = self._download_webpage(url, video_id, fatal=False)
228         if webpage:
229             player = self._extract_player(webpage, url, fatal=False)
230             if player:
231                 return self._extract_regular(url, player, video_id)
232
233         return self._extract_mobile(video_id)
234
235
236 class ZDFChannelIE(ZDFBaseIE):
237     _VALID_URL = r'https?://www\.zdf\.de/(?:[^/]+/)*(?P<id>[^/?#&]+)'
238     _TESTS = [{
239         'url': 'https://www.zdf.de/sport/das-aktuelle-sportstudio',
240         'info_dict': {
241             'id': 'das-aktuelle-sportstudio',
242             'title': 'das aktuelle sportstudio | ZDF',
243         },
244         'playlist_count': 21,
245     }, {
246         'url': 'https://www.zdf.de/dokumentation/planet-e',
247         'info_dict': {
248             'id': 'planet-e',
249             'title': 'planet e.',
250         },
251         'playlist_count': 4,
252     }, {
253         'url': 'https://www.zdf.de/filme/taunuskrimi/',
254         'only_matching': True,
255     }]
256
257     @classmethod
258     def suitable(cls, url):
259         return False if ZDFIE.suitable(url) else super(ZDFChannelIE, cls).suitable(url)
260
261     def _real_extract(self, url):
262         channel_id = self._match_id(url)
263
264         webpage = self._download_webpage(url, channel_id)
265
266         entries = [
267             self.url_result(item_url, ie=ZDFIE.ie_key())
268             for item_url in orderedSet(re.findall(
269                 r'data-plusbar-url=["\'](http.+?\.html)', webpage))]
270
271         return self.playlist_result(
272             entries, channel_id, self._og_search_title(webpage, fatal=False))
273
274         r"""
275         player = self._extract_player(webpage, channel_id)
276
277         channel_id = self._search_regex(
278             r'docId\s*:\s*(["\'])(?P<id>(?!\1).+?)\1', webpage,
279             'channel id', group='id')
280
281         channel = self._call_api(
282             'https://api.zdf.de/content/documents/%s.json' % channel_id,
283             player, url, channel_id)
284
285         items = []
286         for module in channel['module']:
287             for teaser in try_get(module, lambda x: x['teaser'], list) or []:
288                 t = try_get(
289                     teaser, lambda x: x['http://zdf.de/rels/target'], dict)
290                 if not t:
291                     continue
292                 items.extend(try_get(
293                     t,
294                     lambda x: x['resultsWithVideo']['http://zdf.de/rels/search/results'],
295                     list) or [])
296             items.extend(try_get(
297                 module,
298                 lambda x: x['filterRef']['resultsWithVideo']['http://zdf.de/rels/search/results'],
299                 list) or [])
300
301         entries = []
302         entry_urls = set()
303         for item in items:
304             t = try_get(item, lambda x: x['http://zdf.de/rels/target'], dict)
305             if not t:
306                 continue
307             sharing_url = t.get('http://zdf.de/rels/sharing-url')
308             if not sharing_url or not isinstance(sharing_url, compat_str):
309                 continue
310             if sharing_url in entry_urls:
311                 continue
312             entry_urls.add(sharing_url)
313             entries.append(self.url_result(
314                 sharing_url, ie=ZDFIE.ie_key(), video_id=t.get('id')))
315
316         return self.playlist_result(entries, channel_id, channel.get('title'))
317         """