youtube_dl/extractor/cda.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 import codecs
   5 import re
   6
   7 from .common import InfoExtractor
   8 from ..utils import (
   9     ExtractorError,
  10     float_or_none,
  11     int_or_none,
  12     multipart_encode,
  13     parse_duration,
  14     random_birthday,
  15     urljoin,
  16 )
  17
  18
  19 class CDAIE(InfoExtractor):
  20     _VALID_URL = r'https?://(?:(?:www\.)?cda\.pl/video|ebd\.cda\.pl/[0-9]+x[0-9]+)/(?P<id>[0-9a-z]+)'
  21     _BASE_URL = 'http://www.cda.pl/'
  22     _TESTS = [{
  23         'url': 'http://www.cda.pl/video/5749950c',
  24         'md5': '6f844bf51b15f31fae165365707ae970',
  25         'info_dict': {
  26             'id': '5749950c',
  27             'ext': 'mp4',
  28             'height': 720,
  29             'title': 'Oto dlaczego przed zakrętem należy zwolnić.',
  30             'description': 'md5:269ccd135d550da90d1662651fcb9772',
  31             'thumbnail': r're:^https?://.*\.jpg$',
  32             'average_rating': float,
  33             'duration': 39,
  34             'age_limit': 0,
  35         }
  36     }, {
  37         'url': 'http://www.cda.pl/video/57413289',
  38         'md5': 'a88828770a8310fc00be6c95faf7f4d5',
  39         'info_dict': {
  40             'id': '57413289',
  41             'ext': 'mp4',
  42             'title': 'Lądowanie na lotnisku na Maderze',
  43             'description': 'md5:60d76b71186dcce4e0ba6d4bbdb13e1a',
  44             'thumbnail': r're:^https?://.*\.jpg$',
  45             'uploader': 'crash404',
  46             'view_count': int,
  47             'average_rating': float,
  48             'duration': 137,
  49             'age_limit': 0,
  50         }
  51     }, {
  52         # Age-restricted
  53         'url': 'http://www.cda.pl/video/1273454c4',
  54         'info_dict': {
  55             'id': '1273454c4',
  56             'ext': 'mp4',
  57             'title': 'Bronson (2008) napisy HD 1080p',
  58             'description': 'md5:1b6cb18508daf2dc4e0fa4db77fec24c',
  59             'height': 1080,
  60             'uploader': 'boniek61',
  61             'thumbnail': r're:^https?://.*\.jpg$',
  62             'duration': 5554,
  63             'age_limit': 18,
  64             'view_count': int,
  65             'average_rating': float,
  66         },
  67     }, {
  68         'url': 'http://ebd.cda.pl/0x0/5749950c',
  69         'only_matching': True,
  70     }]
  71
  72     def _download_age_confirm_page(self, url, video_id, *args, **kwargs):
  73         form_data = random_birthday('rok', 'miesiac', 'dzien')
  74         form_data.update({'return': url, 'module': 'video', 'module_id': video_id})
  75         data, content_type = multipart_encode(form_data)
  76         return self._download_webpage(
  77             urljoin(url, '/a/validatebirth'), video_id, *args,
  78             data=data, headers={
  79                 'Referer': url,
  80                 'Content-Type': content_type,
  81             }, **kwargs)
  82
  83     def _real_extract(self, url):
  84         video_id = self._match_id(url)
  85         self._set_cookie('cda.pl', 'cda.player', 'html5')
  86         webpage = self._download_webpage(
  87             self._BASE_URL + '/video/' + video_id, video_id)
  88
  89         if 'Ten film jest dostępny dla użytkowników premium' in webpage:
  90             raise ExtractorError('This video is only available for premium users.', expected=True)
  91
  92         need_confirm_age = False
  93         if self._html_search_regex(r'(<form[^>]+action="/a/validatebirth")',
  94                                    webpage, 'birthday validate form', default=None):
  95             webpage = self._download_age_confirm_page(
  96                 url, video_id, note='Confirming age')
  97             need_confirm_age = True
  98
  99         formats = []
 100
 101         uploader = self._search_regex(r'''(?x)
 102             <(span|meta)[^>]+itemprop=(["\'])author\2[^>]*>
 103             (?:<\1[^>]*>[^<]*</\1>|(?!</\1>)(?:.|\n))*?
 104             <(span|meta)[^>]+itemprop=(["\'])name\4[^>]*>(?P<uploader>[^<]+)</\3>
 105         ''', webpage, 'uploader', default=None, group='uploader')
 106         view_count = self._search_regex(
 107             r'Odsłony:(?:\s|&nbsp;)*([0-9]+)', webpage,
 108             'view_count', default=None)
 109         average_rating = self._search_regex(
 110             r'<(?:span|meta)[^>]+itemprop=(["\'])ratingValue\1[^>]*>(?P<rating_value>[0-9.]+)',
 111             webpage, 'rating', fatal=False, group='rating_value')
 112
 113         info_dict = {
 114             'id': video_id,
 115             'title': self._og_search_title(webpage),
 116             'description': self._og_search_description(webpage),
 117             'uploader': uploader,
 118             'view_count': int_or_none(view_count),
 119             'average_rating': float_or_none(average_rating),
 120             'thumbnail': self._og_search_thumbnail(webpage),
 121             'formats': formats,
 122             'duration': None,
 123             'age_limit': 18 if need_confirm_age else 0,
 124         }
 125
 126         def extract_format(page, version):
 127             json_str = self._search_regex(
 128                 r'player_data=(\\?["\'])(?P<player_data>.+?)\1', page,
 129                 '%s player_json' % version, fatal=False, group='player_data')
 130             if not json_str:
 131                 return
 132             player_data = self._parse_json(
 133                 json_str, '%s player_data' % version, fatal=False)
 134             if not player_data:
 135                 return
 136             video = player_data.get('video')
 137             if not video or 'file' not in video:
 138                 self.report_warning('Unable to extract %s version information' % version)
 139                 return
 140             if video['file'].startswith('uggc'):
 141                 video['file'] = codecs.decode(video['file'], 'rot_13')
 142                 if video['file'].endswith('adc.mp4'):
 143                     video['file'] = video['file'].replace('adc.mp4', '.mp4')
 144             f = {
 145                 'url': video['file'],
 146             }
 147             m = re.search(
 148                 r'<a[^>]+data-quality="(?P<format_id>[^"]+)"[^>]+href="[^"]+"[^>]+class="[^"]*quality-btn-active[^"]*">(?P<height>[0-9]+)p',
 149                 page)
 150             if m:
 151                 f.update({
 152                     'format_id': m.group('format_id'),
 153                     'height': int(m.group('height')),
 154                 })
 155             info_dict['formats'].append(f)
 156             if not info_dict['duration']:
 157                 info_dict['duration'] = parse_duration(video.get('duration'))
 158
 159         extract_format(webpage, 'default')
 160
 161         for href, resolution in re.findall(
 162                 r'<a[^>]+data-quality="[^"]+"[^>]+href="([^"]+)"[^>]+class="quality-btn"[^>]*>([0-9]+p)',
 163                 webpage):
 164             if need_confirm_age:
 165                 handler = self._download_age_confirm_page
 166             else:
 167                 handler = self._download_webpage
 168
 169             webpage = handler(
 170                 self._BASE_URL + href, video_id,
 171                 'Downloading %s version information' % resolution, fatal=False)
 172             if not webpage:
 173                 # Manually report warning because empty page is returned when
 174                 # invalid version is requested.
 175                 self.report_warning('Unable to download %s version information' % resolution)
 176                 continue
 177
 178             extract_format(webpage, resolution)
 179
 180         self._sort_formats(formats)
 181
 182         return info_dict