youtube_dl/extractor/safari.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 import re
   5
   6 from .common import InfoExtractor
   7
   8 from ..utils import (
   9     ExtractorError,
  10     sanitized_Request,
  11     std_headers,
  12     urlencode_postdata,
  13     update_url_query,
  14 )
  15
  16
  17 class SafariBaseIE(InfoExtractor):
  18     _LOGIN_URL = 'https://learning.oreilly.com/accounts/login/'
  19     _NETRC_MACHINE = 'safari'
  20
  21     _API_BASE = 'https://learning.oreilly.com/api/v1'
  22     _API_FORMAT = 'json'
  23
  24     LOGGED_IN = False
  25
  26     def _real_initialize(self):
  27         self._login()
  28
  29     def _login(self):
  30         username, password = self._get_login_info()
  31         if username is None:
  32             return
  33
  34         headers = std_headers.copy()
  35         if 'Referer' not in headers:
  36             headers['Referer'] = self._LOGIN_URL
  37
  38         login_page = self._download_webpage(
  39             self._LOGIN_URL, None, 'Downloading login form', headers=headers)
  40
  41         def is_logged(webpage):
  42             return any(re.search(p, webpage) for p in (
  43                 r'href=["\']/accounts/logout/', r'>Sign Out<'))
  44
  45         if is_logged(login_page):
  46             self.LOGGED_IN = True
  47             return
  48
  49         csrf = self._html_search_regex(
  50             r"name='csrfmiddlewaretoken'\s+value='([^']+)'",
  51             login_page, 'csrf token')
  52
  53         login_form = {
  54             'csrfmiddlewaretoken': csrf,
  55             'email': username,
  56             'password1': password,
  57             'login': 'Sign In',
  58             'next': '',
  59         }
  60
  61         request = sanitized_Request(
  62             self._LOGIN_URL, urlencode_postdata(login_form), headers=headers)
  63         login_page = self._download_webpage(
  64             request, None, 'Logging in')
  65
  66         if not is_logged(login_page):
  67             raise ExtractorError(
  68                 'Login failed; make sure your credentials are correct and try again.',
  69                 expected=True)
  70
  71         self.LOGGED_IN = True
  72
  73
  74 class SafariIE(SafariBaseIE):
  75     IE_NAME = 'safari'
  76     IE_DESC = 'safaribooksonline.com online video'
  77     _VALID_URL = r'''(?x)
  78                         https?://
  79                             (?:www\.)?(?:safaribooksonline|learning\.oreilly)\.com/
  80                             (?:
  81                                 library/view/[^/]+/(?P<course_id>[^/]+)/(?P<part>[^/?\#&]+)\.html|
  82                                 videos/[^/]+/[^/]+/(?P<reference_id>[^-]+-[^/?\#&]+)
  83                             )
  84                     '''
  85
  86     _TESTS = [{
  87         'url': 'https://www.safaribooksonline.com/library/view/hadoop-fundamentals-livelessons/9780133392838/part00.html',
  88         'md5': 'dcc5a425e79f2564148652616af1f2a3',
  89         'info_dict': {
  90             'id': '0_qbqx90ic',
  91             'ext': 'mp4',
  92             'title': 'Introduction to Hadoop Fundamentals LiveLessons',
  93             'timestamp': 1437758058,
  94             'upload_date': '20150724',
  95             'uploader_id': 'stork',
  96         },
  97     }, {
  98         # non-digits in course id
  99         'url': 'https://www.safaribooksonline.com/library/view/create-a-nodejs/100000006A0210/part00.html',
 100         'only_matching': True,
 101     }, {
 102         'url': 'https://www.safaribooksonline.com/library/view/learning-path-red/9780134664057/RHCE_Introduction.html',
 103         'only_matching': True,
 104     }, {
 105         'url': 'https://www.safaribooksonline.com/videos/python-programming-language/9780134217314/9780134217314-PYMC_13_00',
 106         'only_matching': True,
 107     }, {
 108         'url': 'https://learning.oreilly.com/videos/hadoop-fundamentals-livelessons/9780133392838/9780133392838-00_SeriesIntro',
 109         'only_matching': True,
 110     }]
 111
 112     _PARTNER_ID = '1926081'
 113     _UICONF_ID = '29375172'
 114
 115     def _real_extract(self, url):
 116         mobj = re.match(self._VALID_URL, url)
 117
 118         reference_id = mobj.group('reference_id')
 119         if reference_id:
 120             video_id = reference_id
 121             partner_id = self._PARTNER_ID
 122             ui_id = self._UICONF_ID
 123         else:
 124             video_id = '%s-%s' % (mobj.group('course_id'), mobj.group('part'))
 125
 126             webpage, urlh = self._download_webpage_handle(url, video_id)
 127
 128             mobj = re.match(self._VALID_URL, urlh.geturl())
 129             reference_id = mobj.group('reference_id')
 130             if not reference_id:
 131                 reference_id = self._search_regex(
 132                     r'data-reference-id=(["\'])(?P<id>(?:(?!\1).)+)\1',
 133                     webpage, 'kaltura reference id', group='id')
 134             partner_id = self._search_regex(
 135                 r'data-partner-id=(["\'])(?P<id>(?:(?!\1).)+)\1',
 136                 webpage, 'kaltura widget id', default=self._PARTNER_ID,
 137                 group='id')
 138             ui_id = self._search_regex(
 139                 r'data-ui-id=(["\'])(?P<id>(?:(?!\1).)+)\1',
 140                 webpage, 'kaltura uiconf id', default=self._UICONF_ID,
 141                 group='id')
 142
 143         query = {
 144             'wid': '_%s' % partner_id,
 145             'uiconf_id': ui_id,
 146             'flashvars[referenceId]': reference_id,
 147         }
 148
 149         if self.LOGGED_IN:
 150             kaltura_session = self._download_json(
 151                 '%s/player/kaltura_session/?reference_id=%s' % (self._API_BASE, reference_id),
 152                 video_id, 'Downloading kaltura session JSON',
 153                 'Unable to download kaltura session JSON', fatal=False)
 154             if kaltura_session:
 155                 session = kaltura_session.get('session')
 156                 if session:
 157                     query['flashvars[ks]'] = session
 158
 159         return self.url_result(update_url_query(
 160             'https://cdnapisec.kaltura.com/html5/html5lib/v2.37.1/mwEmbedFrame.php', query),
 161             'Kaltura')
 162
 163
 164 class SafariApiIE(SafariBaseIE):
 165     IE_NAME = 'safari:api'
 166     _VALID_URL = r'https?://(?:www\.)?(?:safaribooksonline|learning\.oreilly)\.com/api/v1/book/(?P<course_id>[^/]+)/chapter(?:-content)?/(?P<part>[^/?#&]+)\.html'
 167
 168     _TESTS = [{
 169         'url': 'https://www.safaribooksonline.com/api/v1/book/9780133392838/chapter/part00.html',
 170         'only_matching': True,
 171     }, {
 172         'url': 'https://www.safaribooksonline.com/api/v1/book/9780134664057/chapter/RHCE_Introduction.html',
 173         'only_matching': True,
 174     }]
 175
 176     def _real_extract(self, url):
 177         mobj = re.match(self._VALID_URL, url)
 178         part = self._download_json(
 179             url, '%s/%s' % (mobj.group('course_id'), mobj.group('part')),
 180             'Downloading part JSON')
 181         return self.url_result(part['web_url'], SafariIE.ie_key())
 182
 183
 184 class SafariCourseIE(SafariBaseIE):
 185     IE_NAME = 'safari:course'
 186     IE_DESC = 'safaribooksonline.com online courses'
 187
 188     _VALID_URL = r'''(?x)
 189                     https?://
 190                         (?:
 191                             (?:www\.)?(?:safaribooksonline|learning\.oreilly)\.com/
 192                             (?:
 193                                 library/view/[^/]+|
 194                                 api/v1/book|
 195                                 videos/[^/]+
 196                             )|
 197                             techbus\.safaribooksonline\.com
 198                         )
 199                         /(?P<id>[^/]+)
 200                     '''
 201
 202     _TESTS = [{
 203         'url': 'https://www.safaribooksonline.com/library/view/hadoop-fundamentals-livelessons/9780133392838/',
 204         'info_dict': {
 205             'id': '9780133392838',
 206             'title': 'Hadoop Fundamentals LiveLessons',
 207         },
 208         'playlist_count': 22,
 209         'skip': 'Requires safaribooksonline account credentials',
 210     }, {
 211         'url': 'https://www.safaribooksonline.com/api/v1/book/9781449396459/?override_format=json',
 212         'only_matching': True,
 213     }, {
 214         'url': 'http://techbus.safaribooksonline.com/9780134426365',
 215         'only_matching': True,
 216     }, {
 217         'url': 'https://www.safaribooksonline.com/videos/python-programming-language/9780134217314',
 218         'only_matching': True,
 219     }, {
 220         'url': 'https://learning.oreilly.com/videos/hadoop-fundamentals-livelessons/9780133392838',
 221         'only_matching': True,
 222     }]
 223
 224     @classmethod
 225     def suitable(cls, url):
 226         return (False if SafariIE.suitable(url) or SafariApiIE.suitable(url)
 227                 else super(SafariCourseIE, cls).suitable(url))
 228
 229     def _real_extract(self, url):
 230         course_id = self._match_id(url)
 231
 232         course_json = self._download_json(
 233             '%s/book/%s/?override_format=%s' % (self._API_BASE, course_id, self._API_FORMAT),
 234             course_id, 'Downloading course JSON')
 235
 236         if 'chapters' not in course_json:
 237             raise ExtractorError(
 238                 'No chapters found for course %s' % course_id, expected=True)
 239
 240         entries = [
 241             self.url_result(chapter, SafariApiIE.ie_key())
 242             for chapter in course_json['chapters']]
 243
 244         course_title = course_json['title']
 245
 246         return self.playlist_result(entries, course_id, course_title)