youtube_dl/extractor/linuxacademy.py

   1 from __future__ import unicode_literals
   2
   3 import json
   4 import random
   5 import re
   6
   7 from .common import InfoExtractor
   8 from ..compat import (
   9     compat_b64decode,
  10     compat_HTTPError,
  11     compat_str,
  12 )
  13 from ..utils import (
  14     ExtractorError,
  15     orderedSet,
  16     unescapeHTML,
  17     urlencode_postdata,
  18     urljoin,
  19 )
  20
  21
  22 class LinuxAcademyIE(InfoExtractor):
  23     _VALID_URL = r'''(?x)
  24                     https?://
  25                         (?:www\.)?linuxacademy\.com/cp/
  26                         (?:
  27                             courses/lesson/course/(?P<chapter_id>\d+)/lesson/(?P<lesson_id>\d+)|
  28                             modules/view/id/(?P<course_id>\d+)
  29                         )
  30                     '''
  31     _TESTS = [{
  32         'url': 'https://linuxacademy.com/cp/courses/lesson/course/1498/lesson/2/module/154',
  33         'info_dict': {
  34             'id': '1498-2',
  35             'ext': 'mp4',
  36             'title': "Introduction to the Practitioner's Brief",
  37         },
  38         'params': {
  39             'skip_download': True,
  40         },
  41         'skip': 'Requires Linux Academy account credentials',
  42     }, {
  43         'url': 'https://linuxacademy.com/cp/courses/lesson/course/1498/lesson/2',
  44         'only_matching': True,
  45     }, {
  46         'url': 'https://linuxacademy.com/cp/modules/view/id/154',
  47         'info_dict': {
  48             'id': '154',
  49             'title': 'AWS Certified Cloud Practitioner',
  50             'description': 'md5:039db7e60e4aac9cf43630e0a75fa834',
  51         },
  52         'playlist_count': 41,
  53         'skip': 'Requires Linux Academy account credentials',
  54     }]
  55
  56     _AUTHORIZE_URL = 'https://login.linuxacademy.com/authorize'
  57     _ORIGIN_URL = 'https://linuxacademy.com'
  58     _CLIENT_ID = 'KaWxNn1C2Gc7n83W9OFeXltd8Utb5vvx'
  59     _NETRC_MACHINE = 'linuxacademy'
  60
  61     def _real_initialize(self):
  62         self._login()
  63
  64     def _login(self):
  65         username, password = self._get_login_info()
  66         if username is None:
  67             return
  68
  69         def random_string():
  70             return ''.join([
  71                 random.choice('0123456789ABCDEFGHIJKLMNOPQRSTUVXYZabcdefghijklmnopqrstuvwxyz-._~')
  72                 for _ in range(32)])
  73
  74         webpage, urlh = self._download_webpage_handle(
  75             self._AUTHORIZE_URL, None, 'Downloading authorize page', query={
  76                 'client_id': self._CLIENT_ID,
  77                 'response_type': 'token id_token',
  78                 'redirect_uri': self._ORIGIN_URL,
  79                 'scope': 'openid email user_impersonation profile',
  80                 'audience': self._ORIGIN_URL,
  81                 'state': random_string(),
  82                 'nonce': random_string(),
  83             })
  84
  85         login_data = self._parse_json(
  86             self._search_regex(
  87                 r'atob\(\s*(["\'])(?P<value>(?:(?!\1).)+)\1', webpage,
  88                 'login info', group='value'), None,
  89             transform_source=lambda x: compat_b64decode(x).decode('utf-8')
  90         )['extraParams']
  91
  92         login_data.update({
  93             'client_id': self._CLIENT_ID,
  94             'redirect_uri': self._ORIGIN_URL,
  95             'tenant': 'lacausers',
  96             'connection': 'Username-Password-Authentication',
  97             'username': username,
  98             'password': password,
  99             'sso': 'true',
 100         })
 101
 102         login_state_url = compat_str(urlh.geturl())
 103
 104         try:
 105             login_page = self._download_webpage(
 106                 'https://login.linuxacademy.com/usernamepassword/login', None,
 107                 'Downloading login page', data=json.dumps(login_data).encode(),
 108                 headers={
 109                     'Content-Type': 'application/json',
 110                     'Origin': 'https://login.linuxacademy.com',
 111                     'Referer': login_state_url,
 112                 })
 113         except ExtractorError as e:
 114             if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
 115                 error = self._parse_json(e.cause.read(), None)
 116                 message = error.get('description') or error['code']
 117                 raise ExtractorError(
 118                     '%s said: %s' % (self.IE_NAME, message), expected=True)
 119             raise
 120
 121         callback_page, urlh = self._download_webpage_handle(
 122             'https://login.linuxacademy.com/login/callback', None,
 123             'Downloading callback page',
 124             data=urlencode_postdata(self._hidden_inputs(login_page)),
 125             headers={
 126                 'Content-Type': 'application/x-www-form-urlencoded',
 127                 'Origin': 'https://login.linuxacademy.com',
 128                 'Referer': login_state_url,
 129             })
 130
 131         access_token = self._search_regex(
 132             r'access_token=([^=&]+)', compat_str(urlh.geturl()),
 133             'access token')
 134
 135         self._download_webpage(
 136             'https://linuxacademy.com/cp/login/tokenValidateLogin/token/%s'
 137             % access_token, None, 'Downloading token validation page')
 138
 139     def _real_extract(self, url):
 140         mobj = re.match(self._VALID_URL, url)
 141         chapter_id, lecture_id, course_id = mobj.group('chapter_id', 'lesson_id', 'course_id')
 142         item_id = course_id if course_id else '%s-%s' % (chapter_id, lecture_id)
 143
 144         webpage = self._download_webpage(url, item_id)
 145
 146         # course path
 147         if course_id:
 148             entries = [
 149                 self.url_result(
 150                     urljoin(url, lesson_url), ie=LinuxAcademyIE.ie_key())
 151                 for lesson_url in orderedSet(re.findall(
 152                     r'<a[^>]+\bhref=["\'](/cp/courses/lesson/course/\d+/lesson/\d+/module/\d+)',
 153                     webpage))]
 154             title = unescapeHTML(self._html_search_regex(
 155                 (r'class=["\']course-title["\'][^>]*>(?P<value>[^<]+)',
 156                  r'var\s+title\s*=\s*(["\'])(?P<value>(?:(?!\1).)+)\1'),
 157                 webpage, 'title', default=None, group='value'))
 158             description = unescapeHTML(self._html_search_regex(
 159                 r'var\s+description\s*=\s*(["\'])(?P<value>(?:(?!\1).)+)\1',
 160                 webpage, 'description', default=None, group='value'))
 161             return self.playlist_result(entries, course_id, title, description)
 162
 163         # single video path
 164         info = self._extract_jwplayer_data(
 165             webpage, item_id, require_title=False, m3u8_id='hls',)
 166         title = self._search_regex(
 167             (r'>Lecture\s*:\s*(?P<value>[^<]+)',
 168              r'lessonName\s*=\s*(["\'])(?P<value>(?:(?!\1).)+)\1'), webpage,
 169             'title', group='value')
 170         info.update({
 171             'id': item_id,
 172             'title': title,
 173         })
 174         return info