2 from __future__ import unicode_literals
11 from .common import InfoExtractor
12 from ..compat import (
28 def cookie_to_dict(cookie):
31 'value': cookie.value,
33 if cookie.port_specified:
34 cookie_dict['port'] = cookie.port
35 if cookie.domain_specified:
36 cookie_dict['domain'] = cookie.domain
37 if cookie.path_specified:
38 cookie_dict['path'] = cookie.path
39 if cookie.expires is not None:
40 cookie_dict['expires'] = cookie.expires
41 if cookie.secure is not None:
42 cookie_dict['secure'] = cookie.secure
43 if cookie.discard is not None:
44 cookie_dict['discard'] = cookie.discard
46 if (cookie.has_nonstandard_attr('httpOnly')
47 or cookie.has_nonstandard_attr('httponly')
48 or cookie.has_nonstandard_attr('HttpOnly')):
49 cookie_dict['httponly'] = True
55 def cookie_jar_to_list(cookie_jar):
56 return [cookie_to_dict(cookie) for cookie in cookie_jar]
59 class PhantomJSwrapper(object):
60 """PhantomJS wrapper class
62 This class is experimental.
66 phantom.onError = function(msg, trace) {{
67 var msgStack = ['PHANTOM ERROR: ' + msg];
68 if(trace && trace.length) {{
69 msgStack.push('TRACE:');
70 trace.forEach(function(t) {{
71 msgStack.push(' -> ' + (t.file || t.sourceURL) + ': ' + t.line
72 + (t.function ? ' (in function ' + t.function +')' : ''));
75 console.error(msgStack.join('\n'));
78 var page = require('webpage').create();
79 var fs = require('fs');
80 var read = {{ mode: 'r', charset: 'utf-8' }};
81 var write = {{ mode: 'w', charset: 'utf-8' }};
82 JSON.parse(fs.read("{cookies}", read)).forEach(function(x) {{
85 page.settings.resourceTimeout = {timeout};
86 page.settings.userAgent = "{ua}";
87 page.onLoadStarted = function() {{
88 page.evaluate(function() {{
89 delete window._phantom;
90 delete window.callPhantom;
93 var saveAndExit = function() {{
94 fs.write("{html}", page.content, write);
95 fs.write("{cookies}", JSON.stringify(phantom.cookies), write);
98 page.onLoadFinished = function(status) {{
99 if(page.url === "") {{
100 page.setContent(fs.read("{html}", read), "{url}");
109 _TMP_FILE_NAMES = ['script', 'html', 'cookies']
113 return get_exe_version('phantomjs', version_re=r'([0-9.]+)')
115 def __init__(self, extractor, required_version=None, timeout=10000):
118 self.exe = check_executable('phantomjs', ['-v'])
120 raise ExtractorError('PhantomJS executable not found in PATH, '
121 'download it from http://phantomjs.org',
124 self.extractor = extractor
127 version = self._version()
128 if is_outdated_version(version, required_version):
129 self.extractor._downloader.report_warning(
130 'Your copy of PhantomJS is outdated, update it to version '
131 '%s or newer if you encounter any errors.' % required_version)
136 for name in self._TMP_FILE_NAMES:
137 tmp = tempfile.NamedTemporaryFile(delete=False)
139 self._TMP_FILES[name] = tmp
142 for name in self._TMP_FILE_NAMES:
144 os.remove(self._TMP_FILES[name].name)
145 except (IOError, OSError, KeyError):
148 def _save_cookies(self, url):
149 cookies = cookie_jar_to_list(self.extractor._downloader.cookiejar)
150 for cookie in cookies:
151 if 'path' not in cookie:
153 if 'domain' not in cookie:
154 cookie['domain'] = compat_urlparse.urlparse(url).netloc
155 with open(self._TMP_FILES['cookies'].name, 'wb') as f:
156 f.write(json.dumps(cookies).encode('utf-8'))
158 def _load_cookies(self):
159 with open(self._TMP_FILES['cookies'].name, 'rb') as f:
160 cookies = json.loads(f.read().decode('utf-8'))
161 for cookie in cookies:
162 if cookie['httponly'] is True:
163 cookie['rest'] = {'httpOnly': None}
164 if 'expiry' in cookie:
165 cookie['expire_time'] = cookie['expiry']
166 self.extractor._set_cookie(**compat_kwargs(cookie))
168 def get(self, url, html=None, video_id=None, note=None, note2='Executing JS on webpage', headers={}, jscode='saveAndExit();'):
170 Downloads webpage (if needed) and executes JS
174 html: optional, html code of website
176 note: optional, displayed when downloading webpage
177 note2: optional, displayed when executing JS
178 headers: custom http headers
179 jscode: code to be executed when page is loaded
182 * downloaded website (after JS execution)
183 * anything you print with `console.log` (but not inside `page.execute`!)
185 In most cases you don't need to add any `jscode`.
186 It is executed in `page.onLoadFinished`.
187 `saveAndExit();` is mandatory, use it instead of `phantom.exit()`
188 It is possible to wait for some element on the webpage, for example:
189 var check = function() {
190 var elementFound = page.evaluate(function() {
191 return document.querySelector('#b.done') !== null;
196 window.setTimeout(check, 500);
199 page.evaluate(function(){
200 document.querySelector('#a').click();
204 if 'saveAndExit();' not in jscode:
205 raise ExtractorError('`saveAndExit();` not found in `jscode`')
207 html = self.extractor._download_webpage(url, video_id, note=note, headers=headers)
208 with open(self._TMP_FILES['html'].name, 'wb') as f:
209 f.write(html.encode('utf-8'))
211 self._save_cookies(url)
213 replaces = self.options
214 replaces['url'] = url
215 user_agent = headers.get('User-Agent') or std_headers['User-Agent']
216 replaces['ua'] = user_agent.replace('"', '\\"')
217 replaces['jscode'] = jscode
219 for x in self._TMP_FILE_NAMES:
220 replaces[x] = self._TMP_FILES[x].name.replace('\\', '\\\\').replace('"', '\\"')
222 with open(self._TMP_FILES['script'].name, 'wb') as f:
223 f.write(self._TEMPLATE.format(**replaces).encode('utf-8'))
226 self.extractor.to_screen('%s' % (note2,))
228 self.extractor.to_screen('%s: %s' % (video_id, note2))
230 p = subprocess.Popen([
231 self.exe, '--ssl-protocol=any',
232 self._TMP_FILES['script'].name
233 ], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
234 out, err = p.communicate()
235 if p.returncode != 0:
236 raise ExtractorError(
237 'Executing JS failed\n:' + encodeArgument(err))
238 with open(self._TMP_FILES['html'].name, 'rb') as f:
239 html = f.read().decode('utf-8')
243 return (html, encodeArgument(out))
246 class OpenloadIE(InfoExtractor):
247 _DOMAINS = r'(?:openload\.(?:co|io|link|pw)|oload\.(?:tv|stream|site|xyz|win|download|cloud|cc|icu|fun|club|info|press|pw|life|live|space|services|website)|oladblock\.(?:services|xyz|me)|openloed\.co)'
248 _VALID_URL = r'''(?x)
255 (?P<id>[a-zA-Z0-9-_]+)
257 _EMBED_WORD = 'embed'
259 _REDIR_WORD = 'stream'
260 _URL_IDS = ('streamurl', 'streamuri', 'streamurj')
262 'url': 'https://openload.co/f/kUEfGclsU9o',
263 'md5': 'bf1c059b004ebc7a256f89408e65c36e',
267 'title': 'skyrim_no-audio_1080.mp4',
268 'thumbnail': r're:^https?://.*\.jpg$',
271 'url': 'https://openload.co/embed/rjC09fkPLYs',
275 'title': 'movie.mp4',
276 'thumbnail': r're:^https?://.*\.jpg$',
284 'skip_download': True, # test subtitles only
287 'url': 'https://openload.co/embed/kUEfGclsU9o/skyrim_no-audio_1080.mp4',
288 'only_matching': True,
290 'url': 'https://openload.io/f/ZAn6oz-VZGE/',
291 'only_matching': True,
293 'url': 'https://openload.co/f/_-ztPaZtMhM/',
294 'only_matching': True,
296 # unavailable via https://openload.co/f/Sxz5sADo82g/, different layout
298 'url': 'https://openload.co/embed/Sxz5sADo82g/',
299 'only_matching': True,
301 # unavailable via https://openload.co/embed/e-Ixz9ZR5L0/ but available
302 # via https://openload.co/f/e-Ixz9ZR5L0/
303 'url': 'https://openload.co/f/e-Ixz9ZR5L0/',
304 'only_matching': True,
306 'url': 'https://oload.tv/embed/KnG-kKZdcfY/',
307 'only_matching': True,
309 'url': 'http://www.openload.link/f/KnG-kKZdcfY',
310 'only_matching': True,
312 'url': 'https://oload.stream/f/KnG-kKZdcfY',
313 'only_matching': True,
315 'url': 'https://oload.xyz/f/WwRBpzW8Wtk',
316 'only_matching': True,
318 'url': 'https://oload.win/f/kUEfGclsU9o',
319 'only_matching': True,
321 'url': 'https://oload.download/f/kUEfGclsU9o',
322 'only_matching': True,
324 'url': 'https://oload.cloud/f/4ZDnBXRWiB8',
325 'only_matching': True,
327 # Its title has not got its extension but url has it
328 'url': 'https://oload.download/f/N4Otkw39VCw/Tomb.Raider.2018.HDRip.XviD.AC3-EVO.avi.mp4',
329 'only_matching': True,
331 'url': 'https://oload.cc/embed/5NEAbI2BDSk',
332 'only_matching': True,
334 'url': 'https://oload.icu/f/-_i4y_F_Hs8',
335 'only_matching': True,
337 'url': 'https://oload.fun/f/gb6G1H4sHXY',
338 'only_matching': True,
340 'url': 'https://oload.club/f/Nr1L-aZ2dbQ',
341 'only_matching': True,
343 'url': 'https://oload.info/f/5NEAbI2BDSk',
344 'only_matching': True,
346 'url': 'https://openload.pw/f/WyKgK8s94N0',
347 'only_matching': True,
349 'url': 'https://oload.pw/f/WyKgK8s94N0',
350 'only_matching': True,
352 'url': 'https://oload.live/f/-Z58UZ-GR4M',
353 'only_matching': True,
355 'url': 'https://oload.space/f/IY4eZSst3u8/',
356 'only_matching': True,
358 'url': 'https://oload.services/embed/bs1NWj1dCag/',
359 'only_matching': True,
361 'url': 'https://oload.press/embed/drTBl1aOTvk/',
362 'only_matching': True,
364 'url': 'https://oload.website/embed/drTBl1aOTvk/',
365 'only_matching': True,
367 'url': 'https://oload.life/embed/oOzZjNPw9Dc/',
368 'only_matching': True,
370 'url': 'https://oladblock.services/f/b8NWEgkqNLI/',
371 'only_matching': True,
373 'url': 'https://oladblock.xyz/f/b8NWEgkqNLI/',
374 'only_matching': True,
376 'url': 'https://oladblock.me/f/b8NWEgkqNLI/',
377 'only_matching': True,
379 'url': 'https://openloed.co/f/b8NWEgkqNLI/',
380 'only_matching': True,
383 _USER_AGENT_TPL = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/%s Safari/537.36'
1964 def _extract_urls(cls, webpage):
1966 r'<iframe[^>]+src=["\']((?:https?://)?%s/%s/[a-zA-Z0-9-_]+)'
1967 % (cls._DOMAINS, cls._EMBED_WORD), webpage)
1969 def _extract_decrypted_page(self, page_url, webpage, video_id, headers):
1970 phantom = PhantomJSwrapper(self, required_version='2.0')
1971 webpage, _ = phantom.get(page_url, html=webpage, video_id=video_id, headers=headers)
1974 def _real_extract(self, url):
1975 mobj = re.match(self._VALID_URL, url)
1976 host = mobj.group('host')
1977 video_id = mobj.group('id')
1979 url_pattern = 'https://%s/%%s/%s/' % (host, video_id)
1981 'User-Agent': self._USER_AGENT_TPL % random.choice(self._CHROME_VERSIONS),
1984 for path in (self._EMBED_WORD, self._STREAM_WORD):
1985 page_url = url_pattern % path
1986 last = path == self._STREAM_WORD
1987 webpage = self._download_webpage(
1988 page_url, video_id, 'Downloading %s webpage' % path,
1989 headers=headers, fatal=last)
1992 if 'File not found' in webpage or 'deleted by the owner' in webpage:
1995 raise ExtractorError('File not found', expected=True, video_id=video_id)
1998 webpage = self._extract_decrypted_page(page_url, webpage, video_id, headers)
1999 for element_id in self._URL_IDS:
2000 decoded_id = get_element_by_id(element_id, webpage)
2004 decoded_id = self._search_regex(
2005 (r'>\s*([\w-]+~\d{10,}~\d+\.\d+\.0\.0~[\w-]+)\s*<',
2006 r'>\s*([\w~-]+~\d+\.\d+\.\d+\.\d+~[\w~-]+)',
2007 r'>\s*([\w-]+~\d{10,}~(?:[a-f\d]+:){2}:~[\w-]+)\s*<',
2008 r'>\s*([\w~-]+~[a-f0-9:]+~[\w~-]+)\s*<',
2009 r'>\s*([\w~-]+~[a-f0-9:]+~[\w~-]+)'), webpage,
2011 video_url = 'https://%s/%s/%s?mime=true' % (host, self._REDIR_WORD, decoded_id)
2013 title = self._og_search_title(webpage, default=None) or self._search_regex(
2014 r'<span[^>]+class=["\']title["\'][^>]*>([^<]+)', webpage,
2015 'title', default=None) or self._html_search_meta(
2016 'description', webpage, 'title', fatal=True)
2018 entries = self._parse_html5_media_entries(page_url, webpage, video_id)
2019 entry = entries[0] if entries else {}
2020 subtitles = entry.get('subtitles')
2025 'thumbnail': entry.get('thumbnail') or self._og_search_thumbnail(webpage, default=None),
2027 'ext': determine_ext(title, None) or determine_ext(url, 'mp4'),
2028 'subtitles': subtitles,
2029 'http_headers': headers,
2033 class VerystreamIE(OpenloadIE):
2034 IE_NAME = 'verystream'
2036 _DOMAINS = r'(?:verystream\.com)'
2037 _VALID_URL = r'''(?x)
2044 (?P<id>[a-zA-Z0-9-_]+)
2047 _STREAM_WORD = 'stream'
2048 _REDIR_WORD = 'gettoken'
2049 _URL_IDS = ('videolink', )
2051 'url': 'https://verystream.com/stream/c1GWQ9ngBBx/',
2052 'md5': 'd3e8c5628ccb9970b65fd65269886795',
2054 'id': 'c1GWQ9ngBBx',
2056 'title': 'Big Buck Bunny.mp4',
2057 'thumbnail': r're:^https?://.*\.jpg$',
2060 'url': 'https://verystream.com/e/c1GWQ9ngBBx/',
2061 'only_matching': True,
2064 def _extract_decrypted_page(self, page_url, webpage, video_id, headers):
2065 return webpage # for Verystream, the webpage is already decrypted