--- /dev/null
+#!/usr/bin/env python
+
+from __future__ import unicode_literals
+
+# Allow direct execution
+import os
+import sys
+import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from test.helper import FakeYDL
+from youtube_dl.extractor.common import InfoExtractor
+from youtube_dl.extractor import YoutubeIE, get_info_extractor
+
+
+class TestIE(InfoExtractor):
+ pass
+
+
+class TestInfoExtractor(unittest.TestCase):
+ def setUp(self):
+ self.ie = TestIE(FakeYDL())
+
+ def test_ie_key(self):
+ self.assertEqual(get_info_extractor(YoutubeIE.ie_key()), YoutubeIE)
+
+ def test_html_search_regex(self):
+ html = '<p id="foo">Watch this <a href="http://www.youtube.com/watch?v=BaW_jenozKc">video</a></p>'
+ search = lambda re, *args: self.ie._html_search_regex(re, html, *args)
+ self.assertEqual(search(r'<p id="foo">(.+?)</p>', 'foo'), 'Watch this video')
+
+ def test_opengraph(self):
+ ie = self.ie
+ html = '''
+ <meta name="og:title" content='Foo'/>
+ <meta content="Some video's description " name="og:description"/>
+ <meta property='og:image' content='http://domain.com/pic.jpg?key1=val1&key2=val2'/>
+ '''
+ self.assertEqual(ie._og_search_title(html), 'Foo')
+ self.assertEqual(ie._og_search_description(html), 'Some video\'s description ')
+ self.assertEqual(ie._og_search_thumbnail(html), 'http://domain.com/pic.jpg?key1=val1&key2=val2')
+
+if __name__ == '__main__':
+ unittest.main()
Print the message to stderr, it will be prefixed with 'WARNING:'
If stderr is a tty file the 'WARNING:' will be colored
'''
- if self._err_file.isatty() and os.name != 'nt':
- _msg_header = '\033[0;33mWARNING:\033[0m'
+ if self.params.get('logger') is not None:
+ self.params['logger'].warning(message)
else:
- _msg_header = 'WARNING:'
- warning_message = '%s %s' % (_msg_header, message)
- self.to_stderr(warning_message)
+ if self._err_file.isatty() and os.name != 'nt':
+ _msg_header = '\033[0;33mWARNING:\033[0m'
+ else:
+ _msg_header = 'WARNING:'
+ warning_message = '%s %s' % (_msg_header, message)
+ self.to_stderr(warning_message)
def report_error(self, message, tb=None):
'''
return self._extract_liveweb(url, name, lang)
if re.search(self._LIVE_URL, url) is not None:
- raise ExtractorError(u'Arte live streams are not yet supported, sorry')
+ raise ExtractorError('Arte live streams are not yet supported, sorry')
# self.extractLiveStream(url)
# return
+ raise ExtractorError('No video found')
+
def _extract_video(self, url, video_id, lang):
"""Extract from videos.arte.tv"""
ref_xml_url = url.replace('/videos/', '/do_delegate/videos/')
ref_xml_url = ref_xml_url.replace('.html', ',view,asPlayerXml.xml')
- ref_xml_doc = self._download_xml(ref_xml_url, video_id, note=u'Downloading metadata')
+ ref_xml_doc = self._download_xml(
+ ref_xml_url, video_id, note='Downloading metadata')
config_node = find_xpath_attr(ref_xml_doc, './/video', 'lang', lang)
config_xml_url = config_node.attrib['ref']
- config_xml = self._download_webpage(config_xml_url, video_id, note=u'Downloading configuration')
+ config_xml = self._download_webpage(
+ config_xml_url, video_id, note='Downloading configuration')
video_urls = list(re.finditer(r'<url quality="(?P<quality>.*?)">(?P<url>.*?)</url>', config_xml))
def _key(m):
class FacebookIE(InfoExtractor):
- """Information Extractor for Facebook"""
-
_VALID_URL = r'''(?x)
- (?:https?://)?(?:\w+\.)?facebook\.com/
+ https?://(?:\w+\.)?facebook\.com/
(?:[^#?]*\#!/)?
(?:video/video\.php|photo\.php|video/embed)\?(?:.*?)
(?:v|video_id)=(?P<id>[0-9]+)
'id': '120708114770723',
'ext': 'mp4',
'duration': 279,
- 'title': 'PEOPLE ARE AWESOME 2013'
+ 'title': 'PEOPLE ARE AWESOME 2013',
}
}
- def report_login(self):
- """Report attempt to log in."""
- self.to_screen('Logging in')
-
def _login(self):
(useremail, password) = self._get_login_info()
if useremail is None:
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
- if mobj is None:
- raise ExtractorError('Invalid URL: %s' % url)
video_id = mobj.group('id')
url = 'https://www.facebook.com/video/video.php?v=%s' % video_id
video_url = video_data['sd_src']
if not video_url:
raise ExtractorError('Cannot find video URL')
- video_duration = int(video_data['video_duration'])
- thumbnail = video_data['thumbnail_src']
video_title = self._html_search_regex(
r'<h2 class="uiHeaderTitle">([^<]*)</h2>', webpage, 'title')
- info = {
+ return {
'id': video_id,
'title': video_title,
'url': video_url,
- 'ext': 'mp4',
- 'duration': video_duration,
- 'thumbnail': thumbnail,
+ 'duration': int(video_data['video_duration']),
+ 'thumbnail': video_data['thumbnail_src'],
}
- return [info]
-# coding: utf-8
+from __future__ import unicode_literals
+
import re
from .common import InfoExtractor
from ..utils import (
ExtractorError,
+ RegexNotFoundError,
unescapeHTML,
)
+
class JukeboxIE(InfoExtractor):
_VALID_URL = r'^http://www\.jukebox?\..+?\/.+[,](?P<video_id>[a-z0-9\-]+)\.html'
- _IFRAME = r'<iframe .*src="(?P<iframe>[^"]*)".*>'
- _VIDEO_URL = r'"config":{"file":"(?P<video_url>http:[^"]+[.](?P<video_ext>[^.?]+)[?]mdtk=[0-9]+)"'
- _TITLE = r'<h1 class="inline">(?P<title>[^<]+)</h1>.*<span id="infos_article_artist">(?P<artist>[^<]+)</span>'
- _IS_YOUTUBE = r'config":{"file":"(?P<youtube_url>http:[\\][/][\\][/]www[.]youtube[.]com[\\][/]watch[?]v=[^"]+)"'
+ _TEST = {
+ 'url': 'http://www.jukebox.es/kosheen/videoclip,pride,r303r.html',
+ 'md5': '5dc6477e74b1e37042ac5acedd8413e5',
+ 'info_dict': {
+ 'id': 'r303r',
+ 'ext': 'flv',
+ 'title': 'Kosheen-En Vivo Pride',
+ 'uploader': 'Kosheen',
+ },
+ }
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
video_id = mobj.group('video_id')
html = self._download_webpage(url, video_id)
-
- mobj = re.search(self._IFRAME, html)
- if mobj is None:
- raise ExtractorError(u'Cannot extract iframe url')
- iframe_url = unescapeHTML(mobj.group('iframe'))
+ iframe_url = unescapeHTML(self._search_regex(r'<iframe .*src="([^"]*)"', html, 'iframe url'))
iframe_html = self._download_webpage(iframe_url, video_id, 'Downloading iframe')
- mobj = re.search(r'class="jkb_waiting"', iframe_html)
- if mobj is not None:
- raise ExtractorError(u'Video is not available(in your country?)!')
+ if re.search(r'class="jkb_waiting"', iframe_html) is not None:
+ raise ExtractorError('Video is not available(in your country?)!')
self.report_extraction(video_id)
- mobj = re.search(self._VIDEO_URL, iframe_html)
- if mobj is None:
- mobj = re.search(self._IS_YOUTUBE, iframe_html)
- if mobj is None:
- raise ExtractorError(u'Cannot extract video url')
- youtube_url = unescapeHTML(mobj.group('youtube_url')).replace('\/','/')
- self.to_screen(u'Youtube video detected')
- return self.url_result(youtube_url,ie='Youtube')
- video_url = unescapeHTML(mobj.group('video_url')).replace('\/','/')
- video_ext = unescapeHTML(mobj.group('video_ext'))
-
- mobj = re.search(self._TITLE, html)
- if mobj is None:
- raise ExtractorError(u'Cannot extract title')
- title = unescapeHTML(mobj.group('title'))
- artist = unescapeHTML(mobj.group('artist'))
-
- return [{'id': video_id,
- 'url': video_url,
- 'title': artist + '-' + title,
- 'ext': video_ext
- }]
+ try:
+ video_url = self._search_regex(r'"config":{"file":"(?P<video_url>http:[^"]+\?mdtk=[0-9]+)"',
+ iframe_html, 'video url')
+ video_url = unescapeHTML(video_url).replace('\/', '/')
+ except RegexNotFoundError:
+ youtube_url = self._search_regex(
+ r'config":{"file":"(http:\\/\\/www\.youtube\.com\\/watch\?v=[^"]+)"',
+ iframe_html, 'youtube url')
+ youtube_url = unescapeHTML(youtube_url).replace('\/', '/')
+ self.to_screen('Youtube video detected')
+ return self.url_result(youtube_url, ie='Youtube')
+
+ title = self._html_search_regex(r'<h1 class="inline">([^<]+)</h1>',
+ html, 'title')
+ artist = self._html_search_regex(r'<span id="infos_article_artist">([^<]+)</span>',
+ html, 'artist')
+
+ return {
+ 'id': video_id,
+ 'url': video_url,
+ 'title': artist + '-' + title,
+ 'uploader': artist,
+ }
+from __future__ import unicode_literals
+
import binascii
import base64
import hashlib
)
-
class MyVideoIE(InfoExtractor):
- """Information Extractor for myvideo.de."""
-
- _VALID_URL = r'(?:http://)?(?:www\.)?myvideo\.de/(?:[^/]+/)?watch/([0-9]+)/([^?/]+).*'
- IE_NAME = u'myvideo'
+ _VALID_URL = r'http://(?:www\.)?myvideo\.de/(?:[^/]+/)?watch/(?P<id>[0-9]+)/[^?/]+.*'
+ IE_NAME = 'myvideo'
_TEST = {
- u'url': u'http://www.myvideo.de/watch/8229274/bowling_fail_or_win',
- u'file': u'8229274.flv',
- u'md5': u'2d2753e8130479ba2cb7e0a37002053e',
- u'info_dict': {
- u"title": u"bowling-fail-or-win"
+ 'url': 'http://www.myvideo.de/watch/8229274/bowling_fail_or_win',
+ 'md5': '2d2753e8130479ba2cb7e0a37002053e',
+ 'info_dict': {
+ 'id': '8229274',
+ 'ext': 'flv',
+ 'title': 'bowling-fail-or-win',
}
}
def _real_extract(self,url):
mobj = re.match(self._VALID_URL, url)
- if mobj is None:
- raise ExtractorError(u'invalid URL: %s' % url)
-
- video_id = mobj.group(1)
+ video_id = mobj.group('id')
GK = (
b'WXpnME1EZGhNRGhpTTJNM01XVmhOREU0WldNNVpHTTJOakpt'
video_url = mobj.group(1) + '.flv'
video_title = self._html_search_regex('<title>([^<]+)</title>',
- webpage, u'title')
-
- video_ext = self._search_regex('[.](.+?)$', video_url, u'extension')
+ webpage, 'title')
- return [{
- 'id': video_id,
- 'url': video_url,
- 'uploader': None,
- 'upload_date': None,
- 'title': video_title,
- 'ext': video_ext,
- }]
+ return {
+ 'id': video_id,
+ 'url': video_url,
+ 'title': video_title,
+ }
mobj = re.search(r'data-video-service="/service/data/video/%s/config' % video_id, webpage)
if mobj is not None:
request = compat_urllib_request.Request('http://www.myvideo.de/service/data/video/%s/config' % video_id, '')
response = self._download_webpage(request, video_id,
- u'Downloading video info')
+ 'Downloading video info')
info = json.loads(base64.b64decode(response).decode('utf-8'))
- return {'id': video_id,
- 'title': info['title'],
- 'url': info['streaming_url'].replace('rtmpe', 'rtmpt'),
- 'play_path': info['filename'],
- 'ext': 'flv',
- 'thumbnail': info['thumbnail'][0]['url'],
- }
+ return {
+ 'id': video_id,
+ 'title': info['title'],
+ 'url': info['streaming_url'].replace('rtmpe', 'rtmpt'),
+ 'play_path': info['filename'],
+ 'ext': 'flv',
+ 'thumbnail': info['thumbnail'][0]['url'],
+ }
# try encxml
mobj = re.search('var flashvars={(.+?)}', webpage)
if mobj is None:
- raise ExtractorError(u'Unable to extract video')
+ raise ExtractorError('Unable to extract video')
params = {}
encxml = ''
params['domain'] = 'www.myvideo.de'
xmldata_url = '%s?%s' % (encxml, compat_urllib_parse.urlencode(params))
if 'flash_playertype=MTV' in xmldata_url:
- self._downloader.report_warning(u'avoiding MTV player')
+ self._downloader.report_warning('avoiding MTV player')
xmldata_url = (
'http://www.myvideo.de/dynamic/get_player_video_xml.php'
'?flash_playertype=D&ID=%s&_countlimit=4&autorun=yes'
video_url = compat_urllib_parse.unquote(mobj.group(1))
if 'myvideo2flash' in video_url:
self.report_warning(
- u'Rewriting URL to use unencrypted rtmp:// ...',
+ 'Rewriting URL to use unencrypted rtmp:// ...',
video_id)
video_url = video_url.replace('rtmpe://', 'rtmp://')
# extract non rtmp videos
mobj = re.search('path=\'(http.*?)\' source=\'(.*?)\'', dec_data)
if mobj is None:
- raise ExtractorError(u'unable to extract url')
+ raise ExtractorError('unable to extract url')
video_url = compat_urllib_parse.unquote(mobj.group(1)) + compat_urllib_parse.unquote(mobj.group(2))
- video_file = self._search_regex('source=\'(.*?)\'', dec_data, u'video file')
+ video_file = self._search_regex('source=\'(.*?)\'', dec_data, 'video file')
video_file = compat_urllib_parse.unquote(video_file)
if not video_file.endswith('f4m'):
ppath, prefix = video_file.split('.')
video_playpath = '%s:%s' % (prefix, ppath)
- video_hls_playlist = ''
else:
video_playpath = ''
- video_hls_playlist = (
- video_file
- ).replace('.f4m', '.m3u8')
- video_swfobj = self._search_regex('swfobject.embedSWF\(\'(.+?)\'', webpage, u'swfobj')
+ video_swfobj = self._search_regex('swfobject.embedSWF\(\'(.+?)\'', webpage, 'swfobj')
video_swfobj = compat_urllib_parse.unquote(video_swfobj)
video_title = self._html_search_regex("<h1(?: class='globalHd')?>(.*?)</h1>",
- webpage, u'title')
-
- return [{
- 'id': video_id,
- 'url': video_url,
- 'tc_url': video_url,
- 'uploader': None,
- 'upload_date': None,
- 'title': video_title,
- 'ext': u'flv',
- 'play_path': video_playpath,
- 'video_file': video_file,
- 'video_hls_playlist': video_hls_playlist,
- 'player_url': video_swfobj,
- }]
+ webpage, 'title')
+
+ return {
+ 'id': video_id,
+ 'url': video_url,
+ 'tc_url': video_url,
+ 'title': video_title,
+ 'ext': 'flv',
+ 'play_path': video_playpath,
+ 'player_url': video_swfobj,
+ }
+from __future__ import unicode_literals
+
import datetime
import json
import re
from .common import InfoExtractor
-from ..utils import (
- ExtractorError,
-)
class PhotobucketIE(InfoExtractor):
- """Information extractor for photobucket.com."""
-
- # TODO: the original _VALID_URL was:
- # r'(?:http://)?(?:[a-z0-9]+\.)?photobucket\.com/.*[\?\&]current=(.*\.flv)'
- # Check if it's necessary to keep the old extracion process
- _VALID_URL = r'(?:http://)?(?:[a-z0-9]+\.)?photobucket\.com/.*(([\?\&]current=)|_)(?P<id>.*)\.(?P<ext>(flv)|(mp4))'
- IE_NAME = u'photobucket'
+ _VALID_URL = r'http://(?:[a-z0-9]+\.)?photobucket\.com/.*(([\?\&]current=)|_)(?P<id>.*)\.(?P<ext>(flv)|(mp4))'
_TEST = {
u'url': u'http://media.photobucket.com/user/rachaneronas/media/TiredofLinkBuildingTryBacklinkMyDomaincom_zpsc0c3b9fa.mp4.html?filters[term]=search&filters[primary]=videos&filters[secondary]=images&sort=1&o=0',
u'file': u'zpsc0c3b9fa.mp4',
u'md5': u'7dabfb92b0a31f6c16cebc0f8e60ff99',
u'info_dict': {
- u"upload_date": u"20130504",
- u"uploader": u"rachaneronas",
- u"title": u"Tired of Link Building? Try BacklinkMyDomain.com!"
+ 'upload_date': '20130504',
+ 'uploader': 'rachaneronas',
+ 'title': 'Tired of Link Building? Try BacklinkMyDomain.com!',
}
}
def _real_extract(self, url):
- # Extract id from URL
mobj = re.match(self._VALID_URL, url)
- if mobj is None:
- raise ExtractorError(u'Invalid URL: %s' % url)
-
video_id = mobj.group('id')
-
video_extension = mobj.group('ext')
- # Retrieve video webpage to extract further information
webpage = self._download_webpage(url, video_id)
# Extract URL, uploader, and title from webpage
self.report_extraction(video_id)
- # We try first by looking the javascript code:
- mobj = re.search(r'Pb\.Data\.Shared\.put\(Pb\.Data\.Shared\.MEDIA, (?P<json>.*?)\);', webpage)
- if mobj is not None:
- info = json.loads(mobj.group('json'))
- return [{
- 'id': video_id,
- 'url': info[u'downloadUrl'],
- 'uploader': info[u'username'],
- 'upload_date': datetime.date.fromtimestamp(info[u'creationDate']).strftime('%Y%m%d'),
- 'title': info[u'title'],
- 'ext': video_extension,
- 'thumbnail': info[u'thumbUrl'],
- }]
-
- # We try looking in other parts of the webpage
- video_url = self._search_regex(r'<link rel="video_src" href=".*\?file=([^"]+)" />',
- webpage, u'video URL')
-
- mobj = re.search(r'<title>(.*) video by (.*) - Photobucket</title>', webpage)
- if mobj is None:
- raise ExtractorError(u'Unable to extract title')
- video_title = mobj.group(1).decode('utf-8')
- video_uploader = mobj.group(2).decode('utf-8')
-
- return [{
- 'id': video_id.decode('utf-8'),
- 'url': video_url.decode('utf-8'),
- 'uploader': video_uploader,
- 'upload_date': None,
- 'title': video_title,
- 'ext': video_extension.decode('utf-8'),
- }]
+ info_json = self._search_regex(r'Pb\.Data\.Shared\.put\(Pb\.Data\.Shared\.MEDIA, (.*?)\);',
+ webpage, 'info json')
+ info = json.loads(info_json)
+ return {
+ 'id': video_id,
+ 'url': info[u'downloadUrl'],
+ 'uploader': info[u'username'],
+ 'upload_date': datetime.date.fromtimestamp(info[u'creationDate']).strftime('%Y%m%d'),
+ 'title': info[u'title'],
+ 'ext': video_extension,
+ 'thumbnail': info[u'thumbUrl'],
+ }
video_urls = list(map(compat_urllib_parse.unquote , re.findall(r'"quality_[0-9]{3}p":"([^"]+)', webpage)))
if webpage.find('"encrypted":true') != -1:
- password = self._html_search_regex(r'"video_title":"([^"]+)', webpage, 'password').replace('+', ' ')
+ password = compat_urllib_parse.unquote_plus(self._html_search_regex(r'"video_title":"([^"]+)', webpage, 'password'))
video_urls = list(map(lambda s: aes_decrypt_text(s, password, 32).decode('utf-8'), video_urls))
formats = []
+# encoding: utf-8
+from __future__ import unicode_literals
+
import re
from .common import InfoExtractor
from ..utils import (
compat_urllib_parse,
-
- ExtractorError,
)
class XNXXIE(InfoExtractor):
- _VALID_URL = r'^(?:https?://)?(?:video|www)\.xnxx\.com/video([0-9]+)/(.*)'
- VIDEO_URL_RE = r'flv_url=(.*?)&'
- VIDEO_TITLE_RE = r'<title>(.*?)\s+-\s+XNXX.COM'
- VIDEO_THUMB_RE = r'url_bigthumb=(.*?)&'
+ _VALID_URL = r'^https?://(?:video|www)\.xnxx\.com/video(?P<id>[0-9]+)/(.*)'
_TEST = {
- u'url': u'http://video.xnxx.com/video1135332/lida_naked_funny_actress_5_',
- u'file': u'1135332.flv',
- u'md5': u'0831677e2b4761795f68d417e0b7b445',
- u'info_dict': {
- u"title": u"lida \u00bb Naked Funny Actress (5)",
- u"age_limit": 18,
+ 'url': 'http://video.xnxx.com/video1135332/lida_naked_funny_actress_5_',
+ 'md5': '0831677e2b4761795f68d417e0b7b445',
+ 'info_dict': {
+ 'id': '1135332',
+ 'ext': 'flv',
+ 'title': 'lida ยป Naked Funny Actress (5)',
+ 'age_limit': 18,
}
}
def _real_extract(self, url):
mobj = re.match(self._VALID_URL, url)
- if mobj is None:
- raise ExtractorError(u'Invalid URL: %s' % url)
- video_id = mobj.group(1)
+ video_id = mobj.group('id')
# Get webpage content
webpage = self._download_webpage(url, video_id)
- video_url = self._search_regex(self.VIDEO_URL_RE,
- webpage, u'video URL')
+ video_url = self._search_regex(r'flv_url=(.*?)&',
+ webpage, 'video URL')
video_url = compat_urllib_parse.unquote(video_url)
- video_title = self._html_search_regex(self.VIDEO_TITLE_RE,
- webpage, u'title')
+ video_title = self._html_search_regex(r'<title>(.*?)\s+-\s+XNXX.COM',
+ webpage, 'title')
- video_thumbnail = self._search_regex(self.VIDEO_THUMB_RE,
- webpage, u'thumbnail', fatal=False)
+ video_thumbnail = self._search_regex(r'url_bigthumb=(.*?)&',
+ webpage, 'thumbnail', fatal=False)
- return [{
+ return {
'id': video_id,
'url': video_url,
- 'uploader': None,
- 'upload_date': None,
'title': video_title,
'ext': 'flv',
'thumbnail': video_thumbnail,
- 'description': None,
'age_limit': 18,
- }]
+ }
projects / youtube-dl.git / commitdiff