X-Git-Url: http://git.cielonegro.org/gitweb.cgi?a=blobdiff_plain;f=youtube_dl%2FInfoExtractors.py;h=a220de80a4b62610efc483f50e45ffed5eb47279;hb=ae608b8076497d70e2a95e5e939c1fb31e2dde53;hp=742b036d3dd6fd246ffc450671262a8deb76458c;hpb=3a468f2d8b0261d4f45a7c5837f54edc33acdd8f;p=youtube-dl.git diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py index 742b036d3..a220de80a 100755 --- a/youtube_dl/InfoExtractors.py +++ b/youtube_dl/InfoExtractors.py @@ -216,6 +216,10 @@ class YoutubeIE(InfoExtractor): """Report attempt to download video info webpage.""" self._downloader.to_screen(u'[youtube] %s: Downloading video subtitles' % video_id) + def report_video_subtitles_request(self, video_id, lang): + """Report attempt to download video info webpage.""" + self._downloader.to_screen(u'[youtube] %s: Downloading video subtitles for lang: %s' % (video_id,lang)) + def report_information_extraction(self, video_id): """Report attempt to extract video information.""" self._downloader.to_screen(u'[youtube] %s: Extracting video information' % video_id) @@ -228,25 +232,7 @@ class YoutubeIE(InfoExtractor): """Indicate the download will use the RTMP protocol.""" self._downloader.to_screen(u'[youtube] RTMP download detected') - def _closed_captions_xml_to_srt(self, xml_string): - srt = '' - texts = re.findall(r'([^<]+)', xml_string, re.MULTILINE) - # TODO parse xml instead of regex - for n, (start, dur_tag, dur, caption) in enumerate(texts): - if not dur: dur = '4' - start = float(start) - end = start + float(dur) - start = "%02i:%02i:%02i,%03i" %(start/(60*60), start/60%60, start%60, start%1*1000) - end = "%02i:%02i:%02i,%03i" %(end/(60*60), end/60%60, end%60, end%1*1000) - caption = unescapeHTML(caption) - caption = unescapeHTML(caption) # double cycle, intentional - srt += str(n+1) + '\n' - srt += start + ' --> ' + end + '\n' - srt += caption + '\n\n' - return srt - - def _extract_subtitles(self, video_id): - self.report_video_subtitles_download(video_id) + def _get_available_subtitles(self, video_id): request = compat_urllib_request.Request('http://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id) try: srt_list = compat_urllib_request.urlopen(request).read().decode('utf-8') @@ -256,27 +242,49 @@ class YoutubeIE(InfoExtractor): srt_lang_list = dict((l[1], l[0]) for l in srt_lang_list) if not srt_lang_list: return (u'WARNING: video has no closed captions', None) - if self._downloader.params.get('subtitleslang', False): - srt_lang = self._downloader.params.get('subtitleslang') - elif 'en' in srt_lang_list: - srt_lang = 'en' - else: - srt_lang = list(srt_lang_list.keys())[0] - if not srt_lang in srt_lang_list: - return (u'WARNING: no closed captions found in the specified language', None) + return srt_lang_list + + def _request_subtitle(self, str_lang, str_name, video_id, format = 'srt'): + self.report_video_subtitles_request(video_id, str_lang) params = compat_urllib_parse.urlencode({ - 'lang': srt_lang, - 'name': srt_lang_list[srt_lang].encode('utf-8'), + 'lang': str_lang, + 'name': str_name, 'v': video_id, + 'fmt': format, }) url = 'http://www.youtube.com/api/timedtext?' + params try: - srt_xml = compat_urllib_request.urlopen(url).read().decode('utf-8') + srt = compat_urllib_request.urlopen(url).read().decode('utf-8') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: return (u'WARNING: unable to download video subtitles: %s' % compat_str(err), None) - if not srt_xml: + if not srt: return (u'WARNING: Did not fetch video subtitles', None) - return (None, self._closed_captions_xml_to_srt(srt_xml)) + return (None, str_lang, srt) + + def _extract_subtitle(self, video_id): + self.report_video_subtitles_download(video_id) + srt_lang_list = self._get_available_subtitles(video_id) + + if self._downloader.params.get('subtitleslang', False): + srt_lang = self._downloader.params.get('subtitleslang') + elif 'en' in srt_lang_list: + srt_lang = 'en' + else: + srt_lang = list(srt_lang_list.keys())[0] + if not srt_lang in srt_lang_list: + return (u'WARNING: no closed captions found in the specified language "%s"' % srt_lang, None) + + sub = self._request_subtitle(srt_lang, srt_lang_list[srt_lang].encode('utf-8'), video_id) + return [sub] + + def _extract_all_subtitles(self, video_id): + self.report_video_subtitles_download(video_id) + srt_lang_list = self._get_available_subtitles(video_id) + subs = [] + for srt_lang in srt_lang_list: + sub = self._request_subtitle(srt_lang, srt_lang_list[srt_lang].encode('utf-8'), video_id) + subs.append(sub) + return subs def _print_formats(self, formats): print('Available formats:') @@ -499,10 +507,20 @@ class YoutubeIE(InfoExtractor): # closed captions video_subtitles = None + if self._downloader.params.get('writesubtitles', False): - (srt_error, video_subtitles) = self._extract_subtitles(video_id) - if srt_error: - self._downloader.trouble(srt_error) + video_subtitles = self._extract_subtitle(video_id) + if video_subtitles: + (srt_error, srt_lang, srt) = video_subtitles[0] + if srt_error: + self._downloader.trouble(srt_error) + + if self._downloader.params.get('allsubtitles', False): + video_subtitles = self._extract_all_subtitles(video_id) + for video_subtitle in video_subtitles: + (srt_error, srt_lang, srt) = video_subtitle + if srt_error: + self._downloader.trouble(srt_error) if 'length_seconds' not in video_info: self._downloader.trouble(u'WARNING: unable to extract video duration') @@ -718,6 +736,7 @@ class DailymotionIE(InfoExtractor): _VALID_URL = r'(?i)(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/video/([^/]+)' IE_NAME = u'dailymotion' + _WORKING = False def __init__(self, downloader=None): InfoExtractor.__init__(self, downloader) @@ -1329,7 +1348,7 @@ class GenericIE(InfoExtractor): opener = compat_urllib_request.OpenerDirector() for handler in [compat_urllib_request.HTTPHandler, compat_urllib_request.HTTPDefaultErrorHandler, HTTPMethodFallback, HEADRedirectHandler, - compat_urllib_error.HTTPErrorProcessor, compat_urllib_request.HTTPSHandler]: + compat_urllib_request.HTTPErrorProcessor, compat_urllib_request.HTTPSHandler]: opener.add_handler(handler()) response = opener.open(HeadRequest(url)) @@ -1365,6 +1384,9 @@ class GenericIE(InfoExtractor): if mobj is None: # Broaden the search a little bit mobj = re.search(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage) + if mobj is None: + # Broaden the search a little bit: JWPlayer JS loader + mobj = re.search(r'[^A-Za-z0-9]?file:\s*["\'](http[^\'"&]*)', webpage) if mobj is None: self._downloader.trouble(u'ERROR: Invalid URL: %s' % url) return @@ -2097,6 +2119,10 @@ class FacebookIE(InfoExtractor): params_raw = compat_urllib_parse.unquote(data['params']) params = json.loads(params_raw) video_url = params['hd_src'] + if not video_url: + video_url = params['sd_src'] + if not video_url: + raise ExtractorError(u'Cannot find video URL') video_duration = int(params['video_duration']) m = re.search('

([^<]+)

', webpage) @@ -2232,7 +2258,7 @@ class MyVideoIE(InfoExtractor): webpage = self._download_webpage(webpage_url, video_id) self.report_extraction(video_id) - mobj = re.search(r'', + mobj = re.search(r'', webpage) if mobj is None: self._downloader.trouble(u'ERROR: unable to extract media URL') @@ -3724,13 +3750,13 @@ class YouPornIE(InfoExtractor): webpage = self._download_webpage(req, video_id) # Get the video title - result = re.search(r'videoTitleArea">(?P.*)</h1>', webpage) + result = re.search(r'<h1.*?>(?P<title>.*)</h1>', webpage) if result is None: - raise ExtractorError(u'ERROR: unable to extract video title') + raise ExtractorError(u'Unable to extract video title') video_title = result.group('title').strip() # Get the video date - result = re.search(r'Date:</b>(?P<date>.*)</li>', webpage) + result = re.search(r'Date:</label>(?P<date>.*) </li>', webpage) if result is None: self._downloader.to_stderr(u'WARNING: unable to extract video date') upload_date = None @@ -3738,9 +3764,9 @@ class YouPornIE(InfoExtractor): upload_date = result.group('date').strip() # Get the video uploader - result = re.search(r'Submitted:</b>(?P<uploader>.*)</li>', webpage) + result = re.search(r'Submitted:</label>(?P<uploader>.*)</li>', webpage) if result is None: - self._downloader.to_stderr(u'ERROR: unable to extract uploader') + self._downloader.to_stderr(u'WARNING: unable to extract uploader') video_uploader = None else: video_uploader = result.group('uploader').strip() @@ -3912,7 +3938,7 @@ class EightTracksIE(InfoExtractor): webpage = self._download_webpage(url, playlist_id) - m = re.search(r"new TRAX.Mix\((.*?)\);\n*\s*TRAX.initSearchAutocomplete\('#search'\);", webpage, flags=re.DOTALL) + m = re.search(r"PAGE.mix = (.*?);\n", webpage, flags=re.DOTALL) if not m: raise ExtractorError(u'Cannot find trax information') json_like = m.group(1) @@ -3967,15 +3993,63 @@ class KeekIE(InfoExtractor): 'uploader': uploader } return [info] - + class TEDIE(InfoExtractor): - _VALID_URL=r'http://www.ted.com/talks/(?P<videoName>\w+)' + _VALID_URL=r'''http://www.ted.com/ + ( + ((?P<type_playlist>playlists)/(?P<playlist_id>\d+)) # We have a playlist + | + ((?P<type_talk>talks)) # We have a simple talk + ) + /(?P<name>\w+) # Here goes the name and then ".html" + ''' + + def suitable(self, url): + """Receives a URL and returns True if suitable for this IE.""" + return re.match(self._VALID_URL, url, re.VERBOSE) is not None + def _real_extract(self, url): - m=re.match(self._VALID_URL, url) - videoName=m.group('videoName') - webpage=self._download_webpage(url, 0, 'Downloading \"%s\" page' % videoName) - #If the url includes the language we get the title translated - title_RE=r'<h1><span id="altHeadline" >(?P<title>[\s\w:/\.\?=\+-]*)</span></h1>' + m=re.match(self._VALID_URL, url, re.VERBOSE) + if m.group('type_talk'): + return [self._talk_info(url)] + else : + playlist_id=m.group('playlist_id') + name=m.group('name') + self._downloader.to_screen(u'[%s] Getting info of playlist %s: "%s"' % (self.IE_NAME,playlist_id,name)) + return self._playlist_videos_info(url,name,playlist_id) + + def _talk_video_link(self,mediaSlug): + '''Returns the video link for that mediaSlug''' + return 'http://download.ted.com/talks/%s.mp4' % mediaSlug + + def _playlist_videos_info(self,url,name,playlist_id=0): + '''Returns the videos of the playlist''' + video_RE=r''' + <li\ id="talk_(\d+)"([.\s]*?)data-id="(?P<video_id>\d+)" + ([.\s]*?)data-playlist_item_id="(\d+)" + ([.\s]*?)data-mediaslug="(?P<mediaSlug>.+?)" + ''' + video_name_RE=r'<p\ class="talk-title"><a href="/talks/(.+).html">(?P<fullname>.+?)</a></p>' + webpage=self._download_webpage(url, playlist_id, 'Downloading playlist webpage') + m_videos=re.finditer(video_RE,webpage,re.VERBOSE) + m_names=re.finditer(video_name_RE,webpage) + info=[] + for m_video, m_name in zip(m_videos,m_names): + video_dic={ + 'id': m_video.group('video_id'), + 'url': self._talk_video_link(m_video.group('mediaSlug')), + 'ext': 'mp4', + 'title': m_name.group('fullname') + } + info.append(video_dic) + return info + def _talk_info(self, url, video_id=0): + """Return the video for the talk in the url""" + m=re.match(self._VALID_URL, url,re.VERBOSE) + videoName=m.group('name') + webpage=self._download_webpage(url, video_id, 'Downloading \"%s\" page' % videoName) + # If the url includes the language we get the title translated + title_RE=r'<h1><span id="altHeadline" >(?P<title>[\s\w:/\.\?=\+-\\\']*)</span></h1>' title=re.search(title_RE, webpage).group('title') info_RE=r'''<script\ type="text/javascript">var\ talkDetails\ =(.*?) "id":(?P<videoID>[\d]+).*? @@ -3983,12 +4057,68 @@ class TEDIE(InfoExtractor): info_match=re.search(info_RE,webpage,re.VERBOSE) video_id=info_match.group('videoID') mediaSlug=info_match.group('mediaSlug') - video_url='http://download.ted.com/talks/%s.mp4' % mediaSlug + video_url=self._talk_video_link(mediaSlug) info = { - 'id':video_id, - 'url':video_url, + 'id': video_id, + 'url': video_url, 'ext': 'mp4', 'title': title + } + return info + +class MySpassIE(InfoExtractor): + _VALID_URL = r'http://www.myspass.de/.*' + + def _real_extract(self, url): + META_DATA_URL_TEMPLATE = 'http://www.myspass.de/myspass/includes/apps/video/getvideometadataxml.php?id=%s' + + # video id is the last path element of the URL + # usually there is a trailing slash, so also try the second but last + url_path = compat_urllib_parse_urlparse(url).path + url_parent_path, video_id = os.path.split(url_path) + if not video_id: + _, video_id = os.path.split(url_parent_path) + + # get metadata + metadata_url = META_DATA_URL_TEMPLATE % video_id + metadata_text = self._download_webpage(metadata_url, video_id) + metadata = xml.etree.ElementTree.fromstring(metadata_text.encode('utf-8')) + + # extract values from metadata + url_flv_el = metadata.find('url_flv') + if url_flv_el is None: + self._downloader.trouble(u'ERROR: unable to extract download url') + return + video_url = url_flv_el.text + extension = os.path.splitext(video_url)[1][1:] + title_el = metadata.find('title') + if title_el is None: + self._downloader.trouble(u'ERROR: unable to extract title') + return + title = title_el.text + format_id_el = metadata.find('format_id') + if format_id_el is None: + format = ext + else: + format = format_id_el.text + description_el = metadata.find('description') + if description_el is not None: + description = description_el.text + else: + description = None + imagePreview_el = metadata.find('imagePreview') + if imagePreview_el is not None: + thumbnail = imagePreview_el.text + else: + thumbnail = None + info = { + 'id': video_id, + 'url': video_url, + 'title': title, + 'ext': extension, + 'format': format, + 'thumbnail': thumbnail, + 'description': description } return [info] @@ -4040,6 +4170,7 @@ def gen_extractors(): EightTracksIE(), KeekIE(), TEDIE(), + MySpassIE(), GenericIE() ]