X-Git-Url: http://git.cielonegro.org/gitweb.cgi?a=blobdiff_plain;f=youtube_dl%2FInfoExtractors.py;h=a7fdf1607c4c73e4caef623a1598065e84b3a834;hb=d281274bf250065f876bb4f75fb6f711e1a26eba;hp=2881ae67c54df152ece3400f86446256bd21ec17;hpb=c9c8402093e5da5e7e6a3247761af962031cf255;p=youtube-dl.git diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py index 2881ae67c..a7fdf1607 100755 --- a/youtube_dl/InfoExtractors.py +++ b/youtube_dl/InfoExtractors.py @@ -134,6 +134,28 @@ class InfoExtractor(object): encoding = 'utf-8' webpage_bytes = urlh.read() return webpage_bytes.decode(encoding, 'replace') + + #Methods for following #608 + #They set the correct value of the '_type' key + def video_result(self, video_info): + """Returns a video""" + video_info['_type'] = 'video' + return video_info + def url_result(self, url, ie=None): + """Returns a url that points to a page that should be processed""" + #TODO: ie should be the class used for getting the info + video_info = {'_type': 'url', + 'url': url} + return video_info + def playlist_result(self, entries, playlist_id=None, playlist_title=None): + """Returns a playlist""" + video_info = {'_type': 'playlist', + 'entries': entries} + if playlist_id: + video_info['id'] = playlist_id + if playlist_title: + video_info['title'] = playlist_title + return video_info class YoutubeIE(InfoExtractor): @@ -253,11 +275,11 @@ class YoutubeIE(InfoExtractor): try: sub_list = compat_urllib_request.urlopen(request).read().decode('utf-8') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - return (u'WARNING: unable to download video subtitles: %s' % compat_str(err), None) + return (u'unable to download video subtitles: %s' % compat_str(err), None) sub_lang_list = re.findall(r'name="([^"]*)"[^>]+lang_code="([\w\-]+)"', sub_list) sub_lang_list = dict((l[1], l[0]) for l in sub_lang_list) if not sub_lang_list: - return (u'WARNING: video doesn\'t have subtitles', None) + return (u'video doesn\'t have subtitles', None) return sub_lang_list def _list_available_subtitles(self, video_id): @@ -265,6 +287,10 @@ class YoutubeIE(InfoExtractor): self.report_video_subtitles_available(video_id, sub_lang_list) def _request_subtitle(self, sub_lang, sub_name, video_id, format): + """ + Return tuple: + (error_message, sub_lang, sub) + """ self.report_video_subtitles_request(video_id, sub_lang, format) params = compat_urllib_parse.urlencode({ 'lang': sub_lang, @@ -276,14 +302,20 @@ class YoutubeIE(InfoExtractor): try: sub = compat_urllib_request.urlopen(url).read().decode('utf-8') except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - return (u'WARNING: unable to download video subtitles: %s' % compat_str(err), None) + return (u'unable to download video subtitles: %s' % compat_str(err), None, None) if not sub: - return (u'WARNING: Did not fetch video subtitles', None) + return (u'Did not fetch video subtitles', None, None) return (None, sub_lang, sub) def _extract_subtitle(self, video_id): + """ + Return a list with a tuple: + [(error_message, sub_lang, sub)] + """ sub_lang_list = self._get_available_subtitles(video_id) sub_format = self._downloader.params.get('subtitlesformat') + if isinstance(sub_lang_list,tuple): #There was some error, it didn't get the available subtitles + return [(sub_lang_list[0], None, None)] if self._downloader.params.get('subtitleslang', False): sub_lang = self._downloader.params.get('subtitleslang') elif 'en' in sub_lang_list: @@ -291,7 +323,7 @@ class YoutubeIE(InfoExtractor): else: sub_lang = list(sub_lang_list.keys())[0] if not sub_lang in sub_lang_list: - return (u'WARNING: no closed captions found in the specified language "%s"' % sub_lang, None) + return [(u'no closed captions found in the specified language "%s"' % sub_lang, None, None)] subtitle = self._request_subtitle(sub_lang, sub_lang_list[sub_lang].encode('utf-8'), video_id, sub_format) return [subtitle] @@ -299,6 +331,8 @@ class YoutubeIE(InfoExtractor): def _extract_all_subtitles(self, video_id): sub_lang_list = self._get_available_subtitles(video_id) sub_format = self._downloader.params.get('subtitlesformat') + if isinstance(sub_lang_list,tuple): #There was some error, it didn't get the available subtitles + return [(sub_lang_list[0], None, None)] subtitles = [] for sub_lang in sub_lang_list: subtitle = self._request_subtitle(sub_lang, sub_lang_list[sub_lang].encode('utf-8'), video_id, sub_format) @@ -532,14 +566,14 @@ class YoutubeIE(InfoExtractor): if video_subtitles: (sub_error, sub_lang, sub) = video_subtitles[0] if sub_error: - self._downloader.trouble(sub_error) + self._downloader.report_error(sub_error) if self._downloader.params.get('allsubtitles', False): video_subtitles = self._extract_all_subtitles(video_id) for video_subtitle in video_subtitles: (sub_error, sub_lang, sub) = video_subtitle if sub_error: - self._downloader.trouble(sub_error) + self._downloader.report_error(sub_error) if self._downloader.params.get('listsubtitles', False): sub_lang_list = self._list_available_subtitles(video_id) @@ -689,8 +723,7 @@ class MetacafeIE(InfoExtractor): # Check if video comes from YouTube mobj2 = re.match(r'^yt-(.*)$', video_id) if mobj2 is not None: - self._downloader.download(['http://www.youtube.com/watch?v=%s' % mobj2.group(1)]) - return + return [self.url_result('http://www.youtube.com/watch?v=%s' % mobj2.group(1))] # Retrieve video webpage to extract further information request = compat_urllib_request.Request('http://www.metacafe.com/watch/%s/' % video_id) @@ -1331,7 +1364,7 @@ class GenericIE(InfoExtractor): self._downloader.to_screen(u'[redirect] Following redirect to %s' % new_url) def _test_redirect(self, url): - """Check if it is a redirect, like url shorteners, in case restart chain.""" + """Check if it is a redirect, like url shorteners, in case return the new url.""" class HeadRequest(compat_urllib_request.Request): def get_method(self): return "HEAD" @@ -1382,11 +1415,11 @@ class GenericIE(InfoExtractor): return False self.report_following_redirect(new_url) - self._downloader.download([new_url]) - return True + return new_url def _real_extract(self, url): - if self._test_redirect(url): return + new_url = self._test_redirect(url) + if new_url: return [self.url_result(new_url)] video_id = url.split('/')[-1] try: @@ -1710,9 +1743,7 @@ class YoutubePlaylistIE(InfoExtractor): (?: (?:course|view_play_list|my_playlists|artist|playlist|watch) \? (?:.*?&)*? (?:p|a|list)= - | user/.*?/user/ | p/ - | user/.*?#[pg]/c/ ) ((?:PL|EC|UU)?[0-9A-Za-z-_]{10,}) .* @@ -1775,23 +1806,9 @@ class YoutubePlaylistIE(InfoExtractor): page_num += 1 videos = [v[1] for v in sorted(videos)] - total = len(videos) - - playliststart = self._downloader.params.get('playliststart', 1) - 1 - playlistend = self._downloader.params.get('playlistend', -1) - if playlistend == -1: - videos = videos[playliststart:] - else: - videos = videos[playliststart:playlistend] - if len(videos) == total: - self._downloader.to_screen(u'[youtube] PL %s: Found %i videos' % (playlist_id, total)) - else: - self._downloader.to_screen(u'[youtube] PL %s: Found %i videos, downloading %i' % (playlist_id, total, len(videos))) - - for video in videos: - self._downloader.download([video]) - return + url_results = [self.url_result(url) for url in videos] + return [self.playlist_result(url_results, playlist_id)] class YoutubeChannelIE(InfoExtractor): @@ -1841,9 +1858,9 @@ class YoutubeChannelIE(InfoExtractor): self._downloader.to_screen(u'[youtube] Channel %s: Found %i videos' % (channel_id, len(video_ids))) - for id in video_ids: - self._downloader.download(['http://www.youtube.com/watch?v=%s' % id]) - return + urls = ['http://www.youtube.com/watch?v=%s' % id for id in video_ids] + url_entries = [self.url_result(url) for url in urls] + return [self.playlist_result(url_entries, channel_id)] class YoutubeUserIE(InfoExtractor): @@ -1913,20 +1930,9 @@ class YoutubeUserIE(InfoExtractor): pagenum += 1 - all_ids_count = len(video_ids) - playliststart = self._downloader.params.get('playliststart', 1) - 1 - playlistend = self._downloader.params.get('playlistend', -1) - - if playlistend == -1: - video_ids = video_ids[playliststart:] - else: - video_ids = video_ids[playliststart:playlistend] - - self._downloader.to_screen(u"[youtube] user %s: Collected %d video ids (downloading %d of them)" % - (username, all_ids_count, len(video_ids))) - - for video_id in video_ids: - self._downloader.download(['http://www.youtube.com/watch?v=%s' % video_id]) + urls = ['http://www.youtube.com/watch?v=%s' % video_id for video_id in video_ids] + url_results = [self.url_result(url) for url in urls] + return [self.playlist_result(url_results, playlist_title = username)] class BlipTVUserIE(InfoExtractor): @@ -2004,20 +2010,12 @@ class BlipTVUserIE(InfoExtractor): pagenum += 1 - all_ids_count = len(video_ids) - playliststart = self._downloader.params.get('playliststart', 1) - 1 - playlistend = self._downloader.params.get('playlistend', -1) - - if playlistend == -1: - video_ids = video_ids[playliststart:] - else: - video_ids = video_ids[playliststart:playlistend] - self._downloader.to_screen(u"[%s] user %s: Collected %d video ids (downloading %d of them)" % (self.IE_NAME, username, all_ids_count, len(video_ids))) - for video_id in video_ids: - self._downloader.download([u'http://blip.tv/'+video_id]) + urls = [u'http://blip.tv/%s' % video_id for video_id in video_ids] + url_entries = [self.url_result(url) for url in urls] + return [self.playlist_result(url_entries, playlist_title = username)] class DepositFilesIE(InfoExtractor): @@ -3796,7 +3794,7 @@ class WorldStarHipHopIE(InfoExtractor): _title = r"""(.*)""" mobj = re.search(_title, webpage_src) - + if mobj is not None: title = mobj.group(1) else: @@ -3814,7 +3812,7 @@ class WorldStarHipHopIE(InfoExtractor): if mobj is not None: title = mobj.group(1) thumbnail = None - + results = [{ 'id': video_id, 'url' : video_url,