X-Git-Url: http://git.cielonegro.org/gitweb.cgi?a=blobdiff_plain;f=youtube_dl%2FInfoExtractors.py;h=4bcd8b9cd22076c323071a4b810b433410ea1e9e;hb=c1d58e1c67106e2f65fdd38fc4db99af19a647ce;hp=789f9872e29983c20f5d96d2bea5d8757de5bf3f;hpb=59f4fd4dc65f0576fdf02bfb710cddc90262b1ba;p=youtube-dl.git
diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py
index 789f9872e..4bcd8b9cd 100755
--- a/youtube_dl/InfoExtractors.py
+++ b/youtube_dl/InfoExtractors.py
@@ -920,36 +920,57 @@ class YahooIE(InfoExtractor):
if mobj is None:
raise ExtractorError(u'Invalid URL: %s' % url)
video_id = mobj.group('id')
-
- # TODO: Check which url parameters are required
- info_url = 'http://cosmos.bcst.yahoo.com/rest/v2/pops;lmsoverride=1;outputformat=mrss;cb=974419660;id=%s;rd=news.yahoo.com;datacontext=mdb;lg=KCa2IihxG3qE60vQ7HtyUy' % video_id
- webpage = self._download_webpage(info_url, video_id, "Downloading info webpage")
- info_re = r'''
.*?)\]\]>.*
- .*?)\]\]>.*
- .*?)\ .*\]\]>.*
- .+?)";', webpage)
+
+ if m_id is None:
+ # TODO: Check which url parameters are required
+ info_url = 'http://cosmos.bcst.yahoo.com/rest/v2/pops;lmsoverride=1;outputformat=mrss;cb=974419660;id=%s;rd=news.yahoo.com;datacontext=mdb;lg=KCa2IihxG3qE60vQ7HtyUy' % video_id
+ webpage = self._download_webpage(info_url, video_id, u'Downloading info webpage')
+ info_re = r'''.*?)\]\]>.*
+ .*?)\]\]>.*
+ .*?)\ .*\]\]>.*
+ n:
video_ids = video_ids[:n]
videos = [self.url_result('http://www.youtube.com/watch?v=%s' % id, 'Youtube') for id in video_ids]
- return videos
+ return self.playlist_result(videos, query)
class GoogleSearchIE(InfoExtractor):
"""Information Extractor for Google Video search queries."""
_VALID_URL = r'gvsearch(?P|\d+|all):(?P[\s\S]+)'
- _MORE_PAGES_INDICATOR = r'class="pn" id="pnnext"'
+ _MORE_PAGES_INDICATOR = r'id="pnnext" class="pn"'
_max_google_results = 1000
IE_NAME = u'video.google:search'
@@ -1418,7 +1439,7 @@ class GoogleSearchIE(InfoExtractor):
}
for pagenum in itertools.count(1):
- result_url = u'http://video.google.com/videosearch?q=%s&start=%s&hl=en' % (compat_urllib_parse.quote_plus(query), pagenum*10)
+ result_url = u'http://www.google.com/search?tbm=vid&q=%s&start=%s&hl=en' % (compat_urllib_parse.quote_plus(query), pagenum*10)
webpage = self._download_webpage(result_url, u'gvsearch:' + query,
note='Downloading result page ' + str(pagenum))
@@ -1435,18 +1456,10 @@ class GoogleSearchIE(InfoExtractor):
class YahooSearchIE(InfoExtractor):
"""Information Extractor for Yahoo! Video search queries."""
- _WORKING = False
_VALID_URL = r'yvsearch(\d+|all)?:[\s\S]+'
- _TEMPLATE_URL = 'http://video.yahoo.com/search/?p=%s&o=%s'
- _VIDEO_INDICATOR = r'href="http://video\.yahoo\.com/watch/([0-9]+/[0-9]+)"'
- _MORE_PAGES_INDICATOR = r'\s*Next'
- _max_yahoo_results = 1000
- IE_NAME = u'video.yahoo:search'
- def report_download_page(self, query, pagenum):
- """Report attempt to download playlist page with given number."""
- query = query.decode(preferredencoding())
- self.to_screen(u'query "%s": Downloading page %s' % (query, pagenum))
+ _max_yahoo_results = 1000
+ IE_NAME = u'screen.yahoo:search'
def _real_extract(self, query):
mobj = re.match(self._VALID_URL, query)
@@ -1457,11 +1470,9 @@ class YahooSearchIE(InfoExtractor):
prefix = prefix[8:]
query = query.encode('utf-8')
if prefix == '':
- self._download_n_results(query, 1)
- return
+ return self._get_n_results(query, 1)
elif prefix == 'all':
- self._download_n_results(query, self._max_yahoo_results)
- return
+ return self._get_n_results(query, self._max_yahoo_results)
else:
try:
n = int(prefix)
@@ -1470,46 +1481,36 @@ class YahooSearchIE(InfoExtractor):
elif n > self._max_yahoo_results:
self._downloader.report_warning(u'yvsearch returns max %i results (you requested %i)' % (self._max_yahoo_results, n))
n = self._max_yahoo_results
- self._download_n_results(query, n)
- return
+ return self._get_n_results(query, n)
except ValueError: # parsing prefix as integer fails
- self._download_n_results(query, 1)
- return
-
- def _download_n_results(self, query, n):
- """Downloads a specified number of results for a query"""
-
- video_ids = []
- already_seen = set()
- pagenum = 1
+ return self._get_n_results(query, 1)
- while True:
- self.report_download_page(query, pagenum)
- result_url = self._TEMPLATE_URL % (compat_urllib_parse.quote_plus(query), pagenum)
- request = compat_urllib_request.Request(result_url)
- try:
- page = compat_urllib_request.urlopen(request).read()
- except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
- raise ExtractorError(u'Unable to download webpage: %s' % compat_str(err))
+ def _get_n_results(self, query, n):
+ """Get a specified number of results for a query"""
- # Extract video identifiers
- for mobj in re.finditer(self._VIDEO_INDICATOR, page):
- video_id = mobj.group(1)
- if video_id not in already_seen:
- video_ids.append(video_id)
- already_seen.add(video_id)
- if len(video_ids) == n:
- # Specified n videos reached
- for id in video_ids:
- self._downloader.download(['http://video.yahoo.com/watch/%s' % id])
- return
-
- if re.search(self._MORE_PAGES_INDICATOR, page) is None:
- for id in video_ids:
- self._downloader.download(['http://video.yahoo.com/watch/%s' % id])
- return
+ res = {
+ '_type': 'playlist',
+ 'id': query,
+ 'entries': []
+ }
+ for pagenum in itertools.count(0):
+ result_url = u'http://video.search.yahoo.com/search/?p=%s&fr=screen&o=js&gs=0&b=%d' % (compat_urllib_parse.quote_plus(query), pagenum * 30)
+ webpage = self._download_webpage(result_url, query,
+ note='Downloading results page '+str(pagenum+1))
+ info = json.loads(webpage)
+ m = info[u'm']
+ results = info[u'results']
+
+ for (i, r) in enumerate(results):
+ if (pagenum * 30) +i >= n:
+ break
+ mobj = re.search(r'(?Pscreen\.yahoo\.com/.*?-\d*?\.html)"', r)
+ e = self.url_result('http://' + mobj.group('url'), 'Yahoo')
+ res['entries'].append(e)
+ if (pagenum * 30 +i >= n) or (m[u'last'] >= (m[u'total'] -1 )):
+ break
- pagenum = pagenum + 1
+ return res
class YoutubePlaylistIE(InfoExtractor):