[youtube] Add support for search result URLs (Fixes #2495)

diff --git a/test/test_all_urls.py b/test/test_all_urls.py
index 7a78005a3a32f1e25609ac95e6004c39706e72a2..5c17a39fb556da63020dcfab17d29b9862d90488 100644 (file)
--- a/test/test_all_urls.py
+++ b/test/test_all_urls.py
@@ -71,6 +71,10 @@ class TestAllURLsMatching(unittest.TestCase):
     def test_youtube_truncated(self):
         self.assertMatch('http://www.youtube.com/watch?', ['youtube:truncated_url'])
 
+    def test_youtube_search_matching(self):
+        self.assertMatch('http://www.youtube.com/results?search_query=making+mustard', ['youtube:search_url'])
+        self.assertMatch('https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video', ['youtube:search_url'])
+
     def test_justin_tv_channelid_matching(self):
         self.assertTrue(JustinTVIE.suitable(u"justin.tv/vanillatv"))
         self.assertTrue(JustinTVIE.suitable(u"twitch.tv/vanillatv"))

diff --git a/test/test_youtube_lists.py b/test/test_youtube_lists.py
index 5eccc11ffae7d27f036e83564b8f083240ebe4fc..7d3b9c7054726bb773e3ac390222706c0bcec2af 100644 (file)
--- a/test/test_youtube_lists.py
+++ b/test/test_youtube_lists.py
@@ -16,6 +16,7 @@ from youtube_dl.extractor import (
     YoutubeChannelIE,
     YoutubeShowIE,
     YoutubeTopListIE,
+    YoutubeSearchURLIE,
 )
 
 
@@ -133,5 +134,14 @@ class TestYoutubeLists(unittest.TestCase):
         entries = result['entries']
         self.assertTrue(len(entries) >= 5)
 
+    def test_youtube_search_url(self):
+        dl = FakeYDL()
+        ie = YoutubeSearchURLIE(dl)
+        result = ie.extract('https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video')
+        entries = result['entries']
+        self.assertIsPlaylist(result)
+        self.assertEqual(result['title'], 'youtube-dl test video')
+        self.assertTrue(len(entries) >= 5)
+
 if __name__ == '__main__':
     unittest.main()

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 47205efae69393840f6a200e5722d2a1831753d2..bbfdfb444f3ad549fe5ab602a3e12602e82c8bc8 100644 (file)
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -675,7 +675,7 @@ class YoutubeDL(object):
             info_dict['playlist'] = None
             info_dict['playlist_index'] = None
 
-        if 'display_id' not in info_dict:
+        if 'display_id' not in info_dict and 'id' in info_dict:
             info_dict['display_id'] = info_dict['id']
 
         # This extractors handle format selection themselves

diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py
index e6755151c1b3c7766331b8a87839f8929236c4f6..f35ee49416653ec50c95919ece259ff7eee90abb 100644 (file)
--- a/youtube_dl/extractor/__init__.py
+++ b/youtube_dl/extractor/__init__.py
@@ -285,19 +285,20 @@ from .youku import YoukuIE
 from .youporn import YouPornIE
 from .youtube import (
     YoutubeIE,
+    YoutubeChannelIE,
+    YoutubeFavouritesIE,
+    YoutubeHistoryIE,
     YoutubePlaylistIE,
-    YoutubeSearchIE,
+    YoutubeRecommendedIE,
     YoutubeSearchDateIE,
-    YoutubeUserIE,
-    YoutubeChannelIE,
+    YoutubeSearchIE,
+    YoutubeSearchURLIE,
     YoutubeShowIE,
     YoutubeSubscriptionsIE,
-    YoutubeRecommendedIE,
+    YoutubeTopListIE,
     YoutubeTruncatedURLIE,
+    YoutubeUserIE,
     YoutubeWatchLaterIE,
-    YoutubeFavouritesIE,
-    YoutubeHistoryIE,
-    YoutubeTopListIE,
 )
 from .zdf import ZDFIE
 

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index b40a45384f4d9a01d6f1b664601e7b46b0166141..166a0cf70caf5a52b5891763438cbc892b1a9533 100644 (file)
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1645,7 +1645,7 @@ class YoutubeChannelIE(InfoExtractor):
 
 class YoutubeUserIE(InfoExtractor):
     IE_DESC = u'YouTube.com user videos (URL or "ytuser" keyword)'
-    _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:user/)?(?!(?:attribution_link|watch)(?:$|[^a-z_A-Z0-9-])))|ytuser:)(?!feed/)([A-Za-z0-9_-]+)'
+    _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:user/)?(?!(?:attribution_link|watch|results)(?:$|[^a-z_A-Z0-9-])))|ytuser:)(?!feed/)([A-Za-z0-9_-]+)'
     _TEMPLATE_URL = 'https://gdata.youtube.com/feeds/api/users/%s'
     _GDATA_PAGE_SIZE = 50
     _GDATA_URL = 'https://gdata.youtube.com/feeds/api/users/%s/uploads?max-results=%d&start-index=%d&alt=json'
@@ -1744,12 +1744,50 @@ class YoutubeSearchIE(SearchInfoExtractor):
                   for video_id in video_ids]
         return self.playlist_result(videos, query)
 
+
 class YoutubeSearchDateIE(YoutubeSearchIE):
     IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
     _API_URL = 'https://gdata.youtube.com/feeds/api/videos?q=%s&start-index=%i&max-results=50&v=2&alt=jsonc&orderby=published'
     _SEARCH_KEY = 'ytsearchdate'
     IE_DESC = u'YouTube.com searches, newest videos first'
 
+
+class YoutubeSearchURLIE(InfoExtractor):
+    IE_DESC = u'YouTube.com search URLs'
+    IE_NAME = u'youtube:search_url'
+    _VALID_URL = r'https?://(?:www\.)?youtube\.com/results\?(.*?&)?search_query=(?P<query>[^&]+)(?:[&]|$)'
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        query = compat_urllib_parse.unquote_plus(mobj.group('query'))
+
+        webpage = self._download_webpage(url, query)
+        result_code = self._search_regex(
+            r'(?s)<ol id="search-results"(.*?)</ol>', webpage, u'result HTML')
+
+        part_codes = re.findall(
+            r'(?s)<h3 class="yt-lockup-title">(.*?)</h3>', result_code)
+        entries = []
+        for part_code in part_codes:
+            part_title = self._html_search_regex(
+                r'(?s)title="([^"]+)"', part_code, 'item title', fatal=False)
+            part_url_snippet = self._html_search_regex(
+                r'(?s)href="([^"]+)"', part_code, 'item URL')
+            part_url = compat_urlparse.urljoin(
+                'https://www.youtube.com/', part_url_snippet)
+            entries.append({
+                '_type': 'url',
+                'url': part_url,
+                'title': part_title,
+            })
+
+        return {
+            '_type': 'playlist',
+            'entries': entries,
+            'title': query,
+        }
+
+
 class YoutubeShowIE(InfoExtractor):
     IE_DESC = u'YouTube.com (multi-season) shows'
     _VALID_URL = r'https?://www\.youtube\.com/show/(.*)'