From: Philipp Hagemeister Date: Mon, 23 Dec 2013 04:03:32 +0000 (+0100) Subject: Merge remote-tracking branch 'jaimeMF/split-downloaders' X-Git-Url: http://git.cielonegro.org/gitweb.cgi?a=commitdiff_plain;h=9fc3bef87a1009eb48f2012f3eab65761a53ba5e;hp=3bc2ddccc8622379ec11e802dff30a635285a9c8;p=youtube-dl.git Merge remote-tracking branch 'jaimeMF/split-downloaders' --- diff --git a/README.md b/README.md index 68b2e1ae7..91e18e372 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,8 @@ which means you can modify it, redistribute it or use it however you like. /youtube-dl . --no-cache-dir Disable filesystem caching --bidi-workaround Work around terminals that lack bidirectional - text support. Requires fribidi executable in PATH + text support. Requires bidiv or fribidi + executable in PATH ## Video Selection: --playlist-start NUMBER playlist video to start at (default is 1) @@ -56,6 +57,10 @@ which means you can modify it, redistribute it or use it however you like. --date DATE download only videos uploaded in this date --datebefore DATE download only videos uploaded before this date --dateafter DATE download only videos uploaded after this date + --min-views COUNT Do not download any videos with less than COUNT + views + --max-views COUNT Do not download any videos with more than COUNT + views --no-playlist download only the currently playing video --age-limit YEARS download only videos suitable for the given age --download-archive FILE Download only videos not listed in the archive @@ -127,6 +132,7 @@ which means you can modify it, redistribute it or use it however you like. --get-id simulate, quiet but print id --get-thumbnail simulate, quiet but print thumbnail URL --get-description simulate, quiet but print video description + --get-duration simulate, quiet but print video length --get-filename simulate, quiet but print output filename --get-format simulate, quiet but print output format -j, --dump-json simulate, quiet but print JSON information diff --git a/test/test_all_urls.py b/test/test_all_urls.py index e9458b2e3..bd77b7c30 100644 --- a/test/test_all_urls.py +++ b/test/test_all_urls.py @@ -10,6 +10,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from test.helper import get_testcases from youtube_dl.extractor import ( + FacebookIE, gen_extractors, JustinTVIE, YoutubeIE, @@ -87,12 +88,15 @@ class TestAllURLsMatching(unittest.TestCase): assertExtractId('http://www.youtube.com/watch?v=BaW_jenozKcsharePLED17F32AD9753930', 'BaW_jenozKc') assertExtractId('BaW_jenozKc', 'BaW_jenozKc') + def test_facebook_matching(self): + self.assertTrue(FacebookIE.suitable(u'https://www.facebook.com/Shiniknoh#!/photo.php?v=10153317450565268')) + def test_no_duplicates(self): ies = gen_extractors() for tc in get_testcases(): url = tc['url'] for ie in ies: - if type(ie).__name__ in ['GenericIE', tc['name'] + 'IE']: + if type(ie).__name__ in ('GenericIE', tc['name'] + 'IE'): self.assertTrue(ie.suitable(url), '%s should match URL %r' % (type(ie).__name__, url)) else: self.assertFalse(ie.suitable(url), '%s should not match URL %r' % (type(ie).__name__, url)) diff --git a/test/test_playlists.py b/test/test_playlists.py index 87ca401e5..1b7b4e3d8 100644 --- a/test/test_playlists.py +++ b/test/test_playlists.py @@ -12,6 +12,7 @@ from test.helper import FakeYDL from youtube_dl.extractor import ( + AcademicEarthCourseIE, DailymotionPlaylistIE, DailymotionUserIE, VimeoChannelIE, @@ -26,7 +27,8 @@ from youtube_dl.extractor import ( BambuserChannelIE, BandcampAlbumIE, SmotriCommunityIE, - SmotriUserIE + SmotriUserIE, + IviCompilationIE ) @@ -158,5 +160,34 @@ class TestPlaylists(unittest.TestCase): self.assertEqual(result['title'], u'Inspector') self.assertTrue(len(result['entries']) >= 9) + def test_AcademicEarthCourse(self): + dl = FakeYDL() + ie = AcademicEarthCourseIE(dl) + result = ie.extract(u'http://academicearth.org/courses/building-dynamic-websites/') + self.assertIsPlaylist(result) + self.assertEqual(result['id'], u'building-dynamic-websites') + self.assertEqual(result['title'], u'Building Dynamic Websites') + self.assertEqual(result['description'], u"Today's websites are increasingly dynamic. Pages are no longer static HTML files but instead generated by scripts and database calls. User interfaces are more seamless, with technologies like Ajax replacing traditional page reloads. This course teaches students how to build dynamic websites with Ajax and with Linux, Apache, MySQL, and PHP (LAMP), one of today's most popular frameworks. Students learn how to set up domain names with DNS, how to structure pages with XHTML and CSS, how to program in JavaScript and PHP, how to configure Apache and MySQL, how to design and query databases with SQL, how to use Ajax with both XML and JSON, and how to build mashups. The course explores issues of security, scalability, and cross-browser support and also discusses enterprise-level deployments of websites, including third-party hosting, virtualization, colocation in data centers, firewalling, and load-balancing.") + self.assertEqual(len(result['entries']), 10) + + def test_ivi_compilation(self): + dl = FakeYDL() + ie = IviCompilationIE(dl) + result = ie.extract('http://www.ivi.ru/watch/dezhurnyi_angel') + self.assertIsPlaylist(result) + self.assertEqual(result['id'], u'dezhurnyi_angel') + self.assertEqual(result['title'], u'Дежурный ангел (2010 - 2012)') + self.assertTrue(len(result['entries']) >= 36) + + def test_ivi_compilation_season(self): + dl = FakeYDL() + ie = IviCompilationIE(dl) + result = ie.extract('http://www.ivi.ru/watch/dezhurnyi_angel/season2') + self.assertIsPlaylist(result) + self.assertEqual(result['id'], u'dezhurnyi_angel/season2') + self.assertEqual(result['title'], u'Дежурный ангел (2010 - 2012) 2 сезон') + self.assertTrue(len(result['entries']) >= 20) + + if __name__ == '__main__': unittest.main() diff --git a/test/test_utils.py b/test/test_utils.py index 0fa66beec..e5778cd83 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -13,20 +13,21 @@ import xml.etree.ElementTree #from youtube_dl.utils import htmlentity_transform from youtube_dl.utils import ( - timeconvert, - sanitize_filename, - unescapeHTML, - orderedSet, DateRange, - unified_strdate, + encodeFilename, find_xpath_attr, get_meta_content, - xpath_with_ns, - smuggle_url, - unsmuggle_url, + orderedSet, + sanitize_filename, shell_quote, - encodeFilename, + smuggle_url, str_to_int, + timeconvert, + unescapeHTML, + unified_strdate, + unsmuggle_url, + url_basename, + xpath_with_ns, ) if sys.version_info < (3, 0): @@ -181,6 +182,15 @@ class TestUtil(unittest.TestCase): self.assertEqual(str_to_int('123,456'), 123456) self.assertEqual(str_to_int('123.456'), 123456) + def test_url_basename(self): + self.assertEqual(url_basename(u'http://foo.de/'), u'') + self.assertEqual(url_basename(u'http://foo.de/bar/baz'), u'baz') + self.assertEqual(url_basename(u'http://foo.de/bar/baz?x=y'), u'baz') + self.assertEqual(url_basename(u'http://foo.de/bar/baz#x=y'), u'baz') + self.assertEqual(url_basename(u'http://foo.de/bar/baz/'), u'baz') + self.assertEqual( + url_basename(u'http://media.w3.org/2010/05/sintel/trailer.mp4'), + u'trailer.mp4') if __name__ == '__main__': unittest.main() diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 414aa5a80..04771c637 100644 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -34,6 +34,7 @@ from .utils import ( encodeFilename, ExtractorError, format_bytes, + formatSeconds, get_term_width, locked_file, make_HTTPS_handler, @@ -46,6 +47,7 @@ from .utils import ( subtitles_filename, takewhile_inclusive, UnavailableVideoError, + url_basename, write_json_file, write_string, YoutubeDLHandler, @@ -94,6 +96,7 @@ class YoutubeDL(object): forcethumbnail: Force printing thumbnail URL. forcedescription: Force printing description. forcefilename: Force printing final filename. + forceduration: Force printing duration. forcejson: Force printing info_dict as JSON. simulate: Do not download the video files. format: Video format code. @@ -127,7 +130,16 @@ class YoutubeDL(object): noplaylist: Download single video instead of a playlist if in doubt. age_limit: An integer representing the user's age in years. Unsuitable videos for the given age are skipped. - download_archive: File name of a file where all downloads are recorded. + min_views: An integer representing the minimum view count the video + must have in order to not be skipped. + Videos without view count information are always + downloaded. None for no limit. + max_views: An integer representing the maximum view count. + Videos that are more popular than that are not + downloaded. + Videos without view count information are always + downloaded. None for no limit. + download_archive: File name of a file where all downloads are recorded. Videos already present in the file are not downloaded again. cookiefile: File name where cookies should be read from and dumped to. @@ -171,12 +183,18 @@ class YoutubeDL(object): width_args = [] else: width_args = ['-w', str(width)] - self._fribidi = subprocess.Popen( - ['fribidi', '-c', 'UTF-8'] + width_args, + sp_kwargs = dict( stdin=subprocess.PIPE, stdout=slave, stderr=self._err_file) - self._fribidi_channel = os.fdopen(master, 'rb') + try: + self._output_process = subprocess.Popen( + ['bidiv'] + width_args, **sp_kwargs + ) + except OSError: + self._output_process = subprocess.Popen( + ['fribidi', '-c', 'UTF-8'] + width_args, **sp_kwargs) + self._output_channel = os.fdopen(master, 'rb') except OSError as ose: if ose.errno == 2: self.report_warning(u'Could not find fribidi executable, ignoring --bidi-workaround . Make sure that fribidi is an executable file in one of the directories in your $PATH.') @@ -233,14 +251,15 @@ class YoutubeDL(object): self._fd_progress_hooks.append(ph) def _bidi_workaround(self, message): - if not hasattr(self, '_fribidi_channel'): + if not hasattr(self, '_output_channel'): return message + assert hasattr(self, '_output_process') assert type(message) == type(u'') line_count = message.count(u'\n') + 1 - self._fribidi.stdin.write((message + u'\n').encode('utf-8')) - self._fribidi.stdin.flush() - res = u''.join(self._fribidi_channel.readline().decode('utf-8') + self._output_process.stdin.write((message + u'\n').encode('utf-8')) + self._output_process.stdin.flush() + res = u''.join(self._output_channel.readline().decode('utf-8') for _ in range(line_count)) return res[:-len(u'\n')] @@ -357,22 +376,6 @@ class YoutubeDL(object): error_message = u'%s %s' % (_msg_header, message) self.trouble(error_message, tb) - def report_writedescription(self, descfn): - """ Report that the description file is being written """ - self.to_screen(u'[info] Writing video description to: ' + descfn) - - def report_writesubtitles(self, sub_filename): - """ Report that the subtitles file is being written """ - self.to_screen(u'[info] Writing video subtitles to: ' + sub_filename) - - def report_writeinfojson(self, infofn): - """ Report that the metadata file has been written """ - self.to_screen(u'[info] Video description metadata as JSON to: ' + infofn) - - def report_writeannotations(self, annofn): - """ Report that the annotations file has been written. """ - self.to_screen(u'[info] Writing video annotations to: ' + annofn) - def report_file_already_downloaded(self, file_name): """Report file has already been fully downloaded.""" try: @@ -417,13 +420,14 @@ class YoutubeDL(object): def _match_entry(self, info_dict): """ Returns None iff the file should be downloaded """ + video_title = info_dict.get('title', info_dict.get('id', u'video')) if 'title' in info_dict: # This can happen when we're just evaluating the playlist title = info_dict['title'] matchtitle = self.params.get('matchtitle', False) if matchtitle: if not re.search(matchtitle, title, re.IGNORECASE): - return u'[download] "' + title + '" title did not match pattern "' + matchtitle + '"' + return u'"' + title + '" title did not match pattern "' + matchtitle + '"' rejecttitle = self.params.get('rejecttitle', False) if rejecttitle: if re.search(rejecttitle, title, re.IGNORECASE): @@ -432,14 +436,21 @@ class YoutubeDL(object): if date is not None: dateRange = self.params.get('daterange', DateRange()) if date not in dateRange: - return u'[download] %s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange) + return u'%s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange) + view_count = info_dict.get('view_count', None) + if view_count is not None: + min_views = self.params.get('min_views') + if min_views is not None and view_count < min_views: + return u'Skipping %s, because it has not reached minimum view count (%d/%d)' % (video_title, view_count, min_views) + max_views = self.params.get('max_views') + if max_views is not None and view_count > max_views: + return u'Skipping %s, because it has exceeded the maximum view count (%d/%d)' % (video_title, view_count, max_views) age_limit = self.params.get('age_limit') if age_limit is not None: if age_limit < info_dict.get('age_limit', 0): return u'Skipping "' + title + '" because it is age restricted' if self.in_download_archive(info_dict): - return (u'%s has already been recorded in archive' - % info_dict.get('title', info_dict.get('id', u'video'))) + return u'%s has already been recorded in archive' % video_title return None @staticmethod @@ -483,6 +494,7 @@ class YoutubeDL(object): { 'extractor': ie.IE_NAME, 'webpage_url': url, + 'webpage_url_basename': url_basename(url), 'extractor_key': ie.ie_key(), }) if process: @@ -556,16 +568,16 @@ class YoutubeDL(object): n_all_entries = len(ie_result['entries']) playliststart = self.params.get('playliststart', 1) - 1 - playlistend = self.params.get('playlistend', -1) - + playlistend = self.params.get('playlistend', None) + # For backwards compatibility, interpret -1 as whole list if playlistend == -1: - entries = ie_result['entries'][playliststart:] - else: - entries = ie_result['entries'][playliststart:playlistend] + playlistend = None + entries = ie_result['entries'][playliststart:playlistend] n_entries = len(entries) - self.to_screen(u"[%s] playlist '%s': Collected %d video ids (downloading %d of them)" % + self.to_screen( + u"[%s] playlist '%s': Collected %d video ids (downloading %d of them)" % (ie_result['extractor'], playlist, n_all_entries, n_entries)) for i, entry in enumerate(entries, 1): @@ -575,6 +587,7 @@ class YoutubeDL(object): 'playlist_index': i + playliststart, 'extractor': ie_result['extractor'], 'webpage_url': ie_result['webpage_url'], + 'webpage_url_basename': url_basename(ie_result['webpage_url']), 'extractor_key': ie_result['extractor_key'], } @@ -595,6 +608,7 @@ class YoutubeDL(object): { 'extractor': ie_result['extractor'], 'webpage_url': ie_result['webpage_url'], + 'webpage_url_basename': url_basename(ie_result['webpage_url']), 'extractor_key': ie_result['extractor_key'], }) return r @@ -631,7 +645,7 @@ class YoutubeDL(object): info_dict['playlist_index'] = None # This extractors handle format selection themselves - if info_dict['extractor'] in [u'youtube', u'Youku']: + if info_dict['extractor'] in [u'Youku']: if download: self.process_info(info_dict) return info_dict @@ -657,10 +671,6 @@ class YoutubeDL(object): if 'ext' not in format: format['ext'] = determine_ext(format['url']) - if self.params.get('listformats', None): - self.list_formats(info_dict) - return - format_limit = self.params.get('format_limit', None) if format_limit: formats = list(takewhile_inclusive( @@ -673,9 +683,16 @@ class YoutubeDL(object): except ValueError: ext_ord = -1 # We only compare the extension if they have the same height and width - return (f.get('height'), f.get('width'), ext_ord) + return (f.get('height') if f.get('height') is not None else -1, + f.get('width') if f.get('width') is not None else -1, + ext_ord) formats = sorted(formats, key=_free_formats_key) + info_dict['formats'] = formats + if self.params.get('listformats', None): + self.list_formats(info_dict) + return + req_format = self.params.get('format', 'best') if req_format is None: req_format = 'best' @@ -750,6 +767,8 @@ class YoutubeDL(object): self.to_stdout(info_dict['description']) if self.params.get('forcefilename', False) and filename is not None: self.to_stdout(filename) + if self.params.get('forceduration', False) and info_dict.get('duration') is not None: + self.to_stdout(formatSeconds(info_dict['duration'])) if self.params.get('forceformat', False): self.to_stdout(info_dict['format']) if self.params.get('forcejson', False): @@ -772,28 +791,34 @@ class YoutubeDL(object): return if self.params.get('writedescription', False): - try: - descfn = filename + u'.description' - self.report_writedescription(descfn) - with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile: - descfile.write(info_dict['description']) - except (KeyError, TypeError): - self.report_warning(u'There\'s no description to write.') - except (OSError, IOError): - self.report_error(u'Cannot write description file ' + descfn) - return + descfn = filename + u'.description' + if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(descfn)): + self.to_screen(u'[info] Video description is already present') + else: + try: + self.to_screen(u'[info] Writing video description to: ' + descfn) + with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile: + descfile.write(info_dict['description']) + except (KeyError, TypeError): + self.report_warning(u'There\'s no description to write.') + except (OSError, IOError): + self.report_error(u'Cannot write description file ' + descfn) + return if self.params.get('writeannotations', False): - try: - annofn = filename + u'.annotations.xml' - self.report_writeannotations(annofn) - with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile: - annofile.write(info_dict['annotations']) - except (KeyError, TypeError): - self.report_warning(u'There are no annotations to write.') - except (OSError, IOError): - self.report_error(u'Cannot write annotations file: ' + annofn) - return + annofn = filename + u'.annotations.xml' + if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(annofn)): + self.to_screen(u'[info] Video annotations are already present') + else: + try: + self.to_screen(u'[info] Writing video annotations to: ' + annofn) + with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile: + annofile.write(info_dict['annotations']) + except (KeyError, TypeError): + self.report_warning(u'There are no annotations to write.') + except (OSError, IOError): + self.report_error(u'Cannot write annotations file: ' + annofn) + return subtitles_are_requested = any([self.params.get('writesubtitles', False), self.params.get('writeautomaticsub')]) @@ -809,38 +834,48 @@ class YoutubeDL(object): continue try: sub_filename = subtitles_filename(filename, sub_lang, sub_format) - self.report_writesubtitles(sub_filename) - with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile: - subfile.write(sub) + if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(sub_filename)): + self.to_screen(u'[info] Video subtitle %s.%s is already_present' % (sub_lang, sub_format)) + else: + self.to_screen(u'[info] Writing video subtitles to: ' + sub_filename) + with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8') as subfile: + subfile.write(sub) except (OSError, IOError): self.report_error(u'Cannot write subtitles file ' + descfn) return if self.params.get('writeinfojson', False): infofn = os.path.splitext(filename)[0] + u'.info.json' - self.report_writeinfojson(infofn) - try: - json_info_dict = dict((k, v) for k, v in info_dict.items() if not k in ['urlhandle']) - write_json_file(json_info_dict, encodeFilename(infofn)) - except (OSError, IOError): - self.report_error(u'Cannot write metadata to JSON file ' + infofn) - return + if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)): + self.to_screen(u'[info] Video description metadata is already present') + else: + self.to_screen(u'[info] Writing video description metadata as JSON to: ' + infofn) + try: + json_info_dict = dict((k, v) for k, v in info_dict.items() if not k in ['urlhandle']) + write_json_file(json_info_dict, encodeFilename(infofn)) + except (OSError, IOError): + self.report_error(u'Cannot write metadata to JSON file ' + infofn) + return if self.params.get('writethumbnail', False): if info_dict.get('thumbnail') is not None: thumb_format = determine_ext(info_dict['thumbnail'], u'jpg') thumb_filename = os.path.splitext(filename)[0] + u'.' + thumb_format - self.to_screen(u'[%s] %s: Downloading thumbnail ...' % - (info_dict['extractor'], info_dict['id'])) - try: - uf = compat_urllib_request.urlopen(info_dict['thumbnail']) - with open(thumb_filename, 'wb') as thumbf: - shutil.copyfileobj(uf, thumbf) - self.to_screen(u'[%s] %s: Writing thumbnail to: %s' % - (info_dict['extractor'], info_dict['id'], thumb_filename)) - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - self.report_warning(u'Unable to download thumbnail "%s": %s' % - (info_dict['thumbnail'], compat_str(err))) + if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(thumb_filename)): + self.to_screen(u'[%s] %s: Thumbnail is already present' % + (info_dict['extractor'], info_dict['id'])) + else: + self.to_screen(u'[%s] %s: Downloading thumbnail ...' % + (info_dict['extractor'], info_dict['id'])) + try: + uf = compat_urllib_request.urlopen(info_dict['thumbnail']) + with open(thumb_filename, 'wb') as thumbf: + shutil.copyfileobj(uf, thumbf) + self.to_screen(u'[%s] %s: Writing thumbnail to: %s' % + (info_dict['extractor'], info_dict['id'], thumb_filename)) + except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: + self.report_warning(u'Unable to download thumbnail "%s": %s' % + (info_dict['thumbnail'], compat_str(err))) if not self.params.get('skip_download', False): if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(filename)): diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 3e82cd637..c37d28c59 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -37,6 +37,7 @@ __authors__ = ( 'Anton Larionov', 'Takuya Tsuchida', 'Sergey M.', + 'Michael Orlitzky', ) __license__ = 'Public Domain' @@ -55,13 +56,13 @@ from .utils import ( compat_print, DateRange, decodeOption, - determine_ext, get_term_width, DownloadError, get_cachedir, MaxDownloadsReached, preferredencoding, SameFileError, + setproctitle, std_headers, write_string, ) @@ -193,13 +194,17 @@ def parseOpts(overrideArguments=None): type=float, default=None, help=optparse.SUPPRESS_HELP) general.add_option( '--bidi-workaround', dest='bidi_workaround', action='store_true', - help=u'Work around terminals that lack bidirectional text support. Requires fribidi executable in PATH') + help=u'Work around terminals that lack bidirectional text support. Requires bidiv or fribidi executable in PATH') - selection.add_option('--playlist-start', - dest='playliststart', metavar='NUMBER', help='playlist video to start at (default is %default)', default=1) - selection.add_option('--playlist-end', - dest='playlistend', metavar='NUMBER', help='playlist video to end at (default is last)', default=-1) + selection.add_option( + '--playlist-start', + dest='playliststart', metavar='NUMBER', default=1, type=int, + help='playlist video to start at (default is %default)') + selection.add_option( + '--playlist-end', + dest='playlistend', metavar='NUMBER', default=None, type=int, + help='playlist video to end at (default is last)') selection.add_option('--match-title', dest='matchtitle', metavar='REGEX',help='download only matching titles (regex or caseless sub-string)') selection.add_option('--reject-title', dest='rejecttitle', metavar='REGEX',help='skip download for matching titles (regex or caseless sub-string)') selection.add_option('--max-downloads', metavar='NUMBER', @@ -210,6 +215,14 @@ def parseOpts(overrideArguments=None): selection.add_option('--date', metavar='DATE', dest='date', help='download only videos uploaded in this date', default=None) selection.add_option('--datebefore', metavar='DATE', dest='datebefore', help='download only videos uploaded before this date', default=None) selection.add_option('--dateafter', metavar='DATE', dest='dateafter', help='download only videos uploaded after this date', default=None) + selection.add_option( + '--min-views', metavar='COUNT', dest='min_views', + default=None, type=int, + help="Do not download any videos with less than COUNT views",) + selection.add_option( + '--max-views', metavar='COUNT', dest='max_views', + default=None, type=int, + help="Do not download any videos with more than COUNT views",) selection.add_option('--no-playlist', action='store_true', dest='noplaylist', help='download only the currently playing video', default=False) selection.add_option('--age-limit', metavar='YEARS', dest='age_limit', help='download only videos suitable for the given age', @@ -290,6 +303,9 @@ def parseOpts(overrideArguments=None): verbosity.add_option('--get-description', action='store_true', dest='getdescription', help='simulate, quiet but print video description', default=False) + verbosity.add_option('--get-duration', + action='store_true', dest='getduration', + help='simulate, quiet but print video length', default=False) verbosity.add_option('--get-filename', action='store_true', dest='getfilename', help='simulate, quiet but print output filename', default=False) @@ -460,12 +476,15 @@ def parseOpts(overrideArguments=None): return parser, opts, args + def _real_main(argv=None): # Compatibility fixes for Windows if sys.platform == 'win32': # https://github.com/rg3/youtube-dl/issues/820 codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None) + setproctitle(u'youtube-dl') + parser, opts, args = parseOpts(argv) # Set user agent @@ -505,7 +524,6 @@ def _real_main(argv=None): for ie in sorted(extractors, key=lambda ie: ie.IE_NAME.lower()): compat_print(ie.IE_NAME + (' (CURRENTLY BROKEN)' if not ie._WORKING else '')) matchedUrls = [url for url in all_urls if ie.suitable(url)] - all_urls = [url for url in all_urls if url not in matchedUrls] for mu in matchedUrls: compat_print(u' ' + mu) sys.exit(0) @@ -560,18 +578,10 @@ def _real_main(argv=None): if numeric_buffersize is None: parser.error(u'invalid buffer size specified') opts.buffersize = numeric_buffersize - try: - opts.playliststart = int(opts.playliststart) - if opts.playliststart <= 0: - raise ValueError(u'Playlist start must be positive') - except (TypeError, ValueError): - parser.error(u'invalid playlist start number specified') - try: - opts.playlistend = int(opts.playlistend) - if opts.playlistend != -1 and (opts.playlistend <= 0 or opts.playlistend < opts.playliststart): - raise ValueError(u'Playlist end must be greater than playlist start') - except (TypeError, ValueError): - parser.error(u'invalid playlist end number specified') + if opts.playliststart <= 0: + raise ValueError(u'Playlist start must be positive') + if opts.playlistend not in (-1, None) and opts.playlistend < opts.playliststart: + raise ValueError(u'Playlist end must be greater than playlist start') if opts.extractaudio: if opts.audioformat not in ['best', 'aac', 'mp3', 'm4a', 'opus', 'vorbis', 'wav']: parser.error(u'invalid audio format specified') @@ -604,27 +614,30 @@ def _real_main(argv=None): or (opts.useid and u'%(id)s.%(ext)s') or (opts.autonumber and u'%(autonumber)s-%(id)s.%(ext)s') or u'%(title)s-%(id)s.%(ext)s') - if '%(ext)s' not in outtmpl and opts.extractaudio: + if not os.path.splitext(outtmpl)[1] and opts.extractaudio: parser.error(u'Cannot download a video and extract audio into the same' - u' file! Use "%%(ext)s" instead of %r' % - determine_ext(outtmpl, u'')) + u' file! Use "{0}.%(ext)s" instead of "{0}" as the output' + u' template'.format(outtmpl)) + + any_printing = opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.getduration or opts.dumpjson ydl_opts = { 'usenetrc': opts.usenetrc, 'username': opts.username, 'password': opts.password, 'videopassword': opts.videopassword, - 'quiet': (opts.quiet or opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.dumpjson), + 'quiet': (opts.quiet or any_printing), 'forceurl': opts.geturl, 'forcetitle': opts.gettitle, 'forceid': opts.getid, 'forcethumbnail': opts.getthumbnail, 'forcedescription': opts.getdescription, + 'forceduration': opts.getduration, 'forcefilename': opts.getfilename, 'forceformat': opts.getformat, 'forcejson': opts.dumpjson, 'simulate': opts.simulate, - 'skip_download': (opts.skip_download or opts.simulate or opts.geturl or opts.gettitle or opts.getid or opts.getthumbnail or opts.getdescription or opts.getfilename or opts.getformat or opts.dumpjson), + 'skip_download': (opts.skip_download or opts.simulate or any_printing), 'format': opts.format, 'format_limit': opts.format_limit, 'listformats': opts.listformats, @@ -668,6 +681,8 @@ def _real_main(argv=None): 'keepvideo': opts.keepvideo, 'min_filesize': opts.min_filesize, 'max_filesize': opts.max_filesize, + 'min_views': opts.min_views, + 'max_views': opts.max_views, 'daterange': date, 'cachedir': opts.cachedir, 'youtube_print_sig_code': opts.youtube_print_sig_code, diff --git a/youtube_dl/aes.py b/youtube_dl/aes.py index 9a0c93fa6..e9c5e2152 100644 --- a/youtube_dl/aes.py +++ b/youtube_dl/aes.py @@ -1,4 +1,4 @@ -__all__ = ['aes_encrypt', 'key_expansion', 'aes_ctr_decrypt', 'aes_decrypt_text'] +__all__ = ['aes_encrypt', 'key_expansion', 'aes_ctr_decrypt', 'aes_cbc_decrypt', 'aes_decrypt_text'] import base64 from math import ceil @@ -32,6 +32,31 @@ def aes_ctr_decrypt(data, key, counter): return decrypted_data +def aes_cbc_decrypt(data, key, iv): + """ + Decrypt with aes in CBC mode + + @param {int[]} data cipher + @param {int[]} key 16/24/32-Byte cipher key + @param {int[]} iv 16-Byte IV + @returns {int[]} decrypted data + """ + expanded_key = key_expansion(key) + block_count = int(ceil(float(len(data)) / BLOCK_SIZE_BYTES)) + + decrypted_data=[] + previous_cipher_block = iv + for i in range(block_count): + block = data[i*BLOCK_SIZE_BYTES : (i+1)*BLOCK_SIZE_BYTES] + block += [0]*(BLOCK_SIZE_BYTES - len(block)) + + decrypted_block = aes_decrypt(block, expanded_key) + decrypted_data += xor(decrypted_block, previous_cipher_block) + previous_cipher_block = block + decrypted_data = decrypted_data[:len(data)] + + return decrypted_data + def key_expansion(data): """ Generate key schedule @@ -75,7 +100,7 @@ def aes_encrypt(data, expanded_key): @returns {int[]} 16-Byte cipher """ rounds = len(expanded_key) // BLOCK_SIZE_BYTES - 1 - + data = xor(data, expanded_key[:BLOCK_SIZE_BYTES]) for i in range(1, rounds+1): data = sub_bytes(data) @@ -83,6 +108,26 @@ def aes_encrypt(data, expanded_key): if i != rounds: data = mix_columns(data) data = xor(data, expanded_key[i*BLOCK_SIZE_BYTES : (i+1)*BLOCK_SIZE_BYTES]) + + return data + +def aes_decrypt(data, expanded_key): + """ + Decrypt one block with aes + + @param {int[]} data 16-Byte cipher + @param {int[]} expanded_key 176/208/240-Byte expanded key + @returns {int[]} 16-Byte state + """ + rounds = len(expanded_key) // BLOCK_SIZE_BYTES - 1 + + for i in range(rounds, 0, -1): + data = xor(data, expanded_key[i*BLOCK_SIZE_BYTES : (i+1)*BLOCK_SIZE_BYTES]) + if i != rounds: + data = mix_columns_inv(data) + data = shift_rows_inv(data) + data = sub_bytes_inv(data) + data = xor(data, expanded_key[:BLOCK_SIZE_BYTES]) return data @@ -139,14 +184,69 @@ SBOX = (0x63, 0x7C, 0x77, 0x7B, 0xF2, 0x6B, 0x6F, 0xC5, 0x30, 0x01, 0x67, 0x2B, 0x70, 0x3E, 0xB5, 0x66, 0x48, 0x03, 0xF6, 0x0E, 0x61, 0x35, 0x57, 0xB9, 0x86, 0xC1, 0x1D, 0x9E, 0xE1, 0xF8, 0x98, 0x11, 0x69, 0xD9, 0x8E, 0x94, 0x9B, 0x1E, 0x87, 0xE9, 0xCE, 0x55, 0x28, 0xDF, 0x8C, 0xA1, 0x89, 0x0D, 0xBF, 0xE6, 0x42, 0x68, 0x41, 0x99, 0x2D, 0x0F, 0xB0, 0x54, 0xBB, 0x16) -MIX_COLUMN_MATRIX = ((2,3,1,1), - (1,2,3,1), - (1,1,2,3), - (3,1,1,2)) +SBOX_INV = (0x52, 0x09, 0x6a, 0xd5, 0x30, 0x36, 0xa5, 0x38, 0xbf, 0x40, 0xa3, 0x9e, 0x81, 0xf3, 0xd7, 0xfb, + 0x7c, 0xe3, 0x39, 0x82, 0x9b, 0x2f, 0xff, 0x87, 0x34, 0x8e, 0x43, 0x44, 0xc4, 0xde, 0xe9, 0xcb, + 0x54, 0x7b, 0x94, 0x32, 0xa6, 0xc2, 0x23, 0x3d, 0xee, 0x4c, 0x95, 0x0b, 0x42, 0xfa, 0xc3, 0x4e, + 0x08, 0x2e, 0xa1, 0x66, 0x28, 0xd9, 0x24, 0xb2, 0x76, 0x5b, 0xa2, 0x49, 0x6d, 0x8b, 0xd1, 0x25, + 0x72, 0xf8, 0xf6, 0x64, 0x86, 0x68, 0x98, 0x16, 0xd4, 0xa4, 0x5c, 0xcc, 0x5d, 0x65, 0xb6, 0x92, + 0x6c, 0x70, 0x48, 0x50, 0xfd, 0xed, 0xb9, 0xda, 0x5e, 0x15, 0x46, 0x57, 0xa7, 0x8d, 0x9d, 0x84, + 0x90, 0xd8, 0xab, 0x00, 0x8c, 0xbc, 0xd3, 0x0a, 0xf7, 0xe4, 0x58, 0x05, 0xb8, 0xb3, 0x45, 0x06, + 0xd0, 0x2c, 0x1e, 0x8f, 0xca, 0x3f, 0x0f, 0x02, 0xc1, 0xaf, 0xbd, 0x03, 0x01, 0x13, 0x8a, 0x6b, + 0x3a, 0x91, 0x11, 0x41, 0x4f, 0x67, 0xdc, 0xea, 0x97, 0xf2, 0xcf, 0xce, 0xf0, 0xb4, 0xe6, 0x73, + 0x96, 0xac, 0x74, 0x22, 0xe7, 0xad, 0x35, 0x85, 0xe2, 0xf9, 0x37, 0xe8, 0x1c, 0x75, 0xdf, 0x6e, + 0x47, 0xf1, 0x1a, 0x71, 0x1d, 0x29, 0xc5, 0x89, 0x6f, 0xb7, 0x62, 0x0e, 0xaa, 0x18, 0xbe, 0x1b, + 0xfc, 0x56, 0x3e, 0x4b, 0xc6, 0xd2, 0x79, 0x20, 0x9a, 0xdb, 0xc0, 0xfe, 0x78, 0xcd, 0x5a, 0xf4, + 0x1f, 0xdd, 0xa8, 0x33, 0x88, 0x07, 0xc7, 0x31, 0xb1, 0x12, 0x10, 0x59, 0x27, 0x80, 0xec, 0x5f, + 0x60, 0x51, 0x7f, 0xa9, 0x19, 0xb5, 0x4a, 0x0d, 0x2d, 0xe5, 0x7a, 0x9f, 0x93, 0xc9, 0x9c, 0xef, + 0xa0, 0xe0, 0x3b, 0x4d, 0xae, 0x2a, 0xf5, 0xb0, 0xc8, 0xeb, 0xbb, 0x3c, 0x83, 0x53, 0x99, 0x61, + 0x17, 0x2b, 0x04, 0x7e, 0xba, 0x77, 0xd6, 0x26, 0xe1, 0x69, 0x14, 0x63, 0x55, 0x21, 0x0c, 0x7d) +MIX_COLUMN_MATRIX = ((0x2,0x3,0x1,0x1), + (0x1,0x2,0x3,0x1), + (0x1,0x1,0x2,0x3), + (0x3,0x1,0x1,0x2)) +MIX_COLUMN_MATRIX_INV = ((0xE,0xB,0xD,0x9), + (0x9,0xE,0xB,0xD), + (0xD,0x9,0xE,0xB), + (0xB,0xD,0x9,0xE)) +RIJNDAEL_EXP_TABLE = (0x01, 0x03, 0x05, 0x0F, 0x11, 0x33, 0x55, 0xFF, 0x1A, 0x2E, 0x72, 0x96, 0xA1, 0xF8, 0x13, 0x35, + 0x5F, 0xE1, 0x38, 0x48, 0xD8, 0x73, 0x95, 0xA4, 0xF7, 0x02, 0x06, 0x0A, 0x1E, 0x22, 0x66, 0xAA, + 0xE5, 0x34, 0x5C, 0xE4, 0x37, 0x59, 0xEB, 0x26, 0x6A, 0xBE, 0xD9, 0x70, 0x90, 0xAB, 0xE6, 0x31, + 0x53, 0xF5, 0x04, 0x0C, 0x14, 0x3C, 0x44, 0xCC, 0x4F, 0xD1, 0x68, 0xB8, 0xD3, 0x6E, 0xB2, 0xCD, + 0x4C, 0xD4, 0x67, 0xA9, 0xE0, 0x3B, 0x4D, 0xD7, 0x62, 0xA6, 0xF1, 0x08, 0x18, 0x28, 0x78, 0x88, + 0x83, 0x9E, 0xB9, 0xD0, 0x6B, 0xBD, 0xDC, 0x7F, 0x81, 0x98, 0xB3, 0xCE, 0x49, 0xDB, 0x76, 0x9A, + 0xB5, 0xC4, 0x57, 0xF9, 0x10, 0x30, 0x50, 0xF0, 0x0B, 0x1D, 0x27, 0x69, 0xBB, 0xD6, 0x61, 0xA3, + 0xFE, 0x19, 0x2B, 0x7D, 0x87, 0x92, 0xAD, 0xEC, 0x2F, 0x71, 0x93, 0xAE, 0xE9, 0x20, 0x60, 0xA0, + 0xFB, 0x16, 0x3A, 0x4E, 0xD2, 0x6D, 0xB7, 0xC2, 0x5D, 0xE7, 0x32, 0x56, 0xFA, 0x15, 0x3F, 0x41, + 0xC3, 0x5E, 0xE2, 0x3D, 0x47, 0xC9, 0x40, 0xC0, 0x5B, 0xED, 0x2C, 0x74, 0x9C, 0xBF, 0xDA, 0x75, + 0x9F, 0xBA, 0xD5, 0x64, 0xAC, 0xEF, 0x2A, 0x7E, 0x82, 0x9D, 0xBC, 0xDF, 0x7A, 0x8E, 0x89, 0x80, + 0x9B, 0xB6, 0xC1, 0x58, 0xE8, 0x23, 0x65, 0xAF, 0xEA, 0x25, 0x6F, 0xB1, 0xC8, 0x43, 0xC5, 0x54, + 0xFC, 0x1F, 0x21, 0x63, 0xA5, 0xF4, 0x07, 0x09, 0x1B, 0x2D, 0x77, 0x99, 0xB0, 0xCB, 0x46, 0xCA, + 0x45, 0xCF, 0x4A, 0xDE, 0x79, 0x8B, 0x86, 0x91, 0xA8, 0xE3, 0x3E, 0x42, 0xC6, 0x51, 0xF3, 0x0E, + 0x12, 0x36, 0x5A, 0xEE, 0x29, 0x7B, 0x8D, 0x8C, 0x8F, 0x8A, 0x85, 0x94, 0xA7, 0xF2, 0x0D, 0x17, + 0x39, 0x4B, 0xDD, 0x7C, 0x84, 0x97, 0xA2, 0xFD, 0x1C, 0x24, 0x6C, 0xB4, 0xC7, 0x52, 0xF6, 0x01) +RIJNDAEL_LOG_TABLE = (0x00, 0x00, 0x19, 0x01, 0x32, 0x02, 0x1a, 0xc6, 0x4b, 0xc7, 0x1b, 0x68, 0x33, 0xee, 0xdf, 0x03, + 0x64, 0x04, 0xe0, 0x0e, 0x34, 0x8d, 0x81, 0xef, 0x4c, 0x71, 0x08, 0xc8, 0xf8, 0x69, 0x1c, 0xc1, + 0x7d, 0xc2, 0x1d, 0xb5, 0xf9, 0xb9, 0x27, 0x6a, 0x4d, 0xe4, 0xa6, 0x72, 0x9a, 0xc9, 0x09, 0x78, + 0x65, 0x2f, 0x8a, 0x05, 0x21, 0x0f, 0xe1, 0x24, 0x12, 0xf0, 0x82, 0x45, 0x35, 0x93, 0xda, 0x8e, + 0x96, 0x8f, 0xdb, 0xbd, 0x36, 0xd0, 0xce, 0x94, 0x13, 0x5c, 0xd2, 0xf1, 0x40, 0x46, 0x83, 0x38, + 0x66, 0xdd, 0xfd, 0x30, 0xbf, 0x06, 0x8b, 0x62, 0xb3, 0x25, 0xe2, 0x98, 0x22, 0x88, 0x91, 0x10, + 0x7e, 0x6e, 0x48, 0xc3, 0xa3, 0xb6, 0x1e, 0x42, 0x3a, 0x6b, 0x28, 0x54, 0xfa, 0x85, 0x3d, 0xba, + 0x2b, 0x79, 0x0a, 0x15, 0x9b, 0x9f, 0x5e, 0xca, 0x4e, 0xd4, 0xac, 0xe5, 0xf3, 0x73, 0xa7, 0x57, + 0xaf, 0x58, 0xa8, 0x50, 0xf4, 0xea, 0xd6, 0x74, 0x4f, 0xae, 0xe9, 0xd5, 0xe7, 0xe6, 0xad, 0xe8, + 0x2c, 0xd7, 0x75, 0x7a, 0xeb, 0x16, 0x0b, 0xf5, 0x59, 0xcb, 0x5f, 0xb0, 0x9c, 0xa9, 0x51, 0xa0, + 0x7f, 0x0c, 0xf6, 0x6f, 0x17, 0xc4, 0x49, 0xec, 0xd8, 0x43, 0x1f, 0x2d, 0xa4, 0x76, 0x7b, 0xb7, + 0xcc, 0xbb, 0x3e, 0x5a, 0xfb, 0x60, 0xb1, 0x86, 0x3b, 0x52, 0xa1, 0x6c, 0xaa, 0x55, 0x29, 0x9d, + 0x97, 0xb2, 0x87, 0x90, 0x61, 0xbe, 0xdc, 0xfc, 0xbc, 0x95, 0xcf, 0xcd, 0x37, 0x3f, 0x5b, 0xd1, + 0x53, 0x39, 0x84, 0x3c, 0x41, 0xa2, 0x6d, 0x47, 0x14, 0x2a, 0x9e, 0x5d, 0x56, 0xf2, 0xd3, 0xab, + 0x44, 0x11, 0x92, 0xd9, 0x23, 0x20, 0x2e, 0x89, 0xb4, 0x7c, 0xb8, 0x26, 0x77, 0x99, 0xe3, 0xa5, + 0x67, 0x4a, 0xed, 0xde, 0xc5, 0x31, 0xfe, 0x18, 0x0d, 0x63, 0x8c, 0x80, 0xc0, 0xf7, 0x70, 0x07) def sub_bytes(data): return [SBOX[x] for x in data] +def sub_bytes_inv(data): + return [SBOX_INV[x] for x in data] + def rotate(data): return data[1:] + [data[0]] @@ -160,30 +260,31 @@ def key_schedule_core(data, rcon_iteration): def xor(data1, data2): return [x^y for x, y in zip(data1, data2)] -def mix_column(data): +def rijndael_mul(a, b): + if(a==0 or b==0): + return 0 + return RIJNDAEL_EXP_TABLE[(RIJNDAEL_LOG_TABLE[a] + RIJNDAEL_LOG_TABLE[b]) % 0xFF] + +def mix_column(data, matrix): data_mixed = [] for row in range(4): mixed = 0 for column in range(4): - addend = data[column] - if MIX_COLUMN_MATRIX[row][column] in (2,3): - addend <<= 1 - if addend > 0xff: - addend &= 0xff - addend ^= 0x1b - if MIX_COLUMN_MATRIX[row][column] == 3: - addend ^= data[column] - mixed ^= addend & 0xff + # xor is (+) and (-) + mixed ^= rijndael_mul(data[column], matrix[row][column]) data_mixed.append(mixed) return data_mixed -def mix_columns(data): +def mix_columns(data, matrix=MIX_COLUMN_MATRIX): data_mixed = [] for i in range(4): column = data[i*4 : (i+1)*4] - data_mixed += mix_column(column) + data_mixed += mix_column(column, matrix) return data_mixed +def mix_columns_inv(data): + return mix_columns(data, MIX_COLUMN_MATRIX_INV) + def shift_rows(data): data_shifted = [] for column in range(4): @@ -191,6 +292,13 @@ def shift_rows(data): data_shifted.append( data[((column + row) & 0b11) * 4 + row] ) return data_shifted +def shift_rows_inv(data): + data_shifted = [] + for column in range(4): + for row in range(4): + data_shifted.append( data[((column - row) & 0b11) * 4 + row] ) + return data_shifted + def inc(data): data = data[:] # copy for i in range(len(data)-1,-1,-1): diff --git a/youtube_dl/extractor/__init__.py b/youtube_dl/extractor/__init__.py index 1149dc1ec..a39a1e2f4 100644 --- a/youtube_dl/extractor/__init__.py +++ b/youtube_dl/extractor/__init__.py @@ -1,6 +1,8 @@ -from .appletrailers import AppleTrailersIE +from .academicearth import AcademicEarthCourseIE from .addanime import AddAnimeIE from .anitube import AnitubeIE +from .aparat import AparatIE +from .appletrailers import AppleTrailersIE from .archiveorg import ArchiveOrgIE from .ard import ARDIE from .arte import ( @@ -13,6 +15,7 @@ from .arte import ( from .auengine import AUEngineIE from .bambuser import BambuserIE, BambuserChannelIE from .bandcamp import BandcampIE, BandcampAlbumIE +from .blinkx import BlinkxIE from .bliptv import BlipTVIE, BlipTVUserIE from .bloomberg import BloombergIE from .breakcom import BreakIE @@ -20,6 +23,8 @@ from .brightcove import BrightcoveIE from .c56 import C56IE from .canalplus import CanalplusIE from .canalc2 import Canalc2IE +from .cbs import CBSIE +from .channel9 import Channel9IE from .cinemassacre import CinemassacreIE from .clipfish import ClipfishIE from .clipsyndicate import ClipsyndicateIE @@ -28,6 +33,7 @@ from .collegehumor import CollegeHumorIE from .comedycentral import ComedyCentralIE, ComedyCentralShowsIE from .condenast import CondeNastIE from .criterion import CriterionIE +from .crunchyroll import CrunchyrollIE from .cspan import CSpanIE from .d8 import D8IE from .dailymotion import ( @@ -78,6 +84,10 @@ from .ina import InaIE from .infoq import InfoQIE from .instagram import InstagramIE from .internetvideoarchive import InternetVideoArchiveIE +from .ivi import ( + IviIE, + IviCompilationIE +) from .jeuxvideo import JeuxVideoIE from .jukebox import JukeboxIE from .justintv import JustinTVIE @@ -87,6 +97,7 @@ from .kickstarter import KickStarterIE from .keek import KeekIE from .liveleak import LiveLeakIE from .livestream import LivestreamIE, LivestreamOriginalIE +from .mdr import MDRIE from .metacafe import MetacafeIE from .metacritic import MetacriticIE from .mit import TechTVMITIE, MITIE @@ -111,9 +122,11 @@ from .orf import ORFIE from .pbs import PBSIE from .photobucket import PhotobucketIE from .podomatic import PodomaticIE +from .pornhd import PornHdIE from .pornhub import PornHubIE from .pornotube import PornotubeIE from .pyvideo import PyvideoIE +from .radiofrance import RadioFranceIE from .rbmaradio import RBMARadioIE from .redtube import RedTubeIE from .ringtv import RingTVIE diff --git a/youtube_dl/extractor/academicearth.py b/youtube_dl/extractor/academicearth.py new file mode 100644 index 000000000..ac05f8246 --- /dev/null +++ b/youtube_dl/extractor/academicearth.py @@ -0,0 +1,31 @@ +import re + +from .common import InfoExtractor + + +class AcademicEarthCourseIE(InfoExtractor): + _VALID_URL = r'^https?://(?:www\.)?academicearth\.org/(?:courses|playlists)/(?P[^?#/]+)' + IE_NAME = u'AcademicEarth:Course' + + def _real_extract(self, url): + m = re.match(self._VALID_URL, url) + playlist_id = m.group('id') + + webpage = self._download_webpage(url, playlist_id) + title = self._html_search_regex( + r'

(.*?)

', webpage, u'title') + description = self._html_search_regex( + r'

(.*?)

', + webpage, u'description', fatal=False) + urls = re.findall( + r'

', + webpage) + entries = [self.url_result(u) for u in urls] + + return { + '_type': 'playlist', + 'id': playlist_id, + 'title': title, + 'description': description, + 'entries': entries, + } diff --git a/youtube_dl/extractor/aparat.py b/youtube_dl/extractor/aparat.py new file mode 100644 index 000000000..7e93bc4df --- /dev/null +++ b/youtube_dl/extractor/aparat.py @@ -0,0 +1,56 @@ +#coding: utf-8 + +import re + +from .common import InfoExtractor +from ..utils import ( + ExtractorError, + HEADRequest, +) + + +class AparatIE(InfoExtractor): + _VALID_URL = r'^https?://(?:www\.)?aparat\.com/(?:v/|video/video/embed/videohash/)(?P[a-zA-Z0-9]+)' + + _TEST = { + u'url': u'http://www.aparat.com/v/wP8On', + u'file': u'wP8On.mp4', + u'md5': u'6714e0af7e0d875c5a39c4dc4ab46ad1', + u'info_dict': { + u"title": u"تیم گلکسی 11 - زومیت", + }, + #u'skip': u'Extremely unreliable', + } + + def _real_extract(self, url): + m = re.match(self._VALID_URL, url) + video_id = m.group('id') + + # Note: There is an easier-to-parse configuration at + # http://www.aparat.com/video/video/config/videohash/%video_id + # but the URL in there does not work + embed_url = (u'http://www.aparat.com/video/video/embed/videohash/' + + video_id + u'/vt/frame') + webpage = self._download_webpage(embed_url, video_id) + + video_urls = re.findall(r'fileList\[[0-9]+\]\s*=\s*"([^"]+)"', webpage) + for i, video_url in enumerate(video_urls): + req = HEADRequest(video_url) + res = self._request_webpage( + req, video_id, note=u'Testing video URL %d' % i, errnote=False) + if res: + break + else: + raise ExtractorError(u'No working video URLs found') + + title = self._search_regex(r'\s+title:\s*"([^"]+)"', webpage, u'title') + thumbnail = self._search_regex( + r'\s+image:\s*"([^"]+)"', webpage, u'thumbnail', fatal=False) + + return { + 'id': video_id, + 'title': title, + 'url': video_url, + 'ext': 'mp4', + 'thumbnail': thumbnail, + } diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py index 4b7bef775..9254fbfe0 100644 --- a/youtube_dl/extractor/arte.py +++ b/youtube_dl/extractor/arte.py @@ -266,20 +266,6 @@ class ArteTVDDCIE(ArteTVPlus7IE): IE_NAME = u'arte.tv:ddc' _VALID_URL = r'http?://ddc\.arte\.tv/(?Pemission|folge)/(?P.+)' - _TEST = { - u'url': u'http://ddc.arte.tv/folge/neues-aus-mauretanien', - u'file': u'049881-009_PLUS7-D.flv', - u'info_dict': { - u'title': u'Mit offenen Karten', - u'description': u'md5:57929b0eaeddeb8a0c983f58e9ebd3b6', - u'upload_date': u'20131207', - }, - u'params': { - # rtmp download - u'skip_download': True, - }, - } - def _real_extract(self, url): video_id, lang = self._extract_url_info(url) if lang == 'folge': diff --git a/youtube_dl/extractor/blinkx.py b/youtube_dl/extractor/blinkx.py new file mode 100644 index 000000000..144ce64cc --- /dev/null +++ b/youtube_dl/extractor/blinkx.py @@ -0,0 +1,90 @@ +import datetime +import json +import re + +from .common import InfoExtractor +from ..utils import ( + remove_start, +) + + +class BlinkxIE(InfoExtractor): + _VALID_URL = r'^(?:https?://(?:www\.)blinkx\.com/#?ce/|blinkx:)(?P[^?]+)' + _IE_NAME = u'blinkx' + + _TEST = { + u'url': u'http://www.blinkx.com/ce/8aQUy7GVFYgFzpKhT0oqsilwOGFRVXk3R1ZGWWdGenBLaFQwb3FzaWx3OGFRVXk3R1ZGWWdGenB', + u'file': u'8aQUy7GV.mp4', + u'md5': u'2e9a07364af40163a908edbf10bb2492', + u'info_dict': { + u"title": u"Police Car Rolls Away", + u"uploader": u"stupidvideos.com", + u"upload_date": u"20131215", + u"description": u"A police car gently rolls away from a fight. Maybe it felt weird being around a confrontation and just had to get out of there!", + u"duration": 14.886, + u"thumbnails": [{ + "width": 100, + "height": 76, + "url": "http://cdn.blinkx.com/stream/b/41/StupidVideos/20131215/1873969261/1873969261_tn_0.jpg", + }], + }, + } + + def _real_extract(self, url): + m = re.match(self._VALID_URL, url) + video_id = m.group('id') + display_id = video_id[:8] + + api_url = (u'https://apib4.blinkx.com/api.php?action=play_video&' + + u'video=%s' % video_id) + data_json = self._download_webpage(api_url, display_id) + data = json.loads(data_json)['api']['results'][0] + dt = datetime.datetime.fromtimestamp(data['pubdate_epoch']) + upload_date = dt.strftime('%Y%m%d') + + duration = None + thumbnails = [] + formats = [] + for m in data['media']: + if m['type'] == 'jpg': + thumbnails.append({ + 'url': m['link'], + 'width': int(m['w']), + 'height': int(m['h']), + }) + elif m['type'] == 'original': + duration = m['d'] + elif m['type'] == 'youtube': + yt_id = m['link'] + self.to_screen(u'Youtube video detected: %s' % yt_id) + return self.url_result(yt_id, 'Youtube', video_id=yt_id) + elif m['type'] in ('flv', 'mp4'): + vcodec = remove_start(m['vcodec'], 'ff') + acodec = remove_start(m['acodec'], 'ff') + format_id = (u'%s-%sk-%s' % + (vcodec, + (int(m['vbr']) + int(m['abr'])) // 1000, + m['w'])) + formats.append({ + 'format_id': format_id, + 'url': m['link'], + 'vcodec': vcodec, + 'acodec': acodec, + 'abr': int(m['abr']) // 1000, + 'vbr': int(m['vbr']) // 1000, + 'width': int(m['w']), + 'height': int(m['h']), + }) + formats.sort(key=lambda f: (f['width'], f['vbr'], f['abr'])) + + return { + 'id': display_id, + 'fullid': video_id, + 'title': data['title'], + 'formats': formats, + 'uploader': data['channel_name'], + 'upload_date': upload_date, + 'description': data.get('description'), + 'thumbnails': thumbnails, + 'duration': duration, + } diff --git a/youtube_dl/extractor/bliptv.py b/youtube_dl/extractor/bliptv.py index 5e33a69df..0e63208df 100644 --- a/youtube_dl/extractor/bliptv.py +++ b/youtube_dl/extractor/bliptv.py @@ -70,13 +70,14 @@ class BlipTVIE(InfoExtractor): info = None urlh = self._request_webpage(request, None, False, u'unable to download video info webpage') + if urlh.headers.get('Content-Type', '').startswith('video/'): # Direct download basename = url.split('/')[-1] title,ext = os.path.splitext(basename) title = title.decode('UTF-8') ext = ext.replace('.', '') self.report_direct_download(title) - info = { + return { 'id': title, 'url': url, 'uploader': None, @@ -85,49 +86,47 @@ class BlipTVIE(InfoExtractor): 'ext': ext, 'urlhandle': urlh } - if info is None: # Regular URL - try: - json_code_bytes = urlh.read() - json_code = json_code_bytes.decode('utf-8') - except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: - raise ExtractorError(u'Unable to read video info webpage: %s' % compat_str(err)) - - try: - json_data = json.loads(json_code) - if 'Post' in json_data: - data = json_data['Post'] - else: - data = json_data - - upload_date = datetime.datetime.strptime(data['datestamp'], '%m-%d-%y %H:%M%p').strftime('%Y%m%d') - if 'additionalMedia' in data: - formats = sorted(data['additionalMedia'], key=lambda f: int(f['media_height'])) - best_format = formats[-1] - video_url = best_format['url'] - else: - video_url = data['media']['url'] - umobj = re.match(self._URL_EXT, video_url) - if umobj is None: - raise ValueError('Can not determine filename extension') - ext = umobj.group(1) - - info = { - 'id': compat_str(data['item_id']), - 'url': video_url, - 'uploader': data['display_name'], - 'upload_date': upload_date, - 'title': data['title'], - 'ext': ext, - 'format': data['media']['mimeType'], - 'thumbnail': data['thumbnailUrl'], - 'description': data['description'], - 'player_url': data['embedUrl'], - 'user_agent': 'iTunes/10.6.1', - } - except (ValueError,KeyError) as err: - raise ExtractorError(u'Unable to parse video information: %s' % repr(err)) - - return [info] + + try: + json_code_bytes = urlh.read() + json_code = json_code_bytes.decode('utf-8') + except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: + raise ExtractorError(u'Unable to read video info webpage: %s' % compat_str(err)) + + try: + json_data = json.loads(json_code) + if 'Post' in json_data: + data = json_data['Post'] + else: + data = json_data + + upload_date = datetime.datetime.strptime(data['datestamp'], '%m-%d-%y %H:%M%p').strftime('%Y%m%d') + if 'additionalMedia' in data: + formats = sorted(data['additionalMedia'], key=lambda f: int(f['media_height'])) + best_format = formats[-1] + video_url = best_format['url'] + else: + video_url = data['media']['url'] + umobj = re.match(self._URL_EXT, video_url) + if umobj is None: + raise ValueError('Can not determine filename extension') + ext = umobj.group(1) + + return { + 'id': compat_str(data['item_id']), + 'url': video_url, + 'uploader': data['display_name'], + 'upload_date': upload_date, + 'title': data['title'], + 'ext': ext, + 'format': data['media']['mimeType'], + 'thumbnail': data['thumbnailUrl'], + 'description': data['description'], + 'player_url': data['embedUrl'], + 'user_agent': 'iTunes/10.6.1', + } + except (ValueError, KeyError) as err: + raise ExtractorError(u'Unable to parse video information: %s' % repr(err)) class BlipTVUserIE(InfoExtractor): diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py index b1b7526ca..f7f0041c0 100644 --- a/youtube_dl/extractor/brightcove.py +++ b/youtube_dl/extractor/brightcove.py @@ -26,7 +26,7 @@ class BrightcoveIE(InfoExtractor): # From http://www.8tv.cat/8aldia/videos/xavier-sala-i-martin-aquesta-tarda-a-8-al-dia/ u'url': u'http://c.brightcove.com/services/viewer/htmlFederated?playerID=1654948606001&flashID=myExperience&%40videoPlayer=2371591881001', u'file': u'2371591881001.mp4', - u'md5': u'8eccab865181d29ec2958f32a6a754f5', + u'md5': u'5423e113865d26e40624dce2e4b45d95', u'note': u'Test Brightcove downloads and detection in GenericIE', u'info_dict': { u'title': u'Xavier Sala i Martín: “Un banc que no presta és un banc zombi que no serveix per a res”', diff --git a/youtube_dl/extractor/cbs.py b/youtube_dl/extractor/cbs.py new file mode 100644 index 000000000..ac0315853 --- /dev/null +++ b/youtube_dl/extractor/cbs.py @@ -0,0 +1,30 @@ +import re + +from .common import InfoExtractor + + +class CBSIE(InfoExtractor): + _VALID_URL = r'https?://(?:www\.)?cbs\.com/shows/[^/]+/video/(?P[^/]+)/.*' + + _TEST = { + u'url': u'http://www.cbs.com/shows/garth-brooks/video/_u7W953k6la293J7EPTd9oHkSPs6Xn6_/connect-chat-feat-garth-brooks/', + u'file': u'4JUVEwq3wUT7.flv', + u'info_dict': { + u'title': u'Connect Chat feat. Garth Brooks', + u'description': u'Connect with country music singer Garth Brooks, as he chats with fans on Wednesday November 27, 2013. Be sure to tune in to Garth Brooks: Live from Las Vegas, Friday November 29, at 9/8c on CBS!', + u'duration': 1495, + }, + u'params': { + # rtmp download + u'skip_download': True, + }, + } + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + video_id = mobj.group('id') + webpage = self._download_webpage(url, video_id) + real_id = self._search_regex( + r"video\.settings\.pid\s*=\s*'([^']+)';", + webpage, u'real video ID') + return self.url_result(u'theplatform:%s' % real_id) diff --git a/youtube_dl/extractor/channel9.py b/youtube_dl/extractor/channel9.py new file mode 100644 index 000000000..ae70ea229 --- /dev/null +++ b/youtube_dl/extractor/channel9.py @@ -0,0 +1,267 @@ +# encoding: utf-8 + +import re + +from .common import InfoExtractor +from ..utils import ExtractorError + +class Channel9IE(InfoExtractor): + ''' + Common extractor for channel9.msdn.com. + + The type of provided URL (video or playlist) is determined according to + meta Search.PageType from web page HTML rather than URL itself, as it is + not always possible to do. + ''' + IE_DESC = u'Channel 9' + IE_NAME = u'channel9' + _VALID_URL = r'^https?://(?:www\.)?channel9\.msdn\.com/(?P.+)/?' + + _TESTS = [ + { + u'url': u'http://channel9.msdn.com/Events/TechEd/Australia/2013/KOS002', + u'file': u'Events_TechEd_Australia_2013_KOS002.mp4', + u'md5': u'bbd75296ba47916b754e73c3a4bbdf10', + u'info_dict': { + u'title': u'Developer Kick-Off Session: Stuff We Love', + u'description': u'md5:c08d72240b7c87fcecafe2692f80e35f', + u'duration': 4576, + u'thumbnail': u'http://media.ch9.ms/ch9/9d51/03902f2d-fc97-4d3c-b195-0bfe15a19d51/KOS002_220.jpg', + u'session_code': u'KOS002', + u'session_day': u'Day 1', + u'session_room': u'Arena 1A', + u'session_speakers': [ u'Ed Blankenship', u'Andrew Coates', u'Brady Gaster', u'Patrick Klug', u'Mads Kristensen' ], + }, + }, + { + u'url': u'http://channel9.msdn.com/posts/Self-service-BI-with-Power-BI-nuclear-testing', + u'file': u'posts_Self-service-BI-with-Power-BI-nuclear-testing.mp4', + u'md5': u'b43ee4529d111bc37ba7ee4f34813e68', + u'info_dict': { + u'title': u'Self-service BI with Power BI - nuclear testing', + u'description': u'md5:d1e6ecaafa7fb52a2cacdf9599829f5b', + u'duration': 1540, + u'thumbnail': u'http://media.ch9.ms/ch9/87e1/0300391f-a455-4c72-bec3-4422f19287e1/selfservicenuk_512.jpg', + u'authors': [ u'Mike Wilmot' ], + }, + } + ] + + _RSS_URL = 'http://channel9.msdn.com/%s/RSS' + + # Sorted by quality + _known_formats = ['MP3', 'MP4', 'Mid Quality WMV', 'Mid Quality MP4', 'High Quality WMV', 'High Quality MP4'] + + def _restore_bytes(self, formatted_size): + if not formatted_size: + return 0 + m = re.match(r'^(?P\d+(?:\.\d+)?)\s+(?P[a-zA-Z]+)', formatted_size) + if not m: + return 0 + units = m.group('units') + try: + exponent = [u'B', u'KB', u'MB', u'GB', u'TB', u'PB', u'EB', u'ZB', u'YB'].index(units.upper()) + except ValueError: + return 0 + size = float(m.group('size')) + return int(size * (1024 ** exponent)) + + def _formats_from_html(self, html): + FORMAT_REGEX = r''' + (?x) + [^"]+)">(?P[^<]+)\s* + \((?P[^\)]+)\)\s* + (?:\s* +

File\s+size

\s*(?P.*?)\s* + )? # File size part may be missing + ''' + # Extract known formats + formats = [{'url': x.group('url'), + 'format_id': x.group('quality'), + 'format_note': x.group('note'), + 'format': '%s (%s)' % (x.group('quality'), x.group('note')), + 'filesize': self._restore_bytes(x.group('filesize')), # File size is approximate + } for x in list(re.finditer(FORMAT_REGEX, html)) if x.group('quality') in self._known_formats] + # Sort according to known formats list + formats.sort(key=lambda fmt: self._known_formats.index(fmt['format_id'])) + return formats + + def _extract_title(self, html): + title = self._html_search_meta(u'title', html, u'title') + if title is None: + title = self._og_search_title(html) + TITLE_SUFFIX = u' (Channel 9)' + if title is not None and title.endswith(TITLE_SUFFIX): + title = title[:-len(TITLE_SUFFIX)] + return title + + def _extract_description(self, html): + DESCRIPTION_REGEX = r'''(?sx) + \s* + \s* + (?P.+?)\s* + \s* + + ''' + m = re.search(DESCRIPTION_REGEX, html) + if m is not None: + return m.group('description') + return self._html_search_meta(u'description', html, u'description') + + def _extract_duration(self, html): + m = re.search(r'data-video_duration="(?P\d{2}):(?P\d{2}):(?P\d{2})"', html) + return ((int(m.group('hours')) * 60 * 60) + (int(m.group('minutes')) * 60) + int(m.group('seconds'))) if m else None + + def _extract_slides(self, html): + m = re.search(r'Slides', html) + return m.group('slidesurl') if m is not None else None + + def _extract_zip(self, html): + m = re.search(r'Zip', html) + return m.group('zipurl') if m is not None else None + + def _extract_avg_rating(self, html): + m = re.search(r'

Avg Rating: (?P[^<]+)

', html) + return float(m.group('avgrating')) if m is not None else 0 + + def _extract_rating_count(self, html): + m = re.search(r'
\((?P[^<]+)\)
', html) + return int(self._fix_count(m.group('ratingcount'))) if m is not None else 0 + + def _extract_view_count(self, html): + m = re.search(r'
  • \s*(?P[^<]+) Views\s*
    • ', html) + return int(self._fix_count(m.group('viewcount'))) if m is not None else 0 + + def _extract_comment_count(self, html): + m = re.search(r'
  • \s*\s*(?P[^<]+) Comments\s*\s*
    • ', html) + return int(self._fix_count(m.group('commentcount'))) if m is not None else 0 + + def _fix_count(self, count): + return int(str(count).replace(',', '')) if count is not None else None + + def _extract_authors(self, html): + m = re.search(r'(?s)
  • (.*?)
    • ', html) + if m is None: + return None + return re.findall(r'([^<]+)', m.group(1)) + + def _extract_session_code(self, html): + m = re.search(r'
  • \s*(?P.+?)\s*
    • ', html) + return m.group('code') if m is not None else None + + def _extract_session_day(self, html): + m = re.search(r'
  • \s*(?P[^<]+)\s*
    • ', html) + return m.group('day') if m is not None else None + + def _extract_session_room(self, html): + m = re.search(r'
  • \s*(?P.+?)\s*
    • ', html) + return m.group('room') if m is not None else None + + def _extract_session_speakers(self, html): + return re.findall(r'([^<]+)', html) + + def _extract_content(self, html, content_path): + # Look for downloadable content + formats = self._formats_from_html(html) + slides = self._extract_slides(html) + zip_ = self._extract_zip(html) + + # Nothing to download + if len(formats) == 0 and slides is None and zip_ is None: + self._downloader.report_warning(u'None of recording, slides or zip are available for %s' % content_path) + return + + # Extract meta + title = self._extract_title(html) + description = self._extract_description(html) + thumbnail = self._og_search_thumbnail(html) + duration = self._extract_duration(html) + avg_rating = self._extract_avg_rating(html) + rating_count = self._extract_rating_count(html) + view_count = self._extract_view_count(html) + comment_count = self._extract_comment_count(html) + + common = {'_type': 'video', + 'id': content_path, + 'description': description, + 'thumbnail': thumbnail, + 'duration': duration, + 'avg_rating': avg_rating, + 'rating_count': rating_count, + 'view_count': view_count, + 'comment_count': comment_count, + } + + result = [] + + if slides is not None: + d = common.copy() + d.update({ 'title': title + '-Slides', 'url': slides }) + result.append(d) + + if zip_ is not None: + d = common.copy() + d.update({ 'title': title + '-Zip', 'url': zip_ }) + result.append(d) + + if len(formats) > 0: + d = common.copy() + d.update({ 'title': title, 'formats': formats }) + result.append(d) + + return result + + def _extract_entry_item(self, html, content_path): + contents = self._extract_content(html, content_path) + if contents is None: + return contents + + authors = self._extract_authors(html) + + for content in contents: + content['authors'] = authors + + return contents + + def _extract_session(self, html, content_path): + contents = self._extract_content(html, content_path) + if contents is None: + return contents + + session_meta = {'session_code': self._extract_session_code(html), + 'session_day': self._extract_session_day(html), + 'session_room': self._extract_session_room(html), + 'session_speakers': self._extract_session_speakers(html), + } + + for content in contents: + content.update(session_meta) + + return contents + + def _extract_list(self, content_path): + rss = self._download_xml(self._RSS_URL % content_path, content_path, u'Downloading RSS') + entries = [self.url_result(session_url.text, 'Channel9') + for session_url in rss.findall('./channel/item/link')] + title_text = rss.find('./channel/title').text + return self.playlist_result(entries, content_path, title_text) + + def _real_extract(self, url): + mobj = re.match(self._VALID_URL, url) + content_path = mobj.group('contentpath') + + webpage = self._download_webpage(url, content_path, u'Downloading web page') + + page_type_m = re.search(r'', webpage) + if page_type_m is None: + raise ExtractorError(u'Search.PageType not found, don\'t know how to process this page', expected=True) + + page_type = page_type_m.group('pagetype') + if page_type == 'List': # List page, may contain list of 'item'-like objects + return self._extract_list(content_path) + elif page_type == 'Entry.Item': # Any 'item'-like page, may contain downloadable content + return self._extract_entry_item(webpage, content_path) + elif page_type == 'Session': # Event session page, may contain downloadable content + return self._extract_session(webpage, content_path) + else: + raise ExtractorError(u'Unexpected Search.PageType %s' % page_type, expected=True) \ No newline at end of file diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index 69a083b68..ba46a7bc7 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -18,6 +18,7 @@ from ..utils import ( sanitize_filename, unescapeHTML, ) +_NO_DEFAULT = object() class InfoExtractor(object): @@ -34,15 +35,39 @@ class InfoExtractor(object): The dictionaries must include the following fields: id: Video identifier. - url: Final video URL. title: Video title, unescaped. - ext: Video filename extension. - Instead of url and ext, formats can also specified. + Additionally, it must contain either a formats entry or url and ext: + + formats: A list of dictionaries for each format available, it must + be ordered from worst to best quality. Potential fields: + * url Mandatory. The URL of the video file + * ext Will be calculated from url if missing + * format A human-readable description of the format + ("mp4 container with h264/opus"). + Calculated from the format_id, width, height. + and format_note fields if missing. + * format_id A short description of the format + ("mp4_h264_opus" or "19") + * format_note Additional info about the format + ("3D" or "DASH video") + * width Width of the video, if known + * height Height of the video, if known + * abr Average audio bitrate in KBit/s + * acodec Name of the audio codec in use + * vbr Average video bitrate in KBit/s + * vcodec Name of the video codec in use + * filesize The number of bytes, if known in advance + * player_url SWF Player URL (used for rtmpdump). + url: Final video URL. + ext: Video filename extension. + format: The video format, defaults to ext (used for --get-format) + player_url: SWF Player URL (used for rtmpdump). + urlhandle: [internal] The urlHandle to be used to download the file, + like returned by urllib.request.urlopen The following fields are optional: - format: The video format, defaults to ext (used for --get-format) thumbnails: A list of dictionaries (with the entries "resolution" and "url") for the varying thumbnails thumbnail: Full URL to a video thumbnail image. @@ -51,35 +76,14 @@ class InfoExtractor(object): upload_date: Video upload date (YYYYMMDD). uploader_id: Nickname or id of the video uploader. location: Physical location of the video. - player_url: SWF Player URL (used for rtmpdump). subtitles: The subtitle file contents as a dictionary in the format {language: subtitles}. + duration: Length of the video in seconds, as an integer. view_count: How many users have watched the video on the platform. like_count: Number of positive ratings of the video dislike_count: Number of negative ratings of the video comment_count: Number of comments on the video - urlhandle: [internal] The urlHandle to be used to download the file, - like returned by urllib.request.urlopen age_limit: Age restriction for the video, as an integer (years) - formats: A list of dictionaries for each format available, it must - be ordered from worst to best quality. Potential fields: - * url Mandatory. The URL of the video file - * ext Will be calculated from url if missing - * format A human-readable description of the format - ("mp4 container with h264/opus"). - Calculated from the format_id, width, height. - and format_note fields if missing. - * format_id A short description of the format - ("mp4_h264_opus" or "19") - * format_note Additional info about the format - ("3D" or "DASH video") - * width Width of the video, if known - * height Height of the video, if known - * abr Average audio bitrate in KBit/s - * acodec Name of the audio codec in use - * vbr Average video bitrate in KBit/s - * vcodec Name of the video codec in use - * filesize The number of bytes, if known in advance webpage_url: The url to the video webpage, if given to youtube-dl it should allow to get the same result again. (It will be set by YoutubeDL if it's missing) @@ -166,6 +170,8 @@ class InfoExtractor(object): try: return self._downloader.urlopen(url_or_request) except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: + if errnote is False: + return False if errnote is None: errnote = u'Unable to download webpage' errmsg = u'%s: %s' % (errnote, compat_str(err)) @@ -259,7 +265,8 @@ class InfoExtractor(object): self.to_screen(u'Logging in') #Methods for following #608 - def url_result(self, url, ie=None, video_id=None): + @staticmethod + def url_result(url, ie=None, video_id=None): """Returns a url that points to a page that should be processed""" #TODO: ie should be the class used for getting the info video_info = {'_type': 'url', @@ -268,7 +275,8 @@ class InfoExtractor(object): if video_id is not None: video_info['id'] = video_id return video_info - def playlist_result(self, entries, playlist_id=None, playlist_title=None): + @staticmethod + def playlist_result(entries, playlist_id=None, playlist_title=None): """Returns a playlist""" video_info = {'_type': 'playlist', 'entries': entries} @@ -278,7 +286,7 @@ class InfoExtractor(object): video_info['title'] = playlist_title return video_info - def _search_regex(self, pattern, string, name, default=None, fatal=True, flags=0): + def _search_regex(self, pattern, string, name, default=_NO_DEFAULT, fatal=True, flags=0): """ Perform a regex search on the given string, using a single or a list of patterns returning the first matching group. @@ -292,7 +300,7 @@ class InfoExtractor(object): mobj = re.search(p, string, flags) if mobj: break - if sys.stderr.isatty() and os.name != 'nt': + if os.name != 'nt' and sys.stderr.isatty(): _name = u'\033[0;34m%s\033[0m' % name else: _name = name @@ -300,7 +308,7 @@ class InfoExtractor(object): if mobj: # return the first matching group return next(g for g in mobj.groups() if g is not None) - elif default is not None: + elif default is not _NO_DEFAULT: return default elif fatal: raise RegexNotFoundError(u'Unable to extract %s' % _name) @@ -309,7 +317,7 @@ class InfoExtractor(object): u'please report this issue on http://yt-dl.org/bug' % _name) return None - def _html_search_regex(self, pattern, string, name, default=None, fatal=True, flags=0): + def _html_search_regex(self, pattern, string, name, default=_NO_DEFAULT, fatal=True, flags=0): """ Like _search_regex, but strips HTML tags and unescapes entities. """ diff --git a/youtube_dl/extractor/crunchyroll.py b/youtube_dl/extractor/crunchyroll.py new file mode 100644 index 000000000..2b66bddbb --- /dev/null +++ b/youtube_dl/extractor/crunchyroll.py @@ -0,0 +1,171 @@ +# encoding: utf-8 +import re, base64, zlib +from hashlib import sha1 +from math import pow, sqrt, floor +from .common import InfoExtractor +from ..utils import ( + ExtractorError, + compat_urllib_parse, + compat_urllib_request, + bytes_to_intlist, + intlist_to_bytes, + unified_strdate, + clean_html, +) +from ..aes import ( + aes_cbc_decrypt, + inc, +) + +class CrunchyrollIE(InfoExtractor): + _VALID_URL = r'(?:https?://)?(?:www\.)?(?Pcrunchyroll\.com/[^/]*/[^/?&]*?(?P[0-9]+))(?:[/?&]|$)' + _TESTS = [{ + u'url': u'http://www.crunchyroll.com/wanna-be-the-strongest-in-the-world/episode-1-an-idol-wrestler-is-born-645513', + u'file': u'645513.flv', + #u'md5': u'b1639fd6ddfaa43788c85f6d1dddd412', + u'info_dict': { + u'title': u'Wanna be the Strongest in the World Episode 1 – An Idol-Wrestler is Born!', + u'description': u'md5:2d17137920c64f2f49981a7797d275ef', + u'thumbnail': u'http://img1.ak.crunchyroll.com/i/spire1-tmb/20c6b5e10f1a47b10516877d3c039cae1380951166_full.jpg', + u'uploader': u'Yomiuri Telecasting Corporation (YTV)', + u'upload_date': u'20131013', + }, + u'params': { + # rtmp + u'skip_download': True, + }, + }] + + _FORMAT_IDS = { + u'360': (u'60', u'106'), + u'480': (u'61', u'106'), + u'720': (u'62', u'106'), + u'1080': (u'80', u'108'), + } + + def _decrypt_subtitles(self, data, iv, id): + data = bytes_to_intlist(data) + iv = bytes_to_intlist(iv) + id = int(id) + + def obfuscate_key_aux(count, modulo, start): + output = list(start) + for _ in range(count): + output.append(output[-1] + output[-2]) + # cut off start values + output = output[2:] + output = list(map(lambda x: x % modulo + 33, output)) + return output + + def obfuscate_key(key): + num1 = int(floor(pow(2, 25) * sqrt(6.9))) + num2 = (num1 ^ key) << 5 + num3 = key ^ num1 + num4 = num3 ^ (num3 >> 3) ^ num2 + prefix = intlist_to_bytes(obfuscate_key_aux(20, 97, (1, 2))) + shaHash = bytes_to_intlist(sha1(prefix + str(num4).encode(u'ascii')).digest()) + # Extend 160 Bit hash to 256 Bit + return shaHash + [0] * 12 + + key = obfuscate_key(id) + class Counter: + __value = iv + def next_value(self): + temp = self.__value + self.__value = inc(self.__value) + return temp + decrypted_data = intlist_to_bytes(aes_cbc_decrypt(data, key, iv)) + return zlib.decompress(decrypted_data) + + def _convert_subtitles_to_srt(self, subtitles): + i=1 + output = u'' + for start, end, text in re.findall(r']*?start="([^"]+)" [^>]*?end="([^"]+)" [^>]*?text="([^"]+)"[^>]*?>', subtitles): + start = start.replace(u'.', u',') + end = end.replace(u'.', u',') + text = clean_html(text) + text = text.replace(u'\\N', u'\n') + if not text: + continue + output += u'%d\n%s --> %s\n%s\n\n' % (i, start, end, text) + i+=1 + return output + + def _real_extract(self,url): + mobj = re.match(self._VALID_URL, url) + + webpage_url = u'http://www.' + mobj.group('url') + video_id = mobj.group(u'video_id') + webpage = self._download_webpage(webpage_url, video_id) + note_m = self._html_search_regex(r'
    (.+?)
    ', webpage, u'trailer-notice', default=u'') + if note_m: + raise ExtractorError(note_m) + + video_title = self._html_search_regex(r']*>(.+?)

    ', webpage, u'video_title', flags=re.DOTALL) + video_title = re.sub(r' {2,}', u' ', video_title) + video_description = self._html_search_regex(r'"description":"([^"]+)', webpage, u'video_description', default=u'') + if not video_description: + video_description = None + video_upload_date = self._html_search_regex(r'
    Availability for free users:(.+?)
    ', webpage, u'video_upload_date', fatal=False, flags=re.DOTALL) + if video_upload_date: + video_upload_date = unified_strdate(video_upload_date) + video_uploader = self._html_search_regex(r'
    \s*Publisher:(.+?)
    ', webpage, u'video_uploader', fatal=False, flags=re.DOTALL) + + playerdata_url = compat_urllib_parse.unquote(self._html_search_regex(r'"config_url":"([^"]+)', webpage, u'playerdata_url')) + playerdata_req = compat_urllib_request.Request(playerdata_url) + playerdata_req.data = compat_urllib_parse.urlencode({u'current_page': webpage_url}) + playerdata_req.add_header(u'Content-Type', u'application/x-www-form-urlencoded') + playerdata = self._download_webpage(playerdata_req, video_id, note=u'Downloading media info') + + stream_id = self._search_regex(r'([^<]+)', playerdata, u'stream_id') + video_thumbnail = self._search_regex(r'([^<]+)', playerdata, u'thumbnail', fatal=False) + + formats = [] + for fmt in re.findall(r'\?p([0-9]{3,4})=1', webpage): + stream_quality, stream_format = self._FORMAT_IDS[fmt] + video_format = fmt+u'p' + streamdata_req = compat_urllib_request.Request(u'http://www.crunchyroll.com/xml/') + # urlencode doesn't work! + streamdata_req.data = u'req=RpcApiVideoEncode%5FGetStreamInfo&video%5Fencode%5Fquality='+stream_quality+u'&media%5Fid='+stream_id+u'&video%5Fformat='+stream_format + streamdata_req.add_header(u'Content-Type', u'application/x-www-form-urlencoded') + streamdata_req.add_header(u'Content-Length', str(len(streamdata_req.data))) + streamdata = self._download_webpage(streamdata_req, video_id, note=u'Downloading media info for '+video_format) + video_url = self._search_regex(r'([^<]+)', streamdata, u'video_url') + video_play_path = self._search_regex(r'([^<]+)', streamdata, u'video_play_path') + formats.append({ + u'url': video_url, + u'play_path': video_play_path, + u'ext': 'flv', + u'format': video_format, + u'format_id': video_format, + }) + + subtitles = {} + for sub_id, sub_name in re.findall(r'\?ssid=([0-9]+)" title="([^"]+)', webpage): + sub_page = self._download_webpage(u'http://www.crunchyroll.com/xml/?req=RpcApiSubtitle_GetXml&subtitle_script_id='+sub_id,\ + video_id, note=u'Downloading subtitles for '+sub_name) + id = self._search_regex(r'id=\'([0-9]+)', sub_page, u'subtitle_id', fatal=False) + iv = self._search_regex(r'([^<]+)', sub_page, u'subtitle_iv', fatal=False) + data = self._search_regex(r'([^<]+)', sub_page, u'subtitle_data', fatal=False) + if not id or not iv or not data: + continue + id = int(id) + iv = base64.b64decode(iv) + data = base64.b64decode(data) + + subtitle = self._decrypt_subtitles(data, iv, id).decode(u'utf-8') + lang_code = self._search_regex(r'lang_code=\'([^\']+)', subtitle, u'subtitle_lang_code', fatal=False) + if not lang_code: + continue + subtitles[lang_code] = self._convert_subtitles_to_srt(subtitle) + + return { + u'id': video_id, + u'title': video_title, + u'description': video_description, + u'thumbnail': video_thumbnail, + u'uploader': video_uploader, + u'upload_date': video_upload_date, + u'subtitles': subtitles, + u'formats': formats, + } diff --git a/youtube_dl/extractor/dailymotion.py b/youtube_dl/extractor/dailymotion.py index aea7e557e..6685c94a3 100644 --- a/youtube_dl/extractor/dailymotion.py +++ b/youtube_dl/extractor/dailymotion.py @@ -28,7 +28,7 @@ class DailymotionBaseInfoExtractor(InfoExtractor): class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor): """Information Extractor for Dailymotion""" - _VALID_URL = r'(?i)(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/(?:embed/)?video/([^/]+)' + _VALID_URL = r'(?i)(?:https?://)?(?:(www|touch)\.)?dailymotion\.[a-z]{2,3}/(?:(embed|#)/)?video/(?P[^/?_]+)' IE_NAME = u'dailymotion' _FORMATS = [ @@ -81,7 +81,7 @@ class DailymotionIE(DailymotionBaseInfoExtractor, SubtitlesInfoExtractor): # Extract id and simplified title from URL mobj = re.match(self._VALID_URL, url) - video_id = mobj.group(1).split('_')[0].split('?')[0] + video_id = mobj.group('id') url = 'http://www.dailymotion.com/video/%s' % video_id diff --git a/youtube_dl/extractor/daum.py b/youtube_dl/extractor/daum.py index d418ce4a8..4876ecb48 100644 --- a/youtube_dl/extractor/daum.py +++ b/youtube_dl/extractor/daum.py @@ -9,7 +9,7 @@ from ..utils import ( class DaumIE(InfoExtractor): - _VALID_URL = r'https?://tvpot\.daum\.net/.*?clipid=(?P\d+)' + _VALID_URL = r'https?://(?:m\.)?tvpot\.daum\.net/.*?clipid=(?P\d+)' IE_NAME = u'daum.net' _TEST = { diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py index 3b210710e..4556079c8 100644 --- a/youtube_dl/extractor/facebook.py +++ b/youtube_dl/extractor/facebook.py @@ -17,7 +17,7 @@ from ..utils import ( class FacebookIE(InfoExtractor): """Information Extractor for Facebook""" - _VALID_URL = r'^(?:https?://)?(?:\w+\.)?facebook\.com/(?:video/video|photo)\.php\?(?:.*?)v=(?P\d+)(?:.*)' + _VALID_URL = r'^(?:https?://)?(?:\w+\.)?facebook\.com/(?:[^#?]*#!/)?(?:video/video|photo)\.php\?(?:.*?)v=(?P\d+)(?:.*)' _LOGIN_URL = 'https://www.facebook.com/login.php?next=http%3A%2F%2Ffacebook.com%2Fhome.php&login_attempt=1' _CHECKPOINT_URL = 'https://www.facebook.com/checkpoint/?next=http%3A%2F%2Ffacebook.com%2Fhome.php&_fb_noscript=1' _NETRC_MACHINE = 'facebook' @@ -27,7 +27,7 @@ class FacebookIE(InfoExtractor): u'file': u'120708114770723.mp4', u'md5': u'48975a41ccc4b7a581abd68651c1a5a8', u'info_dict': { - u"duration": 279, + u"duration": 279, u"title": u"PEOPLE ARE AWESOME 2013" } } diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py index 216e03218..7a14c98f9 100644 --- a/youtube_dl/extractor/generic.py +++ b/youtube_dl/extractor/generic.py @@ -11,10 +11,14 @@ from ..utils import ( compat_urlparse, ExtractorError, + HEADRequest, smuggle_url, unescapeHTML, + unified_strdate, + url_basename, ) from .brightcove import BrightcoveIE +from .ooyala import OoyalaIE class GenericIE(InfoExtractor): @@ -71,6 +75,27 @@ class GenericIE(InfoExtractor): u'skip_download': True, }, }, + # Direct link to a video + { + u'url': u'http://media.w3.org/2010/05/sintel/trailer.mp4', + u'file': u'trailer.mp4', + u'md5': u'67d406c2bcb6af27fa886f31aa934bbe', + u'info_dict': { + u'id': u'trailer', + u'title': u'trailer', + u'upload_date': u'20100513', + } + }, + # ooyala video + { + u'url': u'http://www.rollingstone.com/music/videos/norwegian-dj-cashmere-cat-goes-spartan-on-with-me-premiere-20131219', + u'md5': u'5644c6ca5d5782c1d0d350dad9bd840c', + u'info_dict': { + u'id': u'BwY2RxaTrTkslxOfcan0UCf0YqyvWysJ', + u'ext': u'mp4', + u'title': u'2cc213299525360.mov', #that's what we get + }, + }, ] def report_download_webpage(self, video_id): @@ -83,23 +108,20 @@ class GenericIE(InfoExtractor): """Report information extraction.""" self._downloader.to_screen(u'[redirect] Following redirect to %s' % new_url) - def _test_redirect(self, url): + def _send_head(self, url): """Check if it is a redirect, like url shorteners, in case return the new url.""" - class HeadRequest(compat_urllib_request.Request): - def get_method(self): - return "HEAD" class HEADRedirectHandler(compat_urllib_request.HTTPRedirectHandler): """ Subclass the HTTPRedirectHandler to make it use our - HeadRequest also on the redirected URL + HEADRequest also on the redirected URL """ def redirect_request(self, req, fp, code, msg, headers, newurl): if code in (301, 302, 303, 307): newurl = newurl.replace(' ', '%20') newheaders = dict((k,v) for k,v in req.headers.items() if k.lower() not in ("content-length", "content-type")) - return HeadRequest(newurl, + return HEADRequest(newurl, headers=newheaders, origin_req_host=req.get_origin_req_host(), unverifiable=True) @@ -128,32 +150,49 @@ class GenericIE(InfoExtractor): compat_urllib_request.HTTPErrorProcessor, compat_urllib_request.HTTPSHandler]: opener.add_handler(handler()) - response = opener.open(HeadRequest(url)) + response = opener.open(HEADRequest(url)) if response is None: raise ExtractorError(u'Invalid URL protocol') - new_url = response.geturl() - - if url == new_url: - return False - - self.report_following_redirect(new_url) - return new_url + return response def _real_extract(self, url): parsed_url = compat_urlparse.urlparse(url) if not parsed_url.scheme: self._downloader.report_warning('The url doesn\'t specify the protocol, trying with http') return self.url_result('http://' + url) + video_id = os.path.splitext(url.split('/')[-1])[0] try: - new_url = self._test_redirect(url) - if new_url: - return [self.url_result(new_url)] + response = self._send_head(url) + + # Check for redirect + new_url = response.geturl() + if url != new_url: + self.report_following_redirect(new_url) + return self.url_result(new_url) + + # Check for direct link to a video + content_type = response.headers.get('Content-Type', '') + m = re.match(r'^(?Paudio|video|application(?=/ogg$))/(?P.+)$', content_type) + if m: + upload_date = response.headers.get('Last-Modified') + if upload_date: + upload_date = unified_strdate(upload_date) + return { + 'id': video_id, + 'title': os.path.splitext(url_basename(url))[0], + 'formats': [{ + 'format_id': m.group('format_id'), + 'url': url, + 'vcodec': u'none' if m.group('type') == 'audio' else None + }], + 'upload_date': upload_date, + } + except compat_urllib_error.HTTPError: # This may be a stupid server that doesn't like HEAD, our UA, or so pass - video_id = url.split('/')[-1] try: webpage = self._download_webpage(url, video_id) except ValueError: @@ -183,7 +222,7 @@ class GenericIE(InfoExtractor): self.to_screen(u'Brightcove video detected.') return self.url_result(bc_url, 'Brightcove') - # Look for embedded Vimeo player + # Look for embedded (iframe) Vimeo player mobj = re.search( r']+?src="(https?://player.vimeo.com/video/.+?)"', webpage) if mobj: @@ -191,9 +230,18 @@ class GenericIE(InfoExtractor): surl = smuggle_url(player_url, {'Referer': url}) return self.url_result(surl, 'Vimeo') + # Look for embedded (swf embed) Vimeo player + mobj = re.search( + r']+?src="(https?://(?:www\.)?vimeo.com/moogaloop.swf.+?)"', webpage) + if mobj: + return self.url_result(mobj.group(1), 'Vimeo') + # Look for embedded YouTube player - matches = re.findall( - r']+?src=(["\'])(?P(?:https?:)?//(?:www\.)?youtube\.com/embed/.+?)\1', webpage) + matches = re.findall(r'''(?x) + (?:]+?src=|embedSWF\(\s*) + (["\'])(?P(?:https?:)?//(?:www\.)?youtube\.com/ + (?:embed|v)/.+?) + \1''', webpage) if matches: urlrs = [self.url_result(unescapeHTML(tuppl[1]), 'Youtube') for tuppl in matches] @@ -222,6 +270,18 @@ class GenericIE(InfoExtractor): 'id': video_id, } + # Look for embedded blip.tv player + mobj = re.search(r']*https?://api.blip.tv/\w+/redirect/\w+/(\d+)', webpage) + if mobj: + return self.url_result('http://blip.tv/seo/-'+mobj.group(1), 'BlipTV') + mobj = re.search(r'<(?:iframe|embed|object)\s[^>]*https?://(?:\w+\.)?blip.tv/(?:play/|api\.swf#)([a-zA-Z0-9]+)', webpage) + if mobj: + player_url = 'http://blip.tv/play/%s.x?p=1' % mobj.group(1) + player_page = self._download_webpage(player_url, mobj.group(1)) + blip_video_id = self._search_regex(r'data-episode-id="(\d+)', player_page, u'blip_video_id', fatal=False) + if blip_video_id: + return self.url_result('http://blip.tv/seo/-'+blip_video_id, 'BlipTV') + # Look for Bandcamp pages with custom domain mobj = re.search(r']*?content="(.*?bandcamp\.com.*?)"', webpage) if mobj is not None: @@ -229,6 +289,22 @@ class GenericIE(InfoExtractor): # Don't set the extractor because it can be a track url or an album return self.url_result(burl) + # Look for embedded Vevo player + mobj = re.search( + r']+?src=(["\'])(?P(?:https?:)?//(?:cache\.)?vevo\.com/.+?)\1', webpage) + if mobj is not None: + return self.url_result(mobj.group('url')) + + # Look for Ooyala videos + mobj = re.search(r'player.ooyala.com/[^"?]+\?[^"]*?(?:embedCode|ec)=([^"&]+)', webpage) + if mobj is not None: + return OoyalaIE._build_url_result(mobj.group(1)) + + # Look for Aparat videos + mobj = re.search(r'