Move Collegehumor IE into its own file

[youtube-dl.git] / youtube_dl / FileDownloader.py

diff --git a/youtube_dl/FileDownloader.py b/youtube_dl/FileDownloader.py
index eb68d94781c210de0fbec2a21b47aacfde81bc58..b3a07617cd9553b4bbaf4ea632f65332e8ff0ebf 100644 (file)
--- a/youtube_dl/FileDownloader.py
+++ b/youtube_dl/FileDownloader.py
@@ -1,8 +1,3 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-from __future__ import absolute_import
-

 import math
 import io
 import os
 import math
 import io
 import os


@@ -83,7 +78,6 @@ class FileDownloader(object):
     writeinfojson:     Write the video description to a .info.json file
     writethumbnail:    Write the thumbnail image to a file
     writesubtitles:    Write the video subtitles to a file
     writeinfojson:     Write the video description to a .info.json file
     writethumbnail:    Write the thumbnail image to a file
     writesubtitles:    Write the video subtitles to a file

-    onlysubtitles:     Downloads only the subtitles of the video

     allsubtitles:      Downloads all the subtitles of the video
     listsubtitles:     Lists all available subtitles for the video
     subtitlesformat:   Subtitle format [sbv/srt] (default=srt)
     allsubtitles:      Downloads all the subtitles of the video
     listsubtitles:     Lists all available subtitles for the video
     subtitlesformat:   Subtitle format [sbv/srt] (default=srt)


@@ -93,6 +87,7 @@ class FileDownloader(object):
     min_filesize:      Skip files smaller than this size
     max_filesize:      Skip files larger than this size
     daterange:         A DateRange object, download only if the upload_date is in the range.
     min_filesize:      Skip files smaller than this size
     max_filesize:      Skip files larger than this size
     daterange:         A DateRange object, download only if the upload_date is in the range.

+    skip_download:     Skip the actual download of the video file

     """
 
     params = None
     """
 
     params = None


@@ -322,6 +317,9 @@ class FileDownloader(object):
         filetime = timeconvert(timestr)
         if filetime is None:
             return filetime
         filetime = timeconvert(timestr)
         if filetime is None:
             return filetime

+        # Ignore obviously invalid dates
+        if filetime == 0:
+            return

         try:
             os.utime(filename, (time.time(), filetime))
         except:
         try:
             os.utime(filename, (time.time(), filetime))
         except:


@@ -436,10 +434,11 @@ class FileDownloader(object):
                 return u'[download] %s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
         return None
         
                 return u'[download] %s upload date is not in range %s' % (date_from_str(date).isoformat(), dateRange)
         return None
         

-    def extract_info(self, url, download=True, ie_key=None):
+    def extract_info(self, url, download=True, ie_key=None, extra_info={}):

         '''
         Returns a list with a dictionary for each video we find.
         If 'download', also downloads the videos.
         '''
         Returns a list with a dictionary for each video we find.
         If 'download', also downloads the videos.

+        extra_info is a dict containing the extra values to add to each result

          '''
         
         if ie_key:
          '''
         
         if ie_key:


@@ -463,10 +462,14 @@ class FileDownloader(object):
                     break
                 if isinstance(ie_result, list):
                     # Backwards compatibility: old IE result format
                     break
                 if isinstance(ie_result, list):
                     # Backwards compatibility: old IE result format

+                    for result in ie_result:
+                        result.update(extra_info)

                     ie_result = {
                         '_type': 'compat_list',
                         'entries': ie_result,
                     }
                     ie_result = {
                         '_type': 'compat_list',
                         'entries': ie_result,
                     }

+                else:
+                    ie_result.update(extra_info)

                 if 'extractor' not in ie_result:
                     ie_result['extractor'] = ie.IE_NAME
                 return self.process_ie_result(ie_result, download=download)
                 if 'extractor' not in ie_result:
                     ie_result['extractor'] = ie.IE_NAME
                 return self.process_ie_result(ie_result, download=download)


@@ -482,7 +485,7 @@ class FileDownloader(object):
         else:
             self.report_error(u'no suitable InfoExtractor: %s' % url)
         
         else:
             self.report_error(u'no suitable InfoExtractor: %s' % url)
         

-    def process_ie_result(self, ie_result, download=True):
+    def process_ie_result(self, ie_result, download=True, extra_info={}):

         """
         Take the result of the ie(may be modified) and resolve all unresolved
         references (URLs, playlist items).
         """
         Take the result of the ie(may be modified) and resolve all unresolved
         references (URLs, playlist items).


@@ -501,7 +504,12 @@ class FileDownloader(object):
                 self.process_info(ie_result)
             return ie_result
         elif result_type == 'url':
                 self.process_info(ie_result)
             return ie_result
         elif result_type == 'url':

-            return self.extract_info(ie_result['url'], download, ie_key=ie_result.get('ie_key'))
+            # We have to add extra_info to the results because it may be
+            # contained in a playlist
+            return self.extract_info(ie_result['url'],
+                                     download,
+                                     ie_key=ie_result.get('ie_key'),
+                                     extra_info=extra_info)

         elif result_type == 'playlist':
             # We process each entry in the playlist
             playlist = ie_result.get('title', None) or ie_result.get('id', None)
         elif result_type == 'playlist':
             # We process each entry in the playlist
             playlist = ie_result.get('title', None) or ie_result.get('id', None)


@@ -525,9 +533,18 @@ class FileDownloader(object):
 
             for i,entry in enumerate(entries,1):
                 self.to_screen(u'[download] Downloading video #%s of %s' %(i, n_entries))
 
             for i,entry in enumerate(entries,1):
                 self.to_screen(u'[download] Downloading video #%s of %s' %(i, n_entries))

-                entry['playlist'] = playlist
-                entry['playlist_index'] = i + playliststart
-                entry_result = self.process_ie_result(entry, download=download)
+                extra = {
+                         'playlist': playlist, 
+                         'playlist_index': i + playliststart,
+                         }
+                if not 'extractor' in entry:
+                    # We set the extractor, if it's an url it will be set then to
+                    # the new extractor, but if it's already a video we must make
+                    # sure it's present: see issue #877
+                    entry['extractor'] = ie_result['extractor']
+                entry_result = self.process_ie_result(entry,
+                                                      download=download,
+                                                      extra_info=extra)

                 playlist_results.append(entry_result)
             ie_result['entries'] = playlist_results
             return ie_result
                 playlist_results.append(entry_result)
             ie_result['entries'] = playlist_results
             return ie_result


@@ -597,7 +614,7 @@ class FileDownloader(object):
 
         try:
             dn = os.path.dirname(encodeFilename(filename))
 
         try:
             dn = os.path.dirname(encodeFilename(filename))

-            if dn != '' and not os.path.exists(dn): # dn is already encoded
+            if dn != '' and not os.path.exists(dn):

                 os.makedirs(dn)
         except (OSError, IOError) as err:
             self.report_error(u'unable to create directory ' + compat_str(err))
                 os.makedirs(dn)
         except (OSError, IOError) as err:
             self.report_error(u'unable to create directory ' + compat_str(err))


@@ -630,8 +647,6 @@ class FileDownloader(object):
                 except (OSError, IOError):
                     self.report_error(u'Cannot write subtitles file ' + descfn)
                     return
                 except (OSError, IOError):
                     self.report_error(u'Cannot write subtitles file ' + descfn)
                     return

-            if self.params.get('onlysubtitles', False):
-                return 

 
         if self.params.get('allsubtitles', False) and 'subtitles' in info_dict and info_dict['subtitles']:
             subtitles = info_dict['subtitles']
 
         if self.params.get('allsubtitles', False) and 'subtitles' in info_dict and info_dict['subtitles']:
             subtitles = info_dict['subtitles']


@@ -649,8 +664,6 @@ class FileDownloader(object):
                     except (OSError, IOError):
                         self.report_error(u'Cannot write subtitles file ' + descfn)
                         return
                     except (OSError, IOError):
                         self.report_error(u'Cannot write subtitles file ' + descfn)
                         return

-            if self.params.get('onlysubtitles', False):
-                return 

 
         if self.params.get('writeinfojson', False):
             infofn = filename + u'.info.json'
 
         if self.params.get('writeinfojson', False):
             infofn = filename + u'.info.json'


@@ -738,7 +751,7 @@ class FileDownloader(object):
             except (IOError, OSError):
                 self.report_warning(u'Unable to remove downloaded video file')
 
             except (IOError, OSError):
                 self.report_warning(u'Unable to remove downloaded video file')
 

-    def _download_with_rtmpdump(self, filename, url, player_url, page_url, play_path):
+    def _download_with_rtmpdump(self, filename, url, player_url, page_url, play_path, tc_url):

         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)
 
         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)
 


@@ -748,18 +761,21 @@ class FileDownloader(object):
         except (OSError, IOError):
             self.report_error(u'RTMP download detected but "rtmpdump" could not be run')
             return False
         except (OSError, IOError):
             self.report_error(u'RTMP download detected but "rtmpdump" could not be run')
             return False

+        verbosity_option = '--verbose' if self.params.get('verbose', False) else '--quiet'

 
         # Download using rtmpdump. rtmpdump returns exit code 2 when
         # the connection was interrumpted and resuming appears to be
         # possible. This is part of rtmpdump's normal usage, AFAIK.
 
         # Download using rtmpdump. rtmpdump returns exit code 2 when
         # the connection was interrumpted and resuming appears to be
         # possible. This is part of rtmpdump's normal usage, AFAIK.

-        basic_args = ['rtmpdump', '-q', '-r', url, '-o', tmpfilename]
+        basic_args = ['rtmpdump', verbosity_option, '-r', url, '-o', tmpfilename]

         if player_url is not None:
         if player_url is not None:

-            basic_args += ['-W', player_url]
+            basic_args += ['--swfVfy', player_url]

         if page_url is not None:
             basic_args += ['--pageUrl', page_url]
         if play_path is not None:
         if page_url is not None:
             basic_args += ['--pageUrl', page_url]
         if play_path is not None:

-            basic_args += ['-y', play_path]
-        args = basic_args + [[], ['-e', '-k', '1']][self.params.get('continuedl', False)]
+            basic_args += ['--playpath', play_path]
+        if tc_url is not None:
+            basic_args += ['--tcUrl', url]
+        args = basic_args + [[], ['--resume', '--skip', '1']][self.params.get('continuedl', False)]

         if self.params.get('verbose', False):
             try:
                 import pipes
         if self.params.get('verbose', False):
             try:
                 import pipes


@@ -797,6 +813,37 @@ class FileDownloader(object):
             self.report_error(u'rtmpdump exited with code %d' % retval)
             return False
 
             self.report_error(u'rtmpdump exited with code %d' % retval)
             return False
 

+    def _download_with_mplayer(self, filename, url):
+        self.report_destination(filename)
+        tmpfilename = self.temp_name(filename)
+
+        args = ['mplayer', '-really-quiet', '-vo', 'null', '-vc', 'dummy', '-dumpstream', '-dumpfile', tmpfilename, url]
+        # Check for mplayer first
+        try:
+            subprocess.call(['mplayer', '-h'], stdout=(open(os.path.devnull, 'w')), stderr=subprocess.STDOUT)
+        except (OSError, IOError):
+            self.report_error(u'MMS or RTSP download detected but "%s" could not be run' % args[0] )
+            return False
+
+        # Download using mplayer. 
+        retval = subprocess.call(args)
+        if retval == 0:
+            fsize = os.path.getsize(encodeFilename(tmpfilename))
+            self.to_screen(u'\r[%s] %s bytes' % (args[0], fsize))
+            self.try_rename(tmpfilename, filename)
+            self._hook_progress({
+                'downloaded_bytes': fsize,
+                'total_bytes': fsize,
+                'filename': filename,
+                'status': 'finished',
+            })
+            return True
+        else:
+            self.to_stderr(u"\n")
+            self.report_error(u'mplayer exited with code %d' % retval)
+            return False
+
+

     def _do_download(self, filename, info_dict):
         url = info_dict['url']
 
     def _do_download(self, filename, info_dict):
         url = info_dict['url']
 


@@ -814,7 +861,12 @@ class FileDownloader(object):
             return self._download_with_rtmpdump(filename, url,
                                                 info_dict.get('player_url', None),
                                                 info_dict.get('page_url', None),
             return self._download_with_rtmpdump(filename, url,
                                                 info_dict.get('player_url', None),
                                                 info_dict.get('page_url', None),

-                                                info_dict.get('play_path', None))
+                                                info_dict.get('play_path', None),
+                                                info_dict.get('tc_url', None))
+
+        # Attempt to download using mplayer
+        if url.startswith('mms') or url.startswith('rtsp'):
+            return self._download_with_mplayer(filename, url)

 
         tmpfilename = self.temp_name(filename)
         stream = None
 
         tmpfilename = self.temp_name(filename)
         stream = None