youtube-dl

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3 # Author: Ricardo Garcia Gonzalez
   4 # Author: Danny Colligan
   5 # License: Public domain code
   6 import htmlentitydefs
   7 import httplib
   8 import locale
   9 import math
  10 import netrc
  11 import os
  12 import os.path
  13 import re
  14 import socket
  15 import string
  16 import sys
  17 import time
  18 import urllib
  19 import urllib2
  20
  21 std_headers = {
  22         'User-Agent': 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US; rv:1.9.1.2) Gecko/20090729 Firefox/3.5.2',
  23         'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
  24         'Accept': 'text/xml,application/xml,application/xhtml+xml,text/html;q=0.9,text/plain;q=0.8,image/png,*/*;q=0.5',
  25         'Accept-Language': 'en-us,en;q=0.5',
  26 }
  27
  28 simple_title_chars = string.ascii_letters.decode('ascii') + string.digits.decode('ascii')
  29
  30 def preferredencoding():
  31         """Get preferred encoding.
  32
  33         Returns the best encoding scheme for the system, based on
  34         locale.getpreferredencoding() and some further tweaks.
  35         """
  36         try:
  37                 pref = locale.getpreferredencoding()
  38                 # Mac OSX systems have this problem sometimes
  39                 if pref == '':
  40                         return 'UTF-8'
  41                 return pref
  42         except:
  43                 sys.stderr.write('WARNING: problem obtaining preferred encoding. Falling back to UTF-8.\n')
  44                 return 'UTF-8'
  45
  46 class DownloadError(Exception):
  47         """Download Error exception.
  48
  49         This exception may be thrown by FileDownloader objects if they are not
  50         configured to continue on errors. They will contain the appropriate
  51         error message.
  52         """
  53         pass
  54
  55 class SameFileError(Exception):
  56         """Same File exception.
  57
  58         This exception will be thrown by FileDownloader objects if they detect
  59         multiple files would have to be downloaded to the same file on disk.
  60         """
  61         pass
  62
  63 class PostProcessingError(Exception):
  64         """Post Processing exception.
  65
  66         This exception may be raised by PostProcessor's .run() method to
  67         indicate an error in the postprocessing task.
  68         """
  69         pass
  70
  71 class UnavailableFormatError(Exception):
  72         """Unavailable Format exception.
  73
  74         This exception will be thrown when a video is requested
  75         in a format that is not available for that video.
  76         """
  77         pass
  78
  79 class ContentTooShortError(Exception):
  80         """Content Too Short exception.
  81
  82         This exception may be raised by FileDownloader objects when a file they
  83         download is too small for what the server announced first, indicating
  84         the connection was probably interrupted.
  85         """
  86         # Both in bytes
  87         downloaded = None
  88         expected = None
  89
  90         def __init__(self, downloaded, expected):
  91                 self.downloaded = downloaded
  92                 self.expected = expected
  93
  94 class FileDownloader(object):
  95         """File Downloader class.
  96
  97         File downloader objects are the ones responsible of downloading the
  98         actual video file and writing it to disk if the user has requested
  99         it, among some other tasks. In most cases there should be one per
 100         program. As, given a video URL, the downloader doesn't know how to
 101         extract all the needed information, task that InfoExtractors do, it
 102         has to pass the URL to one of them.
 103
 104         For this, file downloader objects have a method that allows
 105         InfoExtractors to be registered in a given order. When it is passed
 106         a URL, the file downloader handles it to the first InfoExtractor it
 107         finds that reports being able to handle it. The InfoExtractor extracts
 108         all the information about the video or videos the URL refers to, and
 109         asks the FileDownloader to process the video information, possibly
 110         downloading the video.
 111
 112         File downloaders accept a lot of parameters. In order not to saturate
 113         the object constructor with arguments, it receives a dictionary of
 114         options instead. These options are available through the params
 115         attribute for the InfoExtractors to use. The FileDownloader also
 116         registers itself as the downloader in charge for the InfoExtractors
 117         that are added to it, so this is a "mutual registration".
 118
 119         Available options:
 120
 121         username:       Username for authentication purposes.
 122         password:       Password for authentication purposes.
 123         usenetrc:       Use netrc for authentication instead.
 124         quiet:          Do not print messages to stdout.
 125         forceurl:       Force printing final URL.
 126         forcetitle:     Force printing title.
 127         simulate:       Do not download the video files.
 128         format:         Video format code.
 129         outtmpl:        Template for output names.
 130         ignoreerrors:   Do not stop on download errors.
 131         ratelimit:      Download speed limit, in bytes/sec.
 132         nooverwrites:   Prevent overwriting files.
 133         continuedl:     Try to continue downloads if possible.
 134         """
 135
 136         params = None
 137         _ies = []
 138         _pps = []
 139         _download_retcode = None
 140
 141         def __init__(self, params):
 142                 """Create a FileDownloader object with the given options."""
 143                 self._ies = []
 144                 self._pps = []
 145                 self._download_retcode = 0
 146                 self.params = params
 147
 148         @staticmethod
 149         def pmkdir(filename):
 150                 """Create directory components in filename. Similar to Unix "mkdir -p"."""
 151                 components = filename.split(os.sep)
 152                 aggregate = [os.sep.join(components[0:x]) for x in xrange(1, len(components))]
 153                 aggregate = ['%s%s' % (x, os.sep) for x in aggregate] # Finish names with separator
 154                 for dir in aggregate:
 155                         if not os.path.exists(dir):
 156                                 os.mkdir(dir)
 157
 158         @staticmethod
 159         def format_bytes(bytes):
 160                 if bytes is None:
 161                         return 'N/A'
 162                 if type(bytes) is str:
 163                         bytes = float(bytes)
 164                 if bytes == 0.0:
 165                         exponent = 0
 166                 else:
 167                         exponent = long(math.log(bytes, 1024.0))
 168                 suffix = 'bkMGTPEZY'[exponent]
 169                 converted = float(bytes) / float(1024**exponent)
 170                 return '%.2f%s' % (converted, suffix)
 171
 172         @staticmethod
 173         def calc_percent(byte_counter, data_len):
 174                 if data_len is None:
 175                         return '---.-%'
 176                 return '%6s' % ('%3.1f%%' % (float(byte_counter) / float(data_len) * 100.0))
 177
 178         @staticmethod
 179         def calc_eta(start, now, total, current):
 180                 if total is None:
 181                         return '--:--'
 182                 dif = now - start
 183                 if current == 0 or dif < 0.001: # One millisecond
 184                         return '--:--'
 185                 rate = float(current) / dif
 186                 eta = long((float(total) - float(current)) / rate)
 187                 (eta_mins, eta_secs) = divmod(eta, 60)
 188                 if eta_mins > 99:
 189                         return '--:--'
 190                 return '%02d:%02d' % (eta_mins, eta_secs)
 191
 192         @staticmethod
 193         def calc_speed(start, now, bytes):
 194                 dif = now - start
 195                 if bytes == 0 or dif < 0.001: # One millisecond
 196                         return '%10s' % '---b/s'
 197                 return '%10s' % ('%s/s' % FileDownloader.format_bytes(float(bytes) / dif))
 198
 199         @staticmethod
 200         def best_block_size(elapsed_time, bytes):
 201                 new_min = max(bytes / 2.0, 1.0)
 202                 new_max = min(max(bytes * 2.0, 1.0), 4194304) # Do not surpass 4 MB
 203                 if elapsed_time < 0.001:
 204                         return long(new_max)
 205                 rate = bytes / elapsed_time
 206                 if rate > new_max:
 207                         return long(new_max)
 208                 if rate < new_min:
 209                         return long(new_min)
 210                 return long(rate)
 211
 212         @staticmethod
 213         def parse_bytes(bytestr):
 214                 """Parse a string indicating a byte quantity into a long integer."""
 215                 matchobj = re.match(r'(?i)^(\d+(?:\.\d+)?)([kMGTPEZY]?)$', bytestr)
 216                 if matchobj is None:
 217                         return None
 218                 number = float(matchobj.group(1))
 219                 multiplier = 1024.0 ** 'bkmgtpezy'.index(matchobj.group(2).lower())
 220                 return long(round(number * multiplier))
 221
 222         @staticmethod
 223         def verify_url(url):
 224                 """Verify a URL is valid and data could be downloaded."""
 225                 request = urllib2.Request(url, None, std_headers)
 226                 data = urllib2.urlopen(request)
 227                 data.read(1)
 228                 data.close()
 229
 230         def add_info_extractor(self, ie):
 231                 """Add an InfoExtractor object to the end of the list."""
 232                 self._ies.append(ie)
 233                 ie.set_downloader(self)
 234
 235         def add_post_processor(self, pp):
 236                 """Add a PostProcessor object to the end of the chain."""
 237                 self._pps.append(pp)
 238                 pp.set_downloader(self)
 239
 240         def to_stdout(self, message, skip_eol=False):
 241                 """Print message to stdout if not in quiet mode."""
 242                 if not self.params.get('quiet', False):
 243                         print (u'%s%s' % (message, [u'\n', u''][skip_eol])).encode(preferredencoding()),
 244                         sys.stdout.flush()
 245
 246         def to_stderr(self, message):
 247                 """Print message to stderr."""
 248                 print >>sys.stderr, message.encode(preferredencoding())
 249
 250         def fixed_template(self):
 251                 """Checks if the output template is fixed."""
 252                 return (re.search(ur'(?u)%\(.+?\)s', self.params['outtmpl']) is None)
 253
 254         def trouble(self, message=None):
 255                 """Determine action to take when a download problem appears.
 256
 257                 Depending on if the downloader has been configured to ignore
 258                 download errors or not, this method may throw an exception or
 259                 not when errors are found, after printing the message.
 260                 """
 261                 if message is not None:
 262                         self.to_stderr(message)
 263                 if not self.params.get('ignoreerrors', False):
 264                         raise DownloadError(message)
 265                 self._download_retcode = 1
 266
 267         def slow_down(self, start_time, byte_counter):
 268                 """Sleep if the download speed is over the rate limit."""
 269                 rate_limit = self.params.get('ratelimit', None)
 270                 if rate_limit is None or byte_counter == 0:
 271                         return
 272                 now = time.time()
 273                 elapsed = now - start_time
 274                 if elapsed <= 0.0:
 275                         return
 276                 speed = float(byte_counter) / elapsed
 277                 if speed > rate_limit:
 278                         time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
 279
 280         def report_destination(self, filename):
 281                 """Report destination filename."""
 282                 self.to_stdout(u'[download] Destination: %s' % filename)
 283
 284         def report_progress(self, percent_str, data_len_str, speed_str, eta_str):
 285                 """Report download progress."""
 286                 self.to_stdout(u'\r[download] %s of %s at %s ETA %s' %
 287                                 (percent_str, data_len_str, speed_str, eta_str), skip_eol=True)
 288
 289         def report_resuming_byte(self, resume_len):
 290                 """Report attemtp to resume at given byte."""
 291                 self.to_stdout(u'[download] Resuming download at byte %s' % resume_len)
 292
 293         def report_file_already_downloaded(self, file_name):
 294                 """Report file has already been fully downloaded."""
 295                 self.to_stdout(u'[download] %s has already been downloaded' % file_name)
 296
 297         def report_unable_to_resume(self):
 298                 """Report it was impossible to resume download."""
 299                 self.to_stdout(u'[download] Unable to resume')
 300
 301         def report_finish(self):
 302                 """Report download finished."""
 303                 self.to_stdout(u'')
 304
 305         def process_info(self, info_dict):
 306                 """Process a single dictionary returned by an InfoExtractor."""
 307                 # Do nothing else if in simulate mode
 308                 if self.params.get('simulate', False):
 309                         try:
 310                                 self.verify_url(info_dict['url'])
 311                         except (OSError, IOError, urllib2.URLError, httplib.HTTPException, socket.error), err:
 312                                 raise UnavailableFormatError
 313
 314                         # Forced printings
 315                         if self.params.get('forcetitle', False):
 316                                 print info_dict['title'].encode(preferredencoding())
 317                         if self.params.get('forceurl', False):
 318                                 print info_dict['url'].encode(preferredencoding())
 319
 320                         return
 321
 322                 try:
 323                         template_dict = dict(info_dict)
 324                         template_dict['epoch'] = unicode(long(time.time()))
 325                         filename = self.params['outtmpl'] % template_dict
 326                 except (ValueError, KeyError), err:
 327                         self.trouble('ERROR: invalid output template or system charset: %s' % str(err))
 328                 if self.params['nooverwrites'] and os.path.exists(filename):
 329                         self.to_stderr(u'WARNING: file exists: %s; skipping' % filename)
 330                         return
 331
 332                 try:
 333                         self.pmkdir(filename)
 334                 except (OSError, IOError), err:
 335                         self.trouble('ERROR: unable to create directories: %s' % str(err))
 336                         return
 337
 338                 try:
 339                         success = self._do_download(filename, info_dict['url'])
 340                 except (OSError, IOError), err:
 341                         raise UnavailableFormatError
 342                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 343                         self.trouble('ERROR: unable to download video data: %s' % str(err))
 344                         return
 345                 except (ContentTooShortError, ), err:
 346                         self.trouble('ERROR: content too short (expected %s bytes and served %s)' % (err.expected, err.downloaded))
 347                         return
 348
 349                 if success:
 350                         try:
 351                                 self.post_process(filename, info_dict)
 352                         except (PostProcessingError), err:
 353                                 self.trouble('ERROR: postprocessing: %s' % str(err))
 354                                 return
 355
 356         def download(self, url_list):
 357                 """Download a given list of URLs."""
 358                 if len(url_list) > 1 and self.fixed_template():
 359                         raise SameFileError(self.params['outtmpl'])
 360
 361                 for url in url_list:
 362                         suitable_found = False
 363                         for ie in self._ies:
 364                                 # Go to next InfoExtractor if not suitable
 365                                 if not ie.suitable(url):
 366                                         continue
 367
 368                                 # Suitable InfoExtractor found
 369                                 suitable_found = True
 370
 371                                 # Extract information from URL and process it
 372                                 ie.extract(url)
 373
 374                                 # Suitable InfoExtractor had been found; go to next URL
 375                                 break
 376
 377                         if not suitable_found:
 378                                 self.trouble('ERROR: no suitable InfoExtractor: %s' % url)
 379
 380                 return self._download_retcode
 381
 382         def post_process(self, filename, ie_info):
 383                 """Run the postprocessing chain on the given file."""
 384                 info = dict(ie_info)
 385                 info['filepath'] = filename
 386                 for pp in self._pps:
 387                         info = pp.run(info)
 388                         if info is None:
 389                                 break
 390
 391         def _do_download(self, filename, url):
 392                 stream = None
 393                 open_mode = 'ab'
 394
 395                 basic_request = urllib2.Request(url, None, std_headers)
 396                 request = urllib2.Request(url, None, std_headers)
 397
 398                 # Attempt to resume download with "continuedl" option
 399                 if os.path.isfile(filename):
 400                         resume_len = os.path.getsize(filename)
 401                 else:
 402                         resume_len = 0
 403                 if self.params['continuedl'] and resume_len != 0:
 404                         self.report_resuming_byte(resume_len)
 405                         request.add_header('Range','bytes=%d-' % resume_len)
 406
 407                 # Establish connection
 408                 try:
 409                         data = urllib2.urlopen(request)
 410                 except (urllib2.HTTPError, ), err:
 411                         if err.code != 416: #  416 is 'Requested range not satisfiable'
 412                                 raise
 413                         data = urllib2.urlopen(basic_request)
 414                         content_length = data.info()['Content-Length']
 415                         if content_length is not None and long(content_length) == resume_len:
 416                                 self.report_file_already_downloaded(filename)
 417                                 return True
 418                         else:
 419                                 self.report_unable_to_resume()
 420                                 open_mode = 'wb'
 421
 422                 data_len = data.info().get('Content-length', None)
 423                 data_len_str = self.format_bytes(data_len)
 424                 byte_counter = 0
 425                 block_size = 1024
 426                 start = time.time()
 427                 while True:
 428                         # Download and write
 429                         before = time.time()
 430                         data_block = data.read(block_size)
 431                         after = time.time()
 432                         data_block_len = len(data_block)
 433                         if data_block_len == 0:
 434                                 break
 435                         byte_counter += data_block_len
 436
 437                         # Open file just in time
 438                         if stream is None:
 439                                 try:
 440                                         stream = open(filename, open_mode)
 441                                         self.report_destination(filename)
 442                                 except (OSError, IOError), err:
 443                                         self.trouble('ERROR: unable to open for writing: %s' % str(err))
 444                                         return False
 445                         stream.write(data_block)
 446                         block_size = self.best_block_size(after - before, data_block_len)
 447
 448                         # Progress message
 449                         percent_str = self.calc_percent(byte_counter, data_len)
 450                         eta_str = self.calc_eta(start, time.time(), data_len, byte_counter)
 451                         speed_str = self.calc_speed(start, time.time(), byte_counter)
 452                         self.report_progress(percent_str, data_len_str, speed_str, eta_str)
 453
 454                         # Apply rate limit
 455                         self.slow_down(start, byte_counter)
 456
 457                 self.report_finish()
 458                 if data_len is not None and str(byte_counter) != data_len:
 459                         raise ContentTooShortError(byte_counter, long(data_len))
 460                 return True
 461
 462 class InfoExtractor(object):
 463         """Information Extractor class.
 464
 465         Information extractors are the classes that, given a URL, extract
 466         information from the video (or videos) the URL refers to. This
 467         information includes the real video URL, the video title and simplified
 468         title, author and others. The information is stored in a dictionary
 469         which is then passed to the FileDownloader. The FileDownloader
 470         processes this information possibly downloading the video to the file
 471         system, among other possible outcomes. The dictionaries must include
 472         the following fields:
 473
 474         id:             Video identifier.
 475         url:            Final video URL.
 476         uploader:       Nickname of the video uploader.
 477         title:          Literal title.
 478         stitle:         Simplified title.
 479         ext:            Video filename extension.
 480
 481         Subclasses of this one should re-define the _real_initialize() and
 482         _real_extract() methods, as well as the suitable() static method.
 483         Probably, they should also be instantiated and added to the main
 484         downloader.
 485         """
 486
 487         _ready = False
 488         _downloader = None
 489
 490         def __init__(self, downloader=None):
 491                 """Constructor. Receives an optional downloader."""
 492                 self._ready = False
 493                 self.set_downloader(downloader)
 494
 495         @staticmethod
 496         def suitable(url):
 497                 """Receives a URL and returns True if suitable for this IE."""
 498                 return False
 499
 500         def initialize(self):
 501                 """Initializes an instance (authentication, etc)."""
 502                 if not self._ready:
 503                         self._real_initialize()
 504                         self._ready = True
 505
 506         def extract(self, url):
 507                 """Extracts URL information and returns it in list of dicts."""
 508                 self.initialize()
 509                 return self._real_extract(url)
 510
 511         def set_downloader(self, downloader):
 512                 """Sets the downloader for this IE."""
 513                 self._downloader = downloader
 514
 515         def _real_initialize(self):
 516                 """Real initialization process. Redefine in subclasses."""
 517                 pass
 518
 519         def _real_extract(self, url):
 520                 """Real extraction process. Redefine in subclasses."""
 521                 pass
 522
 523 class YoutubeIE(InfoExtractor):
 524         """Information extractor for youtube.com."""
 525
 526         _VALID_URL = r'^((?:http://)?(?:\w+\.)?youtube\.com/(?:(?:v/)|(?:(?:watch(?:\.php)?)?\?(?:.+&)?v=)))?([0-9A-Za-z_-]+)(?(1).+)?$'
 527         _LANG_URL = r'http://uk.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
 528         _LOGIN_URL = 'http://www.youtube.com/signup?next=/&gl=US&hl=en'
 529         _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
 530         _NETRC_MACHINE = 'youtube'
 531         _available_formats = ['22', '35', '18', '5', '17', '13', None] # listed in order of priority for -b flag
 532         _video_extensions = {
 533                 '13': '3gp',
 534                 '17': 'mp4',
 535                 '18': 'mp4',
 536                 '22': 'mp4',
 537         }
 538
 539         @staticmethod
 540         def suitable(url):
 541                 return (re.match(YoutubeIE._VALID_URL, url) is not None)
 542
 543         @staticmethod
 544         def htmlentity_transform(matchobj):
 545                 """Transforms an HTML entity to a Unicode character."""
 546                 entity = matchobj.group(1)
 547
 548                 # Known non-numeric HTML entity
 549                 if entity in htmlentitydefs.name2codepoint:
 550                         return unichr(htmlentitydefs.name2codepoint[entity])
 551
 552                 # Unicode character
 553                 mobj = re.match(ur'(?u)#(x?\d+)', entity)
 554                 if mobj is not None:
 555                         numstr = mobj.group(1)
 556                         if numstr.startswith(u'x'):
 557                                 base = 16
 558                                 numstr = u'0%s' % numstr
 559                         else:
 560                                 base = 10
 561                         return unichr(long(numstr, base))
 562
 563                 # Unknown entity in name, return its literal representation
 564                 return (u'&%s;' % entity)
 565
 566         def report_lang(self):
 567                 """Report attempt to set language."""
 568                 self._downloader.to_stdout(u'[youtube] Setting language')
 569
 570         def report_login(self):
 571                 """Report attempt to log in."""
 572                 self._downloader.to_stdout(u'[youtube] Logging in')
 573
 574         def report_age_confirmation(self):
 575                 """Report attempt to confirm age."""
 576                 self._downloader.to_stdout(u'[youtube] Confirming age')
 577
 578         def report_video_info_webpage_download(self, video_id):
 579                 """Report attempt to download video info webpage."""
 580                 self._downloader.to_stdout(u'[youtube] %s: Downloading video info webpage' % video_id)
 581
 582         def report_information_extraction(self, video_id):
 583                 """Report attempt to extract video information."""
 584                 self._downloader.to_stdout(u'[youtube] %s: Extracting video information' % video_id)
 585
 586         def report_video_url(self, video_id, video_real_url):
 587                 """Report extracted video URL."""
 588                 self._downloader.to_stdout(u'[youtube] %s: URL: %s' % (video_id, video_real_url))
 589
 590         def report_unavailable_format(self, video_id, format):
 591                 """Report extracted video URL."""
 592                 self._downloader.to_stdout(u'[youtube] %s: Format %s not available' % (video_id, format))
 593
 594         def _real_initialize(self):
 595                 if self._downloader is None:
 596                         return
 597
 598                 username = None
 599                 password = None
 600                 downloader_params = self._downloader.params
 601
 602                 # Attempt to use provided username and password or .netrc data
 603                 if downloader_params.get('username', None) is not None:
 604                         username = downloader_params['username']
 605                         password = downloader_params['password']
 606                 elif downloader_params.get('usenetrc', False):
 607                         try:
 608                                 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
 609                                 if info is not None:
 610                                         username = info[0]
 611                                         password = info[2]
 612                                 else:
 613                                         raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
 614                         except (IOError, netrc.NetrcParseError), err:
 615                                 self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % str(err))
 616                                 return
 617
 618                 # Set language
 619                 request = urllib2.Request(self._LANG_URL, None, std_headers)
 620                 try:
 621                         self.report_lang()
 622                         urllib2.urlopen(request).read()
 623                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 624                         self._downloader.to_stderr(u'WARNING: unable to set language: %s' % str(err))
 625                         return
 626
 627                 # No authentication to be performed
 628                 if username is None:
 629                         return
 630
 631                 # Log in
 632                 login_form = {
 633                                 'current_form': 'loginForm',
 634                                 'next':         '/',
 635                                 'action_login': 'Log In',
 636                                 'username':     username,
 637                                 'password':     password,
 638                                 }
 639                 request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form), std_headers)
 640                 try:
 641                         self.report_login()
 642                         login_results = urllib2.urlopen(request).read()
 643                         if re.search(r'(?i)<form[^>]* name="loginForm"', login_results) is not None:
 644                                 self._downloader.to_stderr(u'WARNING: unable to log in: bad username or password')
 645                                 return
 646                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 647                         self._downloader.to_stderr(u'WARNING: unable to log in: %s' % str(err))
 648                         return
 649
 650                 # Confirm age
 651                 age_form = {
 652                                 'next_url':             '/',
 653                                 'action_confirm':       'Confirm',
 654                                 }
 655                 request = urllib2.Request(self._AGE_URL, urllib.urlencode(age_form), std_headers)
 656                 try:
 657                         self.report_age_confirmation()
 658                         age_results = urllib2.urlopen(request).read()
 659                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 660                         self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
 661                         return
 662
 663         def _real_extract(self, url):
 664                 # Extract video id from URL
 665                 mobj = re.match(self._VALID_URL, url)
 666                 if mobj is None:
 667                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
 668                         return
 669                 video_id = mobj.group(2)
 670
 671                 # Downloader parameters
 672                 best_quality = False
 673                 format_param = None
 674                 quality_index = 0
 675                 if self._downloader is not None:
 676                         params = self._downloader.params
 677                         format_param = params.get('format', None)
 678                         if format_param == '0':
 679                                 format_param = self._available_formats[quality_index]
 680                                 best_quality = True
 681
 682                 while True:
 683                         # Extension
 684                         video_extension = self._video_extensions.get(format_param, 'flv')
 685
 686                         # Get video info
 687                         video_info_url = 'http://www.youtube.com/get_video_info?&video_id=%s&el=detailpage&ps=default&eurl=&gl=US&hl=en' % video_id
 688                         request = urllib2.Request(video_info_url, None, std_headers)
 689                         try:
 690                                 self.report_video_info_webpage_download(video_id)
 691                                 video_info_webpage = urllib2.urlopen(request).read()
 692                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 693                                 self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % str(err))
 694                                 return
 695                         self.report_information_extraction(video_id)
 696
 697                         # "t" param
 698                         mobj = re.search(r'(?m)&token=([^&]+)(?:&|$)', video_info_webpage)
 699                         if mobj is None:
 700                                 # Attempt to see if YouTube has issued an error message
 701                                 mobj = re.search(r'(?m)&reason=([^&]+)(?:&|$)', video_info_webpage)
 702                                 if mobj is None:
 703                                         self._downloader.trouble(u'ERROR: unable to extract "t" parameter for unknown reason')
 704                                         stream = open('reportme-ydl-%s.dat' % time.time(), 'wb')
 705                                         stream.write(video_info_webpage)
 706                                         stream.close()
 707                                 else:
 708                                         reason = urllib.unquote_plus(mobj.group(1))
 709                                         self._downloader.trouble(u'ERROR: YouTube said: %s' % reason.decode('utf-8'))
 710                                 return
 711                         token = urllib.unquote(mobj.group(1))
 712                         video_real_url = 'http://www.youtube.com/get_video?video_id=%s&t=%s&eurl=&el=detailpage&ps=default&gl=US&hl=en' % (video_id, token)
 713                         if format_param is not None:
 714                                 video_real_url = '%s&fmt=%s' % (video_real_url, format_param)
 715                         self.report_video_url(video_id, video_real_url)
 716
 717                         # uploader
 718                         mobj = re.search(r'(?m)&author=([^&]+)(?:&|$)', video_info_webpage)
 719                         if mobj is None:
 720                                 self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
 721                                 return
 722                         video_uploader = urllib.unquote(mobj.group(1))
 723
 724                         # title
 725                         mobj = re.search(r'(?m)&title=([^&]+)(?:&|$)', video_info_webpage)
 726                         if mobj is None:
 727                                 self._downloader.trouble(u'ERROR: unable to extract video title')
 728                                 return
 729                         video_title = urllib.unquote(mobj.group(1))
 730                         video_title = video_title.decode('utf-8')
 731                         video_title = re.sub(ur'(?u)&(.+?);', self.htmlentity_transform, video_title)
 732                         video_title = video_title.replace(os.sep, u'%')
 733
 734                         # simplified title
 735                         simple_title = re.sub(ur'(?u)([^%s]+)' % simple_title_chars, ur'_', video_title)
 736                         simple_title = simple_title.strip(ur'_')
 737
 738                         try:
 739                                 # Process video information
 740                                 self._downloader.process_info({
 741                                         'id':           video_id.decode('utf-8'),
 742                                         'url':          video_real_url.decode('utf-8'),
 743                                         'uploader':     video_uploader.decode('utf-8'),
 744                                         'title':        video_title,
 745                                         'stitle':       simple_title,
 746                                         'ext':          video_extension.decode('utf-8'),
 747                                 })
 748
 749                                 return
 750
 751                         except UnavailableFormatError, err:
 752                                 if best_quality:
 753                                         if quality_index == len(self._available_formats) - 1:
 754                                                 # I don't ever expect this to happen
 755                                                 self._downloader.trouble(u'ERROR: no known formats available for video')
 756                                                 return
 757                                         else:
 758                                                 self.report_unavailable_format(video_id, format_param)
 759                                                 quality_index += 1
 760                                                 format_param = self._available_formats[quality_index]
 761                                                 continue
 762                                 else:
 763                                         self._downloader.trouble('ERROR: format not available for video')
 764                                         return
 765
 766
 767 class MetacafeIE(InfoExtractor):
 768         """Information Extractor for metacafe.com."""
 769
 770         _VALID_URL = r'(?:http://)?(?:www\.)?metacafe\.com/watch/([^/]+)/([^/]+)/.*'
 771         _DISCLAIMER = 'http://www.metacafe.com/family_filter/'
 772         _FILTER_POST = 'http://www.metacafe.com/f/index.php?inputType=filter&controllerGroup=user'
 773         _youtube_ie = None
 774
 775         def __init__(self, youtube_ie, downloader=None):
 776                 InfoExtractor.__init__(self, downloader)
 777                 self._youtube_ie = youtube_ie
 778
 779         @staticmethod
 780         def suitable(url):
 781                 return (re.match(MetacafeIE._VALID_URL, url) is not None)
 782
 783         def report_disclaimer(self):
 784                 """Report disclaimer retrieval."""
 785                 self._downloader.to_stdout(u'[metacafe] Retrieving disclaimer')
 786
 787         def report_age_confirmation(self):
 788                 """Report attempt to confirm age."""
 789                 self._downloader.to_stdout(u'[metacafe] Confirming age')
 790
 791         def report_download_webpage(self, video_id):
 792                 """Report webpage download."""
 793                 self._downloader.to_stdout(u'[metacafe] %s: Downloading webpage' % video_id)
 794
 795         def report_extraction(self, video_id):
 796                 """Report information extraction."""
 797                 self._downloader.to_stdout(u'[metacafe] %s: Extracting information' % video_id)
 798
 799         def _real_initialize(self):
 800                 # Retrieve disclaimer
 801                 request = urllib2.Request(self._DISCLAIMER, None, std_headers)
 802                 try:
 803                         self.report_disclaimer()
 804                         disclaimer = urllib2.urlopen(request).read()
 805                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 806                         self._downloader.trouble(u'ERROR: unable to retrieve disclaimer: %s' % str(err))
 807                         return
 808
 809                 # Confirm age
 810                 disclaimer_form = {
 811                         'filters': '0',
 812                         'submit': "Continue - I'm over 18",
 813                         }
 814                 request = urllib2.Request(self._FILTER_POST, urllib.urlencode(disclaimer_form), std_headers)
 815                 try:
 816                         self.report_age_confirmation()
 817                         disclaimer = urllib2.urlopen(request).read()
 818                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 819                         self._downloader.trouble(u'ERROR: unable to confirm age: %s' % str(err))
 820                         return
 821
 822         def _real_extract(self, url):
 823                 # Extract id and simplified title from URL
 824                 mobj = re.match(self._VALID_URL, url)
 825                 if mobj is None:
 826                         self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
 827                         return
 828
 829                 video_id = mobj.group(1)
 830
 831                 # Check if video comes from YouTube
 832                 mobj2 = re.match(r'^yt-(.*)$', video_id)
 833                 if mobj2 is not None:
 834                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % mobj2.group(1))
 835                         return
 836
 837                 simple_title = mobj.group(2).decode('utf-8')
 838                 video_extension = 'flv'
 839
 840                 # Retrieve video webpage to extract further information
 841                 request = urllib2.Request('http://www.metacafe.com/watch/%s/' % video_id)
 842                 try:
 843                         self.report_download_webpage(video_id)
 844                         webpage = urllib2.urlopen(request).read()
 845                 except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 846                         self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % str(err))
 847                         return
 848
 849                 # Extract URL, uploader and title from webpage
 850                 self.report_extraction(video_id)
 851                 mobj = re.search(r'(?m)&mediaURL=([^&]+)', webpage)
 852                 if mobj is None:
 853                         self._downloader.trouble(u'ERROR: unable to extract media URL')
 854                         return
 855                 mediaURL = urllib.unquote(mobj.group(1))
 856
 857                 #mobj = re.search(r'(?m)&gdaKey=(.*?)&', webpage)
 858                 #if mobj is None:
 859                 #       self._downloader.trouble(u'ERROR: unable to extract gdaKey')
 860                 #       return
 861                 #gdaKey = mobj.group(1)
 862                 #
 863                 #video_url = '%s?__gda__=%s' % (mediaURL, gdaKey)
 864
 865                 video_url = mediaURL
 866
 867                 mobj = re.search(r'(?im)<title>(.*) - Video</title>', webpage)
 868                 if mobj is None:
 869                         self._downloader.trouble(u'ERROR: unable to extract title')
 870                         return
 871                 video_title = mobj.group(1).decode('utf-8')
 872
 873                 mobj = re.search(r'(?ms)<li id="ChnlUsr">.*?Submitter:.*?<a .*?>(.*?)<', webpage)
 874                 if mobj is None:
 875                         self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
 876                         return
 877                 video_uploader = mobj.group(1)
 878
 879                 try:
 880                         # Process video information
 881                         self._downloader.process_info({
 882                                 'id':           video_id.decode('utf-8'),
 883                                 'url':          video_url.decode('utf-8'),
 884                                 'uploader':     video_uploader.decode('utf-8'),
 885                                 'title':        video_title,
 886                                 'stitle':       simple_title,
 887                                 'ext':          video_extension.decode('utf-8'),
 888                         })
 889                 except UnavailableFormatError:
 890                         self._downloader.trouble(u'ERROR: format not available for video')
 891
 892
 893 class YoutubeSearchIE(InfoExtractor):
 894         """Information Extractor for YouTube search queries."""
 895         _VALID_QUERY = r'ytsearch(\d+|all)?:[\s\S]+'
 896         _TEMPLATE_URL = 'http://www.youtube.com/results?search_query=%s&page=%s&gl=US&hl=en'
 897         _VIDEO_INDICATOR = r'href="/watch\?v=.+?"'
 898         _MORE_PAGES_INDICATOR = r'(?m)>\s*Next\s*</a>'
 899         _youtube_ie = None
 900         _max_youtube_results = 1000
 901
 902         def __init__(self, youtube_ie, downloader=None):
 903                 InfoExtractor.__init__(self, downloader)
 904                 self._youtube_ie = youtube_ie
 905
 906         @staticmethod
 907         def suitable(url):
 908                 return (re.match(YoutubeSearchIE._VALID_QUERY, url) is not None)
 909
 910         def report_download_page(self, query, pagenum):
 911                 """Report attempt to download playlist page with given number."""
 912                 self._downloader.to_stdout(u'[youtube] query "%s": Downloading page %s' % (query, pagenum))
 913
 914         def _real_initialize(self):
 915                 self._youtube_ie.initialize()
 916
 917         def _real_extract(self, query):
 918                 mobj = re.match(self._VALID_QUERY, query)
 919                 if mobj is None:
 920                         self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
 921                         return
 922
 923                 prefix, query = query.split(':')
 924                 prefix = prefix[8:]
 925                 if prefix == '':
 926                         self._download_n_results(query, 1)
 927                         return
 928                 elif prefix == 'all':
 929                         self._download_n_results(query, self._max_youtube_results)
 930                         return
 931                 else:
 932                         try:
 933                                 n = long(prefix)
 934                                 if n <= 0:
 935                                         self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
 936                                         return
 937                                 elif n > self._max_youtube_results:
 938                                         self._downloader.to_stderr(u'WARNING: ytsearch returns max %i results (you requested %i)'  % (self._max_youtube_results, n))
 939                                         n = self._max_youtube_results
 940                                 self._download_n_results(query, n)
 941                                 return
 942                         except ValueError: # parsing prefix as integer fails
 943                                 self._download_n_results(query, 1)
 944                                 return
 945
 946         def _download_n_results(self, query, n):
 947                 """Downloads a specified number of results for a query"""
 948
 949                 video_ids = []
 950                 already_seen = set()
 951                 pagenum = 1
 952
 953                 while True:
 954                         self.report_download_page(query, pagenum)
 955                         result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
 956                         request = urllib2.Request(result_url, None, std_headers)
 957                         try:
 958                                 page = urllib2.urlopen(request).read()
 959                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
 960                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
 961                                 return
 962
 963                         # Extract video identifiers
 964                         for mobj in re.finditer(self._VIDEO_INDICATOR, page):
 965                                 video_id = page[mobj.span()[0]:mobj.span()[1]].split('=')[2][:-1]
 966                                 if video_id not in already_seen:
 967                                         video_ids.append(video_id)
 968                                         already_seen.add(video_id)
 969                                         if len(video_ids) == n:
 970                                                 # Specified n videos reached
 971                                                 for id in video_ids:
 972                                                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
 973                                                 return
 974
 975                         if re.search(self._MORE_PAGES_INDICATOR, page) is None:
 976                                 for id in video_ids:
 977                                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
 978                                 return
 979
 980                         pagenum = pagenum + 1
 981
 982 class YoutubePlaylistIE(InfoExtractor):
 983         """Information Extractor for YouTube playlists."""
 984
 985         _VALID_URL = r'(?:http://)?(?:\w+\.)?youtube.com/(?:view_play_list|my_playlists)\?.*?p=([^&]+).*'
 986         _TEMPLATE_URL = 'http://www.youtube.com/view_play_list?p=%s&page=%s&gl=US&hl=en'
 987         _VIDEO_INDICATOR = r'/watch\?v=(.+?)&'
 988         _MORE_PAGES_INDICATOR = r'/view_play_list?p=%s&page=%s'
 989         _youtube_ie = None
 990
 991         def __init__(self, youtube_ie, downloader=None):
 992                 InfoExtractor.__init__(self, downloader)
 993                 self._youtube_ie = youtube_ie
 994
 995         @staticmethod
 996         def suitable(url):
 997                 return (re.match(YoutubePlaylistIE._VALID_URL, url) is not None)
 998
 999         def report_download_page(self, playlist_id, pagenum):
1000                 """Report attempt to download playlist page with given number."""
1001                 self._downloader.to_stdout(u'[youtube] PL %s: Downloading page #%s' % (playlist_id, pagenum))
1002
1003         def _real_initialize(self):
1004                 self._youtube_ie.initialize()
1005
1006         def _real_extract(self, url):
1007                 # Extract playlist id
1008                 mobj = re.match(self._VALID_URL, url)
1009                 if mobj is None:
1010                         self._downloader.trouble(u'ERROR: invalid url: %s' % url)
1011                         return
1012
1013                 # Download playlist pages
1014                 playlist_id = mobj.group(1)
1015                 video_ids = []
1016                 pagenum = 1
1017
1018                 while True:
1019                         self.report_download_page(playlist_id, pagenum)
1020                         request = urllib2.Request(self._TEMPLATE_URL % (playlist_id, pagenum), None, std_headers)
1021                         try:
1022                                 page = urllib2.urlopen(request).read()
1023                         except (urllib2.URLError, httplib.HTTPException, socket.error), err:
1024                                 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1025                                 return
1026
1027                         # Extract video identifiers
1028                         ids_in_page = []
1029                         for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1030                                 if mobj.group(1) not in ids_in_page:
1031                                         ids_in_page.append(mobj.group(1))
1032                         video_ids.extend(ids_in_page)
1033
1034                         if (self._MORE_PAGES_INDICATOR % (playlist_id.upper(), pagenum + 1)) not in page:
1035                                 break
1036                         pagenum = pagenum + 1
1037
1038                 for id in video_ids:
1039                         self._youtube_ie.extract('http://www.youtube.com/watch?v=%s' % id)
1040                 return
1041
1042 class PostProcessor(object):
1043         """Post Processor class.
1044
1045         PostProcessor objects can be added to downloaders with their
1046         add_post_processor() method. When the downloader has finished a
1047         successful download, it will take its internal chain of PostProcessors
1048         and start calling the run() method on each one of them, first with
1049         an initial argument and then with the returned value of the previous
1050         PostProcessor.
1051
1052         The chain will be stopped if one of them ever returns None or the end
1053         of the chain is reached.
1054
1055         PostProcessor objects follow a "mutual registration" process similar
1056         to InfoExtractor objects.
1057         """
1058
1059         _downloader = None
1060
1061         def __init__(self, downloader=None):
1062                 self._downloader = downloader
1063
1064         def set_downloader(self, downloader):
1065                 """Sets the downloader for this PP."""
1066                 self._downloader = downloader
1067
1068         def run(self, information):
1069                 """Run the PostProcessor.
1070
1071                 The "information" argument is a dictionary like the ones
1072                 composed by InfoExtractors. The only difference is that this
1073                 one has an extra field called "filepath" that points to the
1074                 downloaded file.
1075
1076                 When this method returns None, the postprocessing chain is
1077                 stopped. However, this method may return an information
1078                 dictionary that will be passed to the next postprocessing
1079                 object in the chain. It can be the one it received after
1080                 changing some fields.
1081
1082                 In addition, this method may raise a PostProcessingError
1083                 exception that will be taken into account by the downloader
1084                 it was called from.
1085                 """
1086                 return information # by default, do nothing
1087
1088 ### MAIN PROGRAM ###
1089 if __name__ == '__main__':
1090         try:
1091                 # Modules needed only when running the main program
1092                 import getpass
1093                 import optparse
1094
1095                 # General configuration
1096                 urllib2.install_opener(urllib2.build_opener(urllib2.ProxyHandler()))
1097                 urllib2.install_opener(urllib2.build_opener(urllib2.HTTPCookieProcessor()))
1098                 socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words)
1099
1100                 # Parse command line
1101                 parser = optparse.OptionParser(
1102                         usage='Usage: %prog [options] url...',
1103                         version='INTERNAL',
1104                         conflict_handler='resolve',
1105                 )
1106
1107                 parser.add_option('-h', '--help',
1108                                 action='help', help='print this help text and exit')
1109                 parser.add_option('-v', '--version',
1110                                 action='version', help='print program version and exit')
1111                 parser.add_option('-i', '--ignore-errors',
1112                                 action='store_true', dest='ignoreerrors', help='continue on download errors', default=False)
1113                 parser.add_option('-r', '--rate-limit',
1114                                 dest='ratelimit', metavar='L', help='download rate limit (e.g. 50k or 44.6m)')
1115
1116                 authentication = optparse.OptionGroup(parser, 'Authentication Options')
1117                 authentication.add_option('-u', '--username',
1118                                 dest='username', metavar='UN', help='account username')
1119                 authentication.add_option('-p', '--password',
1120                                 dest='password', metavar='PW', help='account password')
1121                 authentication.add_option('-n', '--netrc',
1122                                 action='store_true', dest='usenetrc', help='use .netrc authentication data', default=False)
1123                 parser.add_option_group(authentication)
1124
1125                 video_format = optparse.OptionGroup(parser, 'Video Format Options')
1126                 video_format.add_option('-f', '--format',
1127                                 action='store', dest='format', metavar='FMT', help='video format code')
1128                 video_format.add_option('-b', '--best-quality',
1129                                 action='store_const', dest='format', help='download the best quality video possible', const='0')
1130                 video_format.add_option('-m', '--mobile-version',
1131                                 action='store_const', dest='format', help='alias for -f 17', const='17')
1132                 video_format.add_option('-d', '--high-def',
1133                                 action='store_const', dest='format', help='alias for -f 22', const='22')
1134                 parser.add_option_group(video_format)
1135
1136                 verbosity = optparse.OptionGroup(parser, 'Verbosity / Simulation Options')
1137                 verbosity.add_option('-q', '--quiet',
1138                                 action='store_true', dest='quiet', help='activates quiet mode', default=False)
1139                 verbosity.add_option('-s', '--simulate',
1140                                 action='store_true', dest='simulate', help='do not download video', default=False)
1141                 verbosity.add_option('-g', '--get-url',
1142                                 action='store_true', dest='geturl', help='simulate, quiet but print URL', default=False)
1143                 verbosity.add_option('-e', '--get-title',
1144                                 action='store_true', dest='gettitle', help='simulate, quiet but print title', default=False)
1145                 parser.add_option_group(verbosity)
1146
1147                 filesystem = optparse.OptionGroup(parser, 'Filesystem Options')
1148                 filesystem.add_option('-t', '--title',
1149                                 action='store_true', dest='usetitle', help='use title in file name', default=False)
1150                 filesystem.add_option('-l', '--literal',
1151                                 action='store_true', dest='useliteral', help='use literal title in file name', default=False)
1152                 filesystem.add_option('-o', '--output',
1153                                 dest='outtmpl', metavar='TPL', help='output filename template')
1154                 filesystem.add_option('-a', '--batch-file',
1155                                 dest='batchfile', metavar='F', help='file containing URLs to download')
1156                 filesystem.add_option('-w', '--no-overwrites',
1157                                 action='store_true', dest='nooverwrites', help='do not overwrite files', default=False)
1158                 filesystem.add_option('-c', '--continue',
1159                                 action='store_true', dest='continue_dl', help='resume partially downloaded files', default=False)
1160                 parser.add_option_group(filesystem)
1161
1162                 (opts, args) = parser.parse_args()
1163
1164                 # Batch file verification
1165                 batchurls = []
1166                 if opts.batchfile is not None:
1167                         try:
1168                                 batchurls = open(opts.batchfile, 'r').readlines()
1169                                 batchurls = [x.strip() for x in batchurls]
1170                                 batchurls = [x for x in batchurls if len(x) > 0]
1171                         except IOError:
1172                                 sys.exit(u'ERROR: batch file could not be read')
1173                 all_urls = batchurls + args
1174
1175                 # Conflicting, missing and erroneous options
1176                 if len(all_urls) < 1:
1177                         parser.error(u'you must provide at least one URL')
1178                 if opts.usenetrc and (opts.username is not None or opts.password is not None):
1179                         parser.error(u'using .netrc conflicts with giving username/password')
1180                 if opts.password is not None and opts.username is None:
1181                         parser.error(u'account username missing')
1182                 if opts.outtmpl is not None and (opts.useliteral or opts.usetitle):
1183                         parser.error(u'using output template conflicts with using title or literal title')
1184                 if opts.usetitle and opts.useliteral:
1185                         parser.error(u'using title conflicts with using literal title')
1186                 if opts.username is not None and opts.password is None:
1187                         opts.password = getpass.getpass(u'Type account password and press return:')
1188                 if opts.ratelimit is not None:
1189                         numeric_limit = FileDownloader.parse_bytes(opts.ratelimit)
1190                         if numeric_limit is None:
1191                                 parser.error(u'invalid rate limit specified')
1192                         opts.ratelimit = numeric_limit
1193
1194                 # Information extractors
1195                 youtube_ie = YoutubeIE()
1196                 metacafe_ie = MetacafeIE(youtube_ie)
1197                 youtube_pl_ie = YoutubePlaylistIE(youtube_ie)
1198                 youtube_search_ie = YoutubeSearchIE(youtube_ie)
1199
1200                 # File downloader
1201                 fd = FileDownloader({
1202                         'usenetrc': opts.usenetrc,
1203                         'username': opts.username,
1204                         'password': opts.password,
1205                         'quiet': (opts.quiet or opts.geturl or opts.gettitle),
1206                         'forceurl': opts.geturl,
1207                         'forcetitle': opts.gettitle,
1208                         'simulate': (opts.simulate or opts.geturl or opts.gettitle),
1209                         'format': opts.format,
1210                         'outtmpl': ((opts.outtmpl is not None and opts.outtmpl.decode(preferredencoding()))
1211                                 or (opts.usetitle and u'%(stitle)s-%(id)s.%(ext)s')
1212                                 or (opts.useliteral and u'%(title)s-%(id)s.%(ext)s')
1213                                 or u'%(id)s.%(ext)s'),
1214                         'ignoreerrors': opts.ignoreerrors,
1215                         'ratelimit': opts.ratelimit,
1216                         'nooverwrites': opts.nooverwrites,
1217                         'continuedl': opts.continue_dl,
1218                         })
1219                 fd.add_info_extractor(youtube_search_ie)
1220                 fd.add_info_extractor(youtube_pl_ie)
1221                 fd.add_info_extractor(metacafe_ie)
1222                 fd.add_info_extractor(youtube_ie)
1223                 retcode = fd.download(all_urls)
1224                 sys.exit(retcode)
1225
1226         except DownloadError:
1227                 sys.exit(1)
1228         except SameFileError:
1229                 sys.exit(u'ERROR: fixed output name but more than one file to download')
1230         except KeyboardInterrupt:
1231                 sys.exit(u'\nERROR: Interrupted by user')