import cookielib
import ctypes
import datetime
+import gzip
import htmlentitydefs
import httplib
import locale
import re
import socket
import string
+import StringIO
import subprocess
import sys
import time
import urllib
import urllib2
+import zlib
# parse_qs was moved from the cgi module to the urlparse module recently.
try:
self.downloaded = downloaded
self.expected = expected
+class YoutubeDLHandler(urllib2.HTTPHandler):
+ """Handler for HTTP requests and responses.
+
+ This class, when installed with an OpenerDirector, automatically adds
+ the standard headers to every HTTP request and handles gzipped and
+ deflated responses from web servers. If compression is to be avoided in
+ a particular request, the original request in the program code only has
+ to include the HTTP header "Youtubedl-No-Compression", which will be
+ removed before making the real request.
+
+ Part of this code was copied from:
+
+ http://techknack.net/python-urllib2-handlers/
+
+ Andrew Rowls, the author of that code, agreed to release it to the
+ public domain.
+ """
+
+ @staticmethod
+ def deflate(data):
+ try:
+ return zlib.decompress(data, -zlib.MAX_WBITS)
+ except zlib.error:
+ return zlib.decompress(data)
+
+ def http_request(self, req):
+ for h in std_headers:
+ if h in req.headers:
+ del req.headers[h]
+ req.add_header(h, std_headers[h])
+ if 'Youtubedl-no-compression' in req.headers:
+ if 'Accept-encoding' in req.headers:
+ del req.headers['Accept-encoding']
+ del req.headers['Youtubedl-no-compression']
+ return req
+
+ def http_response(self, req, resp):
+ old_resp = resp
+ # gzip
+ if resp.headers.get('Content-encoding', '') == 'gzip':
+ gz = gzip.GzipFile(fileobj=StringIO.StringIO(resp.read()), mode='r')
+ resp = urllib2.addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code)
+ resp.msg = old_resp.msg
+ # deflate
+ if resp.headers.get('Content-encoding', '') == 'deflate':
+ gz = StringIO.StringIO(self.deflate(resp.read()))
+ resp = urllib2.addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code)
+ resp.msg = old_resp.msg
+ return resp
+
class FileDownloader(object):
"""File Downloader class.
playlistend: Playlist item to end at.
logtostderr: Log messages to stderr instead of stdout.
consoletitle: Display progress in console window's titlebar.
+ nopart: Do not use temporary .part files.
"""
params = None
for dir in aggregate:
if not os.path.exists(dir):
os.mkdir(dir)
-
- @staticmethod
- def temp_name(filename):
- """Returns a temporary filename for the given filename."""
- if filename == u'-' or (os.path.exists(filename) and not os.path.isfile(filename)):
- return filename
- return filename + u'.part'
-
+
@staticmethod
def format_bytes(bytes):
if bytes is None:
speed = float(byte_counter) / elapsed
if speed > rate_limit:
time.sleep((byte_counter - rate_limit * (now - start_time)) / rate_limit)
-
+
+ def temp_name(self, filename):
+ """Returns a temporary filename for the given filename."""
+ if self.params.get('nopart', False) or filename == u'-' or \
+ (os.path.exists(filename) and not os.path.isfile(filename)):
+ return filename
+ return filename + u'.part'
+
+ def undo_temp_name(self, filename):
+ if filename.endswith(u'.part'):
+ return filename[:-len(u'.part')]
+ return filename
+
def try_rename(self, old_filename, new_filename):
try:
if old_filename == new_filename:
def _do_download(self, filename, url, player_url):
# Check file already present
- if self.params.get('continuedl', False) and os.path.isfile(filename):
+ if self.params.get('continuedl', False) and os.path.isfile(filename) and not self.params.get('nopart', False):
self.report_file_already_downloaded(filename)
return True
tmpfilename = self.temp_name(filename)
stream = None
open_mode = 'wb'
- basic_request = urllib2.Request(url, None, std_headers)
- request = urllib2.Request(url, None, std_headers)
+
+ # Do not include the Accept-Encoding header
+ headers = {'Youtubedl-no-compression': 'True'}
+ basic_request = urllib2.Request(url, None, headers)
+ request = urllib2.Request(url, None, headers)
# Establish possible resume length
if os.path.isfile(tmpfilename):
if stream is None:
try:
(stream, tmpfilename) = sanitize_open(tmpfilename, open_mode)
+ filename = self.undo_temp_name(tmpfilename)
self.report_destination(filename)
except (OSError, IOError), err:
self.trouble(u'ERROR: unable to open for writing: %s' % str(err))
class YoutubeIE(InfoExtractor):
"""Information extractor for youtube.com."""
- _VALID_URL = r'^((?:https?://)?(?:youtu\.be/|(?:\w+\.)?youtube(?:-nocookie)?\.com/(?:(?:v/)|(?:(?:watch(?:_popup)?(?:\.php)?)?(?:\?|#!?)(?:.+&)?v=))))?([0-9A-Za-z_-]+)(?(1).+)?$'
+ _VALID_URL = r'^((?:https?://)?(?:youtu\.be/|(?:\w+\.)?youtube(?:-nocookie)?\.com/)(?:(?:v/)|(?:(?:watch(?:_popup)?(?:\.php)?)?(?:\?|#!?)(?:.+&)?v=)))?([0-9A-Za-z_-]+)(?(1).+)?$'
_LANG_URL = r'http://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
_LOGIN_URL = 'https://www.youtube.com/signup?next=/&gl=US&hl=en'
_AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
return
# Set language
- request = urllib2.Request(self._LANG_URL, None, std_headers)
+ request = urllib2.Request(self._LANG_URL)
try:
self.report_lang()
urllib2.urlopen(request).read()
'username': username,
'password': password,
}
- request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form), std_headers)
+ request = urllib2.Request(self._LOGIN_URL, urllib.urlencode(login_form))
try:
self.report_login()
login_results = urllib2.urlopen(request).read()
'next_url': '/',
'action_confirm': 'Confirm',
}
- request = urllib2.Request(self._AGE_URL, urllib.urlencode(age_form), std_headers)
+ request = urllib2.Request(self._AGE_URL, urllib.urlencode(age_form))
try:
self.report_age_confirmation()
age_results = urllib2.urlopen(request).read()
# Get video webpage
self.report_video_webpage_download(video_id)
- request = urllib2.Request('http://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1' % video_id, None, std_headers)
+ request = urllib2.Request('http://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1' % video_id)
try:
video_webpage = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']:
video_info_url = ('http://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
% (video_id, el_type))
- request = urllib2.Request(video_info_url, None, std_headers)
+ request = urllib2.Request(video_info_url)
try:
video_info_webpage = urllib2.urlopen(request).read()
video_info = parse_qs(video_info_webpage)
def _real_initialize(self):
# Retrieve disclaimer
- request = urllib2.Request(self._DISCLAIMER, None, std_headers)
+ request = urllib2.Request(self._DISCLAIMER)
try:
self.report_disclaimer()
disclaimer = urllib2.urlopen(request).read()
'filters': '0',
'submit': "Continue - I'm over 18",
}
- request = urllib2.Request(self._FILTER_POST, urllib.urlencode(disclaimer_form), std_headers)
+ request = urllib2.Request(self._FILTER_POST, urllib.urlencode(disclaimer_form))
try:
self.report_age_confirmation()
disclaimer = urllib2.urlopen(request).read()
while True:
self.report_download_page(query, pagenum)
result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
- request = urllib2.Request(result_url, None, std_headers)
+ request = urllib2.Request(result_url)
try:
page = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
while True:
self.report_download_page(query, pagenum)
result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
- request = urllib2.Request(result_url, None, std_headers)
+ request = urllib2.Request(result_url)
try:
page = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
while True:
self.report_download_page(query, pagenum)
result_url = self._TEMPLATE_URL % (urllib.quote_plus(query), pagenum)
- request = urllib2.Request(result_url, None, std_headers)
+ request = urllib2.Request(result_url)
try:
page = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
class YoutubePlaylistIE(InfoExtractor):
"""Information Extractor for YouTube playlists."""
- _VALID_URL = r'(?:http://)?(?:\w+\.)?youtube.com/(?:(?:view_play_list|my_playlists)\?.*?p=|user/.*?/user/)([^&]+).*'
+ _VALID_URL = r'(?:http://)?(?:\w+\.)?youtube.com/(?:(?:view_play_list|my_playlists)\?.*?p=|user/.*?/user/|p/)([^&]+).*'
_TEMPLATE_URL = 'http://www.youtube.com/view_play_list?p=%s&page=%s&gl=US&hl=en'
_VIDEO_INDICATOR = r'/watch\?v=(.+?)&'
_MORE_PAGES_INDICATOR = r'(?m)>\s*Next\s*</a>'
while True:
self.report_download_page(playlist_id, pagenum)
- request = urllib2.Request(self._TEMPLATE_URL % (playlist_id, pagenum), None, std_headers)
+ request = urllib2.Request(self._TEMPLATE_URL % (playlist_id, pagenum))
try:
page = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
pagenum = 1
self.report_download_page(username)
- request = urllib2.Request(self._TEMPLATE_URL % (username), None, std_headers)
+ request = urllib2.Request(self._TEMPLATE_URL % (username))
try:
page = urllib2.urlopen(request).read()
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
# Retrieve file webpage with 'Free download' button pressed
free_download_indication = { 'gateway_result' : '1' }
- request = urllib2.Request(url, urllib.urlencode(free_download_indication), std_headers)
+ request = urllib2.Request(url, urllib.urlencode(free_download_indication))
try:
self.report_download_webpage(file_id)
webpage = urllib2.urlopen(request).read()
import getpass
import optparse
- # Function to update the program file with the latest version from bitbucket.org
+ # Function to update the program file with the latest version from the repository.
def update_self(downloader, filename):
# Note: downloader only used for options
- if not os.access (filename, os.W_OK):
+ if not os.access(filename, os.W_OK):
sys.exit('ERROR: no write permissions on %s' % filename)
downloader.to_screen('Updating to latest stable version...')
- latest_url = 'http://github.com/rg3/youtube-dl/raw/master/LATEST_VERSION'
- latest_version = urllib.urlopen(latest_url).read().strip()
- prog_url = 'http://github.com/rg3/youtube-dl/raw/%s/youtube-dl' % latest_version
- newcontent = urllib.urlopen(prog_url).read()
- stream = open(filename, 'w')
- stream.write(newcontent)
- stream.close()
+ try:
+ latest_url = 'http://github.com/rg3/youtube-dl/raw/master/LATEST_VERSION'
+ latest_version = urllib.urlopen(latest_url).read().strip()
+ prog_url = 'http://github.com/rg3/youtube-dl/raw/%s/youtube-dl' % latest_version
+ newcontent = urllib.urlopen(prog_url).read()
+ except (IOError, OSError), err:
+ sys.exit('ERROR: unable to download latest version')
+ try:
+ stream = open(filename, 'w')
+ stream.write(newcontent)
+ stream.close()
+ except (IOError, OSError), err:
+ sys.exit('ERROR: unable to overwrite current version')
downloader.to_screen('Updated to version %s' % latest_version)
# Parse command line
action='store_true', dest='continue_dl', help='resume partially downloaded files', default=False)
filesystem.add_option('--cookies',
dest='cookiefile', metavar='FILE', help='file to dump cookie jar to')
+ filesystem.add_option('--no-part',
+ action='store_true', dest='nopart', help='do not use .part files', default=False)
parser.add_option_group(filesystem)
(opts, args) = parser.parse_args()
# General configuration
cookie_processor = urllib2.HTTPCookieProcessor(jar)
- urllib2.install_opener(urllib2.build_opener(urllib2.ProxyHandler()))
- urllib2.install_opener(urllib2.build_opener(cookie_processor))
+ urllib2.install_opener(urllib2.build_opener(urllib2.ProxyHandler(), cookie_processor, YoutubeDLHandler()))
socket.setdefaulttimeout(300) # 5 minutes should be enough (famous last words)
# Batch file verification
'playlistend': opts.playlistend,
'logtostderr': opts.outtmpl == '-',
'consoletitle': opts.consoletitle,
+ 'nopart': opts.nopart,
})
fd.add_info_extractor(youtube_search_ie)
fd.add_info_extractor(youtube_pl_ie)
projects / youtube-dl.git / blobdiff