X-Git-Url: http://git.cielonegro.org/gitweb.cgi?a=blobdiff_plain;f=youtube_dl%2FInfoExtractors.py;h=499d9b1aed8c3d4177539179f515d8abdcf981d8;hb=3210735c499e6a7d0b4df4b8120e17bd7ba8927b;hp=d77154dcb4902422f9297c6b0d0fbf90168f5975;hpb=9e6dd238761e6628d7bd0b6bdb7a997604de3757;p=youtube-dl.git
diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py
index d77154dcb..499d9b1ae 100644
--- a/youtube_dl/InfoExtractors.py
+++ b/youtube_dl/InfoExtractors.py
@@ -12,23 +12,14 @@ import time
import urllib
import urllib2
import email.utils
+import xml.etree.ElementTree
+from urlparse import parse_qs
try:
import cStringIO as StringIO
except ImportError:
import StringIO
-# parse_qs was moved from the cgi module to the urlparse module recently.
-try:
- from urlparse import parse_qs
-except ImportError:
- from cgi import parse_qs
-
-try:
- import xml.etree.ElementTree
-except ImportError: # Python<2.5: Not officially supported, but let it slip
- warnings.warn('xml.etree.ElementTree support is missing. Consider upgrading to Python >= 2.5 if you get related errors.')
-
from utils import *
@@ -48,7 +39,6 @@ class InfoExtractor(object):
url: Final video URL.
uploader: Nickname of the video uploader.
title: Literal title.
- stitle: Simplified title.
ext: Video filename extension.
format: Video format.
player_url: SWF Player URL (may be None).
@@ -112,8 +102,8 @@ class YoutubeIE(InfoExtractor):
_NEXT_URL_RE = r'[\?&]next_url=([^&]+)'
_NETRC_MACHINE = 'youtube'
# Listed in order of quality
- _available_formats = ['38', '37', '22', '45', '35', '44', '34', '18', '43', '6', '5', '17', '13']
- _available_formats_prefer_free = ['38', '37', '45', '22', '44', '35', '43', '34', '18', '6', '5', '17', '13']
+ _available_formats = ['38', '37', '46', '22', '45', '35', '44', '34', '18', '43', '6', '5', '17', '13']
+ _available_formats_prefer_free = ['38', '46', '37', '45', '22', '44', '35', '43', '34', '18', '6', '5', '17', '13']
_video_extensions = {
'13': '3gp',
'17': 'mp4',
@@ -124,6 +114,7 @@ class YoutubeIE(InfoExtractor):
'43': 'webm',
'44': 'webm',
'45': 'webm',
+ '46': 'webm',
}
_video_dimensions = {
'5': '240x400',
@@ -139,6 +130,7 @@ class YoutubeIE(InfoExtractor):
'43': '360x640',
'44': '480x854',
'45': '720x1280',
+ '46': '1080x1920',
}
IE_NAME = u'youtube'
@@ -189,8 +181,8 @@ class YoutubeIE(InfoExtractor):
start = "%02i:%02i:%02i,%03i" %(start/(60*60), start/60%60, start%60, start%1*1000)
end = "%02i:%02i:%02i,%03i" %(end/(60*60), end/60%60, end%60, end%1*1000)
caption = unescapeHTML(caption)
- caption = unescapeHTML(caption) # double cycle, inentional
- srt += str(n) + '\n'
+ caption = unescapeHTML(caption) # double cycle, intentional
+ srt += str(n+1) + '\n'
srt += start + ' --> ' + end + '\n'
srt += caption + '\n\n'
return srt
@@ -319,6 +311,11 @@ class YoutubeIE(InfoExtractor):
self._downloader.trouble(u'ERROR: "token" parameter not in video info for unknown reason')
return
+ # Check for "rental" videos
+ if 'ypc_video_rental_bar_text' in video_info and 'author' not in video_info:
+ self._downloader.trouble(u'ERROR: "rental" videos not supported')
+ return
+
# Start extracting information
self.report_information_extraction(video_id)
@@ -334,10 +331,6 @@ class YoutubeIE(InfoExtractor):
return
video_title = urllib.unquote_plus(video_info['title'][0])
video_title = video_title.decode('utf-8')
- video_title = sanitize_title(video_title)
-
- # simplified title
- simple_title = simplify_title(video_title)
# thumbnail image
if 'thumbnail_url' not in video_info:
@@ -359,40 +352,39 @@ class YoutubeIE(InfoExtractor):
pass
# description
- video_description = get_element_by_id("eow-description", video_webpage)
- if video_description: video_description = clean_html(video_description.decode('utf8'))
+ video_description = get_element_by_id("eow-description", video_webpage.decode('utf8'))
+ if video_description: video_description = clean_html(video_description)
else: video_description = ''
# closed captions
video_subtitles = None
if self._downloader.params.get('writesubtitles', False):
- self.report_video_subtitles_download(video_id)
- request = urllib2.Request('http://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id)
try:
- srt_list = urllib2.urlopen(request).read()
- except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'WARNING: unable to download video subtitles: %s' % str(err))
- else:
+ self.report_video_subtitles_download(video_id)
+ request = urllib2.Request('http://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id)
+ try:
+ srt_list = urllib2.urlopen(request).read()
+ except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+ raise Trouble(u'WARNING: unable to download video subtitles: %s' % str(err))
srt_lang_list = re.findall(r'lang_code="([\w\-]+)"', srt_list)
- if srt_lang_list:
- if self._downloader.params.get('subtitleslang', False):
- srt_lang = self._downloader.params.get('subtitleslang')
- elif 'en' in srt_lang_list:
- srt_lang = 'en'
- else:
- srt_lang = srt_lang_list[0]
- if not srt_lang in srt_lang_list:
- self._downloader.trouble(u'WARNING: no closed captions found in the specified language')
- else:
- request = urllib2.Request('http://video.google.com/timedtext?hl=en&lang=%s&v=%s' % (srt_lang, video_id))
- try:
- srt_xml = urllib2.urlopen(request).read()
- except (urllib2.URLError, httplib.HTTPException, socket.error), err:
- self._downloader.trouble(u'WARNING: unable to download video subtitles: %s' % str(err))
- else:
- video_subtitles = self._closed_captions_xml_to_srt(srt_xml.decode('utf-8'))
+ if not srt_lang_list:
+ raise Trouble(u'WARNING: video has no closed captions')
+ if self._downloader.params.get('subtitleslang', False):
+ srt_lang = self._downloader.params.get('subtitleslang')
+ elif 'en' in srt_lang_list:
+ srt_lang = 'en'
else:
- self._downloader.trouble(u'WARNING: video has no closed captions')
+ srt_lang = srt_lang_list[0]
+ if not srt_lang in srt_lang_list:
+ raise Trouble(u'WARNING: no closed captions found in the specified language')
+ request = urllib2.Request('http://video.google.com/timedtext?hl=en&lang=%s&v=%s' % (srt_lang, video_id))
+ try:
+ srt_xml = urllib2.urlopen(request).read()
+ except (urllib2.URLError, httplib.HTTPException, socket.error), err:
+ raise Trouble(u'WARNING: unable to download video subtitles: %s' % str(err))
+ video_subtitles = self._closed_captions_xml_to_srt(srt_xml.decode('utf-8'))
+ except Trouble as trouble:
+ self._downloader.trouble(trouble[0])
# token
video_token = urllib.unquote_plus(video_info['token'][0])
@@ -455,7 +447,6 @@ class YoutubeIE(InfoExtractor):
'uploader': video_uploader.decode('utf-8'),
'upload_date': upload_date,
'title': video_title,
- 'stitle': simple_title,
'ext': video_extension.decode('utf-8'),
'format': (format_param is None and u'NA' or format_param.decode('utf-8')),
'thumbnail': video_thumbnail.decode('utf-8'),
@@ -531,8 +522,6 @@ class MetacafeIE(InfoExtractor):
self._downloader.download(['http://www.youtube.com/watch?v=%s' % mobj2.group(1)])
return
- simple_title = mobj.group(2).decode('utf-8')
-
# Retrieve video webpage to extract further information
request = urllib2.Request('http://www.metacafe.com/watch/%s/' % video_id)
try:
@@ -578,7 +567,6 @@ class MetacafeIE(InfoExtractor):
self._downloader.trouble(u'ERROR: unable to extract title')
return
video_title = mobj.group(1).decode('utf-8')
- video_title = sanitize_title(video_title)
mobj = re.search(r'(?ms)By:\s*(.+?)<', webpage)
if mobj is None:
@@ -592,7 +580,6 @@ class MetacafeIE(InfoExtractor):
'uploader': video_uploader.decode('utf-8'),
'upload_date': u'NA',
'title': video_title,
- 'stitle': simple_title,
'ext': video_extension.decode('utf-8'),
'format': u'NA',
'player_url': None,
@@ -659,8 +646,6 @@ class DailymotionIE(InfoExtractor):
self._downloader.trouble(u'ERROR: unable to extract title')
return
video_title = unescapeHTML(mobj.group('title').decode('utf-8'))
- video_title = sanitize_title(video_title)
- simple_title = simplify_title(video_title)
mobj = re.search(r'(?im)[^<]+?]+?>([^<]+?)', webpage)
if mobj is None:
@@ -674,7 +659,6 @@ class DailymotionIE(InfoExtractor):
'uploader': video_uploader.decode('utf-8'),
'upload_date': u'NA',
'title': video_title,
- 'stitle': simple_title,
'ext': video_extension.decode('utf-8'),
'format': u'NA',
'player_url': None,
@@ -738,8 +722,6 @@ class GoogleIE(InfoExtractor):
self._downloader.trouble(u'ERROR: unable to extract title')
return
video_title = mobj.group(1).decode('utf-8')
- video_title = sanitize_title(video_title)
- simple_title = simplify_title(video_title)
# Extract video description
mobj = re.search(r'([^<]*)', webpage)
@@ -772,7 +754,6 @@ class GoogleIE(InfoExtractor):
'uploader': u'NA',
'upload_date': u'NA',
'title': video_title,
- 'stitle': simple_title,
'ext': video_extension.decode('utf-8'),
'format': u'NA',
'player_url': None,
@@ -831,8 +812,6 @@ class PhotobucketIE(InfoExtractor):
self._downloader.trouble(u'ERROR: unable to extract title')
return
video_title = mobj.group(1).decode('utf-8')
- video_title = sanitize_title(video_title)
- simple_title = simplify_title(video_title)
video_uploader = mobj.group(2).decode('utf-8')
@@ -842,7 +821,6 @@ class PhotobucketIE(InfoExtractor):
'uploader': video_uploader,
'upload_date': u'NA',
'title': video_title,
- 'stitle': simple_title,
'ext': video_extension.decode('utf-8'),
'format': u'NA',
'player_url': None,
@@ -920,7 +898,6 @@ class YahooIE(InfoExtractor):
self._downloader.trouble(u'ERROR: unable to extract video title')
return
video_title = mobj.group(1).decode('utf-8')
- simple_title = simplify_title(video_title)
mobj = re.search(r'', webpage)
if mobj is None:
@@ -986,7 +963,6 @@ class YahooIE(InfoExtractor):
'uploader': video_uploader,
'upload_date': u'NA',
'title': video_title,
- 'stitle': simple_title,
'ext': video_extension.decode('utf-8'),
'thumbnail': video_thumbnail.decode('utf-8'),
'description': video_description,
@@ -1046,7 +1022,6 @@ class VimeoIE(InfoExtractor):
# Extract title
video_title = config["video"]["title"]
- simple_title = simplify_title(video_title)
# Extract uploader
video_uploader = config["video"]["owner"]["name"]
@@ -1055,8 +1030,8 @@ class VimeoIE(InfoExtractor):
video_thumbnail = config["video"]["thumbnail"]
# Extract video description
- video_description = get_element_by_id("description", webpage)
- if video_description: video_description = clean_html(video_description.decode('utf8'))
+ video_description = get_element_by_id("description", webpage.decode('utf8'))
+ if video_description: video_description = clean_html(video_description)
else: video_description = ''
# Extract upload date
@@ -1092,7 +1067,6 @@ class VimeoIE(InfoExtractor):
'uploader': video_uploader,
'upload_date': video_upload_date,
'title': video_title,
- 'stitle': simple_title,
'ext': video_extension,
'thumbnail': video_thumbnail,
'description': video_description,
@@ -1227,8 +1201,6 @@ class GenericIE(InfoExtractor):
self._downloader.trouble(u'ERROR: unable to extract title')
return
video_title = mobj.group(1).decode('utf-8')
- video_title = sanitize_title(video_title)
- simple_title = simplify_title(video_title)
# video uploader is domain name
mobj = re.match(r'(?:https?://)?([^/]*)/.*', url)
@@ -1243,7 +1215,6 @@ class GenericIE(InfoExtractor):
'uploader': video_uploader,
'upload_date': u'NA',
'title': video_title,
- 'stitle': simple_title,
'ext': video_extension.decode('utf-8'),
'format': u'NA',
'player_url': None,
@@ -1261,7 +1232,7 @@ class YoutubeSearchIE(InfoExtractor):
InfoExtractor.__init__(self, downloader)
def report_download_page(self, query, pagenum):
- """Report attempt to download playlist page with given number."""
+ """Report attempt to download search page with given number."""
query = query.decode(preferredencoding())
self._downloader.to_screen(u'[youtube] query "%s": Downloading page %s' % (query, pagenum))
@@ -1497,8 +1468,8 @@ class YoutubePlaylistIE(InfoExtractor):
_VALID_URL = r'(?:https?://)?(?:\w+\.)?youtube\.com/(?:(?:course|view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)(?:PL)?([0-9A-Za-z-_]+)(?:/.*?/([0-9A-Za-z_-]+))?.*'
_TEMPLATE_URL = 'http://www.youtube.com/%s?%s=%s&page=%s&gl=US&hl=en'
- _VIDEO_INDICATOR_TEMPLATE = r'/watch\?v=(.+?)&list=PL%s&'
- _MORE_PAGES_INDICATOR = r'(?m)>\s*Next\s*'
+ _VIDEO_INDICATOR_TEMPLATE = r'/watch\?v=(.+?)&list=(PL)?%s&'
+ _MORE_PAGES_INDICATOR = r'yt-uix-pager-next'
IE_NAME = u'youtube:playlist'
def __init__(self, downloader=None):
@@ -1708,7 +1679,6 @@ class DepositFilesIE(InfoExtractor):
'uploader': u'NA',
'upload_date': u'NA',
'title': file_title,
- 'stitle': file_title,
'ext': file_extension.decode('utf-8'),
'format': u'NA',
'player_url': None,
@@ -1853,9 +1823,6 @@ class FacebookIE(InfoExtractor):
return
video_title = video_info['title']
video_title = video_title.decode('utf-8')
- video_title = sanitize_title(video_title)
-
- simple_title = simplify_title(video_title)
# thumbnail image
if 'thumbnail' not in video_info:
@@ -1916,7 +1883,6 @@ class FacebookIE(InfoExtractor):
'uploader': video_uploader.decode('utf-8'),
'upload_date': upload_date,
'title': video_title,
- 'stitle': simple_title,
'ext': video_extension.decode('utf-8'),
'format': (format_param is None and u'NA' or format_param.decode('utf-8')),
'thumbnail': video_thumbnail.decode('utf-8'),
@@ -1966,7 +1932,6 @@ class BlipTVIE(InfoExtractor):
'id': title,
'url': url,
'title': title,
- 'stitle': simplify_title(title),
'ext': ext,
'urlhandle': urlh
}
@@ -1986,21 +1951,20 @@ class BlipTVIE(InfoExtractor):
data = json_data['Post']
else:
data = json_data
-
+
upload_date = datetime.datetime.strptime(data['datestamp'], '%m-%d-%y %H:%M%p').strftime('%Y%m%d')
video_url = data['media']['url']
umobj = re.match(self._URL_EXT, video_url)
if umobj is None:
raise ValueError('Can not determine filename extension')
ext = umobj.group(1)
-
+
info = {
'id': data['item_id'],
'url': video_url,
'uploader': data['display_name'],
'upload_date': upload_date,
'title': data['title'],
- 'stitle': simplify_title(data['title']),
'ext': ext,
'format': data['media']['mimeType'],
'thumbnail': data['thumbnailUrl'],
@@ -2062,9 +2026,6 @@ class MyVideoIE(InfoExtractor):
return
video_title = mobj.group(1)
- video_title = sanitize_title(video_title)
-
- simple_title = simplify_title(video_title)
return [{
'id': video_id,
@@ -2072,7 +2033,6 @@ class MyVideoIE(InfoExtractor):
'uploader': u'NA',
'upload_date': u'NA',
'title': video_title,
- 'stitle': simple_title,
'ext': u'flv',
'format': u'NA',
'player_url': None,
@@ -2086,7 +2046,7 @@ class ComedyCentralIE(InfoExtractor):
def report_extraction(self, episode_id):
self._downloader.to_screen(u'[comedycentral] %s: Extracting information' % episode_id)
-
+
def report_config_download(self, episode_id):
self._downloader.to_screen(u'[comedycentral] %s: Downloading configuration' % episode_id)
@@ -2199,7 +2159,6 @@ class ComedyCentralIE(InfoExtractor):
'uploader': showId,
'upload_date': officialDate,
'title': effTitle,
- 'stitle': simplify_title(effTitle),
'ext': 'mp4',
'format': format,
'thumbnail': None,
@@ -2234,7 +2193,10 @@ class EscapistIE(InfoExtractor):
self.report_extraction(showName)
try:
- webPage = urllib2.urlopen(url).read()
+ webPage = urllib2.urlopen(url)
+ webPageBytes = webPage.read()
+ m = re.match(r'text/html; charset="?([^"]+)"?', webPage.headers['Content-Type'])
+ webPage = webPageBytes.decode(m.group(1) if m else 'utf-8')
except (urllib2.URLError, httplib.HTTPException, socket.error), err:
self._downloader.trouble(u'ERROR: unable to download webpage: ' + unicode(err))
return
@@ -2273,7 +2235,6 @@ class EscapistIE(InfoExtractor):
'uploader': showName,
'upload_date': None,
'title': showName,
- 'stitle': simplify_title(showName),
'ext': 'flv',
'format': 'flv',
'thumbnail': imgUrl,
@@ -2337,7 +2298,6 @@ class CollegeHumorIE(InfoExtractor):
videoNode = mdoc.findall('./video')[0]
info['description'] = videoNode.findall('./description')[0].text
info['title'] = videoNode.findall('./caption')[0].text
- info['stitle'] = simplify_title(info['title'])
info['url'] = videoNode.findall('./file')[0].text
info['thumbnail'] = videoNode.findall('./thumbnail')[0].text
info['ext'] = info['url'].rpartition('.')[2]
@@ -2411,7 +2371,6 @@ class XVideosIE(InfoExtractor):
'uploader': None,
'upload_date': None,
'title': video_title,
- 'stitle': simplify_title(video_title),
'ext': 'flv',
'format': 'flv',
'thumbnail': video_thumbnail,
@@ -2455,7 +2414,7 @@ class SoundcloudIE(InfoExtractor):
uploader = mobj.group(1).decode('utf-8')
# extract simple title (uploader + slug of song title)
slug_title = mobj.group(2).decode('utf-8')
- simple_title = uploader + '-' + slug_title
+ simple_title = uploader + u'-' + slug_title
self.report_webpage('%s/%s' % (uploader, slug_title))
@@ -2477,7 +2436,9 @@ class SoundcloudIE(InfoExtractor):
# extract unsimplified title
mobj = re.search('"title":"(.*?)",', webpage)
if mobj:
- title = mobj.group(1)
+ title = mobj.group(1).decode('utf-8')
+ else:
+ title = simple_title
# construct media url (with uid/token)
mediaURL = "http://media.soundcloud.com/stream/%s?stream_token=%s"
@@ -2496,7 +2457,7 @@ class SoundcloudIE(InfoExtractor):
try:
upload_date = datetime.datetime.strptime(mobj.group(1), '%B %d, %Y %H:%M').strftime('%Y%m%d')
except Exception, e:
- print str(e)
+ self._downloader.to_stderr(str(e))
# for soundcloud, a request to a cross domain is required for cookies
request = urllib2.Request('http://media.soundcloud.com/crossdomain.xml', std_headers)
@@ -2506,8 +2467,7 @@ class SoundcloudIE(InfoExtractor):
'url': mediaURL,
'uploader': uploader.decode('utf-8'),
'upload_date': upload_date,
- 'title': simple_title.decode('utf-8'),
- 'stitle': simple_title.decode('utf-8'),
+ 'title': title,
'ext': u'mp3',
'format': u'NA',
'player_url': None,
@@ -2577,7 +2537,6 @@ class InfoQIE(InfoExtractor):
'uploader': None,
'upload_date': None,
'title': video_title,
- 'stitle': simplify_title(video_title),
'ext': extension,
'format': extension, # Extension is always(?) mp4, but seems to be flv
'thumbnail': None,
@@ -2614,7 +2573,6 @@ class MixcloudIE(InfoExtractor):
url_list = jsonData[fmt][bitrate]
except TypeError: # we have no bitrate info.
url_list = jsonData[fmt]
-
return url_list
def check_urls(self, url_list):
@@ -2694,7 +2652,6 @@ class MixcloudIE(InfoExtractor):
'uploader': uploader.decode('utf-8'),
'upload_date': u'NA',
'title': json_data['name'],
- 'stitle': simplify_title(json_data['name']),
'ext': file_url.split('.')[-1].decode('utf-8'),
'format': (format_param is None and u'NA' or format_param.decode('utf-8')),
'thumbnail': json_data['thumbnail_url'],
@@ -2726,9 +2683,9 @@ class StanfordOpenClassroomIE(InfoExtractor):
course = mobj.group('course')
video = mobj.group('video')
info = {
- 'id': simplify_title(course + '_' + video),
+ 'id': course + '_' + video,
}
-
+
self.report_extraction(info['id'])
baseUrl = 'http://openclassroom.stanford.edu/MainFolder/courses/' + course + '/videos/'
xmlUrl = baseUrl + video + '.xml'
@@ -2744,14 +2701,13 @@ class StanfordOpenClassroomIE(InfoExtractor):
except IndexError:
self._downloader.trouble(u'\nERROR: Invalid metadata XML file')
return
- info['stitle'] = simplify_title(info['title'])
info['ext'] = info['url'].rpartition('.')[2]
info['format'] = info['ext']
return [info]
elif mobj.group('course'): # A course page
course = mobj.group('course')
info = {
- 'id': simplify_title(course),
+ 'id': course,
'type': 'playlist',
}
@@ -2767,7 +2723,6 @@ class StanfordOpenClassroomIE(InfoExtractor):
info['title'] = unescapeHTML(m.group(1))
else:
info['title'] = info['id']
- info['stitle'] = simplify_title(info['title'])
m = re.search('([^<]+)', coursepage)
if m:
@@ -2801,7 +2756,6 @@ class StanfordOpenClassroomIE(InfoExtractor):
return
info['title'] = info['id']
- info['stitle'] = simplify_title(info['title'])
links = orderedSet(re.findall('', rootpage))
info['list'] = [
@@ -2900,7 +2854,6 @@ class MTVIE(InfoExtractor):
'url': video_url,
'uploader': performer,
'title': video_title,
- 'stitle': simplify_title(video_title),
'ext': ext,
'format': format,
}