2 # -*- coding: utf-8 -*-
4 from __future__ import absolute_import
13 import xml.etree.ElementTree
20 class InfoExtractor(object):
21 """Information Extractor class.
23 Information extractors are the classes that, given a URL, extract
24 information about the video (or videos) the URL refers to. This
25 information includes the real video URL, the video title, author and
26 others. The information is stored in a dictionary which is then
27 passed to the FileDownloader. The FileDownloader processes this
28 information possibly downloading the video to the file system, among
29 other possible outcomes.
31 The dictionaries must include the following fields:
35 uploader: Nickname of the video uploader, unescaped.
36 upload_date: Video upload date (YYYYMMDD).
37 title: Video title, unescaped.
38 ext: Video filename extension.
40 The following fields are optional:
42 format: The video format, defaults to ext (used for --get-format)
43 thumbnail: Full URL to a video thumbnail image.
44 description: One-line video description.
45 player_url: SWF Player URL (used for rtmpdump).
46 subtitles: The .srt file contents.
47 urlhandle: [internal] The urlHandle to be used to download the file,
48 like returned by urllib.request.urlopen
50 The fields should all be Unicode strings.
52 Subclasses of this one should re-define the _real_initialize() and
53 _real_extract() methods and define a _VALID_URL regexp.
54 Probably, they should also be added to the list of extractors.
56 _real_extract() must return a *list* of information dictionaries as
59 Finally, the _WORKING attribute should be set to False for broken IEs
60 in order to warn the users and skip the tests.
67 def __init__(self, downloader=None):
68 """Constructor. Receives an optional downloader."""
70 self.set_downloader(downloader)
72 def suitable(self, url):
73 """Receives a URL and returns True if suitable for this IE."""
74 return re.match(self._VALID_URL, url) is not None
77 """Getter method for _WORKING."""
81 """Initializes an instance (authentication, etc)."""
83 self._real_initialize()
86 def extract(self, url):
87 """Extracts URL information and returns it in list of dicts."""
89 return self._real_extract(url)
91 def set_downloader(self, downloader):
92 """Sets the downloader for this IE."""
93 self._downloader = downloader
95 def _real_initialize(self):
96 """Real initialization process. Redefine in subclasses."""
99 def _real_extract(self, url):
100 """Real extraction process. Redefine in subclasses."""
104 class YoutubeIE(InfoExtractor):
105 """Information extractor for youtube.com."""
109 (?:https?://)? # http(s):// (optional)
110 (?:youtu\.be/|(?:\w+\.)?youtube(?:-nocookie)?\.com/|
111 tube\.majestyc\.net/) # the various hostnames, with wildcard subdomains
112 (?:.*?\#/)? # handle anchor (#/) redirect urls
113 (?!view_play_list|my_playlists|artist|playlist) # ignore playlist URLs
114 (?: # the various things that can precede the ID:
115 (?:(?:v|embed|e)/) # v/ or embed/ or e/
116 |(?: # or the v= param in all its forms
117 (?:watch(?:_popup)?(?:\.php)?)? # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
118 (?:\?|\#!?) # the params delimiter ? or # or #!
119 (?:.+&)? # any other preceding param (like /?s=tuff&v=xxxx)
122 )? # optional -> youtube.com/xxxx is OK
123 )? # all until now is optional -> you can pass the naked ID
124 ([0-9A-Za-z_-]+) # here is it! the YouTube video ID
125 (?(1).+)? # if we found the ID, everything can follow
127 _LANG_URL = r'http://www.youtube.com/?hl=en&persist_hl=1&gl=US&persist_gl=1&opt_out_ackd=1'
128 _LOGIN_URL = 'https://www.youtube.com/signup?next=/&gl=US&hl=en'
129 _AGE_URL = 'http://www.youtube.com/verify_age?next_url=/&gl=US&hl=en'
130 _NEXT_URL_RE = r'[\?&]next_url=([^&]+)'
131 _NETRC_MACHINE = 'youtube'
132 # Listed in order of quality
133 _available_formats = ['38', '37', '46', '22', '45', '35', '44', '34', '18', '43', '6', '5', '17', '13']
134 _available_formats_prefer_free = ['38', '46', '37', '45', '22', '44', '35', '43', '34', '18', '6', '5', '17', '13']
135 _video_extensions = {
141 '38': 'video', # You actually don't know if this will be MOV, AVI or whatever
147 _video_dimensions = {
165 def suitable(self, url):
166 """Receives a URL and returns True if suitable for this IE."""
167 return re.match(self._VALID_URL, url, re.VERBOSE) is not None
169 def report_lang(self):
170 """Report attempt to set language."""
171 self._downloader.to_screen(u'[youtube] Setting language')
173 def report_login(self):
174 """Report attempt to log in."""
175 self._downloader.to_screen(u'[youtube] Logging in')
177 def report_age_confirmation(self):
178 """Report attempt to confirm age."""
179 self._downloader.to_screen(u'[youtube] Confirming age')
181 def report_video_webpage_download(self, video_id):
182 """Report attempt to download video webpage."""
183 self._downloader.to_screen(u'[youtube] %s: Downloading video webpage' % video_id)
185 def report_video_info_webpage_download(self, video_id):
186 """Report attempt to download video info webpage."""
187 self._downloader.to_screen(u'[youtube] %s: Downloading video info webpage' % video_id)
189 def report_video_subtitles_download(self, video_id):
190 """Report attempt to download video info webpage."""
191 self._downloader.to_screen(u'[youtube] %s: Downloading video subtitles' % video_id)
193 def report_information_extraction(self, video_id):
194 """Report attempt to extract video information."""
195 self._downloader.to_screen(u'[youtube] %s: Extracting video information' % video_id)
197 def report_unavailable_format(self, video_id, format):
198 """Report extracted video URL."""
199 self._downloader.to_screen(u'[youtube] %s: Format %s not available' % (video_id, format))
201 def report_rtmp_download(self):
202 """Indicate the download will use the RTMP protocol."""
203 self._downloader.to_screen(u'[youtube] RTMP download detected')
205 def _closed_captions_xml_to_srt(self, xml_string):
207 texts = re.findall(r'<text start="([\d\.]+)"( dur="([\d\.]+)")?>([^<]+)</text>', xml_string, re.MULTILINE)
208 # TODO parse xml instead of regex
209 for n, (start, dur_tag, dur, caption) in enumerate(texts):
210 if not dur: dur = '4'
212 end = start + float(dur)
213 start = "%02i:%02i:%02i,%03i" %(start/(60*60), start/60%60, start%60, start%1*1000)
214 end = "%02i:%02i:%02i,%03i" %(end/(60*60), end/60%60, end%60, end%1*1000)
215 caption = unescapeHTML(caption)
216 caption = unescapeHTML(caption) # double cycle, intentional
217 srt += str(n+1) + '\n'
218 srt += start + ' --> ' + end + '\n'
219 srt += caption + '\n\n'
222 def _print_formats(self, formats):
223 print('Available formats:')
225 print('%s\t:\t%s\t[%s]' %(x, self._video_extensions.get(x, 'flv'), self._video_dimensions.get(x, '???')))
227 def _real_initialize(self):
228 if self._downloader is None:
233 downloader_params = self._downloader.params
235 # Attempt to use provided username and password or .netrc data
236 if downloader_params.get('username', None) is not None:
237 username = downloader_params['username']
238 password = downloader_params['password']
239 elif downloader_params.get('usenetrc', False):
241 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
246 raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
247 except (IOError, netrc.NetrcParseError) as err:
248 self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % compat_str(err))
252 request = compat_urllib_request.Request(self._LANG_URL)
255 compat_urllib_request.urlopen(request).read()
256 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
257 self._downloader.to_stderr(u'WARNING: unable to set language: %s' % compat_str(err))
260 # No authentication to be performed
266 'current_form': 'loginForm',
268 'action_login': 'Log In',
269 'username': username,
270 'password': password,
272 request = compat_urllib_request.Request(self._LOGIN_URL, compat_urllib_parse.urlencode(login_form))
275 login_results = compat_urllib_request.urlopen(request).read()
276 if re.search(r'(?i)<form[^>]* name="loginForm"', login_results) is not None:
277 self._downloader.to_stderr(u'WARNING: unable to log in: bad username or password')
279 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
280 self._downloader.to_stderr(u'WARNING: unable to log in: %s' % compat_str(err))
286 'action_confirm': 'Confirm',
288 request = compat_urllib_request.Request(self._AGE_URL, compat_urllib_parse.urlencode(age_form))
290 self.report_age_confirmation()
291 age_results = compat_urllib_request.urlopen(request).read()
292 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
293 self._downloader.trouble(u'ERROR: unable to confirm age: %s' % compat_str(err))
296 def _real_extract(self, url):
297 # Extract original video URL from URL with redirection, like age verification, using next_url parameter
298 mobj = re.search(self._NEXT_URL_RE, url)
300 url = 'http://www.youtube.com/' + compat_urllib_parse.unquote(mobj.group(1)).lstrip('/')
302 # Extract video id from URL
303 mobj = re.match(self._VALID_URL, url, re.VERBOSE)
305 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
307 video_id = mobj.group(2)
310 self.report_video_webpage_download(video_id)
311 request = compat_urllib_request.Request('http://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1' % video_id)
313 video_webpage_bytes = compat_urllib_request.urlopen(request).read()
314 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
315 self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
318 video_webpage = video_webpage_bytes.decode('utf-8', 'ignore')
320 # Attempt to extract SWF player URL
321 mobj = re.search(r'swfConfig.*?"(http:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)
323 player_url = re.sub(r'\\(.)', r'\1', mobj.group(1))
328 self.report_video_info_webpage_download(video_id)
329 for el_type in ['&el=embedded', '&el=detailpage', '&el=vevo', '']:
330 video_info_url = ('http://www.youtube.com/get_video_info?&video_id=%s%s&ps=default&eurl=&gl=US&hl=en'
331 % (video_id, el_type))
332 request = compat_urllib_request.Request(video_info_url)
334 video_info_webpage_bytes = compat_urllib_request.urlopen(request).read()
335 video_info_webpage = video_info_webpage_bytes.decode('utf-8', 'ignore')
336 video_info = compat_parse_qs(video_info_webpage)
337 if 'token' in video_info:
339 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
340 self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % compat_str(err))
342 if 'token' not in video_info:
343 if 'reason' in video_info:
344 self._downloader.trouble(u'ERROR: YouTube said: %s' % video_info['reason'][0])
346 self._downloader.trouble(u'ERROR: "token" parameter not in video info for unknown reason')
349 # Check for "rental" videos
350 if 'ypc_video_rental_bar_text' in video_info and 'author' not in video_info:
351 self._downloader.trouble(u'ERROR: "rental" videos not supported')
354 # Start extracting information
355 self.report_information_extraction(video_id)
358 if 'author' not in video_info:
359 self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
361 video_uploader = compat_urllib_parse.unquote_plus(video_info['author'][0])
364 if 'title' not in video_info:
365 self._downloader.trouble(u'ERROR: unable to extract video title')
367 video_title = compat_urllib_parse.unquote_plus(video_info['title'][0])
370 if 'thumbnail_url' not in video_info:
371 self._downloader.trouble(u'WARNING: unable to extract video thumbnail')
373 else: # don't panic if we can't find it
374 video_thumbnail = compat_urllib_parse.unquote_plus(video_info['thumbnail_url'][0])
378 mobj = re.search(r'id="eow-date.*?>(.*?)</span>', video_webpage, re.DOTALL)
380 upload_date = ' '.join(re.sub(r'[/,-]', r' ', mobj.group(1)).split())
381 format_expressions = ['%d %B %Y', '%B %d %Y', '%b %d %Y']
382 for expression in format_expressions:
384 upload_date = datetime.datetime.strptime(upload_date, expression).strftime('%Y%m%d')
389 video_description = get_element_by_id("eow-description", video_webpage)
390 if video_description:
391 video_description = clean_html(video_description)
393 video_description = ''
396 video_subtitles = None
397 if self._downloader.params.get('writesubtitles', False):
399 self.report_video_subtitles_download(video_id)
400 request = compat_urllib_request.Request('http://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id)
402 srt_list = compat_urllib_request.urlopen(request).read()
403 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
404 raise Trouble(u'WARNING: unable to download video subtitles: %s' % compat_str(err))
405 srt_lang_list = re.findall(r'name="([^"]*)"[^>]+lang_code="([\w\-]+)"', srt_list)
406 srt_lang_list = dict((l[1], l[0]) for l in srt_lang_list)
407 if not srt_lang_list:
408 raise Trouble(u'WARNING: video has no closed captions')
409 if self._downloader.params.get('subtitleslang', False):
410 srt_lang = self._downloader.params.get('subtitleslang')
411 elif 'en' in srt_lang_list:
414 srt_lang = srt_lang_list.keys()[0]
415 if not srt_lang in srt_lang_list:
416 raise Trouble(u'WARNING: no closed captions found in the specified language')
417 request = compat_urllib_request.Request('http://www.youtube.com/api/timedtext?lang=%s&name=%s&v=%s' % (srt_lang, srt_lang_list[srt_lang], video_id))
419 srt_xml = compat_urllib_request.urlopen(request).read()
420 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
421 raise Trouble(u'WARNING: unable to download video subtitles: %s' % compat_str(err))
423 raise Trouble(u'WARNING: unable to download video subtitles')
424 video_subtitles = self._closed_captions_xml_to_srt(srt_xml.decode('utf-8'))
425 except Trouble as trouble:
426 self._downloader.trouble(trouble[0])
428 if 'length_seconds' not in video_info:
429 self._downloader.trouble(u'WARNING: unable to extract video duration')
432 video_duration = compat_urllib_parse.unquote_plus(video_info['length_seconds'][0])
435 video_token = compat_urllib_parse.unquote_plus(video_info['token'][0])
437 # Decide which formats to download
438 req_format = self._downloader.params.get('format', None)
440 if 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
441 self.report_rtmp_download()
442 video_url_list = [(None, video_info['conn'][0])]
443 elif 'url_encoded_fmt_stream_map' in video_info and len(video_info['url_encoded_fmt_stream_map']) >= 1:
444 url_data_strs = video_info['url_encoded_fmt_stream_map'][0].split(',')
445 url_data = [compat_parse_qs(uds) for uds in url_data_strs]
446 url_data = filter(lambda ud: 'itag' in ud and 'url' in ud, url_data)
447 url_map = dict((ud['itag'][0], ud['url'][0] + '&signature=' + ud['sig'][0]) for ud in url_data)
449 format_limit = self._downloader.params.get('format_limit', None)
450 available_formats = self._available_formats_prefer_free if self._downloader.params.get('prefer_free_formats', False) else self._available_formats
451 if format_limit is not None and format_limit in available_formats:
452 format_list = available_formats[available_formats.index(format_limit):]
454 format_list = available_formats
455 existing_formats = [x for x in format_list if x in url_map]
456 if len(existing_formats) == 0:
457 self._downloader.trouble(u'ERROR: no known formats available for video')
459 if self._downloader.params.get('listformats', None):
460 self._print_formats(existing_formats)
462 if req_format is None or req_format == 'best':
463 video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality
464 elif req_format == 'worst':
465 video_url_list = [(existing_formats[len(existing_formats)-1], url_map[existing_formats[len(existing_formats)-1]])] # worst quality
466 elif req_format in ('-1', 'all'):
467 video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats
469 # Specific formats. We pick the first in a slash-delimeted sequence.
470 # For example, if '1/2/3/4' is requested and '2' and '4' are available, we pick '2'.
471 req_formats = req_format.split('/')
472 video_url_list = None
473 for rf in req_formats:
475 video_url_list = [(rf, url_map[rf])]
477 if video_url_list is None:
478 self._downloader.trouble(u'ERROR: requested format not available')
481 self._downloader.trouble(u'ERROR: no conn or url_encoded_fmt_stream_map information found in video info')
485 for format_param, video_real_url in video_url_list:
487 video_extension = self._video_extensions.get(format_param, 'flv')
489 video_format = '{0} - {1}'.format(format_param if format_param else video_extension,
490 self._video_dimensions.get(format_param, '???'))
494 'url': video_real_url,
495 'uploader': video_uploader,
496 'upload_date': upload_date,
497 'title': video_title,
498 'ext': video_extension,
499 'format': video_format,
500 'thumbnail': video_thumbnail,
501 'description': video_description,
502 'player_url': player_url,
503 'subtitles': video_subtitles,
504 'duration': video_duration
509 class MetacafeIE(InfoExtractor):
510 """Information Extractor for metacafe.com."""
512 _VALID_URL = r'(?:http://)?(?:www\.)?metacafe\.com/watch/([^/]+)/([^/]+)/.*'
513 _DISCLAIMER = 'http://www.metacafe.com/family_filter/'
514 _FILTER_POST = 'http://www.metacafe.com/f/index.php?inputType=filter&controllerGroup=user'
515 IE_NAME = u'metacafe'
517 def __init__(self, downloader=None):
518 InfoExtractor.__init__(self, downloader)
520 def report_disclaimer(self):
521 """Report disclaimer retrieval."""
522 self._downloader.to_screen(u'[metacafe] Retrieving disclaimer')
524 def report_age_confirmation(self):
525 """Report attempt to confirm age."""
526 self._downloader.to_screen(u'[metacafe] Confirming age')
528 def report_download_webpage(self, video_id):
529 """Report webpage download."""
530 self._downloader.to_screen(u'[metacafe] %s: Downloading webpage' % video_id)
532 def report_extraction(self, video_id):
533 """Report information extraction."""
534 self._downloader.to_screen(u'[metacafe] %s: Extracting information' % video_id)
536 def _real_initialize(self):
537 # Retrieve disclaimer
538 request = compat_urllib_request.Request(self._DISCLAIMER)
540 self.report_disclaimer()
541 disclaimer = compat_urllib_request.urlopen(request).read()
542 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
543 self._downloader.trouble(u'ERROR: unable to retrieve disclaimer: %s' % compat_str(err))
549 'submit': "Continue - I'm over 18",
551 request = compat_urllib_request.Request(self._FILTER_POST, compat_urllib_parse.urlencode(disclaimer_form))
553 self.report_age_confirmation()
554 disclaimer = compat_urllib_request.urlopen(request).read()
555 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
556 self._downloader.trouble(u'ERROR: unable to confirm age: %s' % compat_str(err))
559 def _real_extract(self, url):
560 # Extract id and simplified title from URL
561 mobj = re.match(self._VALID_URL, url)
563 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
566 video_id = mobj.group(1)
568 # Check if video comes from YouTube
569 mobj2 = re.match(r'^yt-(.*)$', video_id)
570 if mobj2 is not None:
571 self._downloader.download(['http://www.youtube.com/watch?v=%s' % mobj2.group(1)])
574 # Retrieve video webpage to extract further information
575 request = compat_urllib_request.Request('http://www.metacafe.com/watch/%s/' % video_id)
577 self.report_download_webpage(video_id)
578 webpage = compat_urllib_request.urlopen(request).read()
579 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
580 self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % compat_str(err))
583 # Extract URL, uploader and title from webpage
584 self.report_extraction(video_id)
585 mobj = re.search(r'(?m)&mediaURL=([^&]+)', webpage)
587 mediaURL = compat_urllib_parse.unquote(mobj.group(1))
588 video_extension = mediaURL[-3:]
590 # Extract gdaKey if available
591 mobj = re.search(r'(?m)&gdaKey=(.*?)&', webpage)
595 gdaKey = mobj.group(1)
596 video_url = '%s?__gda__=%s' % (mediaURL, gdaKey)
598 mobj = re.search(r' name="flashvars" value="(.*?)"', webpage)
600 self._downloader.trouble(u'ERROR: unable to extract media URL')
602 vardict = compat_parse_qs(mobj.group(1))
603 if 'mediaData' not in vardict:
604 self._downloader.trouble(u'ERROR: unable to extract media URL')
606 mobj = re.search(r'"mediaURL":"(http.*?)","key":"(.*?)"', vardict['mediaData'][0])
608 self._downloader.trouble(u'ERROR: unable to extract media URL')
610 mediaURL = mobj.group(1).replace('\\/', '/')
611 video_extension = mediaURL[-3:]
612 video_url = '%s?__gda__=%s' % (mediaURL, mobj.group(2))
614 mobj = re.search(r'(?im)<title>(.*) - Video</title>', webpage)
616 self._downloader.trouble(u'ERROR: unable to extract title')
618 video_title = mobj.group(1).decode('utf-8')
620 mobj = re.search(r'submitter=(.*?);', webpage)
622 self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
624 video_uploader = mobj.group(1)
627 'id': video_id.decode('utf-8'),
628 'url': video_url.decode('utf-8'),
629 'uploader': video_uploader.decode('utf-8'),
631 'title': video_title,
632 'ext': video_extension.decode('utf-8'),
636 class DailymotionIE(InfoExtractor):
637 """Information Extractor for Dailymotion"""
639 _VALID_URL = r'(?i)(?:https?://)?(?:www\.)?dailymotion\.[a-z]{2,3}/video/([^/]+)'
640 IE_NAME = u'dailymotion'
642 def __init__(self, downloader=None):
643 InfoExtractor.__init__(self, downloader)
645 def report_download_webpage(self, video_id):
646 """Report webpage download."""
647 self._downloader.to_screen(u'[dailymotion] %s: Downloading webpage' % video_id)
649 def report_extraction(self, video_id):
650 """Report information extraction."""
651 self._downloader.to_screen(u'[dailymotion] %s: Extracting information' % video_id)
653 def _real_extract(self, url):
654 # Extract id and simplified title from URL
655 mobj = re.match(self._VALID_URL, url)
657 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
660 video_id = mobj.group(1).split('_')[0].split('?')[0]
662 video_extension = 'mp4'
664 # Retrieve video webpage to extract further information
665 request = compat_urllib_request.Request(url)
666 request.add_header('Cookie', 'family_filter=off')
668 self.report_download_webpage(video_id)
669 webpage = compat_urllib_request.urlopen(request).read()
670 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
671 self._downloader.trouble(u'ERROR: unable retrieve video webpage: %s' % compat_str(err))
674 # Extract URL, uploader and title from webpage
675 self.report_extraction(video_id)
676 mobj = re.search(r'\s*var flashvars = (.*)', webpage)
678 self._downloader.trouble(u'ERROR: unable to extract media URL')
680 flashvars = compat_urllib_parse.unquote(mobj.group(1))
682 for key in ['hd1080URL', 'hd720URL', 'hqURL', 'sdURL', 'ldURL', 'video_url']:
685 self._downloader.to_screen(u'[dailymotion] Using %s' % key)
688 self._downloader.trouble(u'ERROR: unable to extract video URL')
691 mobj = re.search(r'"' + max_quality + r'":"(.+?)"', flashvars)
693 self._downloader.trouble(u'ERROR: unable to extract video URL')
696 video_url = compat_urllib_parse.unquote(mobj.group(1)).replace('\\/', '/')
698 # TODO: support choosing qualities
700 mobj = re.search(r'<meta property="og:title" content="(?P<title>[^"]*)" />', webpage)
702 self._downloader.trouble(u'ERROR: unable to extract title')
704 video_title = unescapeHTML(mobj.group('title').decode('utf-8'))
706 video_uploader = None
707 mobj = re.search(r'(?im)<span class="owner[^\"]+?">[^<]+?<a [^>]+?>([^<]+?)</a>', webpage)
709 # lookin for official user
710 mobj_official = re.search(r'<span rel="author"[^>]+?>([^<]+?)</span>', webpage)
711 if mobj_official is None:
712 self._downloader.trouble(u'WARNING: unable to extract uploader nickname')
714 video_uploader = mobj_official.group(1)
716 video_uploader = mobj.group(1)
718 video_upload_date = None
719 mobj = re.search(r'<div class="[^"]*uploaded_cont[^"]*" title="[^"]*">([0-9]{2})-([0-9]{2})-([0-9]{4})</div>', webpage)
721 video_upload_date = mobj.group(3) + mobj.group(2) + mobj.group(1)
724 'id': video_id.decode('utf-8'),
725 'url': video_url.decode('utf-8'),
726 'uploader': video_uploader.decode('utf-8'),
727 'upload_date': video_upload_date,
728 'title': video_title,
729 'ext': video_extension.decode('utf-8'),
733 class GoogleIE(InfoExtractor):
734 """Information extractor for video.google.com."""
736 _VALID_URL = r'(?:http://)?video\.google\.(?:com(?:\.au)?|co\.(?:uk|jp|kr|cr)|ca|de|es|fr|it|nl|pl)/videoplay\?docid=([^\&]+).*'
737 IE_NAME = u'video.google'
739 def __init__(self, downloader=None):
740 InfoExtractor.__init__(self, downloader)
742 def report_download_webpage(self, video_id):
743 """Report webpage download."""
744 self._downloader.to_screen(u'[video.google] %s: Downloading webpage' % video_id)
746 def report_extraction(self, video_id):
747 """Report information extraction."""
748 self._downloader.to_screen(u'[video.google] %s: Extracting information' % video_id)
750 def _real_extract(self, url):
751 # Extract id from URL
752 mobj = re.match(self._VALID_URL, url)
754 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
757 video_id = mobj.group(1)
759 video_extension = 'mp4'
761 # Retrieve video webpage to extract further information
762 request = compat_urllib_request.Request('http://video.google.com/videoplay?docid=%s&hl=en&oe=utf-8' % video_id)
764 self.report_download_webpage(video_id)
765 webpage = compat_urllib_request.urlopen(request).read()
766 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
767 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
770 # Extract URL, uploader, and title from webpage
771 self.report_extraction(video_id)
772 mobj = re.search(r"download_url:'([^']+)'", webpage)
774 video_extension = 'flv'
775 mobj = re.search(r"(?i)videoUrl\\x3d(.+?)\\x26", webpage)
777 self._downloader.trouble(u'ERROR: unable to extract media URL')
779 mediaURL = compat_urllib_parse.unquote(mobj.group(1))
780 mediaURL = mediaURL.replace('\\x3d', '\x3d')
781 mediaURL = mediaURL.replace('\\x26', '\x26')
785 mobj = re.search(r'<title>(.*)</title>', webpage)
787 self._downloader.trouble(u'ERROR: unable to extract title')
789 video_title = mobj.group(1).decode('utf-8')
791 # Extract video description
792 mobj = re.search(r'<span id=short-desc-content>([^<]*)</span>', webpage)
794 self._downloader.trouble(u'ERROR: unable to extract video description')
796 video_description = mobj.group(1).decode('utf-8')
797 if not video_description:
798 video_description = 'No description available.'
800 # Extract video thumbnail
801 if self._downloader.params.get('forcethumbnail', False):
802 request = compat_urllib_request.Request('http://video.google.com/videosearch?q=%s+site:video.google.com&hl=en' % abs(int(video_id)))
804 webpage = compat_urllib_request.urlopen(request).read()
805 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
806 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
808 mobj = re.search(r'<img class=thumbnail-img (?:.* )?src=(http.*)>', webpage)
810 self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
812 video_thumbnail = mobj.group(1)
813 else: # we need something to pass to process_info
817 'id': video_id.decode('utf-8'),
818 'url': video_url.decode('utf-8'),
821 'title': video_title,
822 'ext': video_extension.decode('utf-8'),
826 class PhotobucketIE(InfoExtractor):
827 """Information extractor for photobucket.com."""
829 _VALID_URL = r'(?:http://)?(?:[a-z0-9]+\.)?photobucket\.com/.*[\?\&]current=(.*\.flv)'
830 IE_NAME = u'photobucket'
832 def __init__(self, downloader=None):
833 InfoExtractor.__init__(self, downloader)
835 def report_download_webpage(self, video_id):
836 """Report webpage download."""
837 self._downloader.to_screen(u'[photobucket] %s: Downloading webpage' % video_id)
839 def report_extraction(self, video_id):
840 """Report information extraction."""
841 self._downloader.to_screen(u'[photobucket] %s: Extracting information' % video_id)
843 def _real_extract(self, url):
844 # Extract id from URL
845 mobj = re.match(self._VALID_URL, url)
847 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
850 video_id = mobj.group(1)
852 video_extension = 'flv'
854 # Retrieve video webpage to extract further information
855 request = compat_urllib_request.Request(url)
857 self.report_download_webpage(video_id)
858 webpage = compat_urllib_request.urlopen(request).read()
859 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
860 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
863 # Extract URL, uploader, and title from webpage
864 self.report_extraction(video_id)
865 mobj = re.search(r'<link rel="video_src" href=".*\?file=([^"]+)" />', webpage)
867 self._downloader.trouble(u'ERROR: unable to extract media URL')
869 mediaURL = compat_urllib_parse.unquote(mobj.group(1))
873 mobj = re.search(r'<title>(.*) video by (.*) - Photobucket</title>', webpage)
875 self._downloader.trouble(u'ERROR: unable to extract title')
877 video_title = mobj.group(1).decode('utf-8')
879 video_uploader = mobj.group(2).decode('utf-8')
882 'id': video_id.decode('utf-8'),
883 'url': video_url.decode('utf-8'),
884 'uploader': video_uploader,
886 'title': video_title,
887 'ext': video_extension.decode('utf-8'),
891 class YahooIE(InfoExtractor):
892 """Information extractor for video.yahoo.com."""
894 # _VALID_URL matches all Yahoo! Video URLs
895 # _VPAGE_URL matches only the extractable '/watch/' URLs
896 _VALID_URL = r'(?:http://)?(?:[a-z]+\.)?video\.yahoo\.com/(?:watch|network)/([0-9]+)(?:/|\?v=)([0-9]+)(?:[#\?].*)?'
897 _VPAGE_URL = r'(?:http://)?video\.yahoo\.com/watch/([0-9]+)/([0-9]+)(?:[#\?].*)?'
898 IE_NAME = u'video.yahoo'
900 def __init__(self, downloader=None):
901 InfoExtractor.__init__(self, downloader)
903 def report_download_webpage(self, video_id):
904 """Report webpage download."""
905 self._downloader.to_screen(u'[video.yahoo] %s: Downloading webpage' % video_id)
907 def report_extraction(self, video_id):
908 """Report information extraction."""
909 self._downloader.to_screen(u'[video.yahoo] %s: Extracting information' % video_id)
911 def _real_extract(self, url, new_video=True):
912 # Extract ID from URL
913 mobj = re.match(self._VALID_URL, url)
915 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
918 video_id = mobj.group(2)
919 video_extension = 'flv'
921 # Rewrite valid but non-extractable URLs as
922 # extractable English language /watch/ URLs
923 if re.match(self._VPAGE_URL, url) is None:
924 request = compat_urllib_request.Request(url)
926 webpage = compat_urllib_request.urlopen(request).read()
927 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
928 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
931 mobj = re.search(r'\("id", "([0-9]+)"\);', webpage)
933 self._downloader.trouble(u'ERROR: Unable to extract id field')
935 yahoo_id = mobj.group(1)
937 mobj = re.search(r'\("vid", "([0-9]+)"\);', webpage)
939 self._downloader.trouble(u'ERROR: Unable to extract vid field')
941 yahoo_vid = mobj.group(1)
943 url = 'http://video.yahoo.com/watch/%s/%s' % (yahoo_vid, yahoo_id)
944 return self._real_extract(url, new_video=False)
946 # Retrieve video webpage to extract further information
947 request = compat_urllib_request.Request(url)
949 self.report_download_webpage(video_id)
950 webpage = compat_urllib_request.urlopen(request).read()
951 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
952 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
955 # Extract uploader and title from webpage
956 self.report_extraction(video_id)
957 mobj = re.search(r'<meta name="title" content="(.*)" />', webpage)
959 self._downloader.trouble(u'ERROR: unable to extract video title')
961 video_title = mobj.group(1).decode('utf-8')
963 mobj = re.search(r'<h2 class="ti-5"><a href="http://video\.yahoo\.com/(people|profile)/[0-9]+" beacon=".*">(.*)</a></h2>', webpage)
965 self._downloader.trouble(u'ERROR: unable to extract video uploader')
967 video_uploader = mobj.group(1).decode('utf-8')
969 # Extract video thumbnail
970 mobj = re.search(r'<link rel="image_src" href="(.*)" />', webpage)
972 self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
974 video_thumbnail = mobj.group(1).decode('utf-8')
976 # Extract video description
977 mobj = re.search(r'<meta name="description" content="(.*)" />', webpage)
979 self._downloader.trouble(u'ERROR: unable to extract video description')
981 video_description = mobj.group(1).decode('utf-8')
982 if not video_description:
983 video_description = 'No description available.'
985 # Extract video height and width
986 mobj = re.search(r'<meta name="video_height" content="([0-9]+)" />', webpage)
988 self._downloader.trouble(u'ERROR: unable to extract video height')
990 yv_video_height = mobj.group(1)
992 mobj = re.search(r'<meta name="video_width" content="([0-9]+)" />', webpage)
994 self._downloader.trouble(u'ERROR: unable to extract video width')
996 yv_video_width = mobj.group(1)
998 # Retrieve video playlist to extract media URL
999 # I'm not completely sure what all these options are, but we
1000 # seem to need most of them, otherwise the server sends a 401.
1001 yv_lg = 'R0xx6idZnW2zlrKP8xxAIR' # not sure what this represents
1002 yv_bitrate = '700' # according to Wikipedia this is hard-coded
1003 request = compat_urllib_request.Request('http://cosmos.bcst.yahoo.com/up/yep/process/getPlaylistFOP.php?node_id=' + video_id +
1004 '&tech=flash&mode=playlist&lg=' + yv_lg + '&bitrate=' + yv_bitrate + '&vidH=' + yv_video_height +
1005 '&vidW=' + yv_video_width + '&swf=as3&rd=video.yahoo.com&tk=null&adsupported=v1,v2,&eventid=1301797')
1007 self.report_download_webpage(video_id)
1008 webpage = compat_urllib_request.urlopen(request).read()
1009 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1010 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
1013 # Extract media URL from playlist XML
1014 mobj = re.search(r'<STREAM APP="(http://.*)" FULLPATH="/?(/.*\.flv\?[^"]*)"', webpage)
1016 self._downloader.trouble(u'ERROR: Unable to extract media URL')
1018 video_url = compat_urllib_parse.unquote(mobj.group(1) + mobj.group(2)).decode('utf-8')
1019 video_url = unescapeHTML(video_url)
1022 'id': video_id.decode('utf-8'),
1024 'uploader': video_uploader,
1025 'upload_date': None,
1026 'title': video_title,
1027 'ext': video_extension.decode('utf-8'),
1028 'thumbnail': video_thumbnail.decode('utf-8'),
1029 'description': video_description,
1033 class VimeoIE(InfoExtractor):
1034 """Information extractor for vimeo.com."""
1036 # _VALID_URL matches Vimeo URLs
1037 _VALID_URL = r'(?:https?://)?(?:(?:www|player).)?vimeo\.com/(?:(?:groups|album)/[^/]+/)?(?:videos?/)?([0-9]+)'
1040 def __init__(self, downloader=None):
1041 InfoExtractor.__init__(self, downloader)
1043 def report_download_webpage(self, video_id):
1044 """Report webpage download."""
1045 self._downloader.to_screen(u'[vimeo] %s: Downloading webpage' % video_id)
1047 def report_extraction(self, video_id):
1048 """Report information extraction."""
1049 self._downloader.to_screen(u'[vimeo] %s: Extracting information' % video_id)
1051 def _real_extract(self, url, new_video=True):
1052 # Extract ID from URL
1053 mobj = re.match(self._VALID_URL, url)
1055 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1058 video_id = mobj.group(1)
1060 # Retrieve video webpage to extract further information
1061 request = compat_urllib_request.Request(url, None, std_headers)
1063 self.report_download_webpage(video_id)
1064 webpage = compat_urllib_request.urlopen(request).read()
1065 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1066 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
1069 # Now we begin extracting as much information as we can from what we
1070 # retrieved. First we extract the information common to all extractors,
1071 # and latter we extract those that are Vimeo specific.
1072 self.report_extraction(video_id)
1074 # Extract the config JSON
1076 config = webpage.split(' = {config:')[1].split(',assets:')[0]
1077 config = json.loads(config)
1079 self._downloader.trouble(u'ERROR: unable to extract info section')
1083 video_title = config["video"]["title"]
1086 video_uploader = config["video"]["owner"]["name"]
1088 # Extract video thumbnail
1089 video_thumbnail = config["video"]["thumbnail"]
1091 # Extract video description
1092 video_description = get_element_by_id("description", webpage.decode('utf8'))
1093 if video_description: video_description = clean_html(video_description)
1094 else: video_description = ''
1096 # Extract upload date
1097 video_upload_date = None
1098 mobj = re.search(r'<span id="clip-date" style="display:none">[^:]*: (.*?)( \([^\(]*\))?</span>', webpage)
1099 if mobj is not None:
1100 video_upload_date = mobj.group(1)
1102 # Vimeo specific: extract request signature and timestamp
1103 sig = config['request']['signature']
1104 timestamp = config['request']['timestamp']
1106 # Vimeo specific: extract video codec and quality information
1107 # First consider quality, then codecs, then take everything
1108 # TODO bind to format param
1109 codecs = [('h264', 'mp4'), ('vp8', 'flv'), ('vp6', 'flv')]
1110 files = { 'hd': [], 'sd': [], 'other': []}
1111 for codec_name, codec_extension in codecs:
1112 if codec_name in config["video"]["files"]:
1113 if 'hd' in config["video"]["files"][codec_name]:
1114 files['hd'].append((codec_name, codec_extension, 'hd'))
1115 elif 'sd' in config["video"]["files"][codec_name]:
1116 files['sd'].append((codec_name, codec_extension, 'sd'))
1118 files['other'].append((codec_name, codec_extension, config["video"]["files"][codec_name][0]))
1120 for quality in ('hd', 'sd', 'other'):
1121 if len(files[quality]) > 0:
1122 video_quality = files[quality][0][2]
1123 video_codec = files[quality][0][0]
1124 video_extension = files[quality][0][1]
1125 self._downloader.to_screen(u'[vimeo] %s: Downloading %s file at %s quality' % (video_id, video_codec.upper(), video_quality))
1128 self._downloader.trouble(u'ERROR: no known codec found')
1131 video_url = "http://player.vimeo.com/play_redirect?clip_id=%s&sig=%s&time=%s&quality=%s&codecs=%s&type=moogaloop_local&embed_location=" \
1132 %(video_id, sig, timestamp, video_quality, video_codec.upper())
1137 'uploader': video_uploader,
1138 'upload_date': video_upload_date,
1139 'title': video_title,
1140 'ext': video_extension,
1141 'thumbnail': video_thumbnail,
1142 'description': video_description,
1146 class ArteTvIE(InfoExtractor):
1147 """arte.tv information extractor."""
1149 _VALID_URL = r'(?:http://)?videos\.arte\.tv/(?:fr|de)/videos/.*'
1150 _LIVE_URL = r'index-[0-9]+\.html$'
1152 IE_NAME = u'arte.tv'
1154 def __init__(self, downloader=None):
1155 InfoExtractor.__init__(self, downloader)
1157 def report_download_webpage(self, video_id):
1158 """Report webpage download."""
1159 self._downloader.to_screen(u'[arte.tv] %s: Downloading webpage' % video_id)
1161 def report_extraction(self, video_id):
1162 """Report information extraction."""
1163 self._downloader.to_screen(u'[arte.tv] %s: Extracting information' % video_id)
1165 def fetch_webpage(self, url):
1166 self._downloader.increment_downloads()
1167 request = compat_urllib_request.Request(url)
1169 self.report_download_webpage(url)
1170 webpage = compat_urllib_request.urlopen(request).read()
1171 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1172 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
1174 except ValueError as err:
1175 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1179 def grep_webpage(self, url, regex, regexFlags, matchTuples):
1180 page = self.fetch_webpage(url)
1181 mobj = re.search(regex, page, regexFlags)
1185 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1188 for (i, key, err) in matchTuples:
1189 if mobj.group(i) is None:
1190 self._downloader.trouble(err)
1193 info[key] = mobj.group(i)
1197 def extractLiveStream(self, url):
1198 video_lang = url.split('/')[-4]
1199 info = self.grep_webpage(
1201 r'src="(.*?/videothek_js.*?\.js)',
1204 (1, 'url', u'ERROR: Invalid URL: %s' % url)
1207 http_host = url.split('/')[2]
1208 next_url = 'http://%s%s' % (http_host, compat_urllib_parse.unquote(info.get('url')))
1209 info = self.grep_webpage(
1211 r'(s_artestras_scst_geoFRDE_' + video_lang + '.*?)\'.*?' +
1212 '(http://.*?\.swf).*?' +
1216 (1, 'path', u'ERROR: could not extract video path: %s' % url),
1217 (2, 'player', u'ERROR: could not extract video player: %s' % url),
1218 (3, 'url', u'ERROR: could not extract video url: %s' % url)
1221 video_url = u'%s/%s' % (info.get('url'), info.get('path'))
1223 def extractPlus7Stream(self, url):
1224 video_lang = url.split('/')[-3]
1225 info = self.grep_webpage(
1227 r'param name="movie".*?videorefFileUrl=(http[^\'"&]*)',
1230 (1, 'url', u'ERROR: Invalid URL: %s' % url)
1233 next_url = compat_urllib_parse.unquote(info.get('url'))
1234 info = self.grep_webpage(
1236 r'<video lang="%s" ref="(http[^\'"&]*)' % video_lang,
1239 (1, 'url', u'ERROR: Could not find <video> tag: %s' % url)
1242 next_url = compat_urllib_parse.unquote(info.get('url'))
1244 info = self.grep_webpage(
1246 r'<video id="(.*?)".*?>.*?' +
1247 '<name>(.*?)</name>.*?' +
1248 '<dateVideo>(.*?)</dateVideo>.*?' +
1249 '<url quality="hd">(.*?)</url>',
1252 (1, 'id', u'ERROR: could not extract video id: %s' % url),
1253 (2, 'title', u'ERROR: could not extract video title: %s' % url),
1254 (3, 'date', u'ERROR: could not extract video date: %s' % url),
1255 (4, 'url', u'ERROR: could not extract video url: %s' % url)
1260 'id': info.get('id'),
1261 'url': compat_urllib_parse.unquote(info.get('url')),
1262 'uploader': u'arte.tv',
1263 'upload_date': info.get('date'),
1264 'title': info.get('title'),
1270 def _real_extract(self, url):
1271 video_id = url.split('/')[-1]
1272 self.report_extraction(video_id)
1274 if re.search(self._LIVE_URL, video_id) is not None:
1275 self.extractLiveStream(url)
1278 info = self.extractPlus7Stream(url)
1283 class GenericIE(InfoExtractor):
1284 """Generic last-resort information extractor."""
1287 IE_NAME = u'generic'
1289 def __init__(self, downloader=None):
1290 InfoExtractor.__init__(self, downloader)
1292 def report_download_webpage(self, video_id):
1293 """Report webpage download."""
1294 self._downloader.to_screen(u'WARNING: Falling back on generic information extractor.')
1295 self._downloader.to_screen(u'[generic] %s: Downloading webpage' % video_id)
1297 def report_extraction(self, video_id):
1298 """Report information extraction."""
1299 self._downloader.to_screen(u'[generic] %s: Extracting information' % video_id)
1301 def report_following_redirect(self, new_url):
1302 """Report information extraction."""
1303 self._downloader.to_screen(u'[redirect] Following redirect to %s' % new_url)
1305 def _test_redirect(self, url):
1306 """Check if it is a redirect, like url shorteners, in case restart chain."""
1307 class HeadRequest(compat_urllib_request.Request):
1308 def get_method(self):
1311 class HEADRedirectHandler(compat_urllib_request.HTTPRedirectHandler):
1313 Subclass the HTTPRedirectHandler to make it use our
1314 HeadRequest also on the redirected URL
1316 def redirect_request(self, req, fp, code, msg, headers, newurl):
1317 if code in (301, 302, 303, 307):
1318 newurl = newurl.replace(' ', '%20')
1319 newheaders = dict((k,v) for k,v in req.headers.items()
1320 if k.lower() not in ("content-length", "content-type"))
1321 return HeadRequest(newurl,
1323 origin_req_host=req.get_origin_req_host(),
1326 raise compat_urllib_error.HTTPError(req.get_full_url(), code, msg, headers, fp)
1328 class HTTPMethodFallback(compat_urllib_request.BaseHandler):
1330 Fallback to GET if HEAD is not allowed (405 HTTP error)
1332 def http_error_405(self, req, fp, code, msg, headers):
1336 newheaders = dict((k,v) for k,v in req.headers.items()
1337 if k.lower() not in ("content-length", "content-type"))
1338 return self.parent.open(compat_urllib_request.Request(req.get_full_url(),
1340 origin_req_host=req.get_origin_req_host(),
1344 opener = compat_urllib_request.OpenerDirector()
1345 for handler in [compat_urllib_request.HTTPHandler, compat_urllib_request.HTTPDefaultErrorHandler,
1346 HTTPMethodFallback, HEADRedirectHandler,
1347 compat_urllib_error.HTTPErrorProcessor, compat_urllib_request.HTTPSHandler]:
1348 opener.add_handler(handler())
1350 response = opener.open(HeadRequest(url))
1351 new_url = response.geturl()
1356 self.report_following_redirect(new_url)
1357 self._downloader.download([new_url])
1360 def _real_extract(self, url):
1361 if self._test_redirect(url): return
1363 video_id = url.split('/')[-1]
1364 request = compat_urllib_request.Request(url)
1366 self.report_download_webpage(video_id)
1367 webpage = compat_urllib_request.urlopen(request).read()
1368 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1369 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
1371 except ValueError as err:
1372 # since this is the last-resort InfoExtractor, if
1373 # this error is thrown, it'll be thrown here
1374 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1377 self.report_extraction(video_id)
1378 # Start with something easy: JW Player in SWFObject
1379 mobj = re.search(r'flashvars: [\'"](?:.*&)?file=(http[^\'"&]*)', webpage)
1381 # Broaden the search a little bit
1382 mobj = re.search(r'[^A-Za-z0-9]?(?:file|source)=(http[^\'"&]*)', webpage)
1384 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1387 # It's possible that one of the regexes
1388 # matched, but returned an empty group:
1389 if mobj.group(1) is None:
1390 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
1393 video_url = compat_urllib_parse.unquote(mobj.group(1))
1394 video_id = os.path.basename(video_url)
1396 # here's a fun little line of code for you:
1397 video_extension = os.path.splitext(video_id)[1][1:]
1398 video_id = os.path.splitext(video_id)[0]
1400 # it's tempting to parse this further, but you would
1401 # have to take into account all the variations like
1402 # Video Title - Site Name
1403 # Site Name | Video Title
1404 # Video Title - Tagline | Site Name
1405 # and so on and so forth; it's just not practical
1406 mobj = re.search(r'<title>(.*)</title>', webpage)
1408 self._downloader.trouble(u'ERROR: unable to extract title')
1410 video_title = mobj.group(1).decode('utf-8')
1412 # video uploader is domain name
1413 mobj = re.match(r'(?:https?://)?([^/]*)/.*', url)
1415 self._downloader.trouble(u'ERROR: unable to extract title')
1417 video_uploader = mobj.group(1).decode('utf-8')
1420 'id': video_id.decode('utf-8'),
1421 'url': video_url.decode('utf-8'),
1422 'uploader': video_uploader,
1423 'upload_date': None,
1424 'title': video_title,
1425 'ext': video_extension.decode('utf-8'),
1429 class YoutubeSearchIE(InfoExtractor):
1430 """Information Extractor for YouTube search queries."""
1431 _VALID_URL = r'ytsearch(\d+|all)?:[\s\S]+'
1432 _API_URL = 'https://gdata.youtube.com/feeds/api/videos?q=%s&start-index=%i&max-results=50&v=2&alt=jsonc'
1433 _max_youtube_results = 1000
1434 IE_NAME = u'youtube:search'
1436 def __init__(self, downloader=None):
1437 InfoExtractor.__init__(self, downloader)
1439 def report_download_page(self, query, pagenum):
1440 """Report attempt to download search page with given number."""
1441 query = query.decode(preferredencoding())
1442 self._downloader.to_screen(u'[youtube] query "%s": Downloading page %s' % (query, pagenum))
1444 def _real_extract(self, query):
1445 mobj = re.match(self._VALID_URL, query)
1447 self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
1450 prefix, query = query.split(':')
1452 query = query.encode('utf-8')
1454 self._download_n_results(query, 1)
1456 elif prefix == 'all':
1457 self._download_n_results(query, self._max_youtube_results)
1463 self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
1465 elif n > self._max_youtube_results:
1466 self._downloader.to_stderr(u'WARNING: ytsearch returns max %i results (you requested %i)' % (self._max_youtube_results, n))
1467 n = self._max_youtube_results
1468 self._download_n_results(query, n)
1470 except ValueError: # parsing prefix as integer fails
1471 self._download_n_results(query, 1)
1474 def _download_n_results(self, query, n):
1475 """Downloads a specified number of results for a query"""
1481 while (50 * pagenum) < limit:
1482 self.report_download_page(query, pagenum+1)
1483 result_url = self._API_URL % (compat_urllib_parse.quote_plus(query), (50*pagenum)+1)
1484 request = compat_urllib_request.Request(result_url)
1486 data = compat_urllib_request.urlopen(request).read()
1487 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1488 self._downloader.trouble(u'ERROR: unable to download API page: %s' % compat_str(err))
1490 api_response = json.loads(data)['data']
1492 new_ids = list(video['id'] for video in api_response['items'])
1493 video_ids += new_ids
1495 limit = min(n, api_response['totalItems'])
1498 if len(video_ids) > n:
1499 video_ids = video_ids[:n]
1500 for id in video_ids:
1501 self._downloader.download(['http://www.youtube.com/watch?v=%s' % id])
1505 class GoogleSearchIE(InfoExtractor):
1506 """Information Extractor for Google Video search queries."""
1507 _VALID_URL = r'gvsearch(\d+|all)?:[\s\S]+'
1508 _TEMPLATE_URL = 'http://video.google.com/videosearch?q=%s+site:video.google.com&start=%s&hl=en'
1509 _VIDEO_INDICATOR = r'<a href="http://video\.google\.com/videoplay\?docid=([^"\&]+)'
1510 _MORE_PAGES_INDICATOR = r'class="pn" id="pnnext"'
1511 _max_google_results = 1000
1512 IE_NAME = u'video.google:search'
1514 def __init__(self, downloader=None):
1515 InfoExtractor.__init__(self, downloader)
1517 def report_download_page(self, query, pagenum):
1518 """Report attempt to download playlist page with given number."""
1519 query = query.decode(preferredencoding())
1520 self._downloader.to_screen(u'[video.google] query "%s": Downloading page %s' % (query, pagenum))
1522 def _real_extract(self, query):
1523 mobj = re.match(self._VALID_URL, query)
1525 self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
1528 prefix, query = query.split(':')
1530 query = query.encode('utf-8')
1532 self._download_n_results(query, 1)
1534 elif prefix == 'all':
1535 self._download_n_results(query, self._max_google_results)
1541 self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
1543 elif n > self._max_google_results:
1544 self._downloader.to_stderr(u'WARNING: gvsearch returns max %i results (you requested %i)' % (self._max_google_results, n))
1545 n = self._max_google_results
1546 self._download_n_results(query, n)
1548 except ValueError: # parsing prefix as integer fails
1549 self._download_n_results(query, 1)
1552 def _download_n_results(self, query, n):
1553 """Downloads a specified number of results for a query"""
1559 self.report_download_page(query, pagenum)
1560 result_url = self._TEMPLATE_URL % (compat_urllib_parse.quote_plus(query), pagenum*10)
1561 request = compat_urllib_request.Request(result_url)
1563 page = compat_urllib_request.urlopen(request).read()
1564 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1565 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
1568 # Extract video identifiers
1569 for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1570 video_id = mobj.group(1)
1571 if video_id not in video_ids:
1572 video_ids.append(video_id)
1573 if len(video_ids) == n:
1574 # Specified n videos reached
1575 for id in video_ids:
1576 self._downloader.download(['http://video.google.com/videoplay?docid=%s' % id])
1579 if re.search(self._MORE_PAGES_INDICATOR, page) is None:
1580 for id in video_ids:
1581 self._downloader.download(['http://video.google.com/videoplay?docid=%s' % id])
1584 pagenum = pagenum + 1
1587 class YahooSearchIE(InfoExtractor):
1588 """Information Extractor for Yahoo! Video search queries."""
1589 _VALID_URL = r'yvsearch(\d+|all)?:[\s\S]+'
1590 _TEMPLATE_URL = 'http://video.yahoo.com/search/?p=%s&o=%s'
1591 _VIDEO_INDICATOR = r'href="http://video\.yahoo\.com/watch/([0-9]+/[0-9]+)"'
1592 _MORE_PAGES_INDICATOR = r'\s*Next'
1593 _max_yahoo_results = 1000
1594 IE_NAME = u'video.yahoo:search'
1596 def __init__(self, downloader=None):
1597 InfoExtractor.__init__(self, downloader)
1599 def report_download_page(self, query, pagenum):
1600 """Report attempt to download playlist page with given number."""
1601 query = query.decode(preferredencoding())
1602 self._downloader.to_screen(u'[video.yahoo] query "%s": Downloading page %s' % (query, pagenum))
1604 def _real_extract(self, query):
1605 mobj = re.match(self._VALID_URL, query)
1607 self._downloader.trouble(u'ERROR: invalid search query "%s"' % query)
1610 prefix, query = query.split(':')
1612 query = query.encode('utf-8')
1614 self._download_n_results(query, 1)
1616 elif prefix == 'all':
1617 self._download_n_results(query, self._max_yahoo_results)
1623 self._downloader.trouble(u'ERROR: invalid download number %s for query "%s"' % (n, query))
1625 elif n > self._max_yahoo_results:
1626 self._downloader.to_stderr(u'WARNING: yvsearch returns max %i results (you requested %i)' % (self._max_yahoo_results, n))
1627 n = self._max_yahoo_results
1628 self._download_n_results(query, n)
1630 except ValueError: # parsing prefix as integer fails
1631 self._download_n_results(query, 1)
1634 def _download_n_results(self, query, n):
1635 """Downloads a specified number of results for a query"""
1638 already_seen = set()
1642 self.report_download_page(query, pagenum)
1643 result_url = self._TEMPLATE_URL % (compat_urllib_parse.quote_plus(query), pagenum)
1644 request = compat_urllib_request.Request(result_url)
1646 page = compat_urllib_request.urlopen(request).read()
1647 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1648 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
1651 # Extract video identifiers
1652 for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1653 video_id = mobj.group(1)
1654 if video_id not in already_seen:
1655 video_ids.append(video_id)
1656 already_seen.add(video_id)
1657 if len(video_ids) == n:
1658 # Specified n videos reached
1659 for id in video_ids:
1660 self._downloader.download(['http://video.yahoo.com/watch/%s' % id])
1663 if re.search(self._MORE_PAGES_INDICATOR, page) is None:
1664 for id in video_ids:
1665 self._downloader.download(['http://video.yahoo.com/watch/%s' % id])
1668 pagenum = pagenum + 1
1671 class YoutubePlaylistIE(InfoExtractor):
1672 """Information Extractor for YouTube playlists."""
1674 _VALID_URL = r'(?:(?:https?://)?(?:\w+\.)?youtube\.com/(?:(?:course|view_play_list|my_playlists|artist|playlist)\?.*?(p|a|list)=|user/.*?/user/|p/|user/.*?#[pg]/c/)(?:PL|EC)?|PL|EC)([0-9A-Za-z-_]{10,})(?:/.*?/([0-9A-Za-z_-]+))?.*'
1675 _TEMPLATE_URL = 'http://www.youtube.com/%s?%s=%s&page=%s&gl=US&hl=en'
1676 _VIDEO_INDICATOR_TEMPLATE = r'/watch\?v=(.+?)&([^&"]+&)*list=.*?%s'
1677 _MORE_PAGES_INDICATOR = u"Next \N{RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK}"
1678 IE_NAME = u'youtube:playlist'
1680 def __init__(self, downloader=None):
1681 InfoExtractor.__init__(self, downloader)
1683 def report_download_page(self, playlist_id, pagenum):
1684 """Report attempt to download playlist page with given number."""
1685 self._downloader.to_screen(u'[youtube] PL %s: Downloading page #%s' % (playlist_id, pagenum))
1687 def _real_extract(self, url):
1688 # Extract playlist id
1689 mobj = re.match(self._VALID_URL, url)
1691 self._downloader.trouble(u'ERROR: invalid url: %s' % url)
1695 if mobj.group(3) is not None:
1696 self._downloader.download([mobj.group(3)])
1699 # Download playlist pages
1700 # prefix is 'p' as default for playlists but there are other types that need extra care
1701 playlist_prefix = mobj.group(1)
1702 if playlist_prefix == 'a':
1703 playlist_access = 'artist'
1705 playlist_prefix = 'p'
1706 playlist_access = 'view_play_list'
1707 playlist_id = mobj.group(2)
1712 self.report_download_page(playlist_id, pagenum)
1713 url = self._TEMPLATE_URL % (playlist_access, playlist_prefix, playlist_id, pagenum)
1714 request = compat_urllib_request.Request(url)
1716 page = compat_urllib_request.urlopen(request).read().decode('utf8')
1717 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1718 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
1721 # Extract video identifiers
1723 for mobj in re.finditer(self._VIDEO_INDICATOR_TEMPLATE % playlist_id, page):
1724 if mobj.group(1) not in ids_in_page:
1725 ids_in_page.append(mobj.group(1))
1726 video_ids.extend(ids_in_page)
1728 if self._MORE_PAGES_INDICATOR not in page:
1730 pagenum = pagenum + 1
1732 total = len(video_ids)
1734 playliststart = self._downloader.params.get('playliststart', 1) - 1
1735 playlistend = self._downloader.params.get('playlistend', -1)
1736 if playlistend == -1:
1737 video_ids = video_ids[playliststart:]
1739 video_ids = video_ids[playliststart:playlistend]
1741 if len(video_ids) == total:
1742 self._downloader.to_screen(u'[youtube] PL %s: Found %i videos' % (playlist_id, total))
1744 self._downloader.to_screen(u'[youtube] PL %s: Found %i videos, downloading %i' % (playlist_id, total, len(video_ids)))
1746 for id in video_ids:
1747 self._downloader.download(['http://www.youtube.com/watch?v=%s' % id])
1751 class YoutubeChannelIE(InfoExtractor):
1752 """Information Extractor for YouTube channels."""
1754 _VALID_URL = r"^(?:https?://)?(?:youtu\.be|(?:\w+\.)?youtube(?:-nocookie)?\.com)/channel/([0-9A-Za-z_-]+)(?:/.*)?$"
1755 _TEMPLATE_URL = 'http://www.youtube.com/channel/%s/videos?sort=da&flow=list&view=0&page=%s&gl=US&hl=en'
1756 _MORE_PAGES_INDICATOR = u"Next \N{RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK}"
1757 IE_NAME = u'youtube:channel'
1759 def report_download_page(self, channel_id, pagenum):
1760 """Report attempt to download channel page with given number."""
1761 self._downloader.to_screen(u'[youtube] Channel %s: Downloading page #%s' % (channel_id, pagenum))
1763 def _real_extract(self, url):
1764 # Extract channel id
1765 mobj = re.match(self._VALID_URL, url)
1767 self._downloader.trouble(u'ERROR: invalid url: %s' % url)
1770 # Download channel pages
1771 channel_id = mobj.group(1)
1776 self.report_download_page(channel_id, pagenum)
1777 url = self._TEMPLATE_URL % (channel_id, pagenum)
1778 request = compat_urllib_request.Request(url)
1780 page = compat_urllib_request.urlopen(request).read().decode('utf8')
1781 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1782 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
1785 # Extract video identifiers
1787 for mobj in re.finditer(r'href="/watch\?v=([0-9A-Za-z_-]+)&', page):
1788 if mobj.group(1) not in ids_in_page:
1789 ids_in_page.append(mobj.group(1))
1790 video_ids.extend(ids_in_page)
1792 if self._MORE_PAGES_INDICATOR not in page:
1794 pagenum = pagenum + 1
1796 self._downloader.to_screen(u'[youtube] Channel %s: Found %i videos' % (channel_id, len(video_ids)))
1798 for id in video_ids:
1799 self._downloader.download(['http://www.youtube.com/watch?v=%s' % id])
1803 class YoutubeUserIE(InfoExtractor):
1804 """Information Extractor for YouTube users."""
1806 _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?youtube\.com/user/)|ytuser:)([A-Za-z0-9_-]+)'
1807 _TEMPLATE_URL = 'http://gdata.youtube.com/feeds/api/users/%s'
1808 _GDATA_PAGE_SIZE = 50
1809 _GDATA_URL = 'http://gdata.youtube.com/feeds/api/users/%s/uploads?max-results=%d&start-index=%d'
1810 _VIDEO_INDICATOR = r'/watch\?v=(.+?)[\<&]'
1811 IE_NAME = u'youtube:user'
1813 def __init__(self, downloader=None):
1814 InfoExtractor.__init__(self, downloader)
1816 def report_download_page(self, username, start_index):
1817 """Report attempt to download user page."""
1818 self._downloader.to_screen(u'[youtube] user %s: Downloading video ids from %d to %d' %
1819 (username, start_index, start_index + self._GDATA_PAGE_SIZE))
1821 def _real_extract(self, url):
1823 mobj = re.match(self._VALID_URL, url)
1825 self._downloader.trouble(u'ERROR: invalid url: %s' % url)
1828 username = mobj.group(1)
1830 # Download video ids using YouTube Data API. Result size per
1831 # query is limited (currently to 50 videos) so we need to query
1832 # page by page until there are no video ids - it means we got
1839 start_index = pagenum * self._GDATA_PAGE_SIZE + 1
1840 self.report_download_page(username, start_index)
1842 request = compat_urllib_request.Request(self._GDATA_URL % (username, self._GDATA_PAGE_SIZE, start_index))
1845 page = compat_urllib_request.urlopen(request).read()
1846 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1847 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
1850 # Extract video identifiers
1853 for mobj in re.finditer(self._VIDEO_INDICATOR, page):
1854 if mobj.group(1) not in ids_in_page:
1855 ids_in_page.append(mobj.group(1))
1857 video_ids.extend(ids_in_page)
1859 # A little optimization - if current page is not
1860 # "full", ie. does not contain PAGE_SIZE video ids then
1861 # we can assume that this page is the last one - there
1862 # are no more ids on further pages - no need to query
1865 if len(ids_in_page) < self._GDATA_PAGE_SIZE:
1870 all_ids_count = len(video_ids)
1871 playliststart = self._downloader.params.get('playliststart', 1) - 1
1872 playlistend = self._downloader.params.get('playlistend', -1)
1874 if playlistend == -1:
1875 video_ids = video_ids[playliststart:]
1877 video_ids = video_ids[playliststart:playlistend]
1879 self._downloader.to_screen(u"[youtube] user %s: Collected %d video ids (downloading %d of them)" %
1880 (username, all_ids_count, len(video_ids)))
1882 for video_id in video_ids:
1883 self._downloader.download(['http://www.youtube.com/watch?v=%s' % video_id])
1886 class BlipTVUserIE(InfoExtractor):
1887 """Information Extractor for blip.tv users."""
1889 _VALID_URL = r'(?:(?:(?:https?://)?(?:\w+\.)?blip\.tv/)|bliptvuser:)([^/]+)/*$'
1891 IE_NAME = u'blip.tv:user'
1893 def __init__(self, downloader=None):
1894 InfoExtractor.__init__(self, downloader)
1896 def report_download_page(self, username, pagenum):
1897 """Report attempt to download user page."""
1898 self._downloader.to_screen(u'[%s] user %s: Downloading video ids from page %d' %
1899 (self.IE_NAME, username, pagenum))
1901 def _real_extract(self, url):
1903 mobj = re.match(self._VALID_URL, url)
1905 self._downloader.trouble(u'ERROR: invalid url: %s' % url)
1908 username = mobj.group(1)
1910 page_base = 'http://m.blip.tv/pr/show_get_full_episode_list?users_id=%s&lite=0&esi=1'
1912 request = compat_urllib_request.Request(url)
1915 page = compat_urllib_request.urlopen(request).read().decode('utf-8')
1916 mobj = re.search(r'data-users-id="([^"]+)"', page)
1917 page_base = page_base % mobj.group(1)
1918 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1919 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
1923 # Download video ids using BlipTV Ajax calls. Result size per
1924 # query is limited (currently to 12 videos) so we need to query
1925 # page by page until there are no video ids - it means we got
1932 self.report_download_page(username, pagenum)
1934 request = compat_urllib_request.Request( page_base + "&page=" + str(pagenum) )
1937 page = compat_urllib_request.urlopen(request).read().decode('utf-8')
1938 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
1939 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % str(err))
1942 # Extract video identifiers
1945 for mobj in re.finditer(r'href="/([^"]+)"', page):
1946 if mobj.group(1) not in ids_in_page:
1947 ids_in_page.append(unescapeHTML(mobj.group(1)))
1949 video_ids.extend(ids_in_page)
1951 # A little optimization - if current page is not
1952 # "full", ie. does not contain PAGE_SIZE video ids then
1953 # we can assume that this page is the last one - there
1954 # are no more ids on further pages - no need to query
1957 if len(ids_in_page) < self._PAGE_SIZE:
1962 all_ids_count = len(video_ids)
1963 playliststart = self._downloader.params.get('playliststart', 1) - 1
1964 playlistend = self._downloader.params.get('playlistend', -1)
1966 if playlistend == -1:
1967 video_ids = video_ids[playliststart:]
1969 video_ids = video_ids[playliststart:playlistend]
1971 self._downloader.to_screen(u"[%s] user %s: Collected %d video ids (downloading %d of them)" %
1972 (self.IE_NAME, username, all_ids_count, len(video_ids)))
1974 for video_id in video_ids:
1975 self._downloader.download([u'http://blip.tv/'+video_id])
1978 class DepositFilesIE(InfoExtractor):
1979 """Information extractor for depositfiles.com"""
1981 _VALID_URL = r'(?:http://)?(?:\w+\.)?depositfiles\.com/(?:../(?#locale))?files/(.+)'
1982 IE_NAME = u'DepositFiles'
1984 def __init__(self, downloader=None):
1985 InfoExtractor.__init__(self, downloader)
1987 def report_download_webpage(self, file_id):
1988 """Report webpage download."""
1989 self._downloader.to_screen(u'[DepositFiles] %s: Downloading webpage' % file_id)
1991 def report_extraction(self, file_id):
1992 """Report information extraction."""
1993 self._downloader.to_screen(u'[DepositFiles] %s: Extracting information' % file_id)
1995 def _real_extract(self, url):
1996 file_id = url.split('/')[-1]
1997 # Rebuild url in english locale
1998 url = 'http://depositfiles.com/en/files/' + file_id
2000 # Retrieve file webpage with 'Free download' button pressed
2001 free_download_indication = { 'gateway_result' : '1' }
2002 request = compat_urllib_request.Request(url, compat_urllib_parse.urlencode(free_download_indication))
2004 self.report_download_webpage(file_id)
2005 webpage = compat_urllib_request.urlopen(request).read()
2006 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2007 self._downloader.trouble(u'ERROR: Unable to retrieve file webpage: %s' % compat_str(err))
2010 # Search for the real file URL
2011 mobj = re.search(r'<form action="(http://fileshare.+?)"', webpage)
2012 if (mobj is None) or (mobj.group(1) is None):
2013 # Try to figure out reason of the error.
2014 mobj = re.search(r'<strong>(Attention.*?)</strong>', webpage, re.DOTALL)
2015 if (mobj is not None) and (mobj.group(1) is not None):
2016 restriction_message = re.sub('\s+', ' ', mobj.group(1)).strip()
2017 self._downloader.trouble(u'ERROR: %s' % restriction_message)
2019 self._downloader.trouble(u'ERROR: unable to extract download URL from: %s' % url)
2022 file_url = mobj.group(1)
2023 file_extension = os.path.splitext(file_url)[1][1:]
2025 # Search for file title
2026 mobj = re.search(r'<b title="(.*?)">', webpage)
2028 self._downloader.trouble(u'ERROR: unable to extract title')
2030 file_title = mobj.group(1).decode('utf-8')
2033 'id': file_id.decode('utf-8'),
2034 'url': file_url.decode('utf-8'),
2036 'upload_date': None,
2037 'title': file_title,
2038 'ext': file_extension.decode('utf-8'),
2042 class FacebookIE(InfoExtractor):
2043 """Information Extractor for Facebook"""
2046 _VALID_URL = r'^(?:https?://)?(?:\w+\.)?facebook\.com/(?:video/video|photo)\.php\?(?:.*?)v=(?P<ID>\d+)(?:.*)'
2047 _LOGIN_URL = 'https://login.facebook.com/login.php?m&next=http%3A%2F%2Fm.facebook.com%2Fhome.php&'
2048 _NETRC_MACHINE = 'facebook'
2049 _available_formats = ['video', 'highqual', 'lowqual']
2050 _video_extensions = {
2055 IE_NAME = u'facebook'
2057 def __init__(self, downloader=None):
2058 InfoExtractor.__init__(self, downloader)
2060 def _reporter(self, message):
2061 """Add header and report message."""
2062 self._downloader.to_screen(u'[facebook] %s' % message)
2064 def report_login(self):
2065 """Report attempt to log in."""
2066 self._reporter(u'Logging in')
2068 def report_video_webpage_download(self, video_id):
2069 """Report attempt to download video webpage."""
2070 self._reporter(u'%s: Downloading video webpage' % video_id)
2072 def report_information_extraction(self, video_id):
2073 """Report attempt to extract video information."""
2074 self._reporter(u'%s: Extracting video information' % video_id)
2076 def _parse_page(self, video_webpage):
2077 """Extract video information from page"""
2079 data = {'title': r'\("video_title", "(.*?)"\)',
2080 'description': r'<div class="datawrap">(.*?)</div>',
2081 'owner': r'\("video_owner_name", "(.*?)"\)',
2082 'thumbnail': r'\("thumb_url", "(?P<THUMB>.*?)"\)',
2085 for piece in data.keys():
2086 mobj = re.search(data[piece], video_webpage)
2087 if mobj is not None:
2088 video_info[piece] = compat_urllib_parse.unquote_plus(mobj.group(1).decode("unicode_escape"))
2092 for fmt in self._available_formats:
2093 mobj = re.search(r'\("%s_src\", "(.+?)"\)' % fmt, video_webpage)
2094 if mobj is not None:
2095 # URL is in a Javascript segment inside an escaped Unicode format within
2096 # the generally utf-8 page
2097 video_urls[fmt] = compat_urllib_parse.unquote_plus(mobj.group(1).decode("unicode_escape"))
2098 video_info['video_urls'] = video_urls
2102 def _real_initialize(self):
2103 if self._downloader is None:
2108 downloader_params = self._downloader.params
2110 # Attempt to use provided username and password or .netrc data
2111 if downloader_params.get('username', None) is not None:
2112 useremail = downloader_params['username']
2113 password = downloader_params['password']
2114 elif downloader_params.get('usenetrc', False):
2116 info = netrc.netrc().authenticators(self._NETRC_MACHINE)
2117 if info is not None:
2121 raise netrc.NetrcParseError('No authenticators for %s' % self._NETRC_MACHINE)
2122 except (IOError, netrc.NetrcParseError) as err:
2123 self._downloader.to_stderr(u'WARNING: parsing .netrc: %s' % compat_str(err))
2126 if useremail is None:
2135 request = compat_urllib_request.Request(self._LOGIN_URL, compat_urllib_parse.urlencode(login_form))
2138 login_results = compat_urllib_request.urlopen(request).read()
2139 if re.search(r'<form(.*)name="login"(.*)</form>', login_results) is not None:
2140 self._downloader.to_stderr(u'WARNING: unable to log in: bad username/password, or exceded login rate limit (~3/min). Check credentials or wait.')
2142 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2143 self._downloader.to_stderr(u'WARNING: unable to log in: %s' % compat_str(err))
2146 def _real_extract(self, url):
2147 mobj = re.match(self._VALID_URL, url)
2149 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2151 video_id = mobj.group('ID')
2154 self.report_video_webpage_download(video_id)
2155 request = compat_urllib_request.Request('https://www.facebook.com/video/video.php?v=%s' % video_id)
2157 page = compat_urllib_request.urlopen(request)
2158 video_webpage = page.read()
2159 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2160 self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
2163 # Start extracting information
2164 self.report_information_extraction(video_id)
2166 # Extract information
2167 video_info = self._parse_page(video_webpage)
2170 if 'owner' not in video_info:
2171 self._downloader.trouble(u'ERROR: unable to extract uploader nickname')
2173 video_uploader = video_info['owner']
2176 if 'title' not in video_info:
2177 self._downloader.trouble(u'ERROR: unable to extract video title')
2179 video_title = video_info['title']
2180 video_title = video_title.decode('utf-8')
2183 if 'thumbnail' not in video_info:
2184 self._downloader.trouble(u'WARNING: unable to extract video thumbnail')
2185 video_thumbnail = ''
2187 video_thumbnail = video_info['thumbnail']
2191 if 'upload_date' in video_info:
2192 upload_time = video_info['upload_date']
2193 timetuple = email.utils.parsedate_tz(upload_time)
2194 if timetuple is not None:
2196 upload_date = time.strftime('%Y%m%d', timetuple[0:9])
2201 video_description = video_info.get('description', 'No description available.')
2203 url_map = video_info['video_urls']
2204 if len(url_map.keys()) > 0:
2205 # Decide which formats to download
2206 req_format = self._downloader.params.get('format', None)
2207 format_limit = self._downloader.params.get('format_limit', None)
2209 if format_limit is not None and format_limit in self._available_formats:
2210 format_list = self._available_formats[self._available_formats.index(format_limit):]
2212 format_list = self._available_formats
2213 existing_formats = [x for x in format_list if x in url_map]
2214 if len(existing_formats) == 0:
2215 self._downloader.trouble(u'ERROR: no known formats available for video')
2217 if req_format is None:
2218 video_url_list = [(existing_formats[0], url_map[existing_formats[0]])] # Best quality
2219 elif req_format == 'worst':
2220 video_url_list = [(existing_formats[len(existing_formats)-1], url_map[existing_formats[len(existing_formats)-1]])] # worst quality
2221 elif req_format == '-1':
2222 video_url_list = [(f, url_map[f]) for f in existing_formats] # All formats
2225 if req_format not in url_map:
2226 self._downloader.trouble(u'ERROR: requested format not available')
2228 video_url_list = [(req_format, url_map[req_format])] # Specific format
2231 for format_param, video_real_url in video_url_list:
2233 video_extension = self._video_extensions.get(format_param, 'mp4')
2236 'id': video_id.decode('utf-8'),
2237 'url': video_real_url.decode('utf-8'),
2238 'uploader': video_uploader.decode('utf-8'),
2239 'upload_date': upload_date,
2240 'title': video_title,
2241 'ext': video_extension.decode('utf-8'),
2242 'format': (format_param is None and u'NA' or format_param.decode('utf-8')),
2243 'thumbnail': video_thumbnail.decode('utf-8'),
2244 'description': video_description.decode('utf-8'),
2248 class BlipTVIE(InfoExtractor):
2249 """Information extractor for blip.tv"""
2251 _VALID_URL = r'^(?:https?://)?(?:\w+\.)?blip\.tv(/.+)$'
2252 _URL_EXT = r'^.*\.([a-z0-9]+)$'
2253 IE_NAME = u'blip.tv'
2255 def report_extraction(self, file_id):
2256 """Report information extraction."""
2257 self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, file_id))
2259 def report_direct_download(self, title):
2260 """Report information extraction."""
2261 self._downloader.to_screen(u'[%s] %s: Direct download detected' % (self.IE_NAME, title))
2263 def _real_extract(self, url):
2264 mobj = re.match(self._VALID_URL, url)
2266 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2273 json_url = url + cchar + 'skin=json&version=2&no_wrap=1'
2274 request = compat_urllib_request.Request(json_url)
2275 self.report_extraction(mobj.group(1))
2278 urlh = compat_urllib_request.urlopen(request)
2279 if urlh.headers.get('Content-Type', '').startswith('video/'): # Direct download
2280 basename = url.split('/')[-1]
2281 title,ext = os.path.splitext(basename)
2282 title = title.decode('UTF-8')
2283 ext = ext.replace('.', '')
2284 self.report_direct_download(title)
2289 'upload_date': None,
2294 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2295 self._downloader.trouble(u'ERROR: unable to download video info webpage: %s' % compat_str(err))
2297 if info is None: # Regular URL
2299 json_code_bytes = urlh.read()
2300 json_code = json_code_bytes.decode('utf-8')
2301 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2302 self._downloader.trouble(u'ERROR: unable to read video info webpage: %s' % compat_str(err))
2306 json_data = json.loads(json_code)
2307 if 'Post' in json_data:
2308 data = json_data['Post']
2312 upload_date = datetime.datetime.strptime(data['datestamp'], '%m-%d-%y %H:%M%p').strftime('%Y%m%d')
2313 video_url = data['media']['url']
2314 umobj = re.match(self._URL_EXT, video_url)
2316 raise ValueError('Can not determine filename extension')
2317 ext = umobj.group(1)
2320 'id': data['item_id'],
2322 'uploader': data['display_name'],
2323 'upload_date': upload_date,
2324 'title': data['title'],
2326 'format': data['media']['mimeType'],
2327 'thumbnail': data['thumbnailUrl'],
2328 'description': data['description'],
2329 'player_url': data['embedUrl']
2331 except (ValueError,KeyError) as err:
2332 self._downloader.trouble(u'ERROR: unable to parse video information: %s' % repr(err))
2335 std_headers['User-Agent'] = 'iTunes/10.6.1'
2339 class MyVideoIE(InfoExtractor):
2340 """Information Extractor for myvideo.de."""
2342 _VALID_URL = r'(?:http://)?(?:www\.)?myvideo\.de/watch/([0-9]+)/([^?/]+).*'
2343 IE_NAME = u'myvideo'
2345 def __init__(self, downloader=None):
2346 InfoExtractor.__init__(self, downloader)
2348 def report_download_webpage(self, video_id):
2349 """Report webpage download."""
2350 self._downloader.to_screen(u'[myvideo] %s: Downloading webpage' % video_id)
2352 def report_extraction(self, video_id):
2353 """Report information extraction."""
2354 self._downloader.to_screen(u'[myvideo] %s: Extracting information' % video_id)
2356 def _real_extract(self,url):
2357 mobj = re.match(self._VALID_URL, url)
2359 self._download.trouble(u'ERROR: invalid URL: %s' % url)
2362 video_id = mobj.group(1)
2365 request = compat_urllib_request.Request('http://www.myvideo.de/watch/%s' % video_id)
2367 self.report_download_webpage(video_id)
2368 webpage = compat_urllib_request.urlopen(request).read()
2369 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2370 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
2373 self.report_extraction(video_id)
2374 mobj = re.search(r'<link rel=\'image_src\' href=\'(http://is[0-9].myvideo\.de/de/movie[0-9]+/[a-f0-9]+)/thumbs/[^.]+\.jpg\' />',
2377 self._downloader.trouble(u'ERROR: unable to extract media URL')
2379 video_url = mobj.group(1) + ('/%s.flv' % video_id)
2381 mobj = re.search('<title>([^<]+)</title>', webpage)
2383 self._downloader.trouble(u'ERROR: unable to extract title')
2386 video_title = mobj.group(1)
2392 'upload_date': None,
2393 'title': video_title,
2397 class ComedyCentralIE(InfoExtractor):
2398 """Information extractor for The Daily Show and Colbert Report """
2400 # urls can be abbreviations like :thedailyshow or :colbert
2401 # urls for episodes like:
2402 # or urls for clips like: http://www.thedailyshow.com/watch/mon-december-10-2012/any-given-gun-day
2403 # or: http://www.colbertnation.com/the-colbert-report-videos/421667/november-29-2012/moon-shattering-news
2404 # or: http://www.colbertnation.com/the-colbert-report-collections/422008/festival-of-lights/79524
2405 _VALID_URL = r"""^(:(?P<shortname>tds|thedailyshow|cr|colbert|colbertnation|colbertreport)
2406 |(https?://)?(www\.)?
2407 (?P<showname>thedailyshow|colbertnation)\.com/
2408 (full-episodes/(?P<episode>.*)|
2410 (the-colbert-report-(videos|collections)/(?P<clipID>[0-9]+)/[^/]*/(?P<cntitle>.*?))
2411 |(watch/(?P<date>[^/]*)/(?P<tdstitle>.*)))))
2413 IE_NAME = u'comedycentral'
2415 _available_formats = ['3500', '2200', '1700', '1200', '750', '400']
2417 _video_extensions = {
2425 _video_dimensions = {
2434 def suitable(self, url):
2435 """Receives a URL and returns True if suitable for this IE."""
2436 return re.match(self._VALID_URL, url, re.VERBOSE) is not None
2438 def report_extraction(self, episode_id):
2439 self._downloader.to_screen(u'[comedycentral] %s: Extracting information' % episode_id)
2441 def report_config_download(self, episode_id):
2442 self._downloader.to_screen(u'[comedycentral] %s: Downloading configuration' % episode_id)
2444 def report_index_download(self, episode_id):
2445 self._downloader.to_screen(u'[comedycentral] %s: Downloading show index' % episode_id)
2447 def report_player_url(self, episode_id):
2448 self._downloader.to_screen(u'[comedycentral] %s: Determining player URL' % episode_id)
2451 def _print_formats(self, formats):
2452 print('Available formats:')
2454 print('%s\t:\t%s\t[%s]' %(x, self._video_extensions.get(x, 'mp4'), self._video_dimensions.get(x, '???')))
2457 def _real_extract(self, url):
2458 mobj = re.match(self._VALID_URL, url, re.VERBOSE)
2460 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2463 if mobj.group('shortname'):
2464 if mobj.group('shortname') in ('tds', 'thedailyshow'):
2465 url = u'http://www.thedailyshow.com/full-episodes/'
2467 url = u'http://www.colbertnation.com/full-episodes/'
2468 mobj = re.match(self._VALID_URL, url, re.VERBOSE)
2469 assert mobj is not None
2471 if mobj.group('clip'):
2472 if mobj.group('showname') == 'thedailyshow':
2473 epTitle = mobj.group('tdstitle')
2475 epTitle = mobj.group('cntitle')
2478 dlNewest = not mobj.group('episode')
2480 epTitle = mobj.group('showname')
2482 epTitle = mobj.group('episode')
2484 req = compat_urllib_request.Request(url)
2485 self.report_extraction(epTitle)
2487 htmlHandle = compat_urllib_request.urlopen(req)
2488 html = htmlHandle.read()
2489 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2490 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
2493 url = htmlHandle.geturl()
2494 mobj = re.match(self._VALID_URL, url, re.VERBOSE)
2496 self._downloader.trouble(u'ERROR: Invalid redirected URL: ' + url)
2498 if mobj.group('episode') == '':
2499 self._downloader.trouble(u'ERROR: Redirected URL is still not specific: ' + url)
2501 epTitle = mobj.group('episode')
2503 mMovieParams = re.findall('(?:<param name="movie" value="|var url = ")(http://media.mtvnservices.com/([^"]*(?:episode|video).*?:.*?))"', html)
2505 if len(mMovieParams) == 0:
2506 # The Colbert Report embeds the information in a without
2507 # a URL prefix; so extract the alternate reference
2508 # and then add the URL prefix manually.
2510 altMovieParams = re.findall('data-mgid="([^"]*(?:episode|video).*?:.*?)"', html)
2511 if len(altMovieParams) == 0:
2512 self._downloader.trouble(u'ERROR: unable to find Flash URL in webpage ' + url)
2515 mMovieParams = [("http://media.mtvnservices.com/" + altMovieParams[0], altMovieParams[0])]
2517 playerUrl_raw = mMovieParams[0][0]
2518 self.report_player_url(epTitle)
2520 urlHandle = compat_urllib_request.urlopen(playerUrl_raw)
2521 playerUrl = urlHandle.geturl()
2522 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2523 self._downloader.trouble(u'ERROR: unable to find out player URL: ' + compat_str(err))
2526 uri = mMovieParams[0][1]
2527 indexUrl = 'http://shadow.comedycentral.com/feeds/video_player/mrss/?' + compat_urllib_parse.urlencode({'uri': uri})
2528 self.report_index_download(epTitle)
2530 indexXml = compat_urllib_request.urlopen(indexUrl).read()
2531 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2532 self._downloader.trouble(u'ERROR: unable to download episode index: ' + compat_str(err))
2537 idoc = xml.etree.ElementTree.fromstring(indexXml)
2538 itemEls = idoc.findall('.//item')
2539 for itemEl in itemEls:
2540 mediaId = itemEl.findall('./guid')[0].text
2541 shortMediaId = mediaId.split(':')[-1]
2542 showId = mediaId.split(':')[-2].replace('.com', '')
2543 officialTitle = itemEl.findall('./title')[0].text
2544 officialDate = itemEl.findall('./pubDate')[0].text
2546 configUrl = ('http://www.comedycentral.com/global/feeds/entertainment/media/mediaGenEntertainment.jhtml?' +
2547 compat_urllib_parse.urlencode({'uri': mediaId}))
2548 configReq = compat_urllib_request.Request(configUrl)
2549 self.report_config_download(epTitle)
2551 configXml = compat_urllib_request.urlopen(configReq).read()
2552 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2553 self._downloader.trouble(u'ERROR: unable to download webpage: %s' % compat_str(err))
2556 cdoc = xml.etree.ElementTree.fromstring(configXml)
2558 for rendition in cdoc.findall('.//rendition'):
2559 finfo = (rendition.attrib['bitrate'], rendition.findall('./src')[0].text)
2563 self._downloader.trouble(u'\nERROR: unable to download ' + mediaId + ': No videos found')
2566 if self._downloader.params.get('listformats', None):
2567 self._print_formats([i[0] for i in turls])
2570 # For now, just pick the highest bitrate
2571 format,video_url = turls[-1]
2573 # Get the format arg from the arg stream
2574 req_format = self._downloader.params.get('format', None)
2576 # Select format if we can find one
2579 format, video_url = f, v
2582 # Patch to download from alternative CDN, which does not
2583 # break on current RTMPDump builds
2584 broken_cdn = "rtmpe://viacomccstrmfs.fplive.net/viacomccstrm/gsp.comedystor/"
2585 better_cdn = "rtmpe://cp10740.edgefcs.net/ondemand/mtvnorigin/gsp.comedystor/"
2587 if video_url.startswith(broken_cdn):
2588 video_url = video_url.replace(broken_cdn, better_cdn)
2590 effTitle = showId + u'-' + epTitle
2595 'upload_date': officialDate,
2600 'description': officialTitle,
2601 'player_url': None #playerUrl
2604 results.append(info)
2609 class EscapistIE(InfoExtractor):
2610 """Information extractor for The Escapist """
2612 _VALID_URL = r'^(https?://)?(www\.)?escapistmagazine\.com/videos/view/(?P<showname>[^/]+)/(?P<episode>[^/?]+)[/?]?.*$'
2613 IE_NAME = u'escapist'
2615 def report_extraction(self, showName):
2616 self._downloader.to_screen(u'[escapist] %s: Extracting information' % showName)
2618 def report_config_download(self, showName):
2619 self._downloader.to_screen(u'[escapist] %s: Downloading configuration' % showName)
2621 def _real_extract(self, url):
2622 mobj = re.match(self._VALID_URL, url)
2624 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2626 showName = mobj.group('showname')
2627 videoId = mobj.group('episode')
2629 self.report_extraction(showName)
2631 webPage = compat_urllib_request.urlopen(url)
2632 webPageBytes = webPage.read()
2633 m = re.match(r'text/html; charset="?([^"]+)"?', webPage.headers['Content-Type'])
2634 webPage = webPageBytes.decode(m.group(1) if m else 'utf-8')
2635 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2636 self._downloader.trouble(u'ERROR: unable to download webpage: ' + compat_str(err))
2639 descMatch = re.search('<meta name="description" content="([^"]*)"', webPage)
2640 description = unescapeHTML(descMatch.group(1))
2641 imgMatch = re.search('<meta property="og:image" content="([^"]*)"', webPage)
2642 imgUrl = unescapeHTML(imgMatch.group(1))
2643 playerUrlMatch = re.search('<meta property="og:video" content="([^"]*)"', webPage)
2644 playerUrl = unescapeHTML(playerUrlMatch.group(1))
2645 configUrlMatch = re.search('config=(.*)$', playerUrl)
2646 configUrl = compat_urllib_parse.unquote(configUrlMatch.group(1))
2648 self.report_config_download(showName)
2650 configJSON = compat_urllib_request.urlopen(configUrl).read()
2651 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2652 self._downloader.trouble(u'ERROR: unable to download configuration: ' + compat_str(err))
2655 # Technically, it's JavaScript, not JSON
2656 configJSON = configJSON.replace("'", '"')
2659 config = json.loads(configJSON)
2660 except (ValueError,) as err:
2661 self._downloader.trouble(u'ERROR: Invalid JSON in configuration file: ' + compat_str(err))
2664 playlist = config['playlist']
2665 videoUrl = playlist[1]['url']
2670 'uploader': showName,
2671 'upload_date': None,
2674 'thumbnail': imgUrl,
2675 'description': description,
2676 'player_url': playerUrl,
2682 class CollegeHumorIE(InfoExtractor):
2683 """Information extractor for collegehumor.com"""
2686 _VALID_URL = r'^(?:https?://)?(?:www\.)?collegehumor\.com/video/(?P<videoid>[0-9]+)/(?P<shorttitle>.*)$'
2687 IE_NAME = u'collegehumor'
2689 def report_manifest(self, video_id):
2690 """Report information extraction."""
2691 self._downloader.to_screen(u'[%s] %s: Downloading XML manifest' % (self.IE_NAME, video_id))
2693 def report_extraction(self, video_id):
2694 """Report information extraction."""
2695 self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
2697 def _real_extract(self, url):
2698 mobj = re.match(self._VALID_URL, url)
2700 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2702 video_id = mobj.group('videoid')
2707 'upload_date': None,
2710 self.report_extraction(video_id)
2711 xmlUrl = 'http://www.collegehumor.com/moogaloop/video/' + video_id
2713 metaXml = compat_urllib_request.urlopen(xmlUrl).read()
2714 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2715 self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % compat_str(err))
2718 mdoc = xml.etree.ElementTree.fromstring(metaXml)
2720 videoNode = mdoc.findall('./video')[0]
2721 info['description'] = videoNode.findall('./description')[0].text
2722 info['title'] = videoNode.findall('./caption')[0].text
2723 info['thumbnail'] = videoNode.findall('./thumbnail')[0].text
2724 manifest_url = videoNode.findall('./file')[0].text
2726 self._downloader.trouble(u'\nERROR: Invalid metadata XML file')
2729 manifest_url += '?hdcore=2.10.3'
2730 self.report_manifest(video_id)
2732 manifestXml = compat_urllib_request.urlopen(manifest_url).read()
2733 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2734 self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % compat_str(err))
2737 adoc = xml.etree.ElementTree.fromstring(manifestXml)
2739 media_node = adoc.findall('./{http://ns.adobe.com/f4m/1.0}media')[0]
2740 node_id = media_node.attrib['url']
2741 video_id = adoc.findall('./{http://ns.adobe.com/f4m/1.0}id')[0].text
2742 except IndexError as err:
2743 self._downloader.trouble(u'\nERROR: Invalid manifest file')
2746 url_pr = compat_urllib_parse_urlparse(manifest_url)
2747 url = url_pr.scheme + '://' + url_pr.netloc + '/z' + video_id[:-2] + '/' + node_id + 'Seg1-Frag1'
2754 class XVideosIE(InfoExtractor):
2755 """Information extractor for xvideos.com"""
2757 _VALID_URL = r'^(?:https?://)?(?:www\.)?xvideos\.com/video([0-9]+)(?:.*)'
2758 IE_NAME = u'xvideos'
2760 def report_webpage(self, video_id):
2761 """Report information extraction."""
2762 self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
2764 def report_extraction(self, video_id):
2765 """Report information extraction."""
2766 self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
2768 def _real_extract(self, url):
2769 mobj = re.match(self._VALID_URL, url)
2771 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2773 video_id = mobj.group(1).decode('utf-8')
2775 self.report_webpage(video_id)
2777 request = compat_urllib_request.Request(r'http://www.xvideos.com/video' + video_id)
2779 webpage = compat_urllib_request.urlopen(request).read()
2780 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2781 self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
2784 self.report_extraction(video_id)
2788 mobj = re.search(r'flv_url=(.+?)&', webpage)
2790 self._downloader.trouble(u'ERROR: unable to extract video url')
2792 video_url = compat_urllib_parse.unquote(mobj.group(1).decode('utf-8'))
2796 mobj = re.search(r'<title>(.*?)\s+-\s+XVID', webpage)
2798 self._downloader.trouble(u'ERROR: unable to extract video title')
2800 video_title = mobj.group(1).decode('utf-8')
2803 # Extract video thumbnail
2804 mobj = re.search(r'http://(?:img.*?\.)xvideos.com/videos/thumbs/[a-fA-F0-9]+/[a-fA-F0-9]+/[a-fA-F0-9]+/[a-fA-F0-9]+/([a-fA-F0-9.]+jpg)', webpage)
2806 self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
2808 video_thumbnail = mobj.group(0).decode('utf-8')
2814 'upload_date': None,
2815 'title': video_title,
2817 'thumbnail': video_thumbnail,
2818 'description': None,
2824 class SoundcloudIE(InfoExtractor):
2825 """Information extractor for soundcloud.com
2826 To access the media, the uid of the song and a stream token
2827 must be extracted from the page source and the script must make
2828 a request to media.soundcloud.com/crossdomain.xml. Then
2829 the media can be grabbed by requesting from an url composed
2830 of the stream token and uid
2833 _VALID_URL = r'^(?:https?://)?(?:www\.)?soundcloud\.com/([\w\d-]+)/([\w\d-]+)'
2834 IE_NAME = u'soundcloud'
2836 def __init__(self, downloader=None):
2837 InfoExtractor.__init__(self, downloader)
2839 def report_resolve(self, video_id):
2840 """Report information extraction."""
2841 self._downloader.to_screen(u'[%s] %s: Resolving id' % (self.IE_NAME, video_id))
2843 def report_extraction(self, video_id):
2844 """Report information extraction."""
2845 self._downloader.to_screen(u'[%s] %s: Retrieving stream' % (self.IE_NAME, video_id))
2847 def _real_extract(self, url):
2848 mobj = re.match(self._VALID_URL, url)
2850 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2853 # extract uploader (which is in the url)
2854 uploader = mobj.group(1)
2855 # extract simple title (uploader + slug of song title)
2856 slug_title = mobj.group(2)
2857 simple_title = uploader + u'-' + slug_title
2859 self.report_resolve('%s/%s' % (uploader, slug_title))
2861 url = 'http://soundcloud.com/%s/%s' % (uploader, slug_title)
2862 resolv_url = 'http://api.soundcloud.com/resolve.json?url=' + url + '&client_id=b45b1aa10f1ac2941910a7f0d10f8e28'
2863 request = compat_urllib_request.Request(resolv_url)
2865 info_json_bytes = compat_urllib_request.urlopen(request).read()
2866 info_json = info_json_bytes.decode('utf-8')
2867 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2868 self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
2871 info = json.loads(info_json)
2872 video_id = info['id']
2873 self.report_extraction('%s/%s' % (uploader, slug_title))
2875 streams_url = 'https://api.sndcdn.com/i1/tracks/' + str(video_id) + '/streams?client_id=b45b1aa10f1ac2941910a7f0d10f8e28'
2876 request = compat_urllib_request.Request(streams_url)
2878 stream_json_bytes = compat_urllib_request.urlopen(request).read()
2879 stream_json = stream_json_bytes.decode('utf-8')
2880 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2881 self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
2884 streams = json.loads(stream_json)
2885 mediaURL = streams['http_mp3_128_url']
2890 'uploader': info['user']['username'],
2891 'upload_date': info['created_at'],
2892 'title': info['title'],
2894 'description': info['description'],
2898 class InfoQIE(InfoExtractor):
2899 """Information extractor for infoq.com"""
2901 _VALID_URL = r'^(?:https?://)?(?:www\.)?infoq\.com/[^/]+/[^/]+$'
2904 def report_webpage(self, video_id):
2905 """Report information extraction."""
2906 self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
2908 def report_extraction(self, video_id):
2909 """Report information extraction."""
2910 self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
2912 def _real_extract(self, url):
2913 mobj = re.match(self._VALID_URL, url)
2915 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
2918 self.report_webpage(url)
2920 request = compat_urllib_request.Request(url)
2922 webpage = compat_urllib_request.urlopen(request).read()
2923 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
2924 self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
2927 self.report_extraction(url)
2931 mobj = re.search(r"jsclassref='([^']*)'", webpage)
2933 self._downloader.trouble(u'ERROR: unable to extract video url')
2935 video_url = 'rtmpe://video.infoq.com/cfx/st/' + compat_urllib_parse.unquote(mobj.group(1).decode('base64'))
2939 mobj = re.search(r'contentTitle = "(.*?)";', webpage)
2941 self._downloader.trouble(u'ERROR: unable to extract video title')
2943 video_title = mobj.group(1).decode('utf-8')
2945 # Extract description
2946 video_description = u'No description available.'
2947 mobj = re.search(r'<meta name="description" content="(.*)"(?:\s*/)?>', webpage)
2948 if mobj is not None:
2949 video_description = mobj.group(1).decode('utf-8')
2951 video_filename = video_url.split('/')[-1]
2952 video_id, extension = video_filename.split('.')
2958 'upload_date': None,
2959 'title': video_title,
2960 'ext': extension, # Extension is always(?) mp4, but seems to be flv
2962 'description': video_description,
2967 class MixcloudIE(InfoExtractor):
2968 """Information extractor for www.mixcloud.com"""
2969 _VALID_URL = r'^(?:https?://)?(?:www\.)?mixcloud\.com/([\w\d-]+)/([\w\d-]+)'
2970 IE_NAME = u'mixcloud'
2972 def __init__(self, downloader=None):
2973 InfoExtractor.__init__(self, downloader)
2975 def report_download_json(self, file_id):
2976 """Report JSON download."""
2977 self._downloader.to_screen(u'[%s] Downloading json' % self.IE_NAME)
2979 def report_extraction(self, file_id):
2980 """Report information extraction."""
2981 self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, file_id))
2983 def get_urls(self, jsonData, fmt, bitrate='best'):
2984 """Get urls from 'audio_formats' section in json"""
2987 bitrate_list = jsonData[fmt]
2988 if bitrate is None or bitrate == 'best' or bitrate not in bitrate_list:
2989 bitrate = max(bitrate_list) # select highest
2991 url_list = jsonData[fmt][bitrate]
2992 except TypeError: # we have no bitrate info.
2993 url_list = jsonData[fmt]
2996 def check_urls(self, url_list):
2997 """Returns 1st active url from list"""
2998 for url in url_list:
3000 compat_urllib_request.urlopen(url)
3002 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
3007 def _print_formats(self, formats):
3008 print('Available formats:')
3009 for fmt in formats.keys():
3010 for b in formats[fmt]:
3012 ext = formats[fmt][b][0]
3013 print('%s\t%s\t[%s]' % (fmt, b, ext.split('.')[-1]))
3014 except TypeError: # we have no bitrate info
3015 ext = formats[fmt][0]
3016 print('%s\t%s\t[%s]' % (fmt, '??', ext.split('.')[-1]))
3019 def _real_extract(self, url):
3020 mobj = re.match(self._VALID_URL, url)
3022 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
3024 # extract uploader & filename from url
3025 uploader = mobj.group(1).decode('utf-8')
3026 file_id = uploader + "-" + mobj.group(2).decode('utf-8')
3028 # construct API request
3029 file_url = 'http://www.mixcloud.com/api/1/cloudcast/' + '/'.join(url.split('/')[-3:-1]) + '.json'
3030 # retrieve .json file with links to files
3031 request = compat_urllib_request.Request(file_url)
3033 self.report_download_json(file_url)
3034 jsonData = compat_urllib_request.urlopen(request).read()
3035 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
3036 self._downloader.trouble(u'ERROR: Unable to retrieve file: %s' % compat_str(err))
3040 json_data = json.loads(jsonData)
3041 player_url = json_data['player_swf_url']
3042 formats = dict(json_data['audio_formats'])
3044 req_format = self._downloader.params.get('format', None)
3047 if self._downloader.params.get('listformats', None):
3048 self._print_formats(formats)
3051 if req_format is None or req_format == 'best':
3052 for format_param in formats.keys():
3053 url_list = self.get_urls(formats, format_param)
3055 file_url = self.check_urls(url_list)
3056 if file_url is not None:
3059 if req_format not in formats.keys():
3060 self._downloader.trouble(u'ERROR: format is not available')
3063 url_list = self.get_urls(formats, req_format)
3064 file_url = self.check_urls(url_list)
3065 format_param = req_format
3068 'id': file_id.decode('utf-8'),
3069 'url': file_url.decode('utf-8'),
3070 'uploader': uploader.decode('utf-8'),
3071 'upload_date': None,
3072 'title': json_data['name'],
3073 'ext': file_url.split('.')[-1].decode('utf-8'),
3074 'format': (format_param is None and u'NA' or format_param.decode('utf-8')),
3075 'thumbnail': json_data['thumbnail_url'],
3076 'description': json_data['description'],
3077 'player_url': player_url.decode('utf-8'),
3080 class StanfordOpenClassroomIE(InfoExtractor):
3081 """Information extractor for Stanford's Open ClassRoom"""
3083 _VALID_URL = r'^(?:https?://)?openclassroom.stanford.edu(?P<path>/?|(/MainFolder/(?:HomePage|CoursePage|VideoPage)\.php([?]course=(?P<course>[^&]+)(&video=(?P<video>[^&]+))?(&.*)?)?))$'
3084 IE_NAME = u'stanfordoc'
3086 def report_download_webpage(self, objid):
3087 """Report information extraction."""
3088 self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, objid))
3090 def report_extraction(self, video_id):
3091 """Report information extraction."""
3092 self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
3094 def _real_extract(self, url):
3095 mobj = re.match(self._VALID_URL, url)
3097 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
3100 if mobj.group('course') and mobj.group('video'): # A specific video
3101 course = mobj.group('course')
3102 video = mobj.group('video')
3104 'id': course + '_' + video,
3106 'upload_date': None,
3109 self.report_extraction(info['id'])
3110 baseUrl = 'http://openclassroom.stanford.edu/MainFolder/courses/' + course + '/videos/'
3111 xmlUrl = baseUrl + video + '.xml'
3113 metaXml = compat_urllib_request.urlopen(xmlUrl).read()
3114 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
3115 self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % compat_str(err))
3117 mdoc = xml.etree.ElementTree.fromstring(metaXml)
3119 info['title'] = mdoc.findall('./title')[0].text
3120 info['url'] = baseUrl + mdoc.findall('./videoFile')[0].text
3122 self._downloader.trouble(u'\nERROR: Invalid metadata XML file')
3124 info['ext'] = info['url'].rpartition('.')[2]
3126 elif mobj.group('course'): # A course page
3127 course = mobj.group('course')
3132 'upload_date': None,
3135 self.report_download_webpage(info['id'])
3137 coursepage = compat_urllib_request.urlopen(url).read()
3138 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
3139 self._downloader.trouble(u'ERROR: unable to download course info page: ' + compat_str(err))
3142 m = re.search('<h1>([^<]+)</h1>', coursepage)
3144 info['title'] = unescapeHTML(m.group(1))
3146 info['title'] = info['id']
3148 m = re.search('<description>([^<]+)</description>', coursepage)
3150 info['description'] = unescapeHTML(m.group(1))
3152 links = orderedSet(re.findall('<a href="(VideoPage.php\?[^"]+)">', coursepage))
3155 'type': 'reference',
3156 'url': 'http://openclassroom.stanford.edu/MainFolder/' + unescapeHTML(vpage),
3160 for entry in info['list']:
3161 assert entry['type'] == 'reference'
3162 results += self.extract(entry['url'])
3167 'id': 'Stanford OpenClassroom',
3170 'upload_date': None,
3173 self.report_download_webpage(info['id'])
3174 rootURL = 'http://openclassroom.stanford.edu/MainFolder/HomePage.php'
3176 rootpage = compat_urllib_request.urlopen(rootURL).read()
3177 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
3178 self._downloader.trouble(u'ERROR: unable to download course info page: ' + compat_str(err))
3181 info['title'] = info['id']
3183 links = orderedSet(re.findall('<a href="(CoursePage.php\?[^"]+)">', rootpage))
3186 'type': 'reference',
3187 'url': 'http://openclassroom.stanford.edu/MainFolder/' + unescapeHTML(cpage),
3192 for entry in info['list']:
3193 assert entry['type'] == 'reference'
3194 results += self.extract(entry['url'])
3197 class MTVIE(InfoExtractor):
3198 """Information extractor for MTV.com"""
3200 _VALID_URL = r'^(?P<proto>https?://)?(?:www\.)?mtv\.com/videos/[^/]+/(?P<videoid>[0-9]+)/[^/]+$'
3203 def report_webpage(self, video_id):
3204 """Report information extraction."""
3205 self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
3207 def report_extraction(self, video_id):
3208 """Report information extraction."""
3209 self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
3211 def _real_extract(self, url):
3212 mobj = re.match(self._VALID_URL, url)
3214 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
3216 if not mobj.group('proto'):
3217 url = 'http://' + url
3218 video_id = mobj.group('videoid')
3219 self.report_webpage(video_id)
3221 request = compat_urllib_request.Request(url)
3223 webpage = compat_urllib_request.urlopen(request).read()
3224 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
3225 self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % compat_str(err))
3228 mobj = re.search(r'<meta name="mtv_vt" content="([^"]+)"/>', webpage)
3230 self._downloader.trouble(u'ERROR: unable to extract song name')
3232 song_name = unescapeHTML(mobj.group(1).decode('iso-8859-1'))
3233 mobj = re.search(r'<meta name="mtv_an" content="([^"]+)"/>', webpage)
3235 self._downloader.trouble(u'ERROR: unable to extract performer')
3237 performer = unescapeHTML(mobj.group(1).decode('iso-8859-1'))
3238 video_title = performer + ' - ' + song_name
3240 mobj = re.search(r'<meta name="mtvn_uri" content="([^"]+)"/>', webpage)
3242 self._downloader.trouble(u'ERROR: unable to mtvn_uri')
3244 mtvn_uri = mobj.group(1)
3246 mobj = re.search(r'MTVN.Player.defaultPlaylistId = ([0-9]+);', webpage)
3248 self._downloader.trouble(u'ERROR: unable to extract content id')
3250 content_id = mobj.group(1)
3252 videogen_url = 'http://www.mtv.com/player/includes/mediaGen.jhtml?uri=' + mtvn_uri + '&id=' + content_id + '&vid=' + video_id + '&ref=www.mtvn.com&viewUri=' + mtvn_uri
3253 self.report_extraction(video_id)
3254 request = compat_urllib_request.Request(videogen_url)
3256 metadataXml = compat_urllib_request.urlopen(request).read()
3257 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
3258 self._downloader.trouble(u'ERROR: unable to download video metadata: %s' % compat_str(err))
3261 mdoc = xml.etree.ElementTree.fromstring(metadataXml)
3262 renditions = mdoc.findall('.//rendition')
3264 # For now, always pick the highest quality.
3265 rendition = renditions[-1]
3268 _,_,ext = rendition.attrib['type'].partition('/')
3269 format = ext + '-' + rendition.attrib['width'] + 'x' + rendition.attrib['height'] + '_' + rendition.attrib['bitrate']
3270 video_url = rendition.find('./src').text
3272 self._downloader.trouble('Invalid rendition field.')
3278 'uploader': performer,
3279 'upload_date': None,
3280 'title': video_title,
3288 class YoukuIE(InfoExtractor):
3290 _VALID_URL = r'(?:http://)?v\.youku\.com/v_show/id_(?P<ID>[A-Za-z0-9]+)\.html'
3293 def __init__(self, downloader=None):
3294 InfoExtractor.__init__(self, downloader)
3296 def report_download_webpage(self, file_id):
3297 """Report webpage download."""
3298 self._downloader.to_screen(u'[Youku] %s: Downloading webpage' % file_id)
3300 def report_extraction(self, file_id):
3301 """Report information extraction."""
3302 self._downloader.to_screen(u'[Youku] %s: Extracting information' % file_id)
3305 nowTime = int(time.time() * 1000)
3306 random1 = random.randint(1000,1998)
3307 random2 = random.randint(1000,9999)
3309 return "%d%d%d" %(nowTime,random1,random2)
3311 def _get_file_ID_mix_string(self, seed):
3313 source = list("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ/\:._-1234567890")
3315 for i in range(len(source)):
3316 seed = (seed * 211 + 30031 ) % 65536
3317 index = math.floor(seed / 65536 * len(source) )
3318 mixed.append(source[int(index)])
3319 source.remove(source[int(index)])
3320 #return ''.join(mixed)
3323 def _get_file_id(self, fileId, seed):
3324 mixed = self._get_file_ID_mix_string(seed)
3325 ids = fileId.split('*')
3329 realId.append(mixed[int(ch)])
3330 return ''.join(realId)
3332 def _real_extract(self, url):
3333 mobj = re.match(self._VALID_URL, url)
3335 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
3337 video_id = mobj.group('ID')
3339 info_url = 'http://v.youku.com/player/getPlayList/VideoIDS/' + video_id
3341 request = compat_urllib_request.Request(info_url, None, std_headers)
3343 self.report_download_webpage(video_id)
3344 jsondata = compat_urllib_request.urlopen(request).read()
3345 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
3346 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
3349 self.report_extraction(video_id)
3351 config = json.loads(jsondata)
3353 video_title = config['data'][0]['title']
3354 seed = config['data'][0]['seed']
3356 format = self._downloader.params.get('format', None)
3357 supported_format = config['data'][0]['streamfileids'].keys()
3359 if format is None or format == 'best':
3360 if 'hd2' in supported_format:
3365 elif format == 'worst':
3373 fileid = config['data'][0]['streamfileids'][format]
3374 seg_number = len(config['data'][0]['segs'][format])
3377 for i in xrange(seg_number):
3378 keys.append(config['data'][0]['segs'][format][i]['k'])
3381 #youku only could be viewed from mainland china
3383 self._downloader.trouble(u'ERROR: unable to extract info section')
3387 sid = self._gen_sid()
3388 fileid = self._get_file_id(fileid, seed)
3390 #column 8,9 of fileid represent the segment number
3391 #fileid[7:9] should be changed
3392 for index, key in enumerate(keys):
3394 temp_fileid = '%s%02X%s' % (fileid[0:8], index, fileid[10:])
3395 download_url = 'http://f.youku.com/player/getFlvPath/sid/%s_%02X/st/flv/fileid/%s?k=%s' % (sid, index, temp_fileid, key)
3398 'id': '%s_part%02d' % (video_id, index),
3399 'url': download_url,
3401 'upload_date': None,
3402 'title': video_title,
3405 files_info.append(info)
3410 class XNXXIE(InfoExtractor):
3411 """Information extractor for xnxx.com"""
3413 _VALID_URL = r'^http://video\.xnxx\.com/video([0-9]+)/(.*)'
3415 VIDEO_URL_RE = r'flv_url=(.*?)&'
3416 VIDEO_TITLE_RE = r'<title>(.*?)\s+-\s+XNXX.COM'
3417 VIDEO_THUMB_RE = r'url_bigthumb=(.*?)&'
3419 def report_webpage(self, video_id):
3420 """Report information extraction"""
3421 self._downloader.to_screen(u'[%s] %s: Downloading webpage' % (self.IE_NAME, video_id))
3423 def report_extraction(self, video_id):
3424 """Report information extraction"""
3425 self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
3427 def _real_extract(self, url):
3428 mobj = re.match(self._VALID_URL, url)
3430 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
3432 video_id = mobj.group(1).decode('utf-8')
3434 self.report_webpage(video_id)
3436 # Get webpage content
3438 webpage = compat_urllib_request.urlopen(url).read()
3439 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
3440 self._downloader.trouble(u'ERROR: unable to download video webpage: %s' % err)
3443 result = re.search(self.VIDEO_URL_RE, webpage)
3445 self._downloader.trouble(u'ERROR: unable to extract video url')
3447 video_url = compat_urllib_parse.unquote(result.group(1).decode('utf-8'))
3449 result = re.search(self.VIDEO_TITLE_RE, webpage)
3451 self._downloader.trouble(u'ERROR: unable to extract video title')
3453 video_title = result.group(1).decode('utf-8')
3455 result = re.search(self.VIDEO_THUMB_RE, webpage)
3457 self._downloader.trouble(u'ERROR: unable to extract video thumbnail')
3459 video_thumbnail = result.group(1).decode('utf-8')
3465 'upload_date': None,
3466 'title': video_title,
3468 'thumbnail': video_thumbnail,
3469 'description': None,
3473 class GooglePlusIE(InfoExtractor):
3474 """Information extractor for plus.google.com."""
3476 _VALID_URL = r'(?:https://)?plus\.google\.com/(?:\w+/)*?(\d+)/posts/(\w+)'
3477 IE_NAME = u'plus.google'
3479 def __init__(self, downloader=None):
3480 InfoExtractor.__init__(self, downloader)
3482 def report_extract_entry(self, url):
3483 """Report downloading extry"""
3484 self._downloader.to_screen(u'[plus.google] Downloading entry: %s' % url.decode('utf-8'))
3486 def report_date(self, upload_date):
3487 """Report downloading extry"""
3488 self._downloader.to_screen(u'[plus.google] Entry date: %s' % upload_date)
3490 def report_uploader(self, uploader):
3491 """Report downloading extry"""
3492 self._downloader.to_screen(u'[plus.google] Uploader: %s' % uploader.decode('utf-8'))
3494 def report_title(self, video_title):
3495 """Report downloading extry"""
3496 self._downloader.to_screen(u'[plus.google] Title: %s' % video_title.decode('utf-8'))
3498 def report_extract_vid_page(self, video_page):
3499 """Report information extraction."""
3500 self._downloader.to_screen(u'[plus.google] Extracting video page: %s' % video_page.decode('utf-8'))
3502 def _real_extract(self, url):
3503 # Extract id from URL
3504 mobj = re.match(self._VALID_URL, url)
3506 self._downloader.trouble(u'ERROR: Invalid URL: %s' % url)
3509 post_url = mobj.group(0)
3510 video_id = mobj.group(2)
3512 video_extension = 'flv'
3514 # Step 1, Retrieve post webpage to extract further information
3515 self.report_extract_entry(post_url)
3516 request = compat_urllib_request.Request(post_url)
3518 webpage = compat_urllib_request.urlopen(request).read()
3519 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
3520 self._downloader.trouble(u'ERROR: Unable to retrieve entry webpage: %s' % compat_str(err))
3523 # Extract update date
3525 pattern = 'title="Timestamp">(.*?)</a>'
3526 mobj = re.search(pattern, webpage)
3528 upload_date = mobj.group(1)
3529 # Convert timestring to a format suitable for filename
3530 upload_date = datetime.datetime.strptime(upload_date, "%Y-%m-%d")
3531 upload_date = upload_date.strftime('%Y%m%d')
3532 self.report_date(upload_date)
3536 pattern = r'rel\="author".*?>(.*?)</a>'
3537 mobj = re.search(pattern, webpage)
3539 uploader = mobj.group(1)
3540 self.report_uploader(uploader)
3543 # Get the first line for title
3545 pattern = r'<meta name\=\"Description\" content\=\"(.*?)[\n<"]'
3546 mobj = re.search(pattern, webpage)
3548 video_title = mobj.group(1)
3549 self.report_title(video_title)
3551 # Step 2, Stimulate clicking the image box to launch video
3552 pattern = '"(https\://plus\.google\.com/photos/.*?)",,"image/jpeg","video"\]'
3553 mobj = re.search(pattern, webpage)
3555 self._downloader.trouble(u'ERROR: unable to extract video page URL')
3557 video_page = mobj.group(1)
3558 request = compat_urllib_request.Request(video_page)
3560 webpage = compat_urllib_request.urlopen(request).read()
3561 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
3562 self._downloader.trouble(u'ERROR: Unable to retrieve video webpage: %s' % compat_str(err))
3564 self.report_extract_vid_page(video_page)
3567 # Extract video links on video page
3568 """Extract video links of all sizes"""
3569 pattern = '\d+,\d+,(\d+),"(http\://redirector\.googlevideo\.com.*?)"'
3570 mobj = re.findall(pattern, webpage)
3572 self._downloader.trouble(u'ERROR: unable to extract video links')
3574 # Sort in resolution
3575 links = sorted(mobj)
3577 # Choose the lowest of the sort, i.e. highest resolution
3578 video_url = links[-1]
3579 # Only get the url. The resolution part in the tuple has no use anymore
3580 video_url = video_url[-1]
3581 # Treat escaped \u0026 style hex
3582 video_url = unicode(video_url, "unicode_escape")
3586 'id': video_id.decode('utf-8'),
3588 'uploader': uploader.decode('utf-8'),
3589 'upload_date': upload_date.decode('utf-8'),
3590 'title': video_title.decode('utf-8'),
3591 'ext': video_extension.decode('utf-8'),
3594 class NBAIE(InfoExtractor):
3595 _VALID_URL = r'^(?:https?://)?(?:watch\.|www\.)?nba\.com/(?:nba/)?video(/[^?]*)(\?.*)?$'
3598 def report_extraction(self, video_id):
3599 self._downloader.to_screen(u'[%s] %s: Extracting information' % (self.IE_NAME, video_id))
3601 def _real_extract(self, url):
3602 mobj = re.match(self._VALID_URL, url)
3604 self._downloader.trouble(u'ERROR: invalid URL: %s' % url)
3607 video_id = mobj.group(1)
3608 if video_id.endswith('/index.html'):
3609 video_id = video_id[:-len('/index.html')]
3611 self.report_extraction(video_id)
3613 urlh = compat_urllib_request.urlopen(url)
3614 webpage_bytes = urlh.read()
3615 webpage = webpage_bytes.decode('utf-8', 'ignore')
3616 except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err:
3617 self._downloader.trouble(u'ERROR: unable to download video info XML: %s' % compat_str(err))
3620 video_url = u'http://ht-mobile.cdn.turner.com/nba/big' + video_id + '_nba_1280x720.mp4'
3621 def _findProp(rexp, default=None):
3622 m = re.search(rexp, webpage)
3624 return unescapeHTML(m.group(1))
3628 shortened_video_id = video_id.rpartition('/')[2]
3629 title = _findProp(r'<meta property="og:title" content="(.*?)"', shortened_video_id).replace('NBA.com: ', '')
3631 'id': shortened_video_id,
3635 'uploader_date': _findProp(r'<b>Date:</b> (.*?)</div>'),
3636 'description': _findProp(r'<div class="description">(.*?)</h1>'),