youtube_dl/extractor/abcnews.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 import calendar
   5 import re
   6 import time
   7
   8 from .amp import AMPIE
   9 from .common import InfoExtractor
  10 from ..compat import compat_urlparse
  11
  12
  13 class AbcNewsVideoIE(AMPIE):
  14     IE_NAME = 'abcnews:video'
  15     _VALID_URL = r'https?://abcnews\.go\.com/[^/]+/video/(?P<display_id>[0-9a-z-]+)-(?P<id>\d+)'
  16
  17     _TESTS = [{
  18         'url': 'http://abcnews.go.com/ThisWeek/video/week-exclusive-irans-foreign-minister-zarif-20411932',
  19         'info_dict': {
  20             'id': '20411932',
  21             'ext': 'mp4',
  22             'display_id': 'week-exclusive-irans-foreign-minister-zarif',
  23             'title': '\'This Week\' Exclusive: Iran\'s Foreign Minister Zarif',
  24             'description': 'George Stephanopoulos goes one-on-one with Iranian Foreign Minister Dr. Javad Zarif.',
  25             'duration': 180,
  26             'thumbnail': r're:^https?://.*\.jpg$',
  27         },
  28         'params': {
  29             # m3u8 download
  30             'skip_download': True,
  31         },
  32     }, {
  33         'url': 'http://abcnews.go.com/2020/video/2020-husband-stands-teacher-jail-student-affairs-26119478',
  34         'only_matching': True,
  35     }]
  36
  37     def _real_extract(self, url):
  38         mobj = re.match(self._VALID_URL, url)
  39         display_id = mobj.group('display_id')
  40         video_id = mobj.group('id')
  41         info_dict = self._extract_feed_info(
  42             'http://abcnews.go.com/video/itemfeed?id=%s' % video_id)
  43         info_dict.update({
  44             'id': video_id,
  45             'display_id': display_id,
  46         })
  47         return info_dict
  48
  49
  50 class AbcNewsIE(InfoExtractor):
  51     IE_NAME = 'abcnews'
  52     _VALID_URL = r'https?://abcnews\.go\.com/(?:[^/]+/)+(?P<display_id>[0-9a-z-]+)/story\?id=(?P<id>\d+)'
  53
  54     _TESTS = [{
  55         'url': 'http://abcnews.go.com/Blotter/News/dramatic-video-rare-death-job-america/story?id=10498713#.UIhwosWHLjY',
  56         'info_dict': {
  57             'id': '10498713',
  58             'ext': 'flv',
  59             'display_id': 'dramatic-video-rare-death-job-america',
  60             'title': 'Occupational Hazards',
  61             'description': 'Nightline investigates the dangers that lurk at various jobs.',
  62             'thumbnail': r're:^https?://.*\.jpg$',
  63             'upload_date': '20100428',
  64             'timestamp': 1272412800,
  65         },
  66         'add_ie': ['AbcNewsVideo'],
  67     }, {
  68         'url': 'http://abcnews.go.com/Entertainment/justin-timberlake-performs-stop-feeling-eurovision-2016/story?id=39125818',
  69         'info_dict': {
  70             'id': '39125818',
  71             'ext': 'mp4',
  72             'display_id': 'justin-timberlake-performs-stop-feeling-eurovision-2016',
  73             'title': 'Justin Timberlake Drops Hints For Secret Single',
  74             'description': 'Lara Spencer reports the buzziest stories of the day in "GMA" Pop News.',
  75             'upload_date': '20160515',
  76             'timestamp': 1463329500,
  77         },
  78         'params': {
  79             # m3u8 download
  80             'skip_download': True,
  81             # The embedded YouTube video is blocked due to copyright issues
  82             'playlist_items': '1',
  83         },
  84         'add_ie': ['AbcNewsVideo'],
  85     }, {
  86         'url': 'http://abcnews.go.com/Technology/exclusive-apple-ceo-tim-cook-iphone-cracking-software/story?id=37173343',
  87         'only_matching': True,
  88     }]
  89
  90     def _real_extract(self, url):
  91         mobj = re.match(self._VALID_URL, url)
  92         display_id = mobj.group('display_id')
  93         video_id = mobj.group('id')
  94
  95         webpage = self._download_webpage(url, video_id)
  96         video_url = self._search_regex(
  97             r'window\.abcnvideo\.url\s*=\s*"([^"]+)"', webpage, 'video URL')
  98         full_video_url = compat_urlparse.urljoin(url, video_url)
  99
 100         youtube_url = self._html_search_regex(
 101             r'<iframe[^>]+src="(https://www\.youtube\.com/embed/[^"]+)"',
 102             webpage, 'YouTube URL', default=None)
 103
 104         timestamp = None
 105         date_str = self._html_search_regex(
 106             r'<span[^>]+class="timestamp">([^<]+)</span>',
 107             webpage, 'timestamp', fatal=False)
 108         if date_str:
 109             tz_offset = 0
 110             if date_str.endswith(' ET'):  # Eastern Time
 111                 tz_offset = -5
 112                 date_str = date_str[:-3]
 113             date_formats = ['%b. %d, %Y', '%b %d, %Y, %I:%M %p']
 114             for date_format in date_formats:
 115                 try:
 116                     timestamp = calendar.timegm(time.strptime(date_str.strip(), date_format))
 117                 except ValueError:
 118                     continue
 119             if timestamp is not None:
 120                 timestamp -= tz_offset * 3600
 121
 122         entry = {
 123             '_type': 'url_transparent',
 124             'ie_key': AbcNewsVideoIE.ie_key(),
 125             'url': full_video_url,
 126             'id': video_id,
 127             'display_id': display_id,
 128             'timestamp': timestamp,
 129         }
 130
 131         if youtube_url:
 132             entries = [entry, self.url_result(youtube_url, 'Youtube')]
 133             return self.playlist_result(entries)
 134
 135         return entry