youtube_dl/extractor/bet.py

   1 from __future__ import unicode_literals
   2
   3 from .common import InfoExtractor
   4 from ..compat import compat_urllib_parse
   5 from ..utils import (
   6     xpath_text,
   7     xpath_with_ns,
   8     int_or_none,
   9     parse_iso8601,
  10 )
  11
  12
  13 class BetIE(InfoExtractor):
  14     _VALID_URL = r'https?://(?:www\.)?bet\.com/(?:[^/]+/)+(?P<id>.+?)\.html'
  15     _TESTS = [
  16         {
  17             'url': 'http://www.bet.com/news/politics/2014/12/08/in-bet-exclusive-obama-talks-race-and-racism.html',
  18             'info_dict': {
  19                 'id': '417cd61c-c793-4e8e-b006-e445ecc45add',
  20                 'display_id': 'in-bet-exclusive-obama-talks-race-and-racism',
  21                 'ext': 'flv',
  22                 'title': 'BET News Presents: A Conversation With President Obama',
  23                 'description': 'md5:5a88d8ae912c1b33e090290af7ec33c6',
  24                 'duration': 1534,
  25                 'timestamp': 1418075340,
  26                 'upload_date': '20141208',
  27                 'uploader': 'admin',
  28                 'thumbnail': 're:(?i)^https?://.*\.jpg$',
  29             },
  30             'params': {
  31                 # rtmp download
  32                 'skip_download': True,
  33             },
  34         },
  35         {
  36             'url': 'http://www.bet.com/video/news/national/2014/justice-for-ferguson-a-community-reacts.html',
  37             'info_dict': {
  38                 'id': '4160e53b-ad41-43b1-980f-8d85f63121f4',
  39                 'display_id': 'justice-for-ferguson-a-community-reacts',
  40                 'ext': 'flv',
  41                 'title': 'Justice for Ferguson: A Community Reacts',
  42                 'description': 'A BET News special.',
  43                 'duration': 1696,
  44                 'timestamp': 1416942360,
  45                 'upload_date': '20141125',
  46                 'uploader': 'admin',
  47                 'thumbnail': 're:(?i)^https?://.*\.jpg$',
  48             },
  49             'params': {
  50                 # rtmp download
  51                 'skip_download': True,
  52             },
  53         }
  54     ]
  55
  56     def _real_extract(self, url):
  57         display_id = self._match_id(url)
  58
  59         webpage = self._download_webpage(url, display_id)
  60
  61         media_url = compat_urllib_parse.unquote(self._search_regex(
  62             [r'mediaURL\s*:\s*"([^"]+)"', r"var\s+mrssMediaUrl\s*=\s*'([^']+)'"],
  63             webpage, 'media URL'))
  64
  65         mrss = self._download_xml(media_url, display_id)
  66
  67         item = mrss.find('./channel/item')
  68
  69         NS_MAP = {
  70             'dc': 'http://purl.org/dc/elements/1.1/',
  71             'media': 'http://search.yahoo.com/mrss/',
  72             'ka': 'http://kickapps.com/karss',
  73         }
  74
  75         title = xpath_text(item, './title', 'title')
  76         description = xpath_text(
  77             item, './description', 'description', fatal=False)
  78
  79         video_id = xpath_text(item, './guid', 'video id', fatal=False)
  80
  81         timestamp = parse_iso8601(xpath_text(
  82             item, xpath_with_ns('./dc:date', NS_MAP),
  83             'upload date', fatal=False))
  84         uploader = xpath_text(
  85             item, xpath_with_ns('./dc:creator', NS_MAP),
  86             'uploader', fatal=False)
  87
  88         media_content = item.find(
  89             xpath_with_ns('./media:content', NS_MAP))
  90         duration = int_or_none(media_content.get('duration'))
  91         smil_url = media_content.get('url')
  92
  93         thumbnail = media_content.find(
  94             xpath_with_ns('./media:thumbnail', NS_MAP)).get('url')
  95
  96         formats = self._extract_smil_formats(smil_url, display_id)
  97
  98         return {
  99             'id': video_id,
 100             'display_id': display_id,
 101             'title': title,
 102             'description': description,
 103             'thumbnail': thumbnail,
 104             'timestamp': timestamp,
 105             'uploader': uploader,
 106             'duration': duration,
 107             'formats': formats,
 108         }