youtube_dl/extractor/vidlii.py

   1 # coding: utf-8
   2 from __future__ import unicode_literals
   3
   4 import re
   5
   6 from .common import InfoExtractor
   7 from ..utils import (
   8     float_or_none,
   9     get_element_by_id,
  10     int_or_none,
  11     strip_or_none,
  12     unified_strdate,
  13     urljoin,
  14 )
  15
  16
  17 class VidLiiIE(InfoExtractor):
  18     _VALID_URL = r'https?://(?:www\.)?vidlii\.com/(?:watch|embed)\?.*?\bv=(?P<id>[0-9A-Za-z_-]{11})'
  19     _TESTS = [{
  20         'url': 'https://www.vidlii.com/watch?v=tJluaH4BJ3v',
  21         'md5': '9bf7d1e005dfa909b6efb0a1ff5175e2',
  22         'info_dict': {
  23             'id': 'tJluaH4BJ3v',
  24             'ext': 'mp4',
  25             'title': 'Vidlii is against me',
  26             'description': 'md5:fa3f119287a2bfb922623b52b1856145',
  27             'thumbnail': 're:https://.*.jpg',
  28             'uploader': 'APPle5auc31995',
  29             'uploader_url': 'https://www.vidlii.com/user/APPle5auc31995',
  30             'upload_date': '20171107',
  31             'duration': 212,
  32             'view_count': int,
  33             'comment_count': int,
  34             'average_rating': float,
  35             'categories': ['News & Politics'],
  36             'tags': ['Vidlii', 'Jan', 'Videogames'],
  37         }
  38     }, {
  39         'url': 'https://www.vidlii.com/embed?v=tJluaH4BJ3v&a=0',
  40         'only_matching': True,
  41     }]
  42
  43     def _real_extract(self, url):
  44         video_id = self._match_id(url)
  45
  46         webpage = self._download_webpage(
  47             'https://www.vidlii.com/watch?v=%s' % video_id, video_id)
  48
  49         video_url = self._search_regex(
  50             r'src\s*:\s*(["\'])(?P<url>(?:https?://)?(?:(?!\1).)+)\1', webpage,
  51             'video url', group='url')
  52
  53         title = self._search_regex(
  54             (r'<h1>([^<]+)</h1>', r'<title>([^<]+) - VidLii<'), webpage,
  55             'title')
  56
  57         description = self._html_search_meta(
  58             ('description', 'twitter:description'), webpage,
  59             default=None) or strip_or_none(
  60             get_element_by_id('des_text', webpage))
  61
  62         thumbnail = self._html_search_meta(
  63             'twitter:image', webpage, default=None)
  64         if not thumbnail:
  65             thumbnail_path = self._search_regex(
  66                 r'img\s*:\s*(["\'])(?P<url>(?:(?!\1).)+)\1', webpage,
  67                 'thumbnail', fatal=False, group='url')
  68             if thumbnail_path:
  69                 thumbnail = urljoin(url, thumbnail_path)
  70
  71         uploader = self._search_regex(
  72             r'<div[^>]+class=["\']wt_person[^>]+>\s*<a[^>]+\bhref=["\']/user/[^>]+>([^<]+)',
  73             webpage, 'uploader', fatal=False)
  74         uploader_url = 'https://www.vidlii.com/user/%s' % uploader if uploader else None
  75
  76         upload_date = unified_strdate(self._html_search_meta(
  77             'datePublished', webpage, default=None) or self._search_regex(
  78             r'<date>([^<]+)', webpage, 'upload date', fatal=False))
  79
  80         duration = int_or_none(self._html_search_meta(
  81             'video:duration', webpage, 'duration',
  82             default=None) or self._search_regex(
  83             r'duration\s*:\s*(\d+)', webpage, 'duration', fatal=False))
  84
  85         view_count = int_or_none(self._search_regex(
  86             (r'<strong>(\d+)</strong> views',
  87              r'Views\s*:\s*<strong>(\d+)</strong>'),
  88             webpage, 'view count', fatal=False))
  89
  90         comment_count = int_or_none(self._search_regex(
  91             (r'<span[^>]+id=["\']cmt_num[^>]+>(\d+)',
  92              r'Comments\s*:\s*<strong>(\d+)'),
  93             webpage, 'comment count', fatal=False))
  94
  95         average_rating = float_or_none(self._search_regex(
  96             r'rating\s*:\s*([\d.]+)', webpage, 'average rating', fatal=False))
  97
  98         category = self._html_search_regex(
  99             r'<div>Category\s*:\s*</div>\s*<div>\s*<a[^>]+>([^<]+)', webpage,
 100             'category', fatal=False)
 101         categories = [category] if category else None
 102
 103         tags = [
 104             strip_or_none(tag)
 105             for tag in re.findall(
 106                 r'<a[^>]+\bhref=["\']/results\?.*?q=[^>]*>([^<]+)',
 107                 webpage) if strip_or_none(tag)
 108         ] or None
 109
 110         return {
 111             'id': video_id,
 112             'url': video_url,
 113             'title': title,
 114             'description': description,
 115             'thumbnail': thumbnail,
 116             'uploader': uploader,
 117             'uploader_url': uploader_url,
 118             'upload_date': upload_date,
 119             'duration': duration,
 120             'view_count': view_count,
 121             'comment_count': comment_count,
 122             'average_rating': average_rating,
 123             'categories': categories,
 124             'tags': tags,
 125         }