Add an extractor for CNN (closes #1318)

[youtube-dl.git] / youtube_dl / utils.py

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index e5d756b8b10174a3aa3a2d6dbf94a376e0284dac..ab1049cc0d99b17adcfc83f951d38e89a89a4bc9 100644 (file)
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 

+import errno

 import gzip
 import io
 import json
 import gzip
 import io
 import json


@@ -11,7 +12,7 @@ import sys
 import traceback
 import zlib
 import email.utils
 import traceback
 import zlib
 import email.utils

-import json
+import socket

 import datetime
 
 try:
 import datetime
 
 try:


@@ -34,6 +35,11 @@ try:
 except ImportError: # Python 2
     from urlparse import urlparse as compat_urllib_parse_urlparse
 
 except ImportError: # Python 2
     from urlparse import urlparse as compat_urllib_parse_urlparse
 

+try:
+    import urllib.parse as compat_urlparse
+except ImportError: # Python 2
+    import urlparse as compat_urlparse
+

 try:
     import http.cookiejar as compat_cookiejar
 except ImportError: # Python 2
 try:
     import http.cookiejar as compat_cookiejar
 except ImportError: # Python 2


@@ -149,6 +155,13 @@ try:
 except NameError:
     compat_chr = chr
 
 except NameError:
     compat_chr = chr
 

+def compat_ord(c):
+    if type(c) is int: return c
+    else: return ord(c)
+
+# This is not clearly defined otherwise
+compiled_regex_type = type(re.compile(''))
+

 std_headers = {
     'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 Firefox/10.0',
     'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
 std_headers = {
     'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:10.0) Gecko/20100101 Firefox/10.0',
     'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',


@@ -190,6 +203,20 @@ else:
         with open(fn, 'w', encoding='utf-8') as f:
             json.dump(obj, f)
 
         with open(fn, 'w', encoding='utf-8') as f:
             json.dump(obj, f)
 

+if sys.version_info >= (2,7):
+    def find_xpath_attr(node, xpath, key, val):
+        """ Find the xpath xpath[@key=val] """
+        assert re.match(r'^[a-zA-Z]+$', key)
+        assert re.match(r'^[a-zA-Z@\s]*$', val)
+        expr = xpath + u"[@%s='%s']" % (key, val)
+        return node.find(expr)
+else:
+    def find_xpath_attr(node, xpath, key, val):
+        for f in node.findall(xpath):
+            if f.attrib.get(key) == val:
+                return f
+        return None
+

 def htmlentity_transform(matchobj):
     """Transforms an HTML entity to a character.
 
 def htmlentity_transform(matchobj):
     """Transforms an HTML entity to a character.
 


@@ -334,12 +361,20 @@ def sanitize_open(filename, open_mode):
         stream = open(encodeFilename(filename), open_mode)
         return (stream, filename)
     except (IOError, OSError) as err:
         stream = open(encodeFilename(filename), open_mode)
         return (stream, filename)
     except (IOError, OSError) as err:

-        # In case of error, try to remove win32 forbidden chars
-        filename = re.sub(u'[/<>:"\\|\\\\?\\*]', u'#', filename)
+        if err.errno in (errno.EACCES,):
+            raise

 
 

-        # An exception here should be caught in the caller
-        stream = open(encodeFilename(filename), open_mode)
-        return (stream, filename)
+        # In case of error, try to remove win32 forbidden chars
+        alt_filename = os.path.join(
+                        re.sub(u'[/<>:"\\|\\\\?\\*]', u'#', path_part)
+                        for path_part in os.path.split(filename)
+                       )
+        if alt_filename == filename:
+            raise
+        else:
+            # An exception here should be caught in the caller
+            stream = open(encodeFilename(filename), open_mode)
+            return (stream, alt_filename)

 
 
 def timeconvert(timestr):
 
 
 def timeconvert(timestr):


@@ -430,11 +465,41 @@ def decodeOption(optval):
     assert isinstance(optval, compat_str)
     return optval
 
     assert isinstance(optval, compat_str)
     return optval
 

+def formatSeconds(secs):
+    if secs > 3600:
+        return '%d:%02d:%02d' % (secs // 3600, (secs % 3600) // 60, secs % 60)
+    elif secs > 60:
+        return '%d:%02d' % (secs // 60, secs % 60)
+    else:
+        return '%d' % secs
+
+def make_HTTPS_handler(opts):
+    if sys.version_info < (3,2):
+        # Python's 2.x handler is very simplistic
+        return YoutubeDLHandlerHTTPS()
+    else:
+        import ssl
+        context = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+        context.set_default_verify_paths()
+        
+        context.verify_mode = (ssl.CERT_NONE
+                               if opts.no_check_certificate
+                               else ssl.CERT_REQUIRED)
+        return YoutubeDLHandlerHTTPS(context=context)
+

 class ExtractorError(Exception):
     """Error during info extraction."""
 class ExtractorError(Exception):
     """Error during info extraction."""

-    def __init__(self, msg, tb=None):
-        """ tb, if given, is the original traceback (so that it can be printed out). """
+    def __init__(self, msg, tb=None, expected=False):
+        """ tb, if given, is the original traceback (so that it can be printed out).
+        If expected is set, this is a normal error message and most likely not a bug in youtube-dl.
+        """
+
+        if sys.exc_info()[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError):
+            expected = True
+        if not expected:
+            msg = msg + u'; please report this issue on https://yt-dl.org/bug . Be sure to call youtube-dl with the --verbose flag and include its complete output. Make sure you are using the latest version; type  youtube-dl -U  to update.'

         super(ExtractorError, self).__init__(msg)
         super(ExtractorError, self).__init__(msg)

+

         self.traceback = tb
         self.exc_info = sys.exc_info()  # preserve original exception
 
         self.traceback = tb
         self.exc_info = sys.exc_info()  # preserve original exception
 


@@ -504,7 +569,8 @@ class ContentTooShortError(Exception):
         self.downloaded = downloaded
         self.expected = expected
 
         self.downloaded = downloaded
         self.expected = expected
 

-class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
+
+class YoutubeDLHandler_Template:  # Old-style class, like HTTPHandler

     """Handler for HTTP requests and responses.
 
     This class, when installed with an OpenerDirector, automatically adds
     """Handler for HTTP requests and responses.
 
     This class, when installed with an OpenerDirector, automatically adds


@@ -537,8 +603,8 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
         ret.code = code
         return ret
 
         ret.code = code
         return ret
 

-    def http_request(self, req):
-        for h,v in std_headers.items():
+    def _http_request(self, req):
+        for h, v in std_headers.items():

             if h in req.headers:
                 del req.headers[h]
             req.add_header(h, v)
             if h in req.headers:
                 del req.headers[h]
             req.add_header(h, v)


@@ -553,7 +619,7 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
             del req.headers['Youtubedl-user-agent']
         return req
 
             del req.headers['Youtubedl-user-agent']
         return req
 

-    def http_response(self, req, resp):
+    def _http_response(self, req, resp):

         old_resp = resp
         # gzip
         if resp.headers.get('Content-encoding', '') == 'gzip':
         old_resp = resp
         # gzip
         if resp.headers.get('Content-encoding', '') == 'gzip':


@@ -567,11 +633,66 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
             resp.msg = old_resp.msg
         return resp
 
             resp.msg = old_resp.msg
         return resp
 

-    https_request = http_request
-    https_response = http_response
-    
+
+class YoutubeDLHandler(YoutubeDLHandler_Template, compat_urllib_request.HTTPHandler):
+    http_request = YoutubeDLHandler_Template._http_request
+    http_response = YoutubeDLHandler_Template._http_response
+
+
+class YoutubeDLHandlerHTTPS(YoutubeDLHandler_Template, compat_urllib_request.HTTPSHandler):
+    https_request = YoutubeDLHandler_Template._http_request
+    https_response = YoutubeDLHandler_Template._http_response
+
+
+def unified_strdate(date_str):
+    """Return a string with the date in the format YYYYMMDD"""
+    upload_date = None
+    #Replace commas
+    date_str = date_str.replace(',',' ')
+    # %z (UTC offset) is only supported in python>=3.2
+    date_str = re.sub(r' (\+|-)[\d]*$', '', date_str)
+    format_expressions = ['%d %B %Y', '%B %d %Y', '%b %d %Y', '%Y-%m-%d', '%d/%m/%Y', '%Y/%m/%d %H:%M:%S', '%d.%m.%Y %H:%M']
+    for expression in format_expressions:
+        try:
+            upload_date = datetime.datetime.strptime(date_str, expression).strftime('%Y%m%d')
+        except:
+            pass
+    return upload_date
+
+def determine_ext(url, default_ext=u'unknown_video'):
+    guess = url.partition(u'?')[0].rpartition(u'.')[2]
+    if re.match(r'^[A-Za-z0-9]+$', guess):
+        return guess
+    else:
+        return default_ext
+
+def subtitles_filename(filename, sub_lang, sub_format):
+    return filename.rsplit('.', 1)[0] + u'.' + sub_lang + u'.' + sub_format
+

 def date_from_str(date_str):
 def date_from_str(date_str):

-    """Return a datetime object from a string in the format YYYYMMDD"""
+    """
+    Return a datetime object from a string in the format YYYYMMDD or
+    (now|today)[+-][0-9](day|week|month|year)(s)?"""
+    today = datetime.date.today()
+    if date_str == 'now'or date_str == 'today':
+        return today
+    match = re.match('(now|today)(?P<sign>[+-])(?P<time>\d+)(?P<unit>day|week|month|year)(s)?', date_str)
+    if match is not None:
+        sign = match.group('sign')
+        time = int(match.group('time'))
+        if sign == '-':
+            time = -time
+        unit = match.group('unit')
+        #A bad aproximation?
+        if unit == 'month':
+            unit = 'day'
+            time *= 30
+        elif unit == 'year':
+            unit = 'day'
+            time *= 365
+        unit += 's'
+        delta = datetime.timedelta(**{unit: time})
+        return today + delta

     return datetime.datetime.strptime(date_str, "%Y%m%d").date()
     
 class DateRange(object):
     return datetime.datetime.strptime(date_str, "%Y%m%d").date()
     
 class DateRange(object):


@@ -586,7 +707,7 @@ class DateRange(object):
             self.end = date_from_str(end)
         else:
             self.end = datetime.datetime.max.date()
             self.end = date_from_str(end)
         else:
             self.end = datetime.datetime.max.date()

-        if self.start >= self.end:
+        if self.start > self.end:

             raise ValueError('Date range: "%s" , the start date must be before the end date' % self)
     @classmethod
     def day(cls, day):
             raise ValueError('Date range: "%s" , the start date must be before the end date' % self)
     @classmethod
     def day(cls, day):


@@ -594,7 +715,8 @@ class DateRange(object):
         return cls(day,day)
     def __contains__(self, date):
         """Check if the date is in the range"""
         return cls(day,day)
     def __contains__(self, date):
         """Check if the date is in the range"""

-        date = date_from_str(date)
-        return self.start <= date and date <= self.end
+        if not isinstance(date, datetime.date):
+            date = date_from_str(date)
+        return self.start <= date <= self.end

     def __str__(self):
         return '%s - %s' % ( self.start.isoformat(), self.end.isoformat())
     def __str__(self):
         return '%s - %s' % ( self.start.isoformat(), self.end.isoformat())