X-Git-Url: http://git.cielonegro.org/gitweb.cgi?a=blobdiff_plain;f=youtube_dl%2Fcompat.py;h=e32bef279842d7a5c97d6e6e52b01cee267e69f9;hb=9550ca506fccf9c9d795816cc0a7817ff262ef45;hp=e4b9286c06e12d967f60b5fbcf2c691802684163;hpb=75e8b2ac87e77db5a752317fcf03cef54c1536d0;p=youtube-dl.git

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index e4b9286c0..e32bef279 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -5,6 +5,7 @@ import getpass
 import optparse
 import os
 import re
+import shlex
 import shutil
 import socket
 import subprocess
@@ -227,6 +228,17 @@ except ImportError:  # Python < 3.3
             return "'" + s.replace("'", "'\"'\"'") + "'"
 
 
+if sys.version_info >= (2, 7, 3):
+    compat_shlex_split = shlex.split
+else:
+    # Working around shlex issue with unicode strings on some python 2
+    # versions (see http://bugs.python.org/issue1548891)
+    def compat_shlex_split(s, comments=False, posix=True):
+        if isinstance(s, unicode):
+            s = s.encode('utf-8')
+        return shlex.split(s, comments, posix)
+
+
 def compat_ord(c):
     if type(c) is int:
         return c
@@ -436,6 +448,11 @@ except TypeError:  # Python 2.6
             yield n
             n += step
 
+if sys.version_info >= (3, 0):
+    from tokenize import tokenize as compat_tokenize_tokenize
+else:
+    from tokenize import generate_tokens as compat_tokenize_tokenize
+
 __all__ = [
     'compat_HTTPError',
     'compat_basestring',
@@ -454,9 +471,11 @@ __all__ = [
     'compat_ord',
     'compat_parse_qs',
     'compat_print',
+    'compat_shlex_split',
     'compat_socket_create_connection',
     'compat_str',
     'compat_subprocess_get_DEVNULL',
+    'compat_tokenize_tokenize',
     'compat_urllib_error',
     'compat_urllib_parse',
     'compat_urllib_parse_unquote',