devscripts/buildserver.py

   1 #!/usr/bin/python3
   2
   3 import argparse
   4 import ctypes
   5 import functools
   6 import shutil
   7 import subprocess
   8 import sys
   9 import tempfile
  10 import threading
  11 import traceback
  12 import os.path
  13
  14 sys.path.insert(0, os.path.dirname(os.path.dirname((os.path.abspath(__file__)))))
  15 from youtube_dl.compat import (
  16     compat_input,
  17     compat_http_server,
  18     compat_str,
  19     compat_urlparse,
  20 )
  21
  22 # These are not used outside of buildserver.py thus not in compat.py
  23
  24 try:
  25     import winreg as compat_winreg
  26 except ImportError:  # Python 2
  27     import _winreg as compat_winreg
  28
  29 try:
  30     import socketserver as compat_socketserver
  31 except ImportError:  # Python 2
  32     import SocketServer as compat_socketserver
  33
  34
  35 class BuildHTTPServer(compat_socketserver.ThreadingMixIn, compat_http_server.HTTPServer):
  36     allow_reuse_address = True
  37
  38
  39 advapi32 = ctypes.windll.advapi32
  40
  41 SC_MANAGER_ALL_ACCESS = 0xf003f
  42 SC_MANAGER_CREATE_SERVICE = 0x02
  43 SERVICE_WIN32_OWN_PROCESS = 0x10
  44 SERVICE_AUTO_START = 0x2
  45 SERVICE_ERROR_NORMAL = 0x1
  46 DELETE = 0x00010000
  47 SERVICE_STATUS_START_PENDING = 0x00000002
  48 SERVICE_STATUS_RUNNING = 0x00000004
  49 SERVICE_ACCEPT_STOP = 0x1
  50
  51 SVCNAME = 'youtubedl_builder'
  52
  53 LPTSTR = ctypes.c_wchar_p
  54 START_CALLBACK = ctypes.WINFUNCTYPE(None, ctypes.c_int, ctypes.POINTER(LPTSTR))
  55
  56
  57 class SERVICE_TABLE_ENTRY(ctypes.Structure):
  58     _fields_ = [
  59         ('lpServiceName', LPTSTR),
  60         ('lpServiceProc', START_CALLBACK)
  61     ]
  62
  63
  64 HandlerEx = ctypes.WINFUNCTYPE(
  65     ctypes.c_int,     # return
  66     ctypes.c_int,     # dwControl
  67     ctypes.c_int,     # dwEventType
  68     ctypes.c_void_p,  # lpEventData,
  69     ctypes.c_void_p,  # lpContext,
  70 )
  71
  72
  73 def _ctypes_array(c_type, py_array):
  74     ar = (c_type * len(py_array))()
  75     ar[:] = py_array
  76     return ar
  77
  78
  79 def win_OpenSCManager():
  80     res = advapi32.OpenSCManagerW(None, None, SC_MANAGER_ALL_ACCESS)
  81     if not res:
  82         raise Exception('Opening service manager failed - '
  83                         'are you running this as administrator?')
  84     return res
  85
  86
  87 def win_install_service(service_name, cmdline):
  88     manager = win_OpenSCManager()
  89     try:
  90         h = advapi32.CreateServiceW(
  91             manager, service_name, None,
  92             SC_MANAGER_CREATE_SERVICE, SERVICE_WIN32_OWN_PROCESS,
  93             SERVICE_AUTO_START, SERVICE_ERROR_NORMAL,
  94             cmdline, None, None, None, None, None)
  95         if not h:
  96             raise OSError('Service creation failed: %s' % ctypes.FormatError())
  97
  98         advapi32.CloseServiceHandle(h)
  99     finally:
 100         advapi32.CloseServiceHandle(manager)
 101
 102
 103 def win_uninstall_service(service_name):
 104     manager = win_OpenSCManager()
 105     try:
 106         h = advapi32.OpenServiceW(manager, service_name, DELETE)
 107         if not h:
 108             raise OSError('Could not find service %s: %s' % (
 109                 service_name, ctypes.FormatError()))
 110
 111         try:
 112             if not advapi32.DeleteService(h):
 113                 raise OSError('Deletion failed: %s' % ctypes.FormatError())
 114         finally:
 115             advapi32.CloseServiceHandle(h)
 116     finally:
 117         advapi32.CloseServiceHandle(manager)
 118
 119
 120 def win_service_report_event(service_name, msg, is_error=True):
 121     with open('C:/sshkeys/log', 'a', encoding='utf-8') as f:
 122         f.write(msg + '\n')
 123
 124     event_log = advapi32.RegisterEventSourceW(None, service_name)
 125     if not event_log:
 126         raise OSError('Could not report event: %s' % ctypes.FormatError())
 127
 128     try:
 129         type_id = 0x0001 if is_error else 0x0004
 130         event_id = 0xc0000000 if is_error else 0x40000000
 131         lines = _ctypes_array(LPTSTR, [msg])
 132
 133         if not advapi32.ReportEventW(
 134                 event_log, type_id, 0, event_id, None, len(lines), 0,
 135                 lines, None):
 136             raise OSError('Event reporting failed: %s' % ctypes.FormatError())
 137     finally:
 138         advapi32.DeregisterEventSource(event_log)
 139
 140
 141 def win_service_handler(stop_event, *args):
 142     try:
 143         raise ValueError('Handler called with args ' + repr(args))
 144         TODO
 145     except Exception as e:
 146         tb = traceback.format_exc()
 147         msg = str(e) + '\n' + tb
 148         win_service_report_event(service_name, msg, is_error=True)
 149         raise
 150
 151
 152 def win_service_set_status(handle, status_code):
 153     svcStatus = SERVICE_STATUS()
 154     svcStatus.dwServiceType = SERVICE_WIN32_OWN_PROCESS
 155     svcStatus.dwCurrentState = status_code
 156     svcStatus.dwControlsAccepted = SERVICE_ACCEPT_STOP
 157
 158     svcStatus.dwServiceSpecificExitCode = 0
 159
 160     if not advapi32.SetServiceStatus(handle, ctypes.byref(svcStatus)):
 161         raise OSError('SetServiceStatus failed: %r' % ctypes.FormatError())
 162
 163
 164 def win_service_main(service_name, real_main, argc, argv_raw):
 165     try:
 166         # args = [argv_raw[i].value for i in range(argc)]
 167         stop_event = threading.Event()
 168         handler = HandlerEx(functools.partial(stop_event, win_service_handler))
 169         h = advapi32.RegisterServiceCtrlHandlerExW(service_name, handler, None)
 170         if not h:
 171             raise OSError('Handler registration failed: %s' %
 172                           ctypes.FormatError())
 173
 174         TODO
 175     except Exception as e:
 176         tb = traceback.format_exc()
 177         msg = str(e) + '\n' + tb
 178         win_service_report_event(service_name, msg, is_error=True)
 179         raise
 180
 181
 182 def win_service_start(service_name, real_main):
 183     try:
 184         cb = START_CALLBACK(
 185             functools.partial(win_service_main, service_name, real_main))
 186         dispatch_table = _ctypes_array(SERVICE_TABLE_ENTRY, [
 187             SERVICE_TABLE_ENTRY(
 188                 service_name,
 189                 cb
 190             ),
 191             SERVICE_TABLE_ENTRY(None, ctypes.cast(None, START_CALLBACK))
 192         ])
 193
 194         if not advapi32.StartServiceCtrlDispatcherW(dispatch_table):
 195             raise OSError('ctypes start failed: %s' % ctypes.FormatError())
 196     except Exception as e:
 197         tb = traceback.format_exc()
 198         msg = str(e) + '\n' + tb
 199         win_service_report_event(service_name, msg, is_error=True)
 200         raise
 201
 202
 203 def main(args=None):
 204     parser = argparse.ArgumentParser()
 205     parser.add_argument('-i', '--install',
 206                         action='store_const', dest='action', const='install',
 207                         help='Launch at Windows startup')
 208     parser.add_argument('-u', '--uninstall',
 209                         action='store_const', dest='action', const='uninstall',
 210                         help='Remove Windows service')
 211     parser.add_argument('-s', '--service',
 212                         action='store_const', dest='action', const='service',
 213                         help='Run as a Windows service')
 214     parser.add_argument('-b', '--bind', metavar='<host:port>',
 215                         action='store', default='0.0.0.0:8142',
 216                         help='Bind to host:port (default %default)')
 217     options = parser.parse_args(args=args)
 218
 219     if options.action == 'install':
 220         fn = os.path.abspath(__file__).replace('v:', '\\\\vboxsrv\\vbox')
 221         cmdline = '%s %s -s -b %s' % (sys.executable, fn, options.bind)
 222         win_install_service(SVCNAME, cmdline)
 223         return
 224
 225     if options.action == 'uninstall':
 226         win_uninstall_service(SVCNAME)
 227         return
 228
 229     if options.action == 'service':
 230         win_service_start(SVCNAME, main)
 231         return
 232
 233     host, port_str = options.bind.split(':')
 234     port = int(port_str)
 235
 236     print('Listening on %s:%d' % (host, port))
 237     srv = BuildHTTPServer((host, port), BuildHTTPRequestHandler)
 238     thr = threading.Thread(target=srv.serve_forever)
 239     thr.start()
 240     compat_input('Press ENTER to shut down')
 241     srv.shutdown()
 242     thr.join()
 243
 244
 245 def rmtree(path):
 246     for name in os.listdir(path):
 247         fname = os.path.join(path, name)
 248         if os.path.isdir(fname):
 249             rmtree(fname)
 250         else:
 251             os.chmod(fname, 0o666)
 252             os.remove(fname)
 253     os.rmdir(path)
 254
 255
 256 class BuildError(Exception):
 257     def __init__(self, output, code=500):
 258         self.output = output
 259         self.code = code
 260
 261     def __str__(self):
 262         return self.output
 263
 264
 265 class HTTPError(BuildError):
 266     pass
 267
 268
 269 class PythonBuilder(object):
 270     def __init__(self, **kwargs):
 271         python_version = kwargs.pop('python', '3.4')
 272         python_path = None
 273         for node in ('Wow6432Node\\', ''):
 274             try:
 275                 key = compat_winreg.OpenKey(
 276                     compat_winreg.HKEY_LOCAL_MACHINE,
 277                     r'SOFTWARE\%sPython\PythonCore\%s\InstallPath' % (node, python_version))
 278                 try:
 279                     python_path, _ = compat_winreg.QueryValueEx(key, '')
 280                 finally:
 281                     compat_winreg.CloseKey(key)
 282                 break
 283             except Exception:
 284                 pass
 285
 286         if not python_path:
 287             raise BuildError('No such Python version: %s' % python_version)
 288
 289         self.pythonPath = python_path
 290
 291         super(PythonBuilder, self).__init__(**kwargs)
 292
 293
 294 class GITInfoBuilder(object):
 295     def __init__(self, **kwargs):
 296         try:
 297             self.user, self.repoName = kwargs['path'][:2]
 298             self.rev = kwargs.pop('rev')
 299         except ValueError:
 300             raise BuildError('Invalid path')
 301         except KeyError as e:
 302             raise BuildError('Missing mandatory parameter "%s"' % e.args[0])
 303
 304         path = os.path.join(os.environ['APPDATA'], 'Build archive', self.repoName, self.user)
 305         if not os.path.exists(path):
 306             os.makedirs(path)
 307         self.basePath = tempfile.mkdtemp(dir=path)
 308         self.buildPath = os.path.join(self.basePath, 'build')
 309
 310         super(GITInfoBuilder, self).__init__(**kwargs)
 311
 312
 313 class GITBuilder(GITInfoBuilder):
 314     def build(self):
 315         try:
 316             subprocess.check_output(['git', 'clone', 'git://github.com/%s/%s.git' % (self.user, self.repoName), self.buildPath])
 317             subprocess.check_output(['git', 'checkout', self.rev], cwd=self.buildPath)
 318         except subprocess.CalledProcessError as e:
 319             raise BuildError(e.output)
 320
 321         super(GITBuilder, self).build()
 322
 323
 324 class YoutubeDLBuilder(object):
 325     authorizedUsers = ['fraca7', 'phihag', 'rg3', 'FiloSottile', 'ytdl-org']
 326
 327     def __init__(self, **kwargs):
 328         if self.repoName != 'youtube-dl':
 329             raise BuildError('Invalid repository "%s"' % self.repoName)
 330         if self.user not in self.authorizedUsers:
 331             raise HTTPError('Unauthorized user "%s"' % self.user, 401)
 332
 333         super(YoutubeDLBuilder, self).__init__(**kwargs)
 334
 335     def build(self):
 336         try:
 337             proc = subprocess.Popen([os.path.join(self.pythonPath, 'python.exe'), 'setup.py', 'py2exe'], stdin=subprocess.PIPE, cwd=self.buildPath)
 338             proc.wait()
 339             #subprocess.check_output([os.path.join(self.pythonPath, 'python.exe'), 'setup.py', 'py2exe'],
 340             #                        cwd=self.buildPath)
 341         except subprocess.CalledProcessError as e:
 342             raise BuildError(e.output)
 343
 344         super(YoutubeDLBuilder, self).build()
 345
 346
 347 class DownloadBuilder(object):
 348     def __init__(self, **kwargs):
 349         self.handler = kwargs.pop('handler')
 350         self.srcPath = os.path.join(self.buildPath, *tuple(kwargs['path'][2:]))
 351         self.srcPath = os.path.abspath(os.path.normpath(self.srcPath))
 352         if not self.srcPath.startswith(self.buildPath):
 353             raise HTTPError(self.srcPath, 401)
 354
 355         super(DownloadBuilder, self).__init__(**kwargs)
 356
 357     def build(self):
 358         if not os.path.exists(self.srcPath):
 359             raise HTTPError('No such file', 404)
 360         if os.path.isdir(self.srcPath):
 361             raise HTTPError('Is a directory: %s' % self.srcPath, 401)
 362
 363         self.handler.send_response(200)
 364         self.handler.send_header('Content-Type', 'application/octet-stream')
 365         self.handler.send_header('Content-Disposition', 'attachment; filename=%s' % os.path.split(self.srcPath)[-1])
 366         self.handler.send_header('Content-Length', str(os.stat(self.srcPath).st_size))
 367         self.handler.end_headers()
 368
 369         with open(self.srcPath, 'rb') as src:
 370             shutil.copyfileobj(src, self.handler.wfile)
 371
 372         super(DownloadBuilder, self).build()
 373
 374
 375 class CleanupTempDir(object):
 376     def build(self):
 377         try:
 378             rmtree(self.basePath)
 379         except Exception as e:
 380             print('WARNING deleting "%s": %s' % (self.basePath, e))
 381
 382         super(CleanupTempDir, self).build()
 383
 384
 385 class Null(object):
 386     def __init__(self, **kwargs):
 387         pass
 388
 389     def start(self):
 390         pass
 391
 392     def close(self):
 393         pass
 394
 395     def build(self):
 396         pass
 397
 398
 399 class Builder(PythonBuilder, GITBuilder, YoutubeDLBuilder, DownloadBuilder, CleanupTempDir, Null):
 400     pass
 401
 402
 403 class BuildHTTPRequestHandler(compat_http_server.BaseHTTPRequestHandler):
 404     actionDict = {'build': Builder, 'download': Builder}  # They're the same, no more caching.
 405
 406     def do_GET(self):
 407         path = compat_urlparse.urlparse(self.path)
 408         paramDict = dict([(key, value[0]) for key, value in compat_urlparse.parse_qs(path.query).items()])
 409         action, _, path = path.path.strip('/').partition('/')
 410         if path:
 411             path = path.split('/')
 412             if action in self.actionDict:
 413                 try:
 414                     builder = self.actionDict[action](path=path, handler=self, **paramDict)
 415                     builder.start()
 416                     try:
 417                         builder.build()
 418                     finally:
 419                         builder.close()
 420                 except BuildError as e:
 421                     self.send_response(e.code)
 422                     msg = compat_str(e).encode('UTF-8')
 423                     self.send_header('Content-Type', 'text/plain; charset=UTF-8')
 424                     self.send_header('Content-Length', len(msg))
 425                     self.end_headers()
 426                     self.wfile.write(msg)
 427             else:
 428                 self.send_response(500, 'Unknown build method "%s"' % action)
 429         else:
 430             self.send_response(500, 'Malformed URL')
 431
 432 if __name__ == '__main__':
 433     main()