]> gitweb @ CieloNegro.org - youtube-dl.git/blob - devscripts/buildserver.py
[giantbomb] Extend _VALID_URL (#25222)
[youtube-dl.git] / devscripts / buildserver.py
1 #!/usr/bin/python3
2
3 import argparse
4 import ctypes
5 import functools
6 import shutil
7 import subprocess
8 import sys
9 import tempfile
10 import threading
11 import traceback
12 import os.path
13
14 sys.path.insert(0, os.path.dirname(os.path.dirname((os.path.abspath(__file__)))))
15 from youtube_dl.compat import (
16     compat_input,
17     compat_http_server,
18     compat_str,
19     compat_urlparse,
20 )
21
22 # These are not used outside of buildserver.py thus not in compat.py
23
24 try:
25     import winreg as compat_winreg
26 except ImportError:  # Python 2
27     import _winreg as compat_winreg
28
29 try:
30     import socketserver as compat_socketserver
31 except ImportError:  # Python 2
32     import SocketServer as compat_socketserver
33
34
35 class BuildHTTPServer(compat_socketserver.ThreadingMixIn, compat_http_server.HTTPServer):
36     allow_reuse_address = True
37
38
39 advapi32 = ctypes.windll.advapi32
40
41 SC_MANAGER_ALL_ACCESS = 0xf003f
42 SC_MANAGER_CREATE_SERVICE = 0x02
43 SERVICE_WIN32_OWN_PROCESS = 0x10
44 SERVICE_AUTO_START = 0x2
45 SERVICE_ERROR_NORMAL = 0x1
46 DELETE = 0x00010000
47 SERVICE_STATUS_START_PENDING = 0x00000002
48 SERVICE_STATUS_RUNNING = 0x00000004
49 SERVICE_ACCEPT_STOP = 0x1
50
51 SVCNAME = 'youtubedl_builder'
52
53 LPTSTR = ctypes.c_wchar_p
54 START_CALLBACK = ctypes.WINFUNCTYPE(None, ctypes.c_int, ctypes.POINTER(LPTSTR))
55
56
57 class SERVICE_TABLE_ENTRY(ctypes.Structure):
58     _fields_ = [
59         ('lpServiceName', LPTSTR),
60         ('lpServiceProc', START_CALLBACK)
61     ]
62
63
64 HandlerEx = ctypes.WINFUNCTYPE(
65     ctypes.c_int,     # return
66     ctypes.c_int,     # dwControl
67     ctypes.c_int,     # dwEventType
68     ctypes.c_void_p,  # lpEventData,
69     ctypes.c_void_p,  # lpContext,
70 )
71
72
73 def _ctypes_array(c_type, py_array):
74     ar = (c_type * len(py_array))()
75     ar[:] = py_array
76     return ar
77
78
79 def win_OpenSCManager():
80     res = advapi32.OpenSCManagerW(None, None, SC_MANAGER_ALL_ACCESS)
81     if not res:
82         raise Exception('Opening service manager failed - '
83                         'are you running this as administrator?')
84     return res
85
86
87 def win_install_service(service_name, cmdline):
88     manager = win_OpenSCManager()
89     try:
90         h = advapi32.CreateServiceW(
91             manager, service_name, None,
92             SC_MANAGER_CREATE_SERVICE, SERVICE_WIN32_OWN_PROCESS,
93             SERVICE_AUTO_START, SERVICE_ERROR_NORMAL,
94             cmdline, None, None, None, None, None)
95         if not h:
96             raise OSError('Service creation failed: %s' % ctypes.FormatError())
97
98         advapi32.CloseServiceHandle(h)
99     finally:
100         advapi32.CloseServiceHandle(manager)
101
102
103 def win_uninstall_service(service_name):
104     manager = win_OpenSCManager()
105     try:
106         h = advapi32.OpenServiceW(manager, service_name, DELETE)
107         if not h:
108             raise OSError('Could not find service %s: %s' % (
109                 service_name, ctypes.FormatError()))
110
111         try:
112             if not advapi32.DeleteService(h):
113                 raise OSError('Deletion failed: %s' % ctypes.FormatError())
114         finally:
115             advapi32.CloseServiceHandle(h)
116     finally:
117         advapi32.CloseServiceHandle(manager)
118
119
120 def win_service_report_event(service_name, msg, is_error=True):
121     with open('C:/sshkeys/log', 'a', encoding='utf-8') as f:
122         f.write(msg + '\n')
123
124     event_log = advapi32.RegisterEventSourceW(None, service_name)
125     if not event_log:
126         raise OSError('Could not report event: %s' % ctypes.FormatError())
127
128     try:
129         type_id = 0x0001 if is_error else 0x0004
130         event_id = 0xc0000000 if is_error else 0x40000000
131         lines = _ctypes_array(LPTSTR, [msg])
132
133         if not advapi32.ReportEventW(
134                 event_log, type_id, 0, event_id, None, len(lines), 0,
135                 lines, None):
136             raise OSError('Event reporting failed: %s' % ctypes.FormatError())
137     finally:
138         advapi32.DeregisterEventSource(event_log)
139
140
141 def win_service_handler(stop_event, *args):
142     try:
143         raise ValueError('Handler called with args ' + repr(args))
144         TODO
145     except Exception as e:
146         tb = traceback.format_exc()
147         msg = str(e) + '\n' + tb
148         win_service_report_event(service_name, msg, is_error=True)
149         raise
150
151
152 def win_service_set_status(handle, status_code):
153     svcStatus = SERVICE_STATUS()
154     svcStatus.dwServiceType = SERVICE_WIN32_OWN_PROCESS
155     svcStatus.dwCurrentState = status_code
156     svcStatus.dwControlsAccepted = SERVICE_ACCEPT_STOP
157
158     svcStatus.dwServiceSpecificExitCode = 0
159
160     if not advapi32.SetServiceStatus(handle, ctypes.byref(svcStatus)):
161         raise OSError('SetServiceStatus failed: %r' % ctypes.FormatError())
162
163
164 def win_service_main(service_name, real_main, argc, argv_raw):
165     try:
166         # args = [argv_raw[i].value for i in range(argc)]
167         stop_event = threading.Event()
168         handler = HandlerEx(functools.partial(stop_event, win_service_handler))
169         h = advapi32.RegisterServiceCtrlHandlerExW(service_name, handler, None)
170         if not h:
171             raise OSError('Handler registration failed: %s' %
172                           ctypes.FormatError())
173
174         TODO
175     except Exception as e:
176         tb = traceback.format_exc()
177         msg = str(e) + '\n' + tb
178         win_service_report_event(service_name, msg, is_error=True)
179         raise
180
181
182 def win_service_start(service_name, real_main):
183     try:
184         cb = START_CALLBACK(
185             functools.partial(win_service_main, service_name, real_main))
186         dispatch_table = _ctypes_array(SERVICE_TABLE_ENTRY, [
187             SERVICE_TABLE_ENTRY(
188                 service_name,
189                 cb
190             ),
191             SERVICE_TABLE_ENTRY(None, ctypes.cast(None, START_CALLBACK))
192         ])
193
194         if not advapi32.StartServiceCtrlDispatcherW(dispatch_table):
195             raise OSError('ctypes start failed: %s' % ctypes.FormatError())
196     except Exception as e:
197         tb = traceback.format_exc()
198         msg = str(e) + '\n' + tb
199         win_service_report_event(service_name, msg, is_error=True)
200         raise
201
202
203 def main(args=None):
204     parser = argparse.ArgumentParser()
205     parser.add_argument('-i', '--install',
206                         action='store_const', dest='action', const='install',
207                         help='Launch at Windows startup')
208     parser.add_argument('-u', '--uninstall',
209                         action='store_const', dest='action', const='uninstall',
210                         help='Remove Windows service')
211     parser.add_argument('-s', '--service',
212                         action='store_const', dest='action', const='service',
213                         help='Run as a Windows service')
214     parser.add_argument('-b', '--bind', metavar='<host:port>',
215                         action='store', default='0.0.0.0:8142',
216                         help='Bind to host:port (default %default)')
217     options = parser.parse_args(args=args)
218
219     if options.action == 'install':
220         fn = os.path.abspath(__file__).replace('v:', '\\\\vboxsrv\\vbox')
221         cmdline = '%s %s -s -b %s' % (sys.executable, fn, options.bind)
222         win_install_service(SVCNAME, cmdline)
223         return
224
225     if options.action == 'uninstall':
226         win_uninstall_service(SVCNAME)
227         return
228
229     if options.action == 'service':
230         win_service_start(SVCNAME, main)
231         return
232
233     host, port_str = options.bind.split(':')
234     port = int(port_str)
235
236     print('Listening on %s:%d' % (host, port))
237     srv = BuildHTTPServer((host, port), BuildHTTPRequestHandler)
238     thr = threading.Thread(target=srv.serve_forever)
239     thr.start()
240     compat_input('Press ENTER to shut down')
241     srv.shutdown()
242     thr.join()
243
244
245 def rmtree(path):
246     for name in os.listdir(path):
247         fname = os.path.join(path, name)
248         if os.path.isdir(fname):
249             rmtree(fname)
250         else:
251             os.chmod(fname, 0o666)
252             os.remove(fname)
253     os.rmdir(path)
254
255
256 class BuildError(Exception):
257     def __init__(self, output, code=500):
258         self.output = output
259         self.code = code
260
261     def __str__(self):
262         return self.output
263
264
265 class HTTPError(BuildError):
266     pass
267
268
269 class PythonBuilder(object):
270     def __init__(self, **kwargs):
271         python_version = kwargs.pop('python', '3.4')
272         python_path = None
273         for node in ('Wow6432Node\\', ''):
274             try:
275                 key = compat_winreg.OpenKey(
276                     compat_winreg.HKEY_LOCAL_MACHINE,
277                     r'SOFTWARE\%sPython\PythonCore\%s\InstallPath' % (node, python_version))
278                 try:
279                     python_path, _ = compat_winreg.QueryValueEx(key, '')
280                 finally:
281                     compat_winreg.CloseKey(key)
282                 break
283             except Exception:
284                 pass
285
286         if not python_path:
287             raise BuildError('No such Python version: %s' % python_version)
288
289         self.pythonPath = python_path
290
291         super(PythonBuilder, self).__init__(**kwargs)
292
293
294 class GITInfoBuilder(object):
295     def __init__(self, **kwargs):
296         try:
297             self.user, self.repoName = kwargs['path'][:2]
298             self.rev = kwargs.pop('rev')
299         except ValueError:
300             raise BuildError('Invalid path')
301         except KeyError as e:
302             raise BuildError('Missing mandatory parameter "%s"' % e.args[0])
303
304         path = os.path.join(os.environ['APPDATA'], 'Build archive', self.repoName, self.user)
305         if not os.path.exists(path):
306             os.makedirs(path)
307         self.basePath = tempfile.mkdtemp(dir=path)
308         self.buildPath = os.path.join(self.basePath, 'build')
309
310         super(GITInfoBuilder, self).__init__(**kwargs)
311
312
313 class GITBuilder(GITInfoBuilder):
314     def build(self):
315         try:
316             subprocess.check_output(['git', 'clone', 'git://github.com/%s/%s.git' % (self.user, self.repoName), self.buildPath])
317             subprocess.check_output(['git', 'checkout', self.rev], cwd=self.buildPath)
318         except subprocess.CalledProcessError as e:
319             raise BuildError(e.output)
320
321         super(GITBuilder, self).build()
322
323
324 class YoutubeDLBuilder(object):
325     authorizedUsers = ['fraca7', 'phihag', 'rg3', 'FiloSottile', 'ytdl-org']
326
327     def __init__(self, **kwargs):
328         if self.repoName != 'youtube-dl':
329             raise BuildError('Invalid repository "%s"' % self.repoName)
330         if self.user not in self.authorizedUsers:
331             raise HTTPError('Unauthorized user "%s"' % self.user, 401)
332
333         super(YoutubeDLBuilder, self).__init__(**kwargs)
334
335     def build(self):
336         try:
337             proc = subprocess.Popen([os.path.join(self.pythonPath, 'python.exe'), 'setup.py', 'py2exe'], stdin=subprocess.PIPE, cwd=self.buildPath)
338             proc.wait()
339             #subprocess.check_output([os.path.join(self.pythonPath, 'python.exe'), 'setup.py', 'py2exe'],
340             #                        cwd=self.buildPath)
341         except subprocess.CalledProcessError as e:
342             raise BuildError(e.output)
343
344         super(YoutubeDLBuilder, self).build()
345
346
347 class DownloadBuilder(object):
348     def __init__(self, **kwargs):
349         self.handler = kwargs.pop('handler')
350         self.srcPath = os.path.join(self.buildPath, *tuple(kwargs['path'][2:]))
351         self.srcPath = os.path.abspath(os.path.normpath(self.srcPath))
352         if not self.srcPath.startswith(self.buildPath):
353             raise HTTPError(self.srcPath, 401)
354
355         super(DownloadBuilder, self).__init__(**kwargs)
356
357     def build(self):
358         if not os.path.exists(self.srcPath):
359             raise HTTPError('No such file', 404)
360         if os.path.isdir(self.srcPath):
361             raise HTTPError('Is a directory: %s' % self.srcPath, 401)
362
363         self.handler.send_response(200)
364         self.handler.send_header('Content-Type', 'application/octet-stream')
365         self.handler.send_header('Content-Disposition', 'attachment; filename=%s' % os.path.split(self.srcPath)[-1])
366         self.handler.send_header('Content-Length', str(os.stat(self.srcPath).st_size))
367         self.handler.end_headers()
368
369         with open(self.srcPath, 'rb') as src:
370             shutil.copyfileobj(src, self.handler.wfile)
371
372         super(DownloadBuilder, self).build()
373
374
375 class CleanupTempDir(object):
376     def build(self):
377         try:
378             rmtree(self.basePath)
379         except Exception as e:
380             print('WARNING deleting "%s": %s' % (self.basePath, e))
381
382         super(CleanupTempDir, self).build()
383
384
385 class Null(object):
386     def __init__(self, **kwargs):
387         pass
388
389     def start(self):
390         pass
391
392     def close(self):
393         pass
394
395     def build(self):
396         pass
397
398
399 class Builder(PythonBuilder, GITBuilder, YoutubeDLBuilder, DownloadBuilder, CleanupTempDir, Null):
400     pass
401
402
403 class BuildHTTPRequestHandler(compat_http_server.BaseHTTPRequestHandler):
404     actionDict = {'build': Builder, 'download': Builder}  # They're the same, no more caching.
405
406     def do_GET(self):
407         path = compat_urlparse.urlparse(self.path)
408         paramDict = dict([(key, value[0]) for key, value in compat_urlparse.parse_qs(path.query).items()])
409         action, _, path = path.path.strip('/').partition('/')
410         if path:
411             path = path.split('/')
412             if action in self.actionDict:
413                 try:
414                     builder = self.actionDict[action](path=path, handler=self, **paramDict)
415                     builder.start()
416                     try:
417                         builder.build()
418                     finally:
419                         builder.close()
420                 except BuildError as e:
421                     self.send_response(e.code)
422                     msg = compat_str(e).encode('UTF-8')
423                     self.send_header('Content-Type', 'text/plain; charset=UTF-8')
424                     self.send_header('Content-Length', len(msg))
425                     self.end_headers()
426                     self.wfile.write(msg)
427             else:
428                 self.send_response(500, 'Unknown build method "%s"' % action)
429         else:
430             self.send_response(500, 'Malformed URL')
431
432 if __name__ == '__main__':
433     main()