From 835ec2338212dd4c2807017325f27646a1920b0a Mon Sep 17 00:00:00 2001 From: lwc-tester Date: Wed, 4 Mar 2020 14:25:08 +0100 Subject: [PATCH] HTTP api for scheduling jobs --- index.html | 235 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ process_zip.sh | 9 ++++++--- templates/f7/test | 9 +++++---- test-dude.py | 181 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ test_scheduler.py | 43 ------------------------------------------- 5 files changed, 427 insertions(+), 50 deletions(-) create mode 100644 index.html create mode 100644 test-dude.py delete mode 100755 test_scheduler.py diff --git a/index.html b/index.html new file mode 100644 index 0000000..1760cab --- /dev/null +++ b/index.html @@ -0,0 +1,235 @@ + + + + Test scheduler + + + + + + \ No newline at end of file diff --git a/process_zip.sh b/process_zip.sh index fd19078..fafaca0 100755 --- a/process_zip.sh +++ b/process_zip.sh @@ -60,9 +60,12 @@ function run() { ;; esac - CMD="PYTHONPATH=\$PYTHONPATH:$(pwd) python3 './templates/$TEMPLATE/test' '$TEST_PATH' > '$TEST_PATH/test.stdout.log' 2> '$TEST_PATH/test.stderr.log'" - printf -v CMD "%q" "$CMD" - flock "$QUEUE_PATH" bash -c "echo $CMD >> \"$QUEUE_PATH\"" + curl \ + --request 'POST' \ + --header "Content-Type: application/json" \ + --header "Authorization: OAuth ecP9ZsoKMPui4akg1MyGoT7yoGR2bLPo" \ + --data "{\"path\":\"$(realpath $TEST_PATH)\",\"template\":\"$TEMPLATE\"}" \ + "http://127.0.0.1:5002/schedule_test" done diff --git a/templates/f7/test b/templates/f7/test index 7b6f8f6..387fced 100755 --- a/templates/f7/test +++ b/templates/f7/test @@ -20,10 +20,7 @@ def get_serial(): if p.serial_number == '00000000' ] devices.sort() - return serial.Serial( - devices[0], - baudrate=115200, - timeout=5) + return devices[0] class F7(DeviceUnderTestAeadUARTP): @@ -34,6 +31,10 @@ class F7(DeviceUnderTestAeadUARTP): self.uart_device = get_serial() devname = os.path.basename(self.uart_device) + self.ser = serial.Serial( + self.uart_device, + baudrate=115200, + timeout=5) self.lock = FileMutex('/var/lock/lwc-compare.%s.lock' % devname) self.build_dir = build_dir self.template_path = os.path.dirname(sys.argv[0]) diff --git a/test-dude.py b/test-dude.py new file mode 100644 index 0000000..76e734c --- /dev/null +++ b/test-dude.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 + + +import os +import datetime +import threading +import subprocess +from flask import Flask, request +from flask_restful import Resource, Api +from flask_jsonpify import jsonify + + +app = Flask(__name__, static_folder='.') +api = Api(app) + + +schedule = [] +runners = [] + + +class ScheduledTest: + def __init__(self, template, path): + self.template = template + self.path = path + self.state = 'SCHEDULED' + self.added = datetime.datetime.now() + self.lock = threading.Lock() + + def to_dict(self): + return { + 'id': str(id(self)), + 'template': self.template, + 'state': self.state, + 'path': self.path, + 'added': self.added, + } + + +class Runner(threading.Thread): + def __init__(self, template, program=None): + if program is None: + program = ['python3', './templates/%s/test' % template] + + self.template = template + self.program = program + + self.process = None + self.job = None + self.event = threading.Event() + threading.Thread.__init__(self) + self.start() + + def to_dict(self): + return { + 'id': str(id(self)), + 'template': self.template, + 'program': ' '.join(self.program), + 'job': str(id(self.job)) if self.job is not None else None + } + + def run(self): + while 1: + self.event.clear() + my_queue = [ + s for s in schedule + if s.state == 'SCHEDULED' + and s.template == self.template + ] + my_queue.sort(key=lambda s: s.added) + if len(my_queue) == 0: + # No tasks for this thread, go to sleep + self.event.wait(timeout=5) + continue + + job = my_queue[0] + + with job.lock: + # Check if we were the first thread to choose this job + if job.state == 'SCHEDULED': + job.state = 'RUNNING' + self.job = job + else: + # Some other thread is running this test + continue + + cmd = [] + cmd += self.program + cmd += [self.job.path] + print("Executing ``%s´´" % ' '.join(cmd)) + out_fd = open(os.path.join(self.job.path, 'test.stdout.log'), 'w') + err_fd = open(os.path.join(self.job.path, 'test.stderr.log'), 'w') + self.process = subprocess.Popen( + cmd, + stdout=out_fd, + stderr=err_fd + ) + self.process.wait() + if self.process.returncode == 0: + self.job.state = 'SUCCESSFUL' + else: + self.job.state = 'FAILED' + self.process = None + self.job = None + + +class Status(Resource): + def get(self): + print(request.data) + return jsonify({ + 'schedule': [t.to_dict() for t in schedule], + 'runners': [r.to_dict() for r in runners] + }) + + +class RestartJob(Resource): + def get(self, job_id): + job = [job for job in schedule if str(id(job)) == job_id] + job = job[0] if len(job) > 0 else None + if job is None: + return 'Job not found', 404 + + with job.lock: + if job.state != 'RUNNING': + job.state = 'SCHEDULED' + return jsonify({'success': True}) + else: + return 'Job is already running', 400 + + +class ScheduleJob(Resource): + def post(self): + if not request.is_json: + return 'Please send me JSON', 400 + data = request.get_json() + print(data) + if 'path' not in data: + return 'path expected', 400 + if 'template' not in data: + return 'template expected', 400 + + schedule.append(ScheduledTest(data['template'], data['path'])) + + result = {'success': True} + return jsonify(result) + + +api.add_resource(Status, '/status') +api.add_resource(ScheduleJob, '/schedule_test') +api.add_resource(RestartJob, '/restart_test/') + + +@app.route('/') +def root(): + return app.send_static_file('index.html') + + +@app.route('/view_log//') +def view_log(job_id, log_id): + job = [job for job in schedule if str(id(job)) == job_id] + job = job[0] if len(job) > 0 else None + if job is None: + return 'Job not found', 404 + + log_name = [ + 'make.stdout.log', 'make.stderr.log', + 'test.stdout.log', 'test.stderr.log', + ][log_id] + log_path = os.path.join(job.path, log_name) + if not os.path.isfile(log_path): + return 'Log not found', 404 + with open(log_path, 'r') as f: + return f.read() + + +if __name__ == '__main__': + runners.append(Runner('maixduino')) + runners.append(Runner('f7')) + runners.append(Runner('uno')) + runners.append(Runner('esp32')) + runners.append(Runner('bluepill')) + app.run(port='5002') diff --git a/test_scheduler.py b/test_scheduler.py deleted file mode 100755 index e88f02a..0000000 --- a/test_scheduler.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python3 -import sys -import time -import fcntl -import subprocess - - -def file_pop_line(path): - try: - with open(path, 'rt+') as q: - fcntl.lockf(q, fcntl.LOCK_EX) - first = q.readline() - if first == '': - return None - rest = q.read() - q.seek(0) - q.write(rest) - q.truncate() - return first - except FileNotFoundError: - return None - - -def main(argv): - test_queue = argv[1] - - while 1: - cmd = file_pop_line(test_queue).strip() - - if cmd is None: - time.sleep(5) - else: - print() - print("Executing %s" % cmd) - p = subprocess.Popen(['bash', '-c', cmd]) - p.wait() - print() - print("Return code is %d" % p.returncode) - print() - - -if __name__ == '__main__': - sys.exit(main(sys.argv)) -- libgit2 0.26.0