Skip to content

lwc / compare

Go to a project
Commit 835ec233 by lwc-tester
Options

HTTP api for scheduling jobs

parent 00c79c4b
Inline Side-by-side
Showing with 428 additions and 50 deletions
index.html 0 → 100644
<html>
<head>
<meta charset="UTF-8">
<title>Test scheduler</title>
<style>
body {
font-family: 'Gill Sans', 'Gill Sans MT', Calibri, 'Trebuchet MS', sans-serif;
}
table {
border: 0px;
border-spacing: 0px 3px;
}
td {
padding: 1em;
border: 0px;
margin: 0px;
}
.iconButton {
height: 2.2em;
width: 2.2em;
}
.logView {
overflow-y: auto;
border: 2px solid #ddd;
top: 30px;
bottom: 0px;
right: 0px;
left: 0px;
padding: .4em;
font-family: monospace;
position: absolute;
}
.logViewWindow {
position: fixed;
display: none;
float: left;
left: 10%;
right: 10%;
top: 10%;
bottom: 10%;
background-color: white;
width: 75%;
height: 75%;
}
.logViewTab {
height: 30px;
}
</style>
</head>
<body>
<script>
'use strict';
(() => {
const scheduleTable = document.createElement('table');
const schedule = {};
const logView = document.createElement('div');
function init() {
scheduleTable.style.display = 'none';
document.body.appendChild(scheduleTable);
scheduleTable.appendChild(makeRow(['ID', 'Created At', 'Template', 'Path', 'State', 'Actions'], 'th'));
document.body.appendChild(logView);
logView.className = 'logViewWindow';
const closeLogViewButton = document.createElement('button');
logView.appendChild(closeLogViewButton);
closeLogViewButton.innerText = "🗴";
closeLogViewButton.style.display = 'block';
closeLogViewButton.style.right = '0px';
closeLogViewButton.style.top = '0px';
closeLogViewButton.style.position = 'absolute';
closeLogViewButton.onclick = () => {logView.style.display = 'none'};
document.addEventListener("keydown", function(event) {
if (event.which == 27) {
logView.style.display = 'none';
}
})
}
function onStatusGet(status) {
scheduleTable.style.display = 'block';
for (const s of status.schedule) {
if (!(s.id in schedule)) {
const row = makeRow([s.id, s.added, s.path, s.template, s.state, '']);
s.row = row;
schedule[s.id] = s;
// Find out correct placement in table
let ids = Object.keys(schedule);
ids.sort();
const idx = ids.indexOf(s.id);
if (idx == -1) {
throw new Error(s.id + " NOT FOUND");
} else if (idx == 0) {
scheduleTable.appendChild(row);
} else {
const prevId = ids[idx-1];
scheduleTable.insertBefore(row, schedule[prevId].row);
}
// Add buttons for actions
const lastCell = row.cells[row.cells.length - 1];
function makeButton(icon, onClick) {
const button = document.createElement('button');
button.innerHTML = icon;
button.onclick = onClick;
button.className = 'iconButton'
return button;
}
lastCell.appendChild(makeButton('↺', () => restartJob(s.id)));
lastCell.appendChild(makeButton('🗴', () => cancelJob(s.id)));
lastCell.appendChild(makeButton('🗑️', () => deleteJob(s.id)));
lastCell.appendChild(makeButton('🗎', () => viewJobLogs(s.id)));
}
const row = schedule[s.id].row;
row.cells[4].innerText = s.state;
if (s.state == 'FAILED') {
row.style.backgroundColor = '#fcc';
} else if (s.state == 'SUCCESSFUL') {
row.style.backgroundColor = '#cfc';
} else if (s.state == 'RUNNING') {
row.style.backgroundColor = '#ff9';
} else {
row.style.backgroundColor = '#eee';
}
}
}
function restartJob(jobId) {
const xhttp = new XMLHttpRequest();
xhttp.onreadystatechange = function() {
console.log(this);
if (this.readyState == 4 && this.status == 200) {
}
};
xhttp.open("GET", "/restart_test/" + jobId, true);
xhttp.send();
}
function viewJobLogs(jobId) {
logView.style.display = 'block';
while (logView.childElementCount > 1) {
logView.removeChild(logView.lastElementChild);
}
const logIds = [0,1,2,3];
const logNames = ['make stdout', 'make stderr', 'test stdout', 'test stderr'];
const tabs = logIds.map(logId => {
const v = document.createElement('button');
v.innerText = logNames[logIds.indexOf(logId)];
v.className = 'logViewTab';
logView.appendChild(v);
return v;
});
logView.appendChild(document.createElement('br'));
const views = logIds.map(logId => {
const v = document.createElement('div');
v.style.display = logId == logIds[0] ? 'block' : 'none';
v.className = 'logView';
logView.appendChild(v);
const xhttp = new XMLHttpRequest();
xhttp.onreadystatechange = function() {
if (this.readyState == 4 && this.status == 200) {
let log = this.responseText;
v.innerText = log;
log = v.innerHTML;
log = log.replace(/\n/g, '<br/>');
v.innerHTML = log;
}
};
xhttp.open("GET", "/view_log/" + jobId + "/" + logId, true);
xhttp.send();
return v;
});
for (const i in logIds) {
const tab = tabs[i];
const view = views[i];
tab.onclick = () => {
views.forEach(v => {v.style.display = 'none';});
view.style.display = 'block';
};
}
}
function requestStatus() {
const xhttp = new XMLHttpRequest();
xhttp.onreadystatechange = function() {
if (this.readyState == 4 && this.status == 200) {
onStatusGet(JSON.parse(this.responseText));
setTimeout(requestStatus, 1000);
}
};
xhttp.open("GET", "/status", true);
xhttp.send();
}
function makeCell(text, tagName='td') {
const cell = document.createElement(tagName);
cell.appendChild(document.createTextNode(text));
return cell;
};
function makeRow(list, cellTagName='td') {
const headerRow = document.createElement('tr');
(list.map((n) => makeCell(n, cellTagName))).forEach((n) => headerRow.appendChild(n));
return headerRow;
};
init();
requestStatus();
})();
</script>
</body>
</html>
\ No newline at end of file
process_zip.sh
......@@ -60,9 +60,12 @@ function run() {
;;
esac
CMD="PYTHONPATH=\$PYTHONPATH:$(pwd) python3 './templates/$TEMPLATE/test' '$TEST_PATH' > '$TEST_PATH/test.stdout.log' 2> '$TEST_PATH/test.stderr.log'"
printf -v CMD "%q" "$CMD"
flock "$QUEUE_PATH" bash -c "echo $CMD >> \"$QUEUE_PATH\""
curl \
--request 'POST' \
--header "Content-Type: application/json" \
--header "Authorization: OAuth ecP9ZsoKMPui4akg1MyGoT7yoGR2bLPo" \
--data "{\"path\":\"$(realpath $TEST_PATH)\",\"template\":\"$TEMPLATE\"}" \
"http://127.0.0.1:5002/schedule_test"
done
......
templates/f7/test
......@@ -20,10 +20,7 @@ def get_serial():
if p.serial_number == '00000000'
]
devices.sort()
return serial.Serial(
devices[0],
baudrate=115200,
timeout=5)
return devices[0]
class F7(DeviceUnderTestAeadUARTP):
......@@ -34,6 +31,10 @@ class F7(DeviceUnderTestAeadUARTP):
self.uart_device = get_serial()
devname = os.path.basename(self.uart_device)
self.ser = serial.Serial(
self.uart_device,
baudrate=115200,
timeout=5)
self.lock = FileMutex('/var/lock/lwc-compare.%s.lock' % devname)
self.build_dir = build_dir
self.template_path = os.path.dirname(sys.argv[0])
......
test-dude.py 0 → 100644
#!/usr/bin/env python3
import os
import datetime
import threading
import subprocess
from flask import Flask, request
from flask_restful import Resource, Api
from flask_jsonpify import jsonify
app = Flask(__name__, static_folder='.')
api = Api(app)
schedule = []
runners = []
class ScheduledTest:
def __init__(self, template, path):
self.template = template
self.path = path
self.state = 'SCHEDULED'
self.added = datetime.datetime.now()
self.lock = threading.Lock()
def to_dict(self):
return {
'id': str(id(self)),
'template': self.template,
'state': self.state,
'path': self.path,
'added': self.added,
}
class Runner(threading.Thread):
def __init__(self, template, program=None):
if program is None:
program = ['python3', './templates/%s/test' % template]
self.template = template
self.program = program
self.process = None
self.job = None
self.event = threading.Event()
threading.Thread.__init__(self)
self.start()
def to_dict(self):
return {
'id': str(id(self)),
'template': self.template,
'program': ' '.join(self.program),
'job': str(id(self.job)) if self.job is not None else None
}
def run(self):
while 1:
self.event.clear()
my_queue = [
s for s in schedule
if s.state == 'SCHEDULED'
and s.template == self.template
]
my_queue.sort(key=lambda s: s.added)
if len(my_queue) == 0:
# No tasks for this thread, go to sleep
self.event.wait(timeout=5)
continue
job = my_queue[0]
with job.lock:
# Check if we were the first thread to choose this job
if job.state == 'SCHEDULED':
job.state = 'RUNNING'
self.job = job
else:
# Some other thread is running this test
continue
cmd = []
cmd += self.program
cmd += [self.job.path]
print("Executing ``%s´´" % ' '.join(cmd))
out_fd = open(os.path.join(self.job.path, 'test.stdout.log'), 'w')
err_fd = open(os.path.join(self.job.path, 'test.stderr.log'), 'w')
self.process = subprocess.Popen(
cmd,
stdout=out_fd,
stderr=err_fd
)
self.process.wait()
if self.process.returncode == 0:
self.job.state = 'SUCCESSFUL'
else:
self.job.state = 'FAILED'
self.process = None
self.job = None
class Status(Resource):
def get(self):
print(request.data)
return jsonify({
'schedule': [t.to_dict() for t in schedule],
'runners': [r.to_dict() for r in runners]
})
class RestartJob(Resource):
def get(self, job_id):
job = [job for job in schedule if str(id(job)) == job_id]
job = job[0] if len(job) > 0 else None
if job is None:
return 'Job not found', 404
with job.lock:
if job.state != 'RUNNING':
job.state = 'SCHEDULED'
return jsonify({'success': True})
else:
return 'Job is already running', 400
class ScheduleJob(Resource):
def post(self):
if not request.is_json:
return 'Please send me JSON', 400
data = request.get_json()
print(data)
if 'path' not in data:
return 'path expected', 400
if 'template' not in data:
return 'template expected', 400
schedule.append(ScheduledTest(data['template'], data['path']))
result = {'success': True}
return jsonify(result)
api.add_resource(Status, '/status')
api.add_resource(ScheduleJob, '/schedule_test')
api.add_resource(RestartJob, '/restart_test/<string:job_id>')
@app.route('/')
def root():
return app.send_static_file('index.html')
@app.route('/view_log/<string:job_id>/<int:log_id>')
def view_log(job_id, log_id):
job = [job for job in schedule if str(id(job)) == job_id]
job = job[0] if len(job) > 0 else None
if job is None:
return 'Job not found', 404
log_name = [
'make.stdout.log', 'make.stderr.log',
'test.stdout.log', 'test.stderr.log',
][log_id]
log_path = os.path.join(job.path, log_name)
if not os.path.isfile(log_path):
return 'Log not found', 404
with open(log_path, 'r') as f:
return f.read()
if __name__ == '__main__':
runners.append(Runner('maixduino'))
runners.append(Runner('f7'))
runners.append(Runner('uno'))
runners.append(Runner('esp32'))
runners.append(Runner('bluepill'))
app.run(port='5002')
test_scheduler.py deleted 100755 → 0
#!/usr/bin/env python3
import sys
import time
import fcntl
import subprocess
def file_pop_line(path):
try:
with open(path, 'rt+') as q:
fcntl.lockf(q, fcntl.LOCK_EX)
first = q.readline()
if first == '':
return None
rest = q.read()
q.seek(0)
q.write(rest)
q.truncate()
return first
except FileNotFoundError:
return None
def main(argv):
test_queue = argv[1]
while 1:
cmd = file_pop_line(test_queue).strip()
if cmd is None:
time.sleep(5)
else:
print()
print("Executing %s" % cmd)
p = subprocess.Popen(['bash', '-c', cmd])
p.wait()
print()
print("Return code is %d" % p.returncode)
print()
if __name__ == '__main__':
sys.exit(main(sys.argv))
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or sign in to comment