some compilation and scheduling fixes

67bcb93e · Enrico Pozzobon · 3a0ee47f · 67bcb93e · 67bcb93e · 67bcb93e
Commit 67bcb93e authored 5 years ago by Enrico Pozzobon
Show whitespace changes
Inline Side-by-side

Showing with 253 additions and 65 deletions

compile_all.py
+21 -17

index.html
+189 -36

process_zip.sh
+1 -1

test-dude.py
+42 -8

test_common.py
+0 -3

No files found.
--- a/compile_all.py
+++ b/compile_all.py
@@ -117,27 +117,30 @@ def main(argv):
    subs = os.listdir(submissions_dir)
    # get all the submissions by looking for files named "api.h"
-    subfiles = []
+    implementations = []
    for submission in subs:
-        implementations_dir = os.path.join(
+        variants_dir = os.path.join(
            submissions_dir, submission, "Implementations", "crypto_aead")
-        if not os.path.isdir(implementations_dir):
+        if not os.path.isdir(variants_dir):
            continue
-        if "NOT ACCEPTED" in implementations_dir:
+        if "NOT ACCEPTED" in variants_dir:
            continue
        print()
        print("###  %s  ###" % submission)
        c = 0
-        # r=root, d=directories, f = files
+        for variant in os.listdir(variants_dir):
-        for r, d, f in os.walk(implementations_dir):
+            implementations_dir = os.path.join(
-            for file in f:
+                variants_dir, variant)
-                if file == "api.h":
+            for implementation in os.listdir(implementations_dir):
-                    f = os.path.join(r, file)
+                implementation_dir = os.path.join(
-                    subfiles.append(f)
+                    implementations_dir, implementation)
+                if os.path.isfile(os.path.join(implementation_dir, "api.h")):
+                    implementations.append(
+                        (submission, variant, implementation))
                    c += 1
        if c == 0:
@@ -147,10 +150,16 @@ def main(argv):
        print("Include list has %d entries" % len(include_list))
    files = []
-    for f in subfiles:
+    for submission, variant, implementation in implementations:
+        # base name n (a.k.a. cipher slug)
+        n = '.'.join([submission, variant, implementation])
+        print(n)
        # Source directory d
-        d = os.path.split(f)[0]
+        d = os.path.join(
+            submissions_dir, submission, "Implementations", "crypto_aead",
+            variant, implementation)
        assert os.path.isdir(d)
        print(d)
@@ -158,11 +167,6 @@ def main(argv):
        t = find_test_vectors(d)
        print(t)
-        # base name n
-        pieces = f.split(os.sep)
-        n = pieces[1] + "." + ".".join(pieces[4:-1])
-        print(n)
        # if include_list was provided, skip elements not in the list
        if include_list is not None:
            if n not in include_list:

--- a/index.html
+++ b/index.html
@@ -23,6 +23,26 @@ td {
    width: 2.2em;
 }
+.menuEntry {
+    display: block;
+    color: whitesmoke;
+    text-decoration: none;
+    margin: .3em;
+    padding: 1em;
+}
+.menuEntry:hover {
+    background-color: darkslategray;
+}
+.menuView {
+    position: absolute;
+    display: block;
+    background-color: black;
+    padding: .5em;
+    border-radius: .5em;
+}
 .logView {
    overflow-y: auto;
    border: 2px solid #ddd;
@@ -37,7 +57,6 @@ td {
 .logViewWindow {
    position: fixed;
-    display: none;
    float: left;
    left: 10%;
    right: 10%;
@@ -61,36 +80,60 @@ td {
 const scheduleTable = document.createElement('table');
 const schedule = {};
-const logView = document.createElement('div');
+let logView = null;
+let menuView = null;
 function init() {
    scheduleTable.style.display = 'none';
    document.body.appendChild(scheduleTable);
-    scheduleTable.appendChild(makeRow(['ID', 'Created At', 'Template', 'Path', 'State', 'Actions'], 'th'));
+    scheduleTable.appendChild(makeRow(['ID', 'Created At', 'Path', 'Template', 'State', 'Actions'], 'th'));
-    document.body.appendChild(logView);
-    logView.className = 'logViewWindow';
-    const closeLogViewButton = document.createElement('button');
-    logView.appendChild(closeLogViewButton);
-    closeLogViewButton.innerText = "🗴";
-    closeLogViewButton.style.display = 'block';
-    closeLogViewButton.style.right = '0px';
-    closeLogViewButton.style.top = '0px';
-    closeLogViewButton.style.position = 'absolute';
-    closeLogViewButton.onclick = () => {logView.style.display = 'none'};
    document.addEventListener("keydown", function(event) {
        if (event.which == 27) {
-            logView.style.display = 'none';
+            closeLogView();
+            closeJobMenu();
+        }
+    });
+    const bodyClickListener = event => {
+        const modals = [
+            {element: menuView, callback: closeJobMenu},
+            {element: logView, callback: closeLogView}
+        ];
+        for (const modal of modals) {
+            if (modal.element === null || !document.body.contains(modal.element)) {
+                continue;
            }
-    })
+            if (!modal.element.contains(event.target)) {
+                console.log("Clicked outside of " + modal.element.className);
+                modal.callback();
+            }
+        }
+    }
+    document.addEventListener('click', bodyClickListener);
 }
 function onStatusGet(status) {
    scheduleTable.style.display = 'block';
-    for (const s of status.schedule) {
-        if (!(s.id in schedule)) {
-            const row = makeRow([s.id, s.added, s.path, s.template, s.state, '']);
+    const incomingIds = status.schedule.map(s => s.id);
-            s.row = row;
+    const newTasksIds = incomingIds.filter(i => !(i in schedule));
+    const delTasksIds = Object.keys(schedule).filter(i => incomingIds.indexOf(i) == -1);
+    // Delete tasks that are not on the server anymore
+    for (const i of delTasksIds) {
+        const row = schedule[i].row;
+        row.parentElement.removeChild(row);
+        delete schedule[i];
+    }
+    // Add rows for new incoming tasks
+    for (const i of newTasksIds) {
+        const row = makeRow([i, '', '', '', '', '']);
+        const s = { id: i, row };
        schedule[s.id] = s;
        // Find out correct placement in table
@@ -115,13 +158,19 @@ function onStatusGet(status) {
            button.className = 'iconButton'
            return button;
        }
-            lastCell.appendChild(makeButton('↺', () => restartJob(s.id)));
+        lastCell.appendChild(makeButton('...', (e) => showJobMenu(e, s.id)));
-            lastCell.appendChild(makeButton('🗴', () => cancelJob(s.id)));
+    }
-            lastCell.appendChild(makeButton('🗑️', () => deleteJob(s.id)));
-            lastCell.appendChild(makeButton('🗎', () => viewJobLogs(s.id)));
+    // Update already existing tasks
+    for (const s of status.schedule) {
+        for (const key in s) {
+            schedule[s.id][key] = s[key];
        }
        const row = schedule[s.id].row;
+        row.cells[1].innerText = s.added;
+        row.cells[2].innerText = s.path;
+        row.cells[3].innerText = s.template;
        row.cells[4].innerText = s.state;
        if (s.state == 'FAILED') {
            row.style.backgroundColor = '#fcc';
@@ -137,28 +186,109 @@ function onStatusGet(status) {
 }
-function restartJob(jobId) {
+function showJobMenu(event, jobId) {
-    const xhttp = new XMLHttpRequest();
+    closeJobMenu();
-    xhttp.onreadystatechange = function() {
-        console.log(this);
+    menuView = document.createElement('div');
-        if (this.readyState == 4 && this.status == 200) {
+    menuView.className = 'menuView';
+    menuView.style.left = 0;
+    menuView.style.top = 0;
+    menuView.style.width = 'auto';
+    menuView.style.height = 'auto';
+    function makeEntry(label, onClick) {
+        const button = document.createElement('a');
+        button.innerHTML = label;
+        button.onclick = (e) => {closeJobMenu(); onClick(e);};
+        button.href = '#';
+        button.className = 'menuEntry';
+        return button;
+    }
+    const st = schedule[jobId].state;
+    if (st == 'SUCCESSFUL' || st == 'FAILED') {
+        menuView.appendChild(makeEntry('↺ Retry', () => restartJob(jobId)));
+        menuView.appendChild(makeEntry('🗑️ Delete', () => deleteJob(jobId)));
    }
+    if (st == 'RUNNING' || st == 'SCHEDULED') {
+        menuView.appendChild(makeEntry('🗴 Cancel', () => cancelJob(jobId)));
+    }
+    menuView.appendChild(makeEntry('🗎 View Logs', () => viewJobLogs(jobId)));
+    if (st == 'SUCCESSFUL') {
+        menuView.appendChild(makeEntry('↓ Download results zip', () => getResultZip(jobId)));
+        menuView.appendChild(makeEntry('↓ Download results sql', () => getResultSql(jobId)));
+    }
+    function onLayout() {
+        // This gets executed after the browser did the layout for the menu,
+        // we have the size of the menu view, now we have to place it close to
+        // the bounding box of the button that was clicked, without putting it
+        // ouside the screen:
+        const b = menuView.getBoundingClientRect();
+        const r = event.target.getBoundingClientRect();
+        const vh = window.innerHeight || document.documentElement.clientHeight;
+        const vw = window.innerWidth || document.documentElement.clientWidth;
+        menuView.style.width = b.width + 'px';
+        menuView.style.height = b.height + 'px';
+        menuView.style.top = Math.min(r.y, vh - b.height - 20) + 'px';
+        menuView.style.left = Math.min(r.x, vw - b.width - 20) + 'px';
    };
+    setTimeout(() => {
+        // Execution needs to be delayed to prevent the ouside click event
+        // from closing this window
+        document.body.appendChild(menuView);
+        setTimeout(onLayout, 1);
+    }, 0);
+}
+function closeJobMenu() {
+    if (menuView === null) {
+        return;
+    }
+    while (menuView.childNodes.length > 0)
+        menuView.removeChild(menuView.lastChild);
+    if (menuView.parentElement)
+        menuView.parentElement.removeChild(menuView);
+    menuView = null;
+}
+function restartJob(jobId) {
+    const xhttp = new XMLHttpRequest();
    xhttp.open("GET", "/restart_test/" + jobId, true);
    xhttp.send();
 }
-function viewJobLogs(jobId) {
+function cancelJob(jobId) {
-    logView.style.display = 'block';
+    const xhttp = new XMLHttpRequest();
-    while (logView.childElementCount > 1) {
+    xhttp.open("GET", "/cancel_test/" + jobId, true);
-        logView.removeChild(logView.lastElementChild);
+    xhttp.send();
-    }
+}
+function getResultZip(jobId) {
+    window.open('/results/' + jobId + '/results.zip');
+}
+function viewJobLogs(jobId) {
    const logIds = [0,1,2,3];
    const logNames = ['make stdout', 'make stderr', 'test stdout', 'test stderr'];
+    logView = document.createElement('div');
+    logView.className = 'logViewWindow';
+    const closeLogViewButton = document.createElement('button');
+    logView.appendChild(closeLogViewButton);
+    closeLogViewButton.innerText = "X";
+    closeLogViewButton.style.display = 'block';
+    closeLogViewButton.style.right = '0px';
+    closeLogViewButton.style.top = '0px';
+    closeLogViewButton.style.position = 'absolute';
+    closeLogViewButton.onclick = closeLogView;
    const tabs = logIds.map(logId => {
        const v = document.createElement('button');
        v.innerText = logNames[logIds.indexOf(logId)];
@@ -197,15 +327,37 @@ function viewJobLogs(jobId) {
            view.style.display = 'block';
        };
    }
+    setTimeout(() => {
+        // Execution needs to be delayed to prevent the ouside click event
+        // from closing this window
+        document.body.appendChild(logView);
+    }, 0);
+}
+function closeLogView() {
+    if (logView === null) {
+        return;
+    }
+    while (logView.childElementCount > 1) {
+        logView.removeChild(logView.lastElementChild);
+    }
+    if (logView.parentElement) {
+        logView.parentElement.removeChild(logView);
+    }
+    logView = null;
 }
 function requestStatus() {
    const xhttp = new XMLHttpRequest();
    xhttp.onreadystatechange = function() {
-        if (this.readyState == 4 && this.status == 200) {
+        if (this.readyState == 4) {
-            onStatusGet(JSON.parse(this.responseText));
            setTimeout(requestStatus, 1000);
+            if (this.status == 200) {
+                onStatusGet(JSON.parse(this.responseText));
+            }
        }
    };
    xhttp.open("GET", "/status", true);
@@ -226,6 +378,7 @@ function makeRow(list, cellTagName='td') {
    return headerRow;
 };
 init();
 requestStatus();

--- a/process_zip.sh
+++ b/process_zip.sh
@@ -37,8 +37,8 @@ function run() {
    mkdir -p "./queues"
    QUEUE_PATH="./queues/$TEMPLATE"
-    TEST_PATH="$DESTDIR/$CIPHER_SLUG"
    CIPHER_SLUG=$(basename $cipher)
+    TEST_PATH="$DESTDIR/$CIPHER_SLUG"
    mkdir -p "$TEST_PATH" || exit 1
    mv $cipher/*.log "$TEST_PATH"

--- a/test-dude.py
+++ b/test-dude.py
@@ -2,10 +2,12 @@
 import os
+import sys
+import signal
 import datetime
 import threading
 import subprocess
-from flask import Flask, request
+from flask import Flask, request, Response
 from flask_restful import Resource, Api
 from flask_jsonpify import jsonify
@@ -43,11 +45,12 @@ class Runner(threading.Thread):
        self.template = template
        self.program = program
        self.process = None
        self.job = None
-        self.event = threading.Event()
+        self.stop_event = threading.Event()
        threading.Thread.__init__(self)
+        self.name += "-%s" % template
        self.start()
    def to_dict(self):
@@ -58,9 +61,11 @@ class Runner(threading.Thread):
            'job': str(id(self.job)) if self.job is not None else None
        }
+    def stop(self):
+        self.stop_event.set()
    def run(self):
-        while 1:
+        while not self.stop_event.is_set():
-            self.event.clear()
            my_queue = [
                s for s in schedule
                if s.state == 'SCHEDULED'
@@ -69,7 +74,7 @@ class Runner(threading.Thread):
            my_queue.sort(key=lambda s: s.added)
            if len(my_queue) == 0:
                # No tasks for this thread, go to sleep
-                self.event.wait(timeout=5)
+                self.stop_event.wait(timeout=5)
                continue
            job = my_queue[0]
@@ -96,7 +101,13 @@ class Runner(threading.Thread):
                    stdout=out_fd,
                    stderr=err_fd
                )
-                self.process.wait()
+                while self.process.poll() is None:
+                    if self.stop_event.wait(timeout=1):
+                        self.process.send_signal(signal.SIGINT)
+                        try:
+                            self.process.wait(timeout=1)
+                        except subprocess.TimeoutExpired:
+                            pass
            if self.process.returncode == 0:
                self.job.state = 'SUCCESSFUL'
@@ -108,7 +119,9 @@ class Runner(threading.Thread):
                ['zip', '-r', 'results.zip', '.'],
                cwd=self.job.path)
+            print("Job %d has finished" % id(self.job))
            self.job = None
+        print("Thread %s has finished" % self.name)
 class Status(Resource):
@@ -177,7 +190,17 @@ def view_log(job_id, log_id):
    if not os.path.isfile(log_path):
        return 'Log not found', 404
    with open(log_path, 'r') as f:
-        return f.read()
+        return Response(f.read(), mimetype='text/plain')
+@app.route('/results/<string:job_id>/results.zip')
+def get_results_zip(job_id):
+    job = next(filter(lambda job: str(id(job)) == job_id, schedule), None)
+    if job is None:
+        return 'Job not found', 404
+    zip_path = os.path.join(job.path, 'results.zip')
+    with open(zip_path, 'rb') as zip:
+        return Response(zip.read(), mimetype='application/zip')
 if __name__ == '__main__':
@@ -186,4 +209,15 @@ if __name__ == '__main__':
    runners.append(Runner('uno'))
    runners.append(Runner('esp32'))
    runners.append(Runner('bluepill'))
+    def signal_handler(signal, frame):
+        print("Process interrupted!", file=sys.stderr)
+        for r in runners:
+            print("Stopping runner %s" % r.name, file=sys.stderr)
+            r.stop()
+            r.join()
+            print("Runner %s stopped" % r.name, file=sys.stderr)
+        sys.exit(0)
+    signal.signal(signal.SIGINT, signal_handler)
    app.run(port='5002')
--- a/test_common.py
+++ b/test_common.py
@@ -5,7 +5,6 @@ import re
 import sys
 import time
 import fcntl
-import pickle
 import struct
 import socket
 import subprocess
@@ -392,7 +391,6 @@ class FileMutex:
 class OpenOcd:
    def __init__(self, config_file, tcl_port=6666, verbose=False):
        self.verbose = verbose
        self.tclRpcIp = "127.0.0.1"
@@ -451,7 +449,6 @@ class OpenOcd:
 def run_nist_lws_aead_test(dut, vectors_file, build_dir,
                           logic_mask=0xffff):
    kat = list(parse_nist_aead_test_vectors(vectors_file))
    dut.flash()