some compilation and scheduling fixes

compile_all.py

@@ -117,27 +117,30 @@ def main(argv):
     subs = os.listdir(submissions_dir)
 
     # get all the submissions by looking for files named "api.h"
-    subfiles = []
+    implementations = []
     for submission in subs:
-        implementations_dir = os.path.join(
+        variants_dir = os.path.join(
             submissions_dir, submission, "Implementations", "crypto_aead")
 
-        if not os.path.isdir(implementations_dir):
+        if not os.path.isdir(variants_dir):
             continue
 
-        if "NOT ACCEPTED" in implementations_dir:
+        if "NOT ACCEPTED" in variants_dir:
             continue
 
         print()
         print("###  %s  ###" % submission)
 
         c = 0
-        # r=root, d=directories, f = files
-        for r, d, f in os.walk(implementations_dir):
-            for file in f:
-                if file == "api.h":
-                    f = os.path.join(r, file)
-                    subfiles.append(f)
+        for variant in os.listdir(variants_dir):
+            implementations_dir = os.path.join(
+                variants_dir, variant)
+            for implementation in os.listdir(implementations_dir):
+                implementation_dir = os.path.join(
+                    implementations_dir, implementation)
+                if os.path.isfile(os.path.join(implementation_dir, "api.h")):
+                    implementations.append(
+                        (submission, variant, implementation))
                     c += 1
 
         if c == 0:
@@ -147,10 +150,16 @@ def main(argv):
         print("Include list has %d entries" % len(include_list))
 
     files = []
-    for f in subfiles:
+    for submission, variant, implementation in implementations:
+
+        # base name n (a.k.a. cipher slug)
+        n = '.'.join([submission, variant, implementation])
+        print(n)
 
         # Source directory d
-        d = os.path.split(f)[0]
+        d = os.path.join(
+            submissions_dir, submission, "Implementations", "crypto_aead",
+            variant, implementation)
         assert os.path.isdir(d)
         print(d)
 
@@ -158,11 +167,6 @@ def main(argv):
         t = find_test_vectors(d)
         print(t)
 
-        # base name n
-        pieces = f.split(os.sep)
-        n = pieces[1] + "." + ".".join(pieces[4:-1])
-        print(n)
-
         # if include_list was provided, skip elements not in the list
         if include_list is not None:
             if n not in include_list:

process_zip.sh

@@ -37,8 +37,8 @@ function run() {
 
     mkdir -p "./queues"
     QUEUE_PATH="./queues/$TEMPLATE"
-    TEST_PATH="$DESTDIR/$CIPHER_SLUG"
     CIPHER_SLUG=$(basename $cipher)
+    TEST_PATH="$DESTDIR/$CIPHER_SLUG"
 
     mkdir -p "$TEST_PATH" || exit 1
     mv $cipher/*.log "$TEST_PATH"

test-dude.py

@@ -2,10 +2,12 @@
 
 
 import os
+import sys
+import signal
 import datetime
 import threading
 import subprocess
-from flask import Flask, request
+from flask import Flask, request, Response
 from flask_restful import Resource, Api
 from flask_jsonpify import jsonify
 
@@ -43,11 +45,12 @@ class Runner(threading.Thread):
 
         self.template = template
         self.program = program
-
         self.process = None
         self.job = None
-        self.event = threading.Event()
+        self.stop_event = threading.Event()
+
         threading.Thread.__init__(self)
+        self.name += "-%s" % template
         self.start()
 
     def to_dict(self):
@@ -58,9 +61,11 @@ class Runner(threading.Thread):
             'job': str(id(self.job)) if self.job is not None else None
         }
 
+    def stop(self):
+        self.stop_event.set()
+
     def run(self):
-        while 1:
-            self.event.clear()
+        while not self.stop_event.is_set():
             my_queue = [
                 s for s in schedule
                 if s.state == 'SCHEDULED'
@@ -69,7 +74,7 @@ class Runner(threading.Thread):
             my_queue.sort(key=lambda s: s.added)
             if len(my_queue) == 0:
                 # No tasks for this thread, go to sleep
-                self.event.wait(timeout=5)
+                self.stop_event.wait(timeout=5)
                 continue
 
             job = my_queue[0]
@@ -96,7 +101,13 @@ class Runner(threading.Thread):
                     stdout=out_fd,
                     stderr=err_fd
                 )
-                self.process.wait()
+                while self.process.poll() is None:
+                    if self.stop_event.wait(timeout=1):
+                        self.process.send_signal(signal.SIGINT)
+                        try:
+                            self.process.wait(timeout=1)
+                        except subprocess.TimeoutExpired:
+                            pass
 
             if self.process.returncode == 0:
                 self.job.state = 'SUCCESSFUL'
@@ -108,7 +119,9 @@ class Runner(threading.Thread):
                 ['zip', '-r', 'results.zip', '.'],
                 cwd=self.job.path)
 
+            print("Job %d has finished" % id(self.job))
             self.job = None
+        print("Thread %s has finished" % self.name)
 
 
 class Status(Resource):
@@ -177,7 +190,17 @@ def view_log(job_id, log_id):
     if not os.path.isfile(log_path):
         return 'Log not found', 404
     with open(log_path, 'r') as f:
-        return f.read()
+        return Response(f.read(), mimetype='text/plain')
+
+
+@app.route('/results/<string:job_id>/results.zip')
+def get_results_zip(job_id):
+    job = next(filter(lambda job: str(id(job)) == job_id, schedule), None)
+    if job is None:
+        return 'Job not found', 404
+    zip_path = os.path.join(job.path, 'results.zip')
+    with open(zip_path, 'rb') as zip:
+        return Response(zip.read(), mimetype='application/zip')
 
 
 if __name__ == '__main__':
@@ -186,4 +209,15 @@ if __name__ == '__main__':
     runners.append(Runner('uno'))
     runners.append(Runner('esp32'))
     runners.append(Runner('bluepill'))
+
+    def signal_handler(signal, frame):
+        print("Process interrupted!", file=sys.stderr)
+        for r in runners:
+            print("Stopping runner %s" % r.name, file=sys.stderr)
+            r.stop()
+            r.join()
+            print("Runner %s stopped" % r.name, file=sys.stderr)
+        sys.exit(0)
+    signal.signal(signal.SIGINT, signal_handler)
+
     app.run(port='5002')

test_common.py

@@ -5,7 +5,6 @@ import re
 import sys
 import time
 import fcntl
-import pickle
 import struct
 import socket
 import subprocess
@@ -392,7 +391,6 @@ class FileMutex:
 
 
 class OpenOcd:
-
     def __init__(self, config_file, tcl_port=6666, verbose=False):
         self.verbose = verbose
         self.tclRpcIp = "127.0.0.1"
@@ -451,7 +449,6 @@ class OpenOcd:
 
 def run_nist_lws_aead_test(dut, vectors_file, build_dir,
                            logic_mask=0xffff):
-
     kat = list(parse_nist_aead_test_vectors(vectors_file))
 
     dut.flash()