compile_all.py

#!/usr/bin/env python3

import os
import sys
import stat
import argparse
import shutil
import subprocess


def build(algo_dir, template_dir, build_dir):
    if os.path.isdir(build_dir):
        return None

    print("Building in %s" % build_dir)

    # copy all the files from the submitted algorithm into the build directory
    shutil.copytree(algo_dir, build_dir)

    # remove the test vectors generator if it is there
    c = os.path.join(build_dir, "genkat_aead.c")
    if os.path.exists(c):
        os.remove(c)

    # find all c and h files, since they will be added to the makefile
    hfiles = []
    cfiles = []
    sfiles = []
    for r, d, f in os.walk(build_dir):
        for file in f:
            if file.lower().endswith(".c"):
                cfiles.append(file)
            elif file.lower().endswith(".s"):
                sfiles.append(file)
            elif file.lower().endswith(".h"):
                hfiles.append(file)

    # copy all the files from the template directory into the build directory
    for f in os.listdir(template_dir):
        dst = os.path.join(build_dir, f)
        src = os.path.join(template_dir, f)
        if os.path.isfile(src) or os.path.islink(src):
            shutil.copy2(src, dst)
        elif os.path.isdir(src):
            shutil.copytree(src, dst)
        else:
            raise Exception("I don't know what %s is" % src)

    # prepare the environmental variables for the makefile
    env = os.environ
    env['SRC_FILES'] = ' '.join(cfiles)
    env['ASM_FILES'] = ' '.join(sfiles)
    env['HDR_FILES'] = ' '.join(hfiles)

    # enter the directory and execute the makefile
    wd = os.getcwd()
    os.chdir(build_dir)
    try:
        if os.path.isfile('./configure'):
            subprocess.check_call(["./configure"])

        stdout_path = 'make.stdout.log'
        stderr_path = 'make.stderr.log'
        with open(stdout_path, 'w') as outfile, \
             open(stderr_path, 'w') as errfile:
            subprocess.check_call(['make'], stdout=outfile, stderr=errfile)

        if os.path.isfile('./cleanup'):
            subprocess.check_call(["./cleanup"])

    finally:
        sys.stdout.flush()
        sys.stderr.flush()
        os.chdir(wd)

    # if execution arrives here, the build was successful
    return build_dir


# Find test vectors in directory or one of the parent directories
def find_test_vectors(d):
    kat = None
    while True:
        if d == '':
            raise Exception("Test vector not found")
        for f in os.listdir(d):
            if f.startswith("LWC_AEAD_KAT_") and f.endswith(".txt"):
                if kat is not None:
                    raise Exception("Multiple test vectors?")
                kat = f
        if kat is None:
            d = os.path.split(d)[0]
        else:
            break
    kat = os.path.join(d, kat)
    return kat


def main(argv):
    include_list = None

    # Parse the arguments
    argparser = argparse.ArgumentParser(
        description='Compiles all LWC submissions for a given template')

    argparser.add_argument('-v', '--verbose', action='count')
    argparser.add_argument('-i', '--include', action='append')
    argparser.add_argument('-t', '--template', default='templates/linux')
    argparser.add_argument('-b', '--build-dir', default='build')
    argparser.add_argument('-s', '--submissions-dir',
                           default='all-lwc-submission-files')

    args = argparser.parse_args(argv[1:])
    template_dir = args.template
    build_root_dir = args.build_dir

    include_list = args.include
    submissions_dir = args.submissions_dir

    print("Using template %s" % template_dir)
    subs = os.listdir(submissions_dir)

    # get all the submissions by looking for files named "api.h"
    implementations = []
    for submission in subs:
        variants_dir = os.path.join(
            submissions_dir, submission, "Implementations", "crypto_aead")

        if not os.path.isdir(variants_dir):
            continue

        if "NOT ACCEPTED" in variants_dir:
            continue

        print()
        print("###  %s  ###" % submission)

        c = 0
        for variant in os.listdir(variants_dir):
            implementations_dir = os.path.join(
                variants_dir, variant)
            for implementation in os.listdir(implementations_dir):
                implementation_dir = os.path.join(
                    implementations_dir, implementation)
                if os.path.isfile(os.path.join(implementation_dir, "api.h")):
                    implementations.append(
                        (submission, variant, implementation))
                    c += 1

        if c == 0:
            raise Exception("No implementations found")

    if include_list is not None:
        print("Include list has %d entries" % len(include_list))

    files = []
    for submission, variant, implementation in implementations:

        # base name n (a.k.a. cipher slug)
        n = '.'.join([submission, variant, implementation])
        print(n)

        # Source directory d
        d = os.path.join(
            submissions_dir, submission, "Implementations", "crypto_aead",
            variant, implementation)
        assert os.path.isdir(d)
        print(d)

        # Test vectors file t
        t = find_test_vectors(d)
        print(t)

        # if include_list was provided, skip elements not in the list
        if include_list is not None:
            if n not in include_list:
                continue

        # Find date of last modification in directory
        st_mtime = 0
        for root, dirs, filess in os.walk(d):
            for name in filess:
                path = os.path.join(root, name)
                st_mtime = max(st_mtime, os.stat(path).st_mtime)

        # Put all in a tuple and count
        files.append((t, d, n, st_mtime))

    # Uncomment next line for testing, if we only want to do 1
    # files = files[:1]
    print("%d algorithms will be compiled" % len(files))

    if not os.path.isdir(build_root_dir):
        os.mkdir(build_root_dir)

    print()

    # Build all found algorithms
    for i, (t, d, name, st_mtime) in enumerate(files):
        print()
        print(d)
        try:
            build_dir = os.path.join(build_root_dir, name)

            b = build(d, template_dir, build_dir)

            if b is None:
                continue

            shutil.copyfile(t, os.path.join(b, 'LWC_AEAD_KAT.txt'))

            mdate_path = os.path.join(build_dir, 'cipher_mtime.txt')
            with open(mdate_path, 'wt') as mdate_file:
                print(int(st_mtime), file=mdate_file)

            print("COMPILATION SUCCESS FOR %s" % d)
        except Exception as ex:
            print("COMPILATION FAILED FOR %s" % d)
            print(ex)

    print()


if __name__ == "__main__":
    sys.exit(main(sys.argv))