Use parallelization and origin TLS-Anvil repo

Also applies some suggestions to move logic from
the workflow yml file to ci_tlsanvil_test.py

.github/workflows/nightly.yml

@@ -71,33 +71,15 @@ jobs:
           target: static
           cache-key: linux-gcc-x86_64-static
 
-      - name: Build Botan
-        run: ./configure.py --compiler-cache=ccache --build-targets=static,cli --without-documentation && make -j$(nproc)
-
-      - name: Install Java and Maven
-        uses: actions/setup-java@v3
-        with:
-          java-version: '11'
-          distribution: 'zulu'
-
-      - name: Fetch TLS-Anvil fork for Botan tests
-        uses: actions/checkout@v3
-        with:
-          repository: FAlbertDev/TLS-Anvil
-          path: ./tlsanvil
-
-      - name: Build TLS-Anvil
-        working-directory: ./tlsanvil/
-        run: mvn install -DskipTests -Dspotless.apply.skip
+      - name: Install Boost
+        run: sudo apt-get -qq install libboost-dev
 
-      - name: Test Botan Server with TLS-Anvil
+      - name: Build and Test Botan Server with TLS-Anvil
         run: >
           python3 ./src/scripts/ci/ci_tlsanvil_test.py
-          --botan-exe ./botan
-          --tlsanvil-jar ./tlsanvil/TLS-Testsuite/apps/TLS-Testsuite.jar
+          --botan-dir .
           --test-target server
-        env:
-          DOCKER: 1 # TLS-Anvil specific to disable the loading bar
+          --parallel $(nproc)
 
       - uses: actions/upload-artifact@v3
         with:

src/scripts/ci/ci_tlsanvil_check.py

@@ -10,13 +10,15 @@
 import json
 import logging
 
+
 result_level = {
     "STRICTLY_SUCCEEDED": 0,
     "CONCEPTUALLY_SUCCEEDED": 1,
     "PARTIALLY_FAILED": 2,
     "FULLY_FAILED": 3,
 }
 
+
 def expected_result_for(method_id: str):
     """ Get the expected result for a given test id """
     # TODO: Analyze failing tests and document if/why they are allowed to fail
@@ -72,6 +74,7 @@ def test_result_valid(method_id: str, result: str):
 
 
 def failing_test_info(json_data, method_id) -> str:
+    """ Print debug information about a failing test """
     info_str = ""
     try:
         method_class, method_name = method_id.rsplit('.', 1)
@@ -192,5 +195,6 @@ def main(args=None):
 
     return int(not total_success)
 
+
 if __name__ == "__main__":
     sys.exit(main())

src/scripts/ci/ci_tlsanvil_test.py

@@ -13,16 +13,35 @@
 
 class Config:
     """ Hardcoded configurations for this CI script """
+    tls_anvil_docker_image = "ghcr.io/tls-attacker/tlsanvil"
+    tls_anvil_version_tag = "v1.2.2"
     key_and_cert_storage_path = "/tmp/"
-    test_suite_results_path = "./TestSuiteResults"
+    test_suite_results_dest = "."
+    test_suite_results_dir_name = "TestSuiteResults"
     tmp_key_file_name = "tmp_rsa_key.pem"
     tmp_cert_file_name = "tmp_rsa_cert.pem"
     server_dest_ip = "127.0.0.1"
     server_dest_port = 4433
     botan_server_log = "./logs/botan_server.log"
+    botan_config_args = ["--compiler-cache=ccache", "--build-targets=static,cli",
+                          "--without-documentation", "--with-boost"]
 
+def group_output(group_title: str, func):
+    """
+    Wraps a function to be called within a GitHub actions group, so that
+    the console output is expandable.
+
+    Returns the wrapped function
+    """
+    def wrapped_func(*args, **kwargs):
+        print(f"::group::{group_title}", flush=True)
+        ret = func(*args, **kwargs)
+        print("\n::endgroup::", flush=True)
+        return ret
+    return wrapped_func
 
-def create_cert_and_key(botan_exe_path):
+
+def create_cert_and_key(botan_dir_path):
     """
     Create a X.509 certificate and associated RSA key at Config.key_and_cert_storage_path
     using Botan's CLI.
@@ -34,29 +53,48 @@ def create_cert_and_key(botan_exe_path):
     cert_path = os.path.join(Config.key_and_cert_storage_path, Config.tmp_cert_file_name)
 
     with open(key_path, 'w', encoding='utf-8') as keyfile:
-        subprocess.run([botan_exe_path, "keygen", "--algo=RSA", "--params=2048"], stdout=keyfile, check=True)
+        subprocess.run([botan_dir_path, "keygen", "--algo=RSA", "--params=2048"], stdout=keyfile, check=True)
 
     with open(cert_path, 'w', encoding='utf-8') as certfile:
-        subprocess.run([botan_exe_path, "gen_self_signed", key_path, "localhost"], stdout=certfile, check=True)
+        subprocess.run([botan_dir_path, "gen_self_signed", key_path, "localhost"], stdout=certfile, check=True)
 
     return (cert_path, key_path)
 
 
-def server_test(botan_exe_path: str, tls_anvil_jar_path: str):
-    cert_path, key_path = create_cert_and_key(botan_exe_path)
+def build_botan(botan_dir: str, parallel_jobs: int) -> str:
+    """
+    Configure and build botan.
+
+    Returns the botan executable path
+    """
+    group_output("Configure Botan", subprocess.run)(["python3", "./configure.py"] + Config.botan_config_args, check=True, cwd=botan_dir)
+    group_output("Build Botan with Make", subprocess.run)(["make", f"-j{parallel_jobs}"], check=True, cwd=botan_dir)
+
+    return os.path.join(botan_dir, "botan")
+
+
+def server_test(botan_dir_path: str, parallel: int):
+    """ Test the Botan TLS server """
+    cert_path, key_path = create_cert_and_key(botan_dir_path)
+    docker_img = f"{Config.tls_anvil_docker_image}:{Config.tls_anvil_version_tag}"
+
+    group_output("Pull TLS-Anvil image", subprocess.run)(["docker", "pull", docker_img], check=True)
 
     tls_anvil_cmd = [
-        "java", "-jar", tls_anvil_jar_path,
+        "docker", "run",
+        "--network", "host",
+        "-v", f"{Config.test_suite_results_dest}:/output",
+        docker_img,
         "-strength", "1",
-        "-parallelHandshakes", "1",
+        "-parallelHandshakes", str(parallel),
         "-disableTcpDump",
-        "-outputFolder", Config.test_suite_results_path,
+        "-outputFolder", os.path.join(Config.test_suite_results_dest, Config.test_suite_results_dir_name),
         "-connectionTimeout", "5000",
         "server", "-connect", f"{Config.server_dest_ip}:{Config.server_dest_port}"
     ]
 
     botan_server_cmd = [
-        botan_exe_path, "tls_server", cert_path, key_path, f"--port={Config.server_dest_port}"
+        botan_dir_path, "tls_http_server", cert_path, key_path, f"--port={Config.server_dest_port}"
     ]
 
     os.makedirs(os.path.dirname(Config.botan_server_log), exist_ok=True)
@@ -68,7 +106,8 @@ def server_test(botan_exe_path: str, tls_anvil_jar_path: str):
         botan_server_process.kill()
 
 
-def client_test(botan_exe_path: str, tls_anvil_jar_path: str):
+def client_test(botan_dir_path: str, parallel: int):
+    """ Test the Botan TLS server """
     raise NotImplementedError("Client tests not yet implemented")
 
 
@@ -77,24 +116,24 @@ def main(args=None):
         args = sys.argv[1:]
 
     parser = argparse.ArgumentParser()
-    parser.add_argument("--botan-exe", help="Botan executable file", required=True)
-    parser.add_argument("--tlsanvil-jar", help="TLS-Anvil test suite jar file", required=True)
+    parser.add_argument("--botan-dir", help="Botan base directory", required=True)
     parser.add_argument("--test-target", help="The TLS side to test", choices=['client', 'server'], required=True)
+    parser.add_argument("--parallel", help="The number of parallel handshakes", type=int, default=1)
 
     args = vars(parser.parse_args(args))
 
-    if not os.path.isfile(args["tlsanvil_jar"]):
-        raise FileNotFoundError(f"Unable to find '{args['tlsanvil_jar']}'")
+    if not os.path.isdir(args["botan_dir"]):
+        raise FileNotFoundError(f"Unable to find '{args['botan_dir']}'")
 
-    if not os.path.isfile(args["botan_exe"]):
-        raise FileNotFoundError(f"Unable to find '{args['botan_exe']}'")
+    botan_exe_path = build_botan(args["botan_dir"], args["parallel"])
 
     if args["test_target"] == "server":
-        server_test(args["botan_exe"], args["tlsanvil_jar"])
+        server_test(botan_exe_path, args["parallel"])
     elif args["test_target"] == "client":
-        client_test(args["botan_exe"], args["tlsanvil_jar"])
+        client_test(botan_exe_path, args["parallel"])
 
     return 0
 
+
 if __name__ == "__main__":
     sys.exit(main())