Run riscv-arch-test suite in CI (#513)

.github/workflows/main.yml

@@ -12,8 +12,128 @@ on:
   workflow_dispatch:
 
 jobs:
+  build-core:
+    name: Synthethise full core
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+            python-version: '3.11'
+
+      - name: Install Coreblocks dependencies
+        run: |
+          python3 -m venv venv
+          . venv/bin/activate
+          python3 -m pip install --upgrade pip
+          python3 -m pip install -r requirements-dev.txt
+
+      - name: Generate Verilog
+        run: |
+          . venv/bin/activate
+          PYTHONHASHSEED=0 ./scripts/gen_verilog.py --verbose --config full
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: "verilog-full-core"
+          path: core.v
+
+
+  build-riscof-tests:
+    name: Build regression tests (riscv-arch-test)
+    runs-on: ubuntu-latest
+    container: ghcr.io/kuznia-rdzeni/riscv-toolchain:2023.11.19_v
+    timeout-minutes: 10
+    env:
+      PYENV_ROOT: "/root/.pyenv"
+      LC_ALL: "C.UTF8"
+      LANG: "C.UTF8"
+    defaults:
+      run:
+        working-directory: test/external/riscof/
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Setup PATH
+        run: echo "/.pyenv/bin" >> $GITHUB_PATH
+
+      - name: Setup pyenv python
+        run: |
+          eval "$(pyenv init --path)"
+          pyenv global 3.6.15
+          . /venv3.6/bin/activate
+
+      - name: Setup arch test suite
+        run: |
+          . /venv3.6/bin/activate
+          riscof --verbose info arch-test --clone
+          riscof testlist --config=config.ini --suite=riscv-arch-test/riscv-test-suite/ --env=riscv-arch-test/riscv-test-suite/env
+
+      - name: Build and run tests on reference and generate Makefiles
+        run: |
+          . /venv3.6/bin/activate
+          riscof run --config=config.ini --suite=riscv-arch-test/riscv-test-suite/ --env=riscv-arch-test/riscv-test-suite/env
+
+      - name: Build tests for Coreblocks
+        run: |
+          MAKEFILE_PATH=riscof_work/Makefile.build-DUT-coreblocks ../../../ci/riscof_run_makefile.sh
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: "riscof-tests"
+          path: |
+            test/external/riscof/riscv-arch-test/**/*.elf
+            test/external/riscof/riscof_work/**/*.signature
+            test/external/riscof/**/*Makefile*
+
+  run-riscof-tests:
+    name: Run regression tests (riscv-arch-test)
+    runs-on: ubuntu-latest
+    container: ghcr.io/kuznia-rdzeni/verilator:v5.008-2023.11.19_v
+    needs: [ build-riscof-tests, build-core ]
+    timeout-minutes: 20
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+            python-version: '3.11'
+
+      - name: Install Coreblocks dependencies
+        run: |
+          python3 -m venv venv
+          . venv/bin/activate
+          python3 -m pip install --upgrade pip
+          python3 -m pip install -r requirements-dev.txt
+
+      - uses: actions/download-artifact@v3
+        with:
+          name: "verilog-full-core"
+          path: .
+
+      - uses: actions/download-artifact@v3
+        with:
+          name: "riscof-tests"
+          path: test/external/riscof/
+
+      - name: Run tests on Coreblocks
+        run: |
+          . venv/bin/activate
+          MAKEFILE_PATH=test/external/riscof/riscof_work/Makefile.run-DUT-coreblocks NPROC=1 ./ci/riscof_run_makefile.sh
+
+      - name: Compare signatures (test results)
+        run: MAKEFILE_PATH=test/external/riscof/riscof_work/Makefile.run-DUT-coreblocks ./ci/riscof_compare.sh
+
+
   build-regression-tests:
-    name: Build regression tests
+    name: Build regression tests (riscv-tests)
     runs-on: ubuntu-latest
     container: ghcr.io/kuznia-rdzeni/riscv-toolchain:2023.11.19_v
     outputs:
@@ -31,7 +151,7 @@ jobs:
           cache-name: cache-regression-tests
         with:
           path: test/external/riscv-tests/test-*
-          
+
           key: ${{ env.cache-name }}-${{ runner.os }}-${{ hashFiles(
               '**/test/external/riscv-tests/environment/**',
               '**/test/external/riscv-tests/Makefile',
@@ -51,7 +171,7 @@ jobs:
           path: test/external/riscv-tests
 
   run-regression-tests:
-    name: Run regression tests
+    name: Run regression tests (riscv-tests)
     runs-on: ubuntu-latest
     timeout-minutes: 10
     container: ghcr.io/kuznia-rdzeni/verilator:v5.008-2023.11.19_v
@@ -74,10 +194,10 @@ jobs:
           python3 -m pip install --upgrade pip
           python3 -m pip install -r requirements-dev.txt
 
-      - name: Generate Verilog
-        run: |
-          . venv/bin/activate
-          PYTHONHASHSEED=0 ./scripts/gen_verilog.py --verbose --config full
+      - uses: actions/download-artifact@v3
+        with:
+          name: "verilog-full-core"
+          path: .
 
       - uses: actions/cache@v3
         env:

ci/riscof_compare.sh

@@ -0,0 +1,53 @@
+#!/bin/sh
+
+if [ -z "$MAKEFILE_PATH" ]; then
+  echo "Makefile path not specifed. Exiting... "
+  exit 1
+fi
+
+[ ! -z "$DIFF_QUIET" ] && [ "$DIFF_QUIET" -eq 1 ] && diff_add_args="-q"
+
+RED="\033[1;31m"
+GREEN="\033[1;32m"
+RED_BG="\033[0;41m"
+GREEN_BG="\033[0;42m"
+NO_COLOR="\033[0m"
+
+signature_files="$(cat $MAKEFILE_PATH | sed -n 's/^.*-o=\(.*\.signature\).*$/\1/p')"
+
+REFERENCE_DIR_SUFF="/../ref/Reference-Spike.signature"
+
+echo "> Veryfing signatures"
+target_cnt=0
+fail_cnt=0
+
+for sig in $signature_files
+do
+    ref="$(dirname "$sig")$REFERENCE_DIR_SUFF"
+    echo ">> Comparing $sig (TARGET$target_cnt) to $ref"
+
+    diff -b --strip-trailing-cr $diff_add_args "$sig" "$ref"
+    res=$?
+
+    [ -f "$ref" ] || echo -e "${RED}!${NO_COLOR} Reference signature file not found!"
+    [ -f "$sig" ] || echo -e "${RED}!${NO_COLOR} Coreblocks signature file not found! Check signature run logs"
+    [ -s "$sig" ] || echo -e "${RED}!${NO_COLOR} Coreblock signature file is empty! Check signature run logs"
+
+    if [ $res = 0 ]
+    then
+        echo -e "${GREEN}[PASS] Signature verification passed (TARGET$target_cnt)${NO_COLOR}"
+    else
+        echo -e "${RED}[FAIL] Signature verification failed (TARGET$target_cnt)${NO_COLOR}"
+        fail_cnt=$(( $fail_cnt+1 ))
+    fi
+
+    target_cnt=$(( $target_cnt+1 ))
+done
+
+rc=1
+[ $fail_cnt -eq 0 ] && rc=0
+
+bg=${GREEN_BG}
+[ $rc = 1 ] && bg=$RED_BG
+echo -e "${bg}>>> Compared $target_cnt signatures, FAILED=$fail_cnt, PASSED=$(($target_cnt-$fail_cnt))${NO_COLOR}"
+exit $rc

ci/riscof_run_makefile.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+if [ -z "$MAKEFILE_PATH" ]; then
+  echo "Makefile path not specifed. Exiting... "
+  exit 1
+fi
+
+[ -z "$NPROC" ] && NPROC=$(nproc)
+
+target_cnt=$(cat $MAKEFILE_PATH | grep TARGET | tail -n 1 | tr -d -c 0-9)
+
+echo "> Running for $target_cnt Makefile targets"
+
+targets=""
+
+for i in $(seq 0 $target_cnt)
+do
+    targets="$targets TARGET$i"
+done
+
+echo "Starting for targets: $targets"
+make -f $MAKEFILE_PATH -i -j $NPROC $targets

scripts/run_signature.py

@@ -19,7 +19,7 @@ def run_with_cocotb(test_name: str, traces: bool, output: str) -> bool:
     arglist = [
         "make",
         "-C",
-        parent + "/" if parent else "" + "test/regression/cocotb",
+        (parent + "/" if parent else "") + "test/regression/cocotb",
         "-f",
         "signature.Makefile",
         "--no-print-directory",

test/external/riscof/config.ini

@@ -0,0 +1,19 @@
+[RISCOF]
+ReferencePlugin=spike_simple
+ReferencePluginPath=./spike_simple
+DUTPlugin=coreblocks
+DUTPluginPath=./coreblocks
+
+[coreblocks]
+pluginpath=./coreblocks
+PATH=../../../../
+ispec=./coreblocks/coreblocks_isa.yaml
+pspec=./coreblocks/coreblocks_platform.yaml
+target_run=0
+
+[spike_simple]
+pluginpath=./spike_simple
+PATH=/spike/install/bin/
+ispec=./spike_simple/spike_simple_isa.yaml
+pspec=./spike_simple/spike_simple_platform.yaml
+compare_run=0

test/external/riscof/coreblocks/coreblocks_isa.yaml

@@ -0,0 +1,7 @@
+hart_ids: [0]
+hart0:
+  ISA: RV32I
+  physical_addr_sz: 32
+
+  User_Spec_Version: '2.3'
+  supported_xlen: [32]

test/external/riscof/coreblocks/coreblocks_platform.yaml

@@ -0,0 +1,4 @@
+reset:
+  label: reset_vector
+nmi:
+  label: nmi_vector