banshee: Fix Banshee runtime support and add tests

.github/workflows/ci.yml

@@ -34,7 +34,7 @@ jobs:
       - uses: actions/checkout@v2
         with:
           submodules: 'recursive'
-      - name: Build Software
+      - name: Build MUSL dependency
         run: |
           cd sw/deps
           mkdir install
@@ -45,30 +45,47 @@ jobs:
           make -j4
           make install
           cd ../../../
+      - name: Build Software
+        run: |
           make -C target/snitch_cluster sw
       - name: Build Hardware
         run: |
           make -C target/snitch_cluster bin/snitch_cluster.vlt
       - name: Run Unit Tests
         working-directory: target/snitch_cluster
         run: |-
-          ./sw/tests/run.py sw/tests/passing-apps.list --simulator verilator
-          ./sw/apps/run.py sw/apps/passing-apps.list --simulator verilator
+          ./sw/tests/run.py sw/tests/passing-apps.list --simulator verilator \
+          2>&1 | tee tests.log
+      - name: Run Test Applications
+        working-directory: target/snitch_cluster
+        env:
+          SNITCH_LOG: info
+        run: |-
+          ./sw/apps/run.py sw/apps/passing-apps.list --simulator verilator \
+          2>&1 | tee apps.log
+      - name: Check Unit Tests
+        working-directory: target/snitch_cluster
+        run: |-
+          grep -zoP 'All tests passed' tests.log
+      - name: Check Test Applications
+        working-directory: target/snitch_cluster
+        run: |-
+          grep -zoP 'All tests passed' apps.log
 
   ############################################
   # Build SW on Snitch Cluster w/ Banshee #
   ############################################
 
   sw-snitch-cluster-banshee:
-    name: Build SW on Snitch Cluster w/ Banshee
+    name: Simulate SW on Snitch Cluster w/ Banshee
     runs-on: ubuntu-22.04
     container:
       image: ghcr.io/pulp-platform/snitch_cluster
     steps:
       - uses: actions/checkout@v2
         with:
           submodules: 'recursive'
-      - name: Build Software
+      - name: Build MUSL dependency
         run: |
           cd sw/deps
           mkdir install
@@ -79,4 +96,28 @@ jobs:
           make -j4
           make install
           cd ../../../
+      - name: Build Software
+        run: |
           make -C target/snitch_cluster SELECT_RUNTIME=banshee sw
+      - name: Run Unit Tests
+        working-directory: target/snitch_cluster
+        env:
+          SNITCH_LOG: info
+        run: |-
+          ./sw/tests/run.py sw/tests/banshee-apps.list --simulator banshee \
+          2>&1 | tee tests.log
+      - name: Run Test Applications
+        working-directory: target/snitch_cluster
+        env:
+          SNITCH_LOG: info
+        run: |-
+          ./sw/apps/run.py sw/apps/passing-apps.list --simulator banshee \
+          2>&1 | tee  apps.log
+      - name: Check Unit Tests
+        working-directory: target/snitch_cluster
+        run: |-
+          grep -zoP 'All tests passed' tests.log
+      - name: Check Test Applications
+        working-directory: target/snitch_cluster
+        run: |-
+          grep -zoP 'All tests passed' apps.log

.gitlab-ci.yml

@@ -14,6 +14,10 @@ variables:
   QUESTA: questa-2022.3
   LLVM_BINROOT: /usr/pack/riscv-1.0-kgf/pulp-llvm-0.12.0/bin
   CLANG: /usr/pack/riscv-1.0-kgf/pulp-llvm-0.12.0/bin/clang
+  CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER: /usr/pack/gcc-9.2.0-af/linux-x64/bin/gcc
+  LLVM_SYS_120_PREFIX: /usr/pack/llvm-12.0.1-af
+  SNITCH_LOG: info
+  CMAKE: cmake-3.18.1
 
 before_script:
   - $PYTHON -m venv .venv
@@ -32,7 +36,7 @@ docs:
 # Build Snitch cluster software #
 #################################
 
-snitch-cluster-sw:
+snitch-cluster-sw-musl:
   script:
     - cd sw/deps
     - mkdir install
@@ -42,14 +46,33 @@ snitch-cluster-sw:
       CFLAGS="-mcpu=snitch -menable-experimental-extensions"
     - make -j4
     - make install
-    - cd ../../../
+  artifacts:
+    paths:
+      - sw/deps/install/bin/*
+      - sw/deps/install/include/*
+      - sw/deps/install/lib/*
+    expire_in: 1 day
+
+snitch-cluster-sw:
+  needs: [snitch-cluster-sw-musl]
+  script:
     - cd target/snitch_cluster
     - make sw
   artifacts:
     paths:
       - target/snitch_cluster/sw/**/build/*.elf
     expire_in: 1 day
 
+snitch-cluster-sw-banshee:
+  needs: [snitch-cluster-sw-musl]
+  script:
+    - cd target/snitch_cluster
+    - make SELECT_RUNTIME=banshee sw
+  artifacts:
+    paths:
+      - target/snitch_cluster/sw/**/build/*.elf
+    expire_in: 1 day
+
 #######################
 # Standalone IP tests #
 #######################
@@ -88,24 +111,77 @@ snitch-cluster-vlt:
   script:
     - cd target/snitch_cluster
     - $VERILATOR make bin/snitch_cluster.vlt
-    - $VERILATOR ./sw/test/run.py sw/tests/passing-apps.list --simulator verilator
-    - $VERILATOR ./sw/apps/run.py sw/apps/passing-apps.list --simulator verilator
+    - $VERILATOR ./sw/tests/run.py sw/tests/passing-apps.list --simulator verilator 2>&1 | tee tests.log
+    - $VERILATOR ./sw/apps/run.py sw/apps/passing-apps.list --simulator verilator 2>&1 | tee apps.log
+    - grep -zoP 'All tests passed' tests.log
+    - grep -zoP 'All tests passed' apps.log
   # yamllint enable rule:line-length
+  artifacts:
+    when: on_failure
+    paths:
+      - target/snitch_cluster/tests.log
+      - target/snitch_cluster/apps.log
+    expire_in: 1 day
 
 # VCS
 snitch-cluster-vcs:
   needs: [snitch-cluster-sw]
+  # yamllint disable rule:line-length
   script:
     - cd target/snitch_cluster
     - $VCS make bin/snitch_cluster.vcs
-    - $VCS ./sw/test/run.py sw/tests/passing-apps.list --simulator vcs
-    - $VCS ./sw/apps/run.py sw/apps/passing-apps.list --simulator vcs
+    - $VCS ./sw/tests/run.py sw/tests/passing-apps.list --simulator vcs 2>&1 | tee tests.log
+    - $VCS ./sw/apps/run.py sw/apps/passing-apps.list --simulator vcs 2>&1 | tee apps.log
+    - grep -zoP 'All tests passed' tests.log
+    - grep -zoP 'All tests passed' apps.log
+  # yamllint enable rule:line-length
+  artifacts:
+    when: on_failure
+    paths:
+      - target/snitch_cluster/tests.log
+      - target/snitch_cluster/apps.log
+    expire_in: 1 day
 
 # Questa
 snitch-cluster-vsim:
   needs: [snitch-cluster-sw]
+  # yamllint disable rule:line-length
   script:
     - cd target/snitch_cluster
     - $QUESTA make bin/snitch_cluster.vsim
-    - $QUESTA ./sw/test/run.py sw/tests/passing-apps.list --simulator vsim
-    - $QUESTA ./sw/apps/run.py sw/apps/passing-apps.list --simulator vsim
+    - $QUESTA ./sw/tests/run.py sw/tests/passing-apps.list --simulator vsim 2>&1 | tee tests.log
+    - $QUESTA ./sw/apps/run.py sw/apps/passing-apps.list --simulator vsim 2>&1 | tee apps.log
+    - grep -zoP 'All tests passed' tests.log
+    - grep -zoP 'All tests passed' apps.log
+  # yamllint enable rule:line-length
+  artifacts:
+    when: on_failure
+    paths:
+      - target/snitch_cluster/tests.log
+      - target/snitch_cluster/apps.log
+    expire_in: 1 day
+
+# Banshee
+snitch-cluster-banshee:
+  needs: [snitch-cluster-sw-banshee]
+  # yamllint disable rule:line-length
+  script:
+    - rustup install 1.63.0
+    - rustup override set 1.63.0
+    - rustc --version
+    - rustup --version
+    - git clone [email protected]:pulp-platform/banshee.git --recurse-submodules
+    - cd banshee
+    - cargo install --path .
+    - cd ../target/snitch_cluster
+    - ./sw/tests/run.py sw/tests/banshee-apps.list --simulator banshee 2>&1 | tee tests.log
+    - ./sw/apps/run.py sw/apps/passing-apps.list --simulator banshee 2>&1 | tee apps.log
+    - grep -zoP 'All tests passed' tests.log
+    - grep -zoP 'All tests passed' apps.log
+  # yamllint enable rule:line-length
+  artifacts:
+    when: on_failure
+    paths:
+      - target/snitch_cluster/tests.log
+      - target/snitch_cluster/apps.log
+    expire_in: 1 day

docs/ug/getting_started.md

@@ -74,3 +74,15 @@ Install the required packages in the currently active virtual environment:
 ```shell
 pip install -r python-requirements.txt
 ```
+
+## Dependencies
+
+This repository has a couple of tool dependencies:
+
+- Dependency management tool `bender`:
+   ```bash
+   # Bender tool
+   cargo install bender --version 0.23.2
+   ```
+- Instruction-accurate simulator `banshee`:
+  Follow instructions on [Banshee Github Repo](https://github.com/pulp-platform/banshee)

sw/snRuntime/src/start.c

@@ -84,12 +84,6 @@ static inline void snrt_init_cls() {
 
 static inline void snrt_init_libs() { snrt_alloc_init(); }
 
-static inline void snrt_exit(int exit_code) {
-    extern volatile uint32_t tohost;
-
-    if (snrt_global_core_idx() == 0) tohost = (exit_code << 1) | 1;
-}
-
 void snrt_main() {
     int exit_code = 0;
 
@@ -160,6 +154,6 @@ void snrt_main() {
 #endif
 
 #ifdef SNRT_CRT0_CALLBACK8
-    snrt_crt0_callback8(exit_code);
+    snrt_crt0_callback8();
 #endif
 }

target/common/common.mk

@@ -52,7 +52,7 @@ VLT_FLAGS    += -Wno-UNOPTFLAT
 VLT_FLAGS    += -Wno-fatal
 VLT_FLAGS    += --unroll-count 1024
 VLT_CFLAGS   += -std=c++14 -pthread
-VLT_CFLAGS   +=-I ${VLT_BUILDDIR} -I $(VLT_ROOT)/include -I $(VLT_ROOT)/include/vltstd -I $(VLT_FESVR)/include -I $(TB_DIR)
+VLT_CFLAGS   +=-I ${VLT_BUILDDIR} -I $(VLT_ROOT)/include -I $(VLT_ROOT)/include/vltstd -I $(VLT_FESVR)/include -I $(TB_DIR) -I ${MKFILE_DIR}/test
 
 ANNOTATE_FLAGS ?= -q --keep-time
 

target/common/test/common_lib.cc

@@ -17,7 +17,7 @@ extern const uint8_t tb_bootrom_end;
 
 asm(".global tb_bootrom_start \n"
     ".global tb_bootrom_end \n"
-    "tb_bootrom_start: .incbin \"test/bootrom.bin\" \n"
+    "tb_bootrom_start: .incbin \"bootrom.bin\" \n"
     "tb_bootrom_end: \n");
 
 // The global memory all memory ports write into.

target/snitch_cluster/Makefile

@@ -253,8 +253,8 @@ clean-vcs: clean-work
 bin/snitch_cluster.vcs: ${VCS_SOURCES} ${TB_SRCS} $(TB_CC_SOURCES) $(VCS_BUILDDIR)/compile.sh work/lib/libfesvr.a
 	mkdir -p bin
 	vcs -Mlib=$(VCS_BUILDDIR) -Mdir=$(VCS_BUILDDIR) -o bin/snitch_cluster.vcs -cc $(CC) -cpp $(CXX) \
-		-assert disable_cover -override_timescale=1ns/1ps -full64 tb_bin $(TB_CC_SRCS) \
-		-CFLAGS $(TB_CC_FLAGS) -LDFLAGS "-L${FESVR}/lib" -lfesvr
+		-assert disable_cover -override_timescale=1ns/1ps -full64 tb_bin $(TB_CC_SOURCES) \
+		-CFLAGS "$(TB_CC_FLAGS)" -LDFLAGS "-L${FESVR}/lib" -lfesvr
 
 ##########
 # Traces #

target/snitch_cluster/README.md

@@ -31,6 +31,9 @@ vcs-2020.12 make bin/snitch_cluster.vcs
 
 These commands compile the RTL sources respectively in `work-vlt`, `work-vsim` and `work-vcs`. Additionally, common C++ testbench sources (e.g. the [frontend server (fesvr)](https://github.com/riscv-software-src/riscv-isa-sim)) are compiled under `work`. Each command will also generate a script or an executable (e.g. `bin/snitch_cluster.vsim`) which you can invoke to simulate the hardware. We will see how to do this in a later section.
 
+### Building the Banshee simulator
+Instead of running an RTL simulation, you can use our instruction-accuracte simulator called `banshee`. To install the simulator, please follow the instructions of the Banshee repository: [https://github.com/pulp-platform/banshee](https://github.com/pulp-platform/banshee).
+
 ### Cluster configuration
 
 Note that the Snitch cluster RTL sources are partly automatically generated from a configuration file provided in `.hjson` format. Several RTL files are templated and use the `.hjson` configuration file to fill the template entries. An example is `/hw/snitch_cluster/src/snitch_cluster_wrapper.sv.tpl`.
@@ -45,18 +48,26 @@ make CFG_OVERRIDE=cfg/custom.hjson bin/snitch_cluster.vlt
 
 ___Note:__ whenever you override the configuration file on the `make` command-line, the configuration will be stored in the `cfg/lru.hjson` file. Successive invocations of `make` will automatically pick up the `cfg/lru.hjson` file. You can therefore omit the `CFG_OVERRIDE` definition in successive commands unless you want to override the least-recently used configuration._
 
+Banshee uses also a cluster configuration file, however, that is given directly when simulating a specific binary with banshee with the help of the `--configuration <cluster_config.yaml>`.
+
 ### Building the software
 
 To build all of the software for the Snitch cluster, run the following command:
 
 ```bash
+# for RTL simulation
 make DEBUG=ON sw
+
+# for Banshee simulation (requires slightly different runtime)
+make SELECT_RUNTIME=banshee DEBUG=ON sw
 ```
 
 The `sw` target first generates some C header files which depend on the hardware configuration. Hence, the need to generate the software for the same configuration as your hardware. Afterwards, it recursively invokes the `make` target in the `sw` subdirectory to build the apps/kernels which have been developed in that directory.
 
 The `DEBUG=ON` flag is used to tell the compiler to produce debugging symbols. It is necessary for the `annotate` target, showcased in the Debugging section of this guide, to work.
 
+The `SELECT_RUNTIME` is set by default to `rtl`. To use the Banshee runtime, it requires to be set to be set to `banshee` with `SELECT_RUNTIME=banshee`.
+
 ___Note:__ the RTL is not the only source which is generated from the configuration file. The software stack also depends on the configuration file. Make sure you always build the software with the same configuration of the hardware you are going to run it on._
 
 ### Running a simulation
@@ -81,6 +92,9 @@ questa-2022.3 bin/snitch_cluster.vsim sw/apps/blas/axpy/build/axpy.elf
 
 # VCS (for IIS users)
 vcs-2020.12 bin/snitch_cluster.vcs sw/apps/blas/axpy/build/axpy.elf
+
+# Banshee
+banshee --no-opt-llvm --no-opt-jit --configuration src/banshee.yaml --trace sw/apps/blas/axpy/build/axpy.elf
 ```
 
 The previous commands will run the simulation in your current terminal. You can also run the simulation in the QuestaSim GUI by adapting the previous command to:
@@ -90,6 +104,9 @@ The previous commands will run the simulation in your current terminal. You can
 questa-2022.3 bin/snitch_cluster.vsim.gui sw/apps/blas/axpy/build/axpy.elf
 ```
 
+For Banshee, you need to give a specific cluster configuration to the simulator with the flag `--configuration <cluster_config.yaml>`. A default Snitch cluster configuration is given (`src/banshee.yaml`). The flag `--trace` enables the printing of the traces similar to the RTL simulation.
+For more information and debug options, please have a look at the Banshee repository: [https://github.com/pulp-platform/banshee](https://github.com/pulp-platform/banshee).
+
 ### Creating your first Snitch app
 
 In the following you will create your own AXPY kernel implementation as an example how to develop software for Snitch.