Improve Test Suite CI (#40)

Update pytest and plugins.

Log HTML and Junit.

Improve test parallelization.

Rerun failed tests.

Load dependencies in separate job.

Try to increase stack size instead of running tests with --forked.

Try to fix last remaining failing tests (upsampler and Alveo end2end).

.gitlab-ci.yml

@@ -1,21 +1,26 @@
 stages:
   - sync
   - singularity_build
+  - load_deps
   - test
 
 variables:
+  PIPELINE_NAME:
+    description: "Optional name to better identify this pipeline"
+    value: ""
   TEST_SUITE:
     description: "Select test suite to run"
     value: "full"
     options:
+      - "none"
       - "quicktest"
-      - "main"
-      - "rtlsim"
-      - "end2end"
       - "full"
   CPU_CORES:
     description: "Select number of CPU cores and test workers"
     value: "8"
+  PARALLEL_JOBS:
+    description: "Number of parallel Slurm array jobs per Benchmark job"
+    value: "2"
   SLURM_TIMEOUT:
     description: "Select SLURM timeout"
     value: "3-0" # [days-hours]
@@ -25,10 +30,14 @@ variables:
   SLURM_QOS:
     description: "Optional QoS option (include --qos, e.g., --qos express)"
     value: ""
-  SINGULARITY_IMG_SELECT:
-    value: "finn_dev.sif"
+  MANUAL_CFG_PATH:
+    description: "Use this config file instead of configs stored in the repo. Path must be accessible to runner"
+    value: ""
+  FINN_XILINX_VERSION:
+    value: "2022.2"
 
 workflow:
+  name: '$PIPELINE_NAME'
   rules:
     # Run pipeline for GitHub PRs to dev (does not support PRs from forks)
     - if: $CI_PIPELINE_SOURCE == "external_pull_request_event" && $CI_EXTERNAL_PULL_REQUEST_TARGET_BRANCH_NAME == "dev"
@@ -84,10 +93,44 @@ Singularity Image Build:
     - docker build --no-cache -f docker/Dockerfile.finn --tag=finn_docker_export .
     - apptainer build --force finn_singularity_image.sif docker-daemon://finn_docker_export:latest
     - rsync -vh finn_singularity_image.sif $PATH_SINGULARITY_IMG_BUILD/finn-plus/finn_$CI_COMMIT_REF_SLUG.sif
+    - echo SINGULARITY_IMG_SELECT=finn_$CI_COMMIT_REF_SLUG.sif > FINN_environment.env
   after_script: # Clean caches
     - echo 'y' | docker image prune
     - echo 'y' | docker builder prune
     - echo 'y' | apptainer cache clean
+  # Save env var selecting Singularity image to be used in subsequent jobs
+  artifacts:
+    reports:
+      dotenv: FINN_environment.env
+
+Fetch Repos:
+  id_tokens:
+    CI_JOB_JWT:
+      aud: https://git.uni-paderborn.de
+  stage: load_deps
+  tags:
+    - login
+  rules:
+    # Do not run on a schedule
+    - if: $CI_PIPELINE_SOURCE == "schedule"
+      when: never
+    # Otherwise run
+    - when: always
+  cache:
+    key: $CI_COMMIT_SHA
+    paths:
+      - deps
+  variables:
+    SINGULARITY_IMG_SELECT: "finn_dev.sif" # default, may be overwritten by dotenv artifact
+  script:
+    - ./fetch-repos.sh
+    # Workaround for https://gitlab.com/gitlab-org/gitlab/-/issues/349538
+    # Passing artifacts from optional parent jobs to child pipelines is not supported
+    # Therefore, we pass the dotenv artifact from "Singularity Image Build" through this job
+    - echo SINGULARITY_IMG_SELECT=$SINGULARITY_IMG_SELECT > FINN_environment_passthrough.env
+  artifacts:
+    reports:
+      dotenv: FINN_environment_passthrough.env
 
 FINN Test Suite 2022.2:
   id_tokens:
@@ -98,29 +141,36 @@ FINN Test Suite 2022.2:
     # Do not run on a schedule
     - if: $CI_PIPELINE_SOURCE == "schedule"
       when: never
-    # Select different Singularity image if it deviates from default (dev branch)
-    - changes:
-        paths:
-          - requirements.txt
-          - docker/Dockerfile.finn
-          - docker/finn_entrypoint.sh
-          - docker/quicktest.sh
-        compare_to: "dev"
-      variables:
-        SINGULARITY_IMG_SELECT: "finn_$CI_COMMIT_REF_SLUG.sif"
+    # Do not run if test suite has been deselected
+    - if: $TEST_SUITE == "none"
+      when: never
     # Always run, as long as there was no prior failure
     - when: on_success
+  cache:
+    key: $CI_COMMIT_SHA
+    policy: pull
+    paths:
+      - deps
   variables:
     SCHEDULER_PARAMETERS: "-A $PROJECT_ACCOUNT -p $SLURM_PARTITION -t $SLURM_TIMEOUT $SLURM_QOS --nodes 1 --ntasks 1 --cpus-per-task $CPU_CORES --exclusive"
     PYTEST_PARALLEL: "$CPU_CORES"
-    FINN_SINGULARITY: "$PATH_SINGULARITY_IMG/finn-plus/$SINGULARITY_IMG_SELECT"
     FINN_XILINX_VERSION: "2022.2"
   before_script:
     - cp -dfR .. $PATH_WORKDIR # Copy to working directory (e.g. RAMdisk)
     - cd $PATH_WORKDIR/finn-plus
+    - chmod 755 ./test.sh
     - module load system singularity
+    - ulimit -s unlimited # Increase stack size limit
+    - export FINN_SINGULARITY=$PATH_SINGULARITY_IMG/finn-plus/$SINGULARITY_IMG_SELECT
   script:
-    - ./run-docker.sh quicktest.sh $TEST_SUITE
+    - ./run-docker.sh ./test.sh $TEST_SUITE
+  artifacts:
+    name: "test_reports"
+    when: always
+    paths:
+      - reports/
+    reports:
+      junit: reports/*.xml
 
 FINN Test Suite 2024.1:
   extends: FINN Test Suite 2022.2

docker/Dockerfile.finn

@@ -103,21 +103,22 @@ RUN pip install ipykernel==6.21.2
 RUN pip install jupyter==1.0.0 --ignore-installed
 RUN pip install markupsafe==2.0.1
 RUN pip install matplotlib==3.7.0  --ignore-installed
-RUN pip install pytest-dependency==0.5.1
-RUN pip install pytest-xdist[setproctitle]==3.2.0
-RUN pip install pytest-parallel==0.1.1
+RUN pip install pytest-dependency==0.6.0
+RUN pip install pytest-xdist[setproctitle]==3.6.1
 RUN pip install "netron>=5.0.0"
 RUN pip install pandas==1.5.3
 RUN pip install scikit-learn==1.2.1
 RUN pip install tqdm==4.64.1
 RUN pip install -e git+https://github.com/fbcotter/[email protected]#egg=dataset_loading
 # these versions of pytest and associated plugins allow for stable collection of
 # test reports and code coverage reports in HTML
-RUN pip install pytest==6.2.5
-RUN pip install pytest-metadata==1.7.0
-RUN pip install pytest-html==3.0.0
-RUN pip install pytest-html-merger==0.0.8
-RUN pip install pytest-cov==4.1.0
+RUN pip install pytest==8.3.4
+RUN pip install pytest-metadata==3.1.1
+RUN pip install pytest-html==4.1.1
+RUN pip install pytest-html-merger==0.1.0
+RUN pip install pytest-cov==6.0.0
+RUN pip install pytest-forked==1.6.0
+RUN pip install pytest-rerunfailures==15.0
 RUN pip install pyyaml==6.0.1
 
 # extra dependencies from other FINN deps

docker/quicktest.sh

@@ -4,25 +4,23 @@
 
 cd $FINN_ROOT
 # check if command line argument is empty or not present
-if [ -z $1 ] || [ $1 = "quicktest" ]; then
+if [ -z $1 ]; then
   echo "Running quicktest: not (vivado or slow or board) with pytest-xdist"
-  pytest -m 'not (vivado or slow or vitis or board or notebooks or bnn_pynq)' --forked --dist=loadfile -n $PYTEST_PARALLEL
+  pytest -m 'not (vivado or slow or vitis or board or notebooks or bnn_pynq)' --dist=loadfile -n $PYTEST_PARALLEL
 elif [ $1 = "main" ]; then
   echo "Running main test suite: not (rtlsim or end2end) with pytest-xdist"
-  pytest -k 'not (rtlsim or end2end)' --forked --dist=loadfile -n $PYTEST_PARALLEL
+  pytest -k 'not (rtlsim or end2end)' --dist=loadfile -n $PYTEST_PARALLEL
 elif [ $1 = "rtlsim" ]; then
-  echo "Running rtlsim test suite with pytest-parallel"
-# there are end2end tests which also have rtlsim in their name, but not vice versa
-  pytest -k 'rtlsim and not end2end' --forked --workers $PYTEST_PARALLEL
+  echo "Running rtlsim test suite with pytest-xdist"
+  pytest -k rtlsim --dist=loadfile -n $PYTEST_PARALLEL
 elif [ $1 = "end2end" ]; then
   echo "Running end2end test suite with no parallelism"
-# filtering by name "end2end" is not sufficient, as the bnn tests need to be selected by a marker
-  pytest -k end2end --forked -m 'sanity_bnn or end2end or fpgadataflow or notebooks'
+  pytest -k end2end
 elif [ $1 = "full" ]; then
   echo "Running full test suite, each step with appropriate parallelism"
   $0 main;
   $0 rtlsim;
   $0 end2end;
 else
-  echo "Unrecognized argument to quicktest.sh: $1"
+  echo "Unrecognized argument to quicktest.sh"
 fi

requirements.txt

@@ -12,7 +12,6 @@ pre-commit==3.3.2
 protobuf==3.20.3
 psutil==5.9.4
 pyscaffold==4.4
-pytest-forked==1.6.0
 scipy==1.10.1
 setupext-janitor>=1.1.2
 sigtools==4.0.1

src/finn/custom_op/fpgadataflow/hls/upsampler_hls.py

@@ -35,7 +35,7 @@
 
 class UpsampleNearestNeighbour_hls(UpsampleNearestNeighbour, HLSBackend):
     """
-    Corresponds to finn-hlslib UpsampleNearestNeighbour_Batch function.
+    Corresponds to finn-hlslib UpsampleNearestNeighbour function.
     Upsampling is done with the Nearest Neighbour algorithm.
     The layer expects square feature maps for the in and output.
     """
@@ -78,7 +78,7 @@ def docompute(self):
         batch = self.get_nodeattr("numInputVectors")
         if is_2d:
             self.code_gen_dict["$DOCOMPUTE$"] = [
-                """UpsampleNearestNeighbour_Batch<OFMDim, IFMDim, IFMChannels,
+                """UpsampleNearestNeighbour<OFMDim, IFMDim, IFMChannels,
                 ap_uint<Input_precision> > (in0_%s, out_%s, numReps);"""
                 % (self.hls_sname(), self.hls_sname())
             ]

src/finn/util/basic.py

@@ -33,7 +33,7 @@
 from qonnx.util.basic import roundup_to_integer_multiple
 
 # test boards
-test_board_map = ["Pynq-Z1", "KV260_SOM", "ZCU104", "U250"]
+test_board_map = ["Pynq-Z1", "KV260_SOM", "ZCU104", "U280"]
 
 # mapping from PYNQ board names to FPGA part names
 pynq_part_map = dict()

test.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+: ${PYTEST_PARALLEL=auto}
+: ${CI_PROJECT_DIR="."}
+
+cd $FINN_ROOT
+
+if [ -z $1 ] || [ $1 = "quicktest" ]; then
+  pytest -m 'not (vivado or slow or vitis or board or notebooks or bnn_pynq)' --junitxml=$CI_PROJECT_DIR/reports/quick.xml --html=$CI_PROJECT_DIR/reports/quick.html --reruns 1 --dist worksteal -n $PYTEST_PARALLEL
+elif [ $1 = "full" ]; then
+  pytest -k 'not (rtlsim or end2end)' --junitxml=$CI_PROJECT_DIR/reports/main.xml --html=$CI_PROJECT_DIRreports/main.html --reruns 1 --dist worksteal -n $PYTEST_PARALLEL
+  pytest -k 'rtlsim and not end2end' --junitxml=$CI_PROJECT_DIR/reports/rtlsim.xml --html=$CI_PROJECT_DIR/reports/rtlsim.html --reruns 1 --dist worksteal -n $PYTEST_PARALLEL
+  pytest -k 'end2end' -m 'sanity_bnn or end2end or fpgadataflow or notebooks' --junitxml=$CI_PROJECT_DIR/reports/end2end.xml --html=$CI_PROJECT_DIR/reports/end2end.html --reruns 1 --dist loadfile -n $PYTEST_PARALLEL
+  pytest_html_merger -i $CI_PROJECT_DIR/reports/ -o $CI_PROJECT_DIR/reports/full_test_suite.html
+else
+  echo "Unrecognized argument to test.sh: $1"
+fi

tests/fpgadataflow/test_fpgadataflow_ipstitch.py

@@ -322,7 +322,7 @@ def test_fpgadataflow_ipstitch_iodma_floorplan():
 
 
 # board
-@pytest.mark.parametrize("board", ["U250"])
+@pytest.mark.parametrize("board", ["U280"])
 # clock period
 @pytest.mark.parametrize("period_ns", [5])
 # override mem_mode to external