Unit tests (using pFunit)

.github/workflows/test_suite_ubuntu.yml

@@ -62,24 +62,44 @@ jobs:
           . ftorch/bin/activate
           pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu
 
+      # MPI is required by pFUnit
+      - name: Install an MPI distribution
+        run: |
+          sudo apt update
+          sudo apt install mpich
+
+      - name: Install pFUnit
+        run: |
+          export FC=/usr/bin/gfortran
+          export MPIF90=/usr/bin/mpif90
+          # TODO: Avoid version pinning (needed because version appears in install path)
+          git clone -b v4.10.0 https://github.com/Goddard-Fortran-Ecosystem/pFUnit.git
+          mkdir pFUnit/build
+          cd pFUnit/build
+          cmake ..
+          make -j 4 install
+
       - name: Build FTorch
         run: |
           . ftorch/bin/activate
           VN=$(python -c "import sys; print('.'.join(sys.version.split('.')[:2]))")
           export Torch_DIR=${VIRTUAL_ENV}/lib/python${VN}/site-packages
           export BUILD_DIR=$(pwd)/src/build
+          # NOTE: The pFUnit version (pinned during installation above) is used in the install path.
+          export PFUNIT_DIR=$(pwd)/pFUnit/build/installed/PFUNIT-4.10
           mkdir ${BUILD_DIR}
           cd ${BUILD_DIR}
           cmake .. \
             -DPython_EXECUTABLE="$(which python)" \
             -DCMAKE_BUILD_TYPE=Release \
             -DCMAKE_INSTALL_PREFIX=${BUILD_DIR} \
             -DCMAKE_BUILD_TESTS=TRUE \
-            -DCMAKE_Fortran_FLAGS="-std=${{matrix.std}}"
+            -DCMAKE_PREFIX_PATH="${PFUNIT_DIR};${Torch_DIR}" \
+            -DCMAKE_Fortran_FLAGS="-std=${{ matrix.std }}"
           cmake --build .
           cmake --install .
 
-      - name: Integration tests
+      - name: Test suite
         run: |
           . ftorch/bin/activate
-          ./run_integration_tests.sh -V
+          ./run_test_suite.sh -V

.github/workflows/test_suite_windows.yml

@@ -92,4 +92,4 @@ jobs:
           for /f "tokens=2*" %%i in ('pip show torch ^| findstr /R "^Location"') do set torch_path=%%i
           set PATH=C:\Program Files (x86)\FTorch\bin;%PATH%
           set PATH=%torch_path%\torch\lib;%PATH%
-          run_integration_tests.bat
+          run_test_suite.bat

pages/testing.md

@@ -4,35 +4,38 @@ title: FTorch test suite
 
 ## Testing
 
-FTorch's test suite currently comprises of integration tests based on a subset
-of the [examples](examples.html). These tests are built and run and their
-outputs are analysed to check they contain expected regular expressions.
+FTorch's test suite is currently comprised of unit tests, as well as integration
+tests based on a subset of the [examples](examples.html). These tests are built
+and run and their outputs are analysed.
 
-### Building the integration tests
+### Building
 
-To enable FTorch's integration tests, ensure that the `CMAKE_BUILD_TESTS` option
-is set to `TRUE` for the build i.e., `-DCMAKE_BUILD_TESTS=True`.
+To enable FTorch's test suite, ensure that the `CMAKE_BUILD_TESTS` option
+is set to `TRUE` for the build,  i.e., `-DCMAKE_BUILD_TESTS=True`.. For the unit
+tests, you will also need to install
+[pFUnit](https://github.com/Goddard-Fortran-Ecosystem/pFUnit) and pass its
+install directory to the `CMAKE_PREFIX_PATH` when building FTorch.
 
-### Running the integration tests
+### Running
 
 Once the build is complete, activate the Python virtual environment you created
 for FTorch<sup>1</sup> and simply run the
 [helper script](https://github.com/Cambridge-ICCS/FTorch/blob/main/run_integration_tests.sh)
 in the root FTorch directory. Depending on the OS you are running you will need
 to use either:
-
-- `./run_integration_tests.sh` for unix (mac and linux)
-- `run_integration_tests.bat` for windows
+```
+- `./run_test_suite.sh` for Unix (Mac and Linux)
+- `run_test_suite.bat` for Windows (note this will only run the integration tests, not unit tests).
 
 This will automatically install any additional Python dependencies for the
 examples.
 
-Alternatively, individual tests may be run by going to the corresponding
-subdirectory of `${BUILD_DIR}/test/examples` (where `${BUILD_DIR}` is the build
-directory for FTorch) and calling `ctest`. This will produce a report on which
-tests passed and which failed for your build. Note that some of the examples
-have additional dependencies, which may need installing into your virtual
-environment.
+Alternatively, individual tests may be run by going to `${BUILD_DIR}/test/unit`
+or the corresponding subdirectory of `${BUILD_DIR}/test/examples` (where
+`${BUILD_DIR}` is the build directory for FTorch) and calling `ctest`. This
+will produce a report on which tests passed and which failed for your build.
+Note that some of the examples have additional dependencies, which may need
+installing into your virtual environment.
 
 > <sup>1</sup> _If you built FTorch against libtorch (rather than creating a
 virtual environment) then either

run_integration_tests.bat → run_test_suite.bat

@@ -8,6 +8,9 @@ rem
 rem See `src/test/README.md` for more details on integration testing.
 rem ---
 
+rem NOTE: This version of run_test_suite only runs the integration tests, not
+rem       the unit tests. These are not currently supported on Windows.
+
 for /d %%i in (1_SimpleNet 2_ResNet18 4_MultiIO) do (
 pushd src\build\test\examples\%%i
 rem run the tests

run_test_suite.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+# ---
+# Execute this shell script to run FTorch's test suite. This includes both unit
+# tests and integration tests.
+#
+# Assumes FTorch has been built with the `-DCMAKE_BUILD_TESTS=TRUE` option.
+# The `BUILD_DIR` variable in this script should be updated as appropriate for
+# your configuration.
+#
+# See `src/test/README.md` for more details on the test suite.
+# ---
+
+set -eu
+
+# Function to display help text
+show_help() {
+  echo "Usage: $0 [BUILD_DIR=<build_dir>] [--verbose | -V]"
+  echo
+  echo "Options:"
+  echo "  BUILD_DIR=<build_dir> Specify the build directory (default: src/build)."
+  echo "  --verbose | -V        Run with verbose ctest output."
+  echo "  --help | -h           Show this help message and exit."
+}
+
+# Parse command line arguments
+BUILD_DIR="src/build"
+VERBOSE=false
+HELP=false
+for ARG in "$@"; do
+  case ${ARG} in
+  BUILD_DIR=*)
+    BUILD_DIR="${ARG#*=}"
+    ;;
+  --verbose | -V)
+    VERBOSE=true
+    shift
+    ;;
+  --help | -h)
+    HELP=true
+    shift
+    ;;
+  *)
+    echo "Unknown argument: ${ARG}"
+    show_help
+    exit 1
+    ;;
+  esac
+done
+
+# Check for --help option
+if [ "${HELP}" = true ]; then
+  show_help
+  exit 0
+fi
+
+# Process command line arguments
+if [ "${VERBOSE}" = true ]; then
+  CTEST_ARGS="-V"
+else
+  CTEST_ARGS=""
+fi
+
+# Run unit tests
+cd "${BUILD_DIR}/test/unit"
+ctest "${CTEST_ARGS}"
+cd -
+
+# Run integration tests
+EXAMPLES="1_SimpleNet 2_ResNet18 4_MultiIO 6_Autograd"
+for EXAMPLE in ${EXAMPLES}; do
+  pip -q install -r examples/"${EXAMPLE}"/requirements.txt
+  cd "${BUILD_DIR}"/test/examples/"${EXAMPLE}"
+  ctest "${CTEST_ARGS}"
+  cd -
+done

src/CMakeLists.txt

@@ -109,6 +109,15 @@ if(CMAKE_BUILD_TESTS)
     COMPONENTS Interpreter
     REQUIRED)
 
+  enable_testing()
+
+  # Unit tests
+  # NOTE: We do not currently support unit testing on Windows
+  if(UNIX)
+    add_subdirectory(test/unit)
+  endif()
+
+  # Integration tests
   file(MAKE_DIRECTORY test/examples)
   file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/../examples/CMakeLists.txt
        DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/test/examples)