Run the integrated tests on GHA

.devcontainer/devcontainer.json

@@ -2,7 +2,7 @@
     "build": {
         "dockerfile": "Dockerfile",
         "args": {
-            "GEOS_TPL_TAG": "249-98"
+            "GEOS_TPL_TAG": "251-99"
         }
     },
     "runArgs": [

.github/workflows/build_and_test.yml

@@ -72,68 +72,71 @@ jobs:
 
     - name: Build, test, deploy.
       run: |
+        # Those two bash arrays will be populated depending on the required options,
+        # and expended as CLI arguments for the docker and scripts calls.
+        docker_args=()
+        script_args=()
+
         COMMIT=${{ github.event.pull_request.head.sha }}
         SHORT_COMMIT=${COMMIT:0:7}
-
+        script_args+=(--install-dir-basename GEOSX-${SHORT_COMMIT})
+
         # All the data exchanged with the docker container is eventually meant to be send to the cloud. 
         if [[ ! -z "${{ inputs.GCP_BUCKET }}" ]]; then
           if [ "${{ inputs.BUILD_TYPE }}" = "build" ]; then
             DATA_BASENAME=GEOSX-and-TPL-${SHORT_COMMIT}.tar.gz
           elif [ "${{ inputs.BUILD_TYPE }}" = "integrated_tests" ]; then
             DATA_BASENAME=integratedTests-pr${{ github.event.number }}-${{ github.run_number }}-${SHORT_COMMIT}.tar.gz
-            RUN_INTEGRATED_TESTS_CLI_ARGS=--run-integrated-tests
+            script_args+=(--run-integrated-tests)
           fi
 
-          DATA_BASENAME_CLI_ARGS="--data-basename ${DATA_BASENAME}"
-          
+          script_args+=(--data-basename ${DATA_BASENAME})
+
           DATA_EXCHANGE_DIR=/mnt/geos-exchange  # Exchange folder outside of the container
           sudo mkdir -p ${DATA_EXCHANGE_DIR}
           DATA_EXCHANGE_MOUNT_POINT=/tmp/exchange  # Exchange folder inside of the container
-          DATA_EXCHANGE_DOCKER_CLI_ARGS="--volume=${DATA_EXCHANGE_DIR}:${DATA_EXCHANGE_MOUNT_POINT}"
-          DATA_EXCHANGE_SCRIPT_CLI_ARGS="--exchange-dir ${DATA_EXCHANGE_MOUNT_POINT}"
+          docker_args+=(--volume=${DATA_EXCHANGE_DIR}:${DATA_EXCHANGE_MOUNT_POINT})
+          script_args+=(--exchange-dir ${DATA_EXCHANGE_MOUNT_POINT})
         fi
 
         HOST_CONFIG=${{ inputs.HOST_CONFIG }}
-        HOST_CONFIG_CLI_ARGS=${HOST_CONFIG:+"--host-config ${HOST_CONFIG}"}
-        
+        script_args+=(${HOST_CONFIG:+"--host-config ${HOST_CONFIG}"})
+
         if ${{ inputs.USE_SCCACHE }} == 'true'; then
-          SCCACHE_CLI_ARGS="--sccache-credentials $(basename ${GOOGLE_GHA_CREDS_PATH})"
+          script_args+=(--sccache-credentials $(basename ${GOOGLE_GHA_CREDS_PATH}))
         fi
 
         # We need to know where the code folder is mounted inside the container so we can run the script at the proper location!
         # Since this information is repeated twice, we use a variable.
         GITHUB_WORKSPACE_MOUNT_POINT=/tmp/geos
+        docker_args+=(--volume=${GITHUB_WORKSPACE}:${GITHUB_WORKSPACE_MOUNT_POINT})
+        script_args+=(--repository ${GITHUB_WORKSPACE_MOUNT_POINT})
 
         # The linear algebra environment variables (ENABLE_HYPRE, ENABLE_HYPRE_DEVICE & ENABLE_TRILINOS)
         # could be passed as scripts parameters as well, but a specific care must be taken to be sure
         # there's no conflict with the host-config files.
         ENABLE_HYPRE=${{ inputs.ENABLE_HYPRE }}
         ENABLE_HYPRE_DEVICE=${{ inputs.ENABLE_HYPRE_DEVICE }}
         ENABLE_TRILINOS=${{ inputs.ENABLE_TRILINOS }}
+        docker_args+=(-e ENABLE_HYPRE=${ENABLE_HYPRE:-OFF})
+        docker_args+=(-e ENABLE_HYPRE_DEVICE=${ENABLE_HYPRE_DEVICE:-CPU})
+        docker_args+=(-e ENABLE_TRILINOS=${ENABLE_TRILINOS:-ON})
+
+        docker_args+=(--cap-add=SYS_PTRACE)
+
+        script_args+=(--cmake-build-type ${{ inputs.CMAKE_BUILD_TYPE }})
+        script_args+=(${{ inputs.BUILD_AND_TEST_CLI_ARGS }})
 
         # In case of integrated tests run, we still want to send the results to the cloud for inspection.
         # While for standard build (if even possible), pushing a failed build would be pointless.
         # GHA set `-e` to bash scripts by default to fail asap,
         # but for this precise call, we want to deal with it more precisely
         set +e
         docker run \
-          --cap-add=SYS_PTRACE \
-          --volume=${GITHUB_WORKSPACE}:${GITHUB_WORKSPACE_MOUNT_POINT} \
-          ${DATA_EXCHANGE_DOCKER_CLI_ARGS} \
-          -e ENABLE_HYPRE=${ENABLE_HYPRE:-OFF} \
-          -e ENABLE_HYPRE_DEVICE=${ENABLE_HYPRE_DEVICE:-CPU} \
-          -e ENABLE_TRILINOS=${ENABLE_TRILINOS:-ON} \
+          ${docker_args[@]} \
           ${{ inputs.DOCKER_REPOSITORY }}:${{ inputs.DOCKER_IMAGE_TAG }} \
           ${GITHUB_WORKSPACE_MOUNT_POINT}/scripts/ci_build_and_test_in_container.sh \
-            --cmake-build-type ${{ inputs.CMAKE_BUILD_TYPE }} \
-            ${DATA_BASENAME_CLI_ARGS} \
-            ${DATA_EXCHANGE_SCRIPT_CLI_ARGS} \
-            ${HOST_CONFIG_CLI_ARGS} \
-            --install-dir-basename GEOSX-${SHORT_COMMIT} \
-            --repository ${GITHUB_WORKSPACE_MOUNT_POINT} \
-            ${SCCACHE_CLI_ARGS} \
-            ${RUN_INTEGRATED_TESTS_CLI_ARGS} \
-            ${{ inputs.BUILD_AND_TEST_CLI_ARGS }}
+            ${script_args[@]}
         EXIT_STATUS=$?
         echo "Received exit status ${EXIT_STATUS} from the build process."
         set -e
@@ -142,7 +145,7 @@ jobs:
         if [[ ! -z "${{ inputs.GCP_BUCKET }}" ]]; then
           if [[ "${{ inputs.BUILD_TYPE }}" = "integrated_tests" || ${EXIT_STATUS} -eq 0 ]]; then
             CLOUDSDK_PYTHON=python3 gsutil cp -a public-read ${DATA_EXCHANGE_DIR}/${DATA_BASENAME} gs://${{ inputs.GCP_BUCKET }}/
-            echo "Download the integrated tests at https://storage.googleapis.com/${{ inputs.GCP_BUCKET }}/${DATA_BASENAME}"
+            echo "Download the bundle at https://storage.googleapis.com/${{ inputs.GCP_BUCKET }}/${DATA_BASENAME}"
           fi
         fi
 

.github/workflows/ci_tests.yml

@@ -73,8 +73,6 @@ jobs:
   are_integrated_tests_requested:
     needs: [is_pull_request_a_draft]
     runs-on: ubuntu-22.04
-    # Temporarily deactivate
-    if: ${{ ! always() }}
     steps:
     - name: Check if running the integrated tests is requested
       run: |
@@ -146,45 +144,55 @@ jobs:
       matrix:
         include:
           - name: Ubuntu (20.04, gcc 9.3.0, open-mpi 4.0.3)
-            DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc9
             CMAKE_BUILD_TYPE: Release
+            DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc9
 
           - name: Ubuntu debug (20.04, gcc 10.3.0, open-mpi 4.0.3) - github codespaces
-            DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc10
             CMAKE_BUILD_TYPE: Debug
+            DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc10
 
           - name: Ubuntu (20.04, gcc 10.3.0, open-mpi 4.0.3) - github codespaces
+            CMAKE_BUILD_TYPE: Release
             DOCKER_REPOSITORY: geosx/ubuntu20.04-gcc10
+
+          - name: Ubuntu (22.04, gcc 11.2.0, open-mpi 4.1.2)
             CMAKE_BUILD_TYPE: Release
+            DOCKER_REPOSITORY: geosx/ubuntu22.04-gcc11
+            ENABLE_HYPRE: ON
+            ENABLE_TRILINOS: OFF
+            GCP_BUCKET: geosx/ubuntu22.04-gcc11
+
+          - name: Ubuntu debug (22.04, gcc 12.3.0, open-mpi 4.1.2)
+            BUILD_AND_TEST_CLI_ARGS: --no-run-unit-tests
+            CMAKE_BUILD_TYPE: Debug
+            DOCKER_REPOSITORY: geosx/ubuntu22.04-gcc12
+            ENABLE_HYPRE: ON
+            ENABLE_TRILINOS: OFF
 
-          # Below this line, jobs that deploy to Google Cloud.
           - name: Pecan CPU (centos 7.7, gcc 8.2.0, open-mpi 4.0.1, mkl 2019.5)
-            DOCKER_REPOSITORY: geosx/pecan-cpu-gcc8.2.0-openmpi4.0.1-mkl2019.5
             CMAKE_BUILD_TYPE: Release
+            DOCKER_REPOSITORY: geosx/pecan-cpu-gcc8.2.0-openmpi4.0.1-mkl2019.5
             HOST_CONFIG: host-configs/TOTAL/pecan-CPU.cmake
             GCP_BUCKET: geosx/Pecan-CPU
 
           - name: Pangea 2 (centos 7.6, gcc 8.3.0, open-mpi 2.1.5, mkl 2019.3)
-            DOCKER_REPOSITORY: geosx/pangea2-gcc8.3.0-openmpi2.1.5-mkl2019.3
             CMAKE_BUILD_TYPE: Release
-            GCP_BUCKET: geosx/Pangea2
+            DOCKER_REPOSITORY: geosx/pangea2-gcc8.3.0-openmpi2.1.5-mkl2019.3
             ENABLE_HYPRE: ON
             ENABLE_TRILINOS: OFF
+            GCP_BUCKET: geosx/Pangea2
 
           - name: Sherlock CPU (centos 7.9.2009, gcc 10.1.0, open-mpi 4.1.2, openblas 0.3.10)
-            DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-openblas0.3.10-zlib1.2.11
             CMAKE_BUILD_TYPE: Release
-            HOST_CONFIG: host-configs/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10.cmake
-            GCP_BUCKET: geosx/Sherlock-CPU
+            DOCKER_REPOSITORY: geosx/sherlock-gcc10.1.0-openmpi4.1.2-openblas0.3.10-zlib1.2.11
             ENABLE_HYPRE: ON
             ENABLE_TRILINOS: OFF
+            HOST_CONFIG: host-configs/Stanford/sherlock-gcc10-ompi4.1.2-openblas0.3.10.cmake
+            GCP_BUCKET: geosx/Sherlock-CPU
 
-          - name: Ubuntu (22.04, gcc 11.2.0, open-mpi 4.1.2)
-            DOCKER_REPOSITORY: geosx/ubuntu22.04-gcc11
-            CMAKE_BUILD_TYPE: Release
-            GCP_BUCKET: geosx/ubuntu22.04-gcc11
     uses: ./.github/workflows/build_and_test.yml
     with:
+      BUILD_AND_TEST_CLI_ARGS: ${{ matrix.BUILD_AND_TEST_CLI_ARGS }}
       CMAKE_BUILD_TYPE: ${{ matrix.CMAKE_BUILD_TYPE }}
       DOCKER_IMAGE_TAG: ${{ needs.is_pull_request_a_draft.outputs.DOCKER_IMAGE_TAG }}
       DOCKER_REPOSITORY: ${{ matrix.DOCKER_REPOSITORY }}
@@ -208,6 +216,8 @@ jobs:
       CMAKE_BUILD_TYPE: Release
       DOCKER_IMAGE_TAG: ${{ needs.is_pull_request_a_draft.outputs.DOCKER_IMAGE_TAG }}
       DOCKER_REPOSITORY: geosx/ubuntu22.04-gcc11
+      ENABLE_HYPRE: ON
+      ENABLE_TRILINOS: OFF
       GCP_BUCKET: geosx/integratedTests
       RUNS_ON: ubuntu-22.04
 

scripts/ci_build_and_test_in_container.sh

@@ -146,6 +146,7 @@ fi
 if [[ "${RUN_INTEGRATED_TESTS}" = true ]]; then
   echo "Running the integrated tests has been requested."
   # We install the python environment required by ATS to run the integrated tests.
+  or_die apt-get update
   or_die apt-get install -y virtualenv python3-dev python-is-python3
   ATS_PYTHON_HOME=/tmp/run_integrated_tests_virtualenv
   or_die virtualenv ${ATS_PYTHON_HOME}
@@ -216,7 +217,10 @@ if [[ "${RUN_INTEGRATED_TESTS}" = true ]]; then
   or_die ninja ats_environment
   # The tests are not run using ninja (`ninja --verbose ats_run`) because it swallows the output while all the simulations are running.
   # We directly use the script instead...
-  integratedTests/geos_ats.sh --failIfTestsFail
+  # Temporarily, we are not adding the `--failIfTestsFail` options to `geos_ats.sh`.
+  # Therefore, `ats` will exit with error code 0, even if some tests fail.
+  # Add `--failIfTestsFail` when you want `failIfTestsFail` to reflect the content of the tests.
+  integratedTests/geos_ats.sh
   # Even (and even moreover) if the integrated tests fail, we want to pack the results for further investigations.
   # So we store the status code for further use.
   INTEGRATED_TEST_EXIT_STATUS=$?