Merge branch 'feature/buildgroup' into feature/ci_update

.github/workflows/ci.yml

@@ -3,8 +3,6 @@
 name: CI
 
 on:
-  push:
-    branches: [develop]
   pull_request:
     branches: [develop]
 
@@ -14,83 +12,154 @@ concurrency:
 
 env:
   REPO: ${{ github.event.repository.name }}
-  BUNDLE: pr-${{ github.event.number }}  
+  PR: pr-${{ github.event.number }}
+  GH_TOKEN: ${{ secrets.GH_PAT }}
+
 
 jobs:
   build1:
-    if: github.repository_owner == 'MetOffice'
-    name: gnu-openmpi (JCSDA)
-    runs-on: ubuntu-latest
+    if: |
+      ( github.repository_owner == 'MetOffice' && github.event.pull_request.draft == false )
+    name: gnu-openmpi (UKMO)
+    runs-on: jedi-self-hosted-rg
 
     steps:
+      - name: Pre-submit cleanup
+        run: |
+          rm -fr ./${{ env.PR }} || true
+          ls -la ./
       - name: Checkout current
         uses: actions/checkout@v4
         with:
-          path: ./${{ env.BUNDLE }}/${{ env.REPO }}
-      - name: initiate bundle
-        run: cp ./${{ env.BUNDLE }}/${{ env.REPO }}/ci/* ./${{ env.BUNDLE }}/
+          path: ./${{ env.PR }}/${{ env.REPO }}
+      - name: Initiate bundle
+        run: cp ./${{ env.PR }}/${{ env.REPO }}/ci/* ./${{ env.PR }}/
 
       - name: Checkout jedicmake
         uses: actions/checkout@v4
         with:
-          path: ./${{ env.BUNDLE }}/jedicmake
+          path: ./${{ env.PR }}/jedicmake
           repository: JCSDA-internal/jedi-cmake
-          submodules: recursive
+          submodules: true
           token: ${{ secrets.GH_PAT }}
 
       - name: Checkout oops
         uses: actions/checkout@v4
         with:
-          path: ./${{ env.BUNDLE }}/oops
+          path: ./${{ env.PR }}/oops
           repository: JCSDA-internal/oops
           token: ${{ secrets.GH_PAT }}
-      - name: Checkout attempt for oops@${{ github.head_ref }}
+      - name: Check build-group for oops
         if: github.event_name == 'pull_request'
-        run: ./${{ env.BUNDLE }}/get-ref-branch ./${{ env.BUNDLE }}/oops
+        run: ./${{ env.PR }}/get-ref-branch ./${{ env.PR }}/oops
+
+      - name: Build and Test
+        env:
+          WORKDIR: /var/tmp/${{ env.REPO }}/${{ env.PR }}
+          MOBB_DATA: /var/tmp/mobb-data
+        shell: bash
+        run: |
+          az acr login --name ngmssboxjediacr
+          ln -sf /testdata/mobb-data ${{ github.workspace }}/mobb-data
+
+          docker run --rm \
+            --name ${REPO}_${PR}_${GITHUB_SHA::7} \
+            --volume ${{ github.workspace }}/${PR}:${WORKDIR} \
+            --volume ${{ github.workspace }}/mobb-data:${MOBB_DATA} \
+            --env CI_TESTS=1 \
+            --workdir=${WORKDIR} \
+            --entrypoint=/usr/local/src/${REPO}/${PR}/build-and-test.sh \
+            'ngmssboxjediacr.azurecr.io/jedibase:alma9'
+
+      - name: Cleanup
+          working-directory: ${{github.workspace}}
+          run: rm -fr ${{ env.PR }} || true
+
+
+  build2:
+    if: |
+      ( github.repository_owner == 'MetOffice' && github.event.pull_request.draft == false )
+    name: gnu-openmpi (JCSDA)
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: checkout current
+        uses: actions/checkout@v4
+        with:
+          path: ./${{ env.PR }}/${{ env.REPO }}
+      - name: Initiate bundle
+        run: cp ./${{ env.PR }}/${{ env.REPO }}/ci/* ./${{ env.PR }}/
+
+      - name: checkout jedicmake
+        uses: actions/checkout@v4
+        with:
+          path: ./${{ env.PR }}/jedicmake
+          repository: JCSDA-internal/jedi-cmake
+          submodules: recursive
+          token: ${{ secrets.GH_PAT }}
+
+      - name: checkout oops
+        uses: actions/checkout@v4
+        with:
+          path: ./${{ env.PR }}/oops
+          repository: JCSDA-internal/oops
+          token: ${{ secrets.GH_PAT }}
+      - name: Check build-group for oops
+        if: github.event_name == 'pull_request'
+        run: ./${{ env.PR }}/check-build-group.sh ./${{ env.PR }}/oops
 
       - name: Build and Lint
+        env:
+          WORKDIR: /usr/local/src/${{ env.REPO }}/${{ env.PR }}
+
         run: |
           docker run --rm \
-            --entrypoint=/usr/local/src/${REPO}/${BUNDLE}/build-and-lint \
-            --workdir=/usr/local/src/${REPO}/${BUNDLE} \
-            --volume ${PWD}/${BUNDLE}:/usr/local/src/${REPO}/${BUNDLE} \
+            --volume ${PWD}/${PR}:${WORKDIR} \
+            --env CI_TESTS=0 \
+            --workdir=${WORKDIR} \
+            --entrypoint=${WORKDIR}/build-and-test.sh \
             'jcsda/docker-gnu-openmpi-dev:latest'
 
-  build2:
-    if: github.repository_owner == 'MetOffice'
+  build3:
+    if: |
+      ( github.repository_owner == 'MetOffice' && github.event.pull_request.draft == false )
     name: clang-mpich (JCSDA)
     runs-on: ubuntu-latest
 
     steps:
-      - name: Checkout current
+      - name: checkout current
         uses: actions/checkout@v4
         with:
-          path: ./${{ env.BUNDLE }}/${{ env.REPO }}
-      - name: initiate bundle
-        run: cp ./${{ env.BUNDLE }}/${{ env.REPO }}/ci/* ./${{ env.BUNDLE }}/
+          path: ./${{ env.PR }}/${{ env.REPO }}
+      - name: initiate PR
+        run: cp ./${{ env.PR }}/${{ env.REPO }}/ci/* ./${{ env.PR }}/
 
-      - name: Checkout jedicmake
+      - name: checkout jedicmake
         uses: actions/checkout@v4
         with:
-          path: ./${{ env.BUNDLE }}/jedicmake
+          path: ./${{ env.PR }}/jedicmake
           repository: JCSDA-internal/jedi-cmake
           submodules: recursive
           token: ${{ secrets.GH_PAT }}
 
       - name: Checkout oops
         uses: actions/checkout@v4
         with:
-          path: ./${{ env.BUNDLE }}/oops
+          path: ./${{ env.PR }}/oops
           repository: JCSDA-internal/oops
           token: ${{ secrets.GH_PAT }}
-      - name: Checkout attempt for oops@${{ github.head_ref }}}
+      - name: Check build-group for oops
         if: github.event_name == 'pull_request'
-        run: ./${{ env.BUNDLE }}/get-ref-branch ./${{ env.BUNDLE }}/oops
+        run: ./${{ env.PR }}/check-build-group.sh ./${{ env.PR }}/oops
 
       - name: Build and Lint
+        env:
+          WORKDIR: /usr/local/src/${{ env.REPO }}/${{ env.PR }}
+
         run: |
           docker run --rm \
-            --entrypoint=/usr/local/src/${REPO}/${BUNDLE}/build-and-lint \
-            --workdir=/usr/local/src/${REPO}/${BUNDLE} \
-            --volume ${PWD}/${BUNDLE}:/usr/local/src/${REPO}/${BUNDLE} \
+            --volume ${PWD}/${PR}:${WORKDIR} \
+            --env CI_TESTS=0 \
+            --workdir=${WORKDIR} \
+            --entrypoint=${WORKDIR}/build-and-test.sh \
             'jcsda/docker-clang-mpich-dev:latest'

CMakeLists.txt

@@ -30,7 +30,7 @@ set(MONIO_LINKER_LANGUAGE CXX)
 find_package(jedicmake QUIET)  # Prefer find modules from jedi-cmake
 find_package(MPI REQUIRED COMPONENTS CXX)
 find_package(HDF5 REQUIRED COMPONENTS)
-find_package(NetCDF COMPONENTS CXX)
+find_package(NetCDF COMPONENTS Fortran CXX)
 find_package(eckit 1.16.1 REQUIRED COMPONENTS MPI)
 find_package(atlas 0.20.2 REQUIRED)
 find_package(oops 1.0.0 REQUIRED)

ci/build-and-test.sh

@@ -0,0 +1,72 @@
+#!/bin/bash
+#
+# (C) Crown Copyright 2024, the Met Office. All rights reserved.
+#
+# shellcheck disable=SC2317
+# shellcheck source=/dev/null
+set -euo pipefail
+
+finally() {
+    trap '' ERR
+    trap '' EXIT
+    if [[ -d "${WORKD:-}" ]]; then
+        cd /
+        rm -fr "${WORKD}"
+    fi
+    if [[ -d ${BASE:-} ]]; then
+        cd /
+        rm -fr "$BASE" 2>/dev/null || true
+    fi
+}
+
+# -- HERE is /usr/local/src/<REPONAME>/pr-<#> (cf ../.github/workflow/ci.yml)
+HERE="$(cd "$(dirname "$0")" && pwd)"
+THIS="$(basename "$0")"
+NPROC=${NPROC:-$(nproc)}
+WORKD="$(mktemp -d "${THIS}-XXXXXX" -t)"
+BASE="${HERE%/*}"
+TESTDIR="${BASE##*/}"
+export CI_TESTS=${CI_TESTS:-0}
+export OMPI_ALLOW_RUN_AS_ROOT=1
+export OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
+CI_TESTS=${CI_TESTS//$'\r'}  # remove those pesky invisible EOL characters
+
+trap finally ERR EXIT
+
+cd "${WORKD}"
+
+# -- Activate spack env if using JCSDA Docker container
+if [[ -f /opt/spack-environment/activate.sh ]]; then
+    source /opt/spack-environment/activate.sh
+fi
+
+# -- Enable OpenMPI over subscription -----------------------------------------
+if command -v ompi_info &>/dev/null; then
+    ompi_vn=$(ompi_info | awk '/Ident string:/ {print $3}')
+    case $ompi_vn in
+        4.*) export OMPI_MCA_rmaps_base_oversubscribe=1 ;;
+        5.*) export PRTE_MCA_rmaps_default_mapping_policy=:oversubscribe ;;
+    esac
+fi
+
+if command -v ninja &>/dev/null; then
+    GENERATOR=Ninja;
+else
+    GENERATOR=Unix\ Makefiles;
+fi
+
+# -- Configure
+cmake -B . -S "${HERE}" -G "${GENERATOR}" -DCMAKE_BUILD_TYPE=Release
+
+# -- Build
+cmake --build . -j "${NPROC}"
+
+# -- Test
+case "$CI_TESTS" in
+    0) ctest -j"${NPROC}" --test-dir "${TESTDIR}" --output-on-failure -R 'coding_norms' ;;
+    1) ctest -j"${NPROC}" --test-dir "${TESTDIR}" --output-on-failure ;;
+    *) echo "** Error: CI_TESTS=$CI_TESTS is not a valid option **";
+       exit 1 ;;
+esac
+
+exit

ci/check-build-group.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+#
+# (C) Copyright 2024 MetOffice Crown Copyright
+#
+# This script attempts to checkout a branch for a particular repository
+# annotated in the body of the PR from which the GitHub action is triggered.
+#
+set -euo pipefail
+
+action_repo="${GITHUB_REPOSITORY}"  # "owner/repository"
+action_branch="${GITHUB_HEAD_REF}"
+if [[ "${action_branch}" == 'develop' ]]; then
+    exit
+fi
+
+target_dir="$1"
+target_repo="${target_dir##*/}"
+found=$(gh pr view "${action_branch}" -R "${action_repo}" \
+        | grep 'build-group=' \
+        | grep -E "/${target_repo}/|/${target_repo}#") || {
+            echo "-- No build-group"; exit 0; }
+echo "$found"
+
+cd "$target_dir"
+target_pr=${found##*/}
+if [[ $target_pr =~ '#' ]]; then
+    target_pr=${target_pr##*#}
+fi
+target_pr=${target_pr//$'\r'}  # remove those pesky invisible EOL characters
+target_branch=$(gh pr list --search "$target_pr" \
+                --json number,headRefName \
+                -q ".[]|select(.number|contains($target_pr))|.headRefName")
+git fetch origin "pull/${target_pr}/head:${target_branch}"
+gh pr checkout "$target_pr"
+echo "-- Switched to $(git branch --show-current)"
+
+exit