Follow-up to #1363 (update from spack dev 20241031): set correct mpiifx/mpiifort in MPI meta module, update Intel MPI in site configs

.github/workflows/ubuntu-ci-x86_64-gnu.yaml

@@ -217,3 +217,6 @@ jobs:
           module load ewok-env
           module load soca-env
           module list
+
+          echo "Contents of MPI metamodule:"
+          cat ${ENVDIR}/install/modulefiles/gcc/11.4.0/stack-openmpi/5.0.5

.github/workflows/ubuntu-ci-x86_64-intel.yaml

@@ -249,3 +249,6 @@ jobs:
           module load ewok-env
           module load soca-env
           module list
+
+          echo "Contents of MPI metamodule:"
+          cat ${ENVDIR}/install/modulefiles/intel/2021.10.0/stack-intel-oneapi-mpi/2021.10.0

.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml

@@ -0,0 +1,248 @@
+name: ubuntu-ci-c6a-x86_64-oneapi-ifx-build
+on:
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  ubuntu-ci-c6a-x86_64-oneapi-ifx-build:
+    runs-on: [ubuntu-ci-c6a-x86_64]
+
+    steps:
+      - name: cleanup
+        run: |
+          pwd
+          ls -lart
+          set +e
+          find ./* -type d -exec chmod u+xw {} \;
+          set -e
+          rm -fr *
+
+      - name: checkout
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+
+      - name: prepare-directories
+        run: |
+          mkdir -p /home/ubuntu/spack-stack/build-cache/
+          mkdir -p /home/ubuntu/spack-stack/source-cache/
+
+      - name: create-buildcache
+        run: |
+          # Get day of week to decide whether to use build caches or not
+          DOW=$(date +%u)
+          # Monday is 1 ... Sunday is 7
+          if [[ $DOW == 7 ]]; then
+            export USE_BINARY_CACHE=false
+            echo "Ignore existing binary cache for creating buildcache environment"
+          else
+            export USE_BINARY_CACHE=true
+            echo "Use existing binary cache for creating buildcache environment"
+          fi
+
+          # Set up spack-stack
+          source ./setup.sh
+          export ENVNAME=ue-oneapi-ifx-2024.2.0-buildcache
+          export ENVDIR=$PWD/envs/${ENVNAME}
+          spack stack create env --site linux.default --template unified-dev --name ${ENVNAME} --compiler oneapi
+          spack env activate ${ENVDIR}
+          export SPACK_SYSTEM_CONFIG_PATH="${ENVDIR}/site"
+
+          # Find external packages
+          spack external find --scope system \
+              --exclude bison --exclude openssl \
+              --exclude curl --exclude python \
+              --exclude gmake
+          spack external find --scope system sed
+          spack external find --scope system perl
+          spack external find --scope system wget
+          spack external find --scope system texlive
+          spack external find --scope system mysql
+
+          # Find compilers
+          spack compiler find --scope system
+
+          # For Intel oneAPI
+          echo "" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
+          echo "- compiler:" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
+          echo "    spec: [email protected]" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
+          echo "    paths:" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
+          echo "      cc: /opt/intel/oneapi/compiler/2024.2/bin/icx" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
+          echo "      cxx: /opt/intel/oneapi/compiler/2024.2/bin/icpx" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
+          echo "      f77: /opt/intel/oneapi/compiler/2024.2/bin/ifx" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
+          echo "      fc: /opt/intel/oneapi/compiler/2024.2/bin/ifx" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
+          echo "    flags: {}" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
+          echo "    operating_system: ubuntu22.04" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
+          echo "    target: x86_64" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
+          echo "    modules: []" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
+          echo "    environment: {}" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
+          echo "    extra_rpaths: []" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml
+
+          # Need to find external Intel MPI and annotate with the
+          # correct compiler, no way to do that with spack commands.
+          echo "" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+          echo "  intel-oneapi-mpi:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+          echo "    buildable: false" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+          echo "    externals:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+          echo "    - spec: [email protected]%[email protected]" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+          echo "      prefix: /opt/intel/oneapi" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+
+          # Add external Intel MKL and oneAPI runtime
+          echo "" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+          echo "  intel-oneapi-mkl:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+          echo "    externals:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+          echo "    - spec: [email protected]%[email protected]" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+          echo "      prefix: /opt/intel/oneapi" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+          echo "  intel-oneapi-runtime:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+          echo "    externals:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+          echo "    - spec: [email protected]%[email protected]" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+          echo "      prefix: /opt/intel/oneapi" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+
+          export -n SPACK_SYSTEM_CONFIG_PATH
+
+          # For buildcaches
+          spack config add config:install_tree:padded_length:200
+
+          # Set compiler and MPI specs
+          spack config add "packages:all:providers:mpi:[[email protected]]"
+          spack config add "packages:all:compiler:[[email protected],[email protected]]"
+
+          # Add additional variants for MET packages, different from config/common/packages.yaml
+          spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis"
+
+          # Switch providers for blas, lapack, fftw-api
+          spack config add "packages:all:providers:mpi:[intel-oneapi-mpi]"
+          spack config add "packages:all:providers:blas:[intel-oneapi-mkl]"
+          spack config add "packages:all:providers:fftw-api:[intel-oneapi-mkl]"
+          spack config add "packages:all:providers:lapack:[intel-oneapi-mkl]"
+          spack config add "packages:ectrans:require:'+mkl ~fftw'"
+          spack config add "packages:gsibec:require:'+mkl'"
+          spack config add "packages:py-numpy:require:['^intel-oneapi-mkl']"
+
+          # Pin gmake to avoid duplicate packages (excluded gmake from spack external find above)
+          spack config add "packages:gmake:require:'@:4.2'"
+
+          # Remove wgrib2 from all virtual packages, since it doesn't build with oneAPI
+          sed -i 's/depends_on("wgrib2")/#depends_on("wgrib2")/g' `grep -lRie wgrib2 spack-ext/`
+
+          # Don't generate ecflow module when using external package
+          spack config add "modules:default:tcl:exclude:[ecflow]"
+
+          # Concretize and check for duplicates
+          spack concretize 2>&1 | tee log.concretize.oneapi-ifx-2024.2.0-buildcache
+          ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.oneapi-ifx-2024.2.0-buildcache -i fms -i crtm -i esmf -i mapl
+
+          # Add and update source cache
+          spack mirror add local-source file:///home/ubuntu/spack-stack/source-cache/
+          spack mirror create -a -d /home/ubuntu/spack-stack/source-cache/
+
+          # Add binary cache if requested
+          if [ "$USE_BINARY_CACHE" = true ] ; then
+            set +e
+            spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/
+            spack buildcache update-index local-binary || (echo "No valid binary cache found, proceed without" && spack mirror rm local-binary)
+            set +e
+            echo "Packages in spack binary cache:"
+            spack buildcache list
+          fi
+
+          # Break installation up in pieces and create build caches in between
+          # This allows us to "spin up" builds that altogether take longer than
+          # six hours, and/or fail later in the build process.
+
+          # base-env
+          echo "base-env ..."
+          spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.oneapi-ifx-2024.2.0-buildcache.base-env
+          spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ base-env
+
+          # jedi-base-env
+          echo "jedi-base-env ..."
+          spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.oneapi-ifx-2024.2.0-buildcache.jedi-base-env
+          spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env
+
+          # jedi-ufs-env
+          echo "jedi-ufs-env ..."
+          spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.oneapi-ifx-2024.2.0-buildcache.jedi-ufs-env
+          spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-ufs-env
+
+          # the rest
+          echo "unified-env ..."
+          spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.oneapi-ifx-2024.2.0-buildcache.unified-env
+          spack buildcache create -u /home/ubuntu/spack-stack/build-cache/
+
+          # Remove binary cache for next round of concretization
+          if [ "$USE_BINARY_CACHE" = true ] ; then
+            spack mirror rm local-binary
+          fi
+
+          # Remove buildcache config settings
+          spack config remove config:install_tree:padded_length
+
+          # Next steps: synchronize source and build cache to a central/combined mirror?
+
+          # Cleanup
+          spack clean -a
+          spack env deactivate
+
+      - name: create-env
+        run: |
+          # Set up spack-stack
+          source ./setup.sh
+          export BUILDCACHE_ENVNAME=ue-oneapi-ifx-2024.2.0-buildcache
+          export BUILDCACHE_ENVDIR=$PWD/envs/${BUILDCACHE_ENVNAME}
+          export ENVNAME=ue-oneapi-ifx-2024.2.0
+          export ENVDIR=$PWD/envs/${ENVNAME}
+          rsync -av --exclude='install' --exclude='spack.lock' --exclude='.spack_db' ${BUILDCACHE_ENVDIR}/ ${ENVDIR}/
+          spack env activate ${ENVDIR}
+
+          # Concretize and check for duplicates
+          spack concretize --force 2>&1 | tee log.concretize.oneapi-ifx-2024.2.0
+          ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.oneapi-ifx-2024.2.0 -i fms -i crtm -i esmf -i mapl
+
+          # Add binary cache back in
+          spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/
+          echo "Packages in combined spack build caches:"
+          spack buildcache list
+
+          # Install from cache
+          spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.oneapi-ifx-2024.2.0.unified-env
+
+          # Check shared libraries
+          ${SPACK_STACK_DIR}/util/ldd_check.py $SPACK_ENV 2>&1 | tee log.ldd_check
+
+          # Create modules
+          spack clean -a
+          spack module tcl refresh -y
+          spack stack setup-meta-modules
+          spack env deactivate
+
+      - name: test-env
+        run: |
+          source /etc/profile.d/modules.sh
+          module use /home/ubuntu/spack-stack/modulefiles
+
+          export ENVNAME=ue-oneapi-ifx-2024.2.0
+          export ENVDIR=$PWD/envs/${ENVNAME}
+          ls -l ${ENVDIR}/install/modulefiles/Core
+
+          module use ${ENVDIR}/install/modulefiles/Core
+          module load stack-oneapi/2024.2.0
+          module load stack-intel-oneapi-mpi/2021.13
+          module load stack-python/3.11.7
+          module available
+
+          module load jedi-ufs-env
+          module load ewok-env
+          module load soca-env
+          module list
+
+          echo "Contents of MPI metamodule:"
+          cat ${ENVDIR}/install/modulefiles/oneapi/2024.2.0/stack-intel-oneapi-mpi/2021.13

.github/workflows/ubuntu-ci-x86_64-oneapi.yaml

@@ -243,3 +243,6 @@ jobs:
           module load ewok-env
           module load soca-env
           module list
+
+          echo "Contents of MPI metamodule:"
+          cat ${ENVDIR}/install/modulefiles/oneapi/2024.2.0/stack-intel-oneapi-mpi/2021.13

configs/sites/tier1/atlantis/packages_intel.yaml

@@ -7,7 +7,7 @@ packages:
     buildable: False
   intel-oneapi-mpi:
     externals:
-    - spec: [email protected]%[email protected]
+    - spec: [email protected]%[email protected] +classic-names
       prefix: /cm/shared/apps/intel/oneapi
       modules:
       - mpi/2021.6.0

configs/sites/tier1/aws-pcluster/packages_intel.yaml

@@ -11,7 +11,7 @@ packages:
     buildable: False
   intel-oneapi-mpi:
     externals:
-    - spec: [email protected]%[email protected]
+    - spec: [email protected]%[email protected] +classic-names
       prefix: /opt/intel/oneapi
       modules:
       - libfabric-aws/1.19.0amzn4.0

configs/sites/tier1/discover-scu16/packages_intel.yaml

@@ -7,7 +7,7 @@ packages:
     buildable: False
   intel-oneapi-mpi:
     externals:
-    - spec: [email protected]%[email protected]
+    - spec: [email protected]%[email protected] +classic-names
       prefix: /usr/local/intel/oneapi/2021
       modules:
       - mpi/impi/2021.6.0

configs/sites/tier1/discover-scu17/packages_intel.yaml

@@ -7,7 +7,7 @@ packages:
     buildable: False
   intel-oneapi-mpi:
     externals:
-    - spec: [email protected]%intel@=2021.10.0
+    - spec: [email protected]%intel@=2021.10.0 +classic-names
       prefix: /usr/local/intel/oneapi/2021
       modules:
       - mpi/impi/2021.10.0

configs/sites/tier1/hera/packages_intel.yaml

@@ -11,7 +11,7 @@ packages:
     buildable: False
   intel-oneapi-mpi:
     externals:
-    - spec: [email protected]%[email protected]
+    - spec: [email protected]%[email protected] +classic-names
       modules:
       - impi/2022.1.2
       prefix: /apps/oneapi

configs/sites/tier1/hercules/packages_intel.yaml

@@ -11,7 +11,7 @@ packages:
     buildable: False
   intel-oneapi-mpi:
     externals:
-    - spec: [email protected]%[email protected]
+    - spec: [email protected]%[email protected] +classic-names
       modules:
       - intel-oneapi-mpi/2021.9.0
   intel-oneapi-mkl:

configs/sites/tier1/jet/packages_intel.yaml

@@ -7,7 +7,7 @@ packages:
     buildable: False
   intel-oneapi-mpi:
     externals:
-    - spec: [email protected]%[email protected]
+    - spec: [email protected]%[email protected] +classic-names
       modules:
       - impi/2022.1.2
       prefix: /apps/oneapi

configs/sites/tier1/noaa-aws/packages_intel.yaml

@@ -11,7 +11,7 @@ packages:
     buildable: False
   intel-oneapi-mpi:
     externals:
-    - spec: [email protected]%[email protected]
+    - spec: [email protected]%[email protected] +classic-names
       prefix: /apps/oneapi
       modules:
       - impi/2023.2.0

configs/sites/tier1/noaa-azure/packages_intel.yaml

@@ -11,7 +11,7 @@ packages:
     buildable: False
   intel-oneapi-mpi:
     externals:
-    - spec: [email protected]%[email protected]
+    - spec: [email protected]%[email protected] +classic-names
       prefix: /apps/oneapi
       modules:
       - impi/2023.2.0

configs/sites/tier1/noaa-gcloud/packages_intel.yaml

@@ -11,7 +11,7 @@ packages:
     buildable: False
   intel-oneapi-mpi:
     externals:
-    - spec: [email protected]%[email protected]
+    - spec: [email protected]%[email protected] +classic-names
       prefix: /apps/oneapi
       modules:
       - impi/2023.2.0

configs/sites/tier1/orion/packages_intel.yaml

@@ -11,7 +11,7 @@ packages:
     buildable: False
   intel-oneapi-mpi:
     externals:
-    - spec: [email protected]%[email protected]
+    - spec: [email protected]%[email protected] +classic-names
       modules:
       - intel-oneapi-mpi/2021.9.0
   intel-oneapi-mkl:

configs/sites/tier1/s4/packages.yaml

@@ -13,7 +13,7 @@ packages:
     buildable: False
   intel-oneapi-mpi:
     externals:
-    - spec: [email protected]%[email protected]
+    - spec: [email protected]%[email protected] +classic-names
       prefix: /opt/intel/oneapi/2023.2
       modules:
       - intel/2023.2