Merge branch 'develop' into warn_upstream_config_mismatch

.github/workflows/macos-ci-aarch64.yaml

@@ -28,12 +28,28 @@ jobs:
           rm -fr *
 
       - name: checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           submodules: true
 
+      - name: prepare-directories
+        run: |
+          mkdir -p /Users/ec2-user/spack-stack/build-cache
+          mkdir -p /Users/ec2-user/spack-stack/source-cache
+
       - name: create-env
         run: |
+          # Get day of week to decide whether to use build caches or not
+          DOW=$(date +%u)
+          # Monday is 1 ... Sunday is 7
+          if [[ $DOW == 7 ]]; then
+            export USE_BINARY_CACHE=false
+            echo "Ignore existing binary cache for creating buildcache environment"
+          else
+            export USE_BINARY_CACHE=true
+            echo "Use existing binary cache for creating buildcache environment"
+          fi
+
           # Set up homebrew and lmod support
           eval "$(/opt/homebrew/bin/brew shellenv)"
           source /opt/homebrew/opt/lmod/init/profile
@@ -42,23 +58,21 @@ jobs:
           source ./setup.sh
           # Important!
           export SPACK_PYTHON=/usr/bin/python3
-
-          # DH* 20230302 - to avoid using padded_length for build caches,
-          # always build in the same environment so that the length of the
-          # path doesn't change - see also other 'DH* 20230302' changes below
-          #export ENVNAME=${{ inputs.template || 'unified-dev' }}.macos-ci-aarch64
-          export ENVNAME=ci-env.macos-ci-aarch64
-          # *DH
+          export ENVNAME=ue-apple-clang-14.0.3
           export ENVDIR=$PWD/envs/${ENVNAME}
-          spack stack create env --site macos.default --template ${{ inputs.template || 'unified-dev' }} --name ${ENVNAME}
+
+          spack clean -ab
+          spack bootstrap now
+
+          spack stack create env --site macos.default --template unified-dev --name ${ENVNAME}
           spack env activate ${ENVDIR}
           spack add ${{ inputs.specs || '' }}
           export SPACK_SYSTEM_CONFIG_PATH="${ENVDIR}/site"
 
           # Find external packages
           spack external find --scope system \
               --exclude bison --exclude openssl \
-              --exclude python
+              --exclude curl --exclude python
           spack external find --scope system perl
           spack external find --scope system libiconv
           spack external find --scope system wget
@@ -74,11 +88,6 @@ jobs:
 
           export -n SPACK_SYSTEM_CONFIG_PATH
 
-          # DH* 20230302 - Don't use yet, too many problems
-          ## For buildcaches
-          #spack config add config:install_tree:padded_length:true
-          # *DH
-
           # Set compiler and MPI
           spack config add "packages:all:providers:mpi:[[email protected]]"
           spack config add "packages:all:compiler:[[email protected]]"
@@ -98,11 +107,15 @@ jobs:
           spack mirror add local-source file:///Users/ec2-user/spack-stack/source-cache/
           spack mirror create -a -d /Users/ec2-user/spack-stack/source-cache/
 
-          # Add binary cache and reindex it
-          spack mirror add local-binary file:///Users/ec2-user/spack-stack/build-cache/
-          spack buildcache update-index local-binary
-          echo "Packages in combined spack build caches:"
-          spack buildcache list
+          # Add binary cache if requested
+          if [ "$USE_BINARY_CACHE" = true ] ; then
+            set +e
+            spack mirror add local-binary file:///Users/ec2-user/spack-stack/build-cache/
+            spack buildcache update-index local-binary || (echo "No valid binary cache found, proceed without" && spack mirror rm local-binary)
+            set +e
+            echo "Packages in spack binary cache:"
+            spack buildcache list
+          fi
 
           # Break installation up in pieces and create build caches in between
           # This allows us to "spin up" builds that altogether take longer than
@@ -113,16 +126,39 @@ jobs:
           spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.apple-clang-14.0.3.base-env
           spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/ base-env
 
+          # jedi-base-env
+          echo "jedi-base-env ..."
+          spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.apple-clang-14.0.3.jedi-base-env
+          spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/ jedi-base-env
+
           # the rest
-          echo "${{ inputs.template || 'unified-dev' }} ..."
-          spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.apple-clang-14.0.3.${{ inputs.template || 'unified-dev' }}
+          echo "unified-env ..."
+          spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.apple-clang-14.0.3.unified-env
           spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/
 
-          # Next steps: synchronize source and build cache to a central/combined mirror?
-          echo "Next steps ..."
+          # Remove binary cache for next round of concretization
+          if [ "$USE_BINARY_CACHE" = true ] ; then
+            spack mirror rm local-binary
+          fi
+
+          # Create modules
           spack clean -a
           spack module lmod refresh -y
           spack stack setup-meta-modules
+
+          # Next steps: synchronize source and build cache to a central/combined mirror?
+          spack env deactivate
+
+          # Test environment chaining
+          echo "Test environment chaining"
+          spack stack create env --name chaintest --template empty --site macos.default --upstream $(realpath envs/${ENVNAME}/install)
+          # Retain config from upstream so we don't have to rebuild:
+          cp -r $PWD/envs/${ENVNAME}/{site,common} $PWD/envs/chaintest/.
+          spack env activate ${PWD}/envs/chaintest
+          spack add [email protected]%apple-clang
+          spack concretize | tee envs/chaintest/log.concretize
+          unwanted_duplicates=$(( cat envs/chaintest/log.concretize | grep -E '^ - ' | grep -Fv '[email protected]' || true ) | wc -l)
+          if [ ${unwanted_duplicates} -gt 0 ]; then echo "Environment chaining test failed"; exit 1; fi
           spack env deactivate
 
       - name: test-env
@@ -131,12 +167,7 @@ jobs:
           eval "$(/opt/homebrew/bin/brew shellenv)"
           source /opt/homebrew/opt/lmod/init/profile
 
-          # DH* 20230302 - to avoid using padded_length for build caches,
-          # always build in the same environment so that the length of the
-          # path doesn't change - see also other 'DH* 20230302' changes below
-          #export ENVNAME=${{ inputs.template || 'unified-dev' }}.macos-ci-aarch64
-          export ENVNAME=ci-env.macos-ci-aarch64
-          # *DH
+          export ENVNAME=ue-apple-clang-14.0.3
           export ENVDIR=$PWD/envs/${ENVNAME}
           ls -l ${ENVDIR}/install/modulefiles/Core
 
@@ -146,7 +177,7 @@ jobs:
           module load stack-python/3.10.13
           module available
 
-          module load jedi-ufs-env/1.0.0
-          module load ewok-env/1.0.0
-          module load soca-env/1.0.0
+          module load jedi-ufs-env
+          module load ewok-env
+          module load soca-env
           module list

.github/workflows/ubuntu-ci-containers-x86_64.yaml

@@ -71,10 +71,6 @@ jobs:
           spack stack create ctr --container ${CONTAINER} --specs ${SPECS}
 
           cd ${ENVDIR}
-          # mapl@:2.41 doesn't build with mpich@4 - https://github.com/JCSDA/spack-stack/issues/608
-          if [[ "${CONTAINER}" == *"mpich"* ]]; then
-              sed -i 's/- mapl@/#- mapl@/g' spack.yaml
-          fi
           spack containerize > Dockerfile
           docker build -t ${CONTAINER}-${SPECS} .
 

.github/workflows/ubuntu-ci-x86_64-gnu.yaml

@@ -28,10 +28,15 @@ jobs:
           rm -fr *
 
       - name: checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           submodules: true
 
+      - name: prepare-directories
+        run: |
+          mkdir -p /home/ubuntu/spack-stack/build-cache/
+          mkdir -p /home/ubuntu/spack-stack/source-cache/
+
       - name: create-buildcache
         run: |
           # Get day of week to decide whether to use build caches or not
@@ -45,7 +50,7 @@ jobs:
             echo "Use existing binary cache for creating buildcache environment"
           fi
 
-          # spack-stack setup
+          # Set up spack-stack
           source ./setup.sh
           export ENVNAME=ue-gcc-11.4.0-buildcache
           export ENVDIR=$PWD/envs/${ENVNAME}
@@ -63,7 +68,7 @@ jobs:
           spack external find --scope system texlive
           spack external find --scope system mysql
 
-          # For GNU
+          # Find compilers
           spack compiler find --scope system
 
           export -n SPACK_SYSTEM_CONFIG_PATH
@@ -89,8 +94,10 @@ jobs:
 
           # Add binary cache if requested
           if [ "$USE_BINARY_CACHE" = true ] ; then
+            set +e
             spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/
-            spack buildcache update-index local-binary
+            spack buildcache update-index local-binary || (echo "No valid binary cache found, proceed without" && spack mirror rm local-binary)
+            set +e
             echo "Packages in spack binary cache:"
             spack buildcache list
           fi

.github/workflows/ubuntu-ci-x86_64-intel.yaml

@@ -28,10 +28,15 @@ jobs:
           rm -fr *
 
       - name: checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           submodules: true
 
+      - name: prepare-directories
+        run: |
+          mkdir -p /home/ubuntu/spack-stack/build-cache/
+          mkdir -p /home/ubuntu/spack-stack/source-cache/
+
       - name: create-buildcache
         run: |
           # Get day of week to decide whether to use build caches or not
@@ -45,7 +50,7 @@ jobs:
             echo "Use existing binary cache for creating buildcache environment"
           fi
 
-          # spack-stack setup
+          # Set up spack-stack
           source ./setup.sh
           export ENVNAME=ue-intel-2021.10.0-buildcache
           export ENVDIR=$PWD/envs/${ENVNAME}
@@ -63,7 +68,7 @@ jobs:
           spack external find --scope system texlive
           spack external find --scope system mysql
 
-          # For GNU - need to for bootstrapping
+          # Find compilers
           spack compiler find --scope system
 
           # For Intel
@@ -131,8 +136,10 @@ jobs:
 
           # Add binary cache if requested
           if [ "$USE_BINARY_CACHE" = true ] ; then
+            set +e
             spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/
-            spack buildcache update-index local-binary
+            spack buildcache update-index local-binary || (echo "No valid binary cache found, proceed without" && spack mirror rm local-binary)
+            set +e
             echo "Packages in spack binary cache:"
             spack buildcache list
           fi

configs/common/modules_lmod.yaml

@@ -31,6 +31,7 @@ modules:
       - fixesproto
       - flex
       - freetype
+      - gcc-runtime
       - gdbm
       - googletest
       - hwloc

configs/common/modules_tcl.yaml

@@ -33,6 +33,7 @@ modules:
       - fixesproto
       - flex
       - freetype
+      - gcc-runtime
       - gdbm
       - googletest
       - hwloc

configs/common/packages.yaml

@@ -59,19 +59,16 @@
     # config and update the projections for lmod/tcl.
     # Also, check the acorn and derecho site configs which have esmf modifications.
     esmf:
-      version: ['8.6.0']
-      variants: ~xerces ~pnetcdf snapshot=none +shared +external-parallelio
-      #version: ['8.6.1b04']
-      #variants: ~xerces ~pnetcdf snapshot=b04 +shared +external-parallelio
-      #version: ['8.7.0b04']
-      #variants: ~xerces ~pnetcdf snapshot=b04 +shared +external-parallelio
       require:
-        - any_of: ['fflags="-fp-model precise" cxxflags="-fp-model precise"']
+        - any_of: ['@=8.6.1 ~xerces ~pnetcdf snapshot=none +shared +external-parallelio fflags="-fp-model precise" cxxflags="-fp-model precise"']
           when: "%intel"
           message: "Extra ESMF compile options for Intel"
-        - any_of: ['']
+        - any_of: ['@=8.6.1 ~xerces ~pnetcdf snapshot=none +shared +external-parallelio']
           when: "%gcc"
           message: "Extra ESMF compile options for GCC"
+        - any_of: ['@=8.6.1 ~xerces ~pnetcdf snapshot=none +shared +external-parallelio']
+          when: "%apple-clang"
+          message: "Extra ESMF compile options for GCC"
     fckit:
       version: ['0.11.0']
       variants: +eckit
@@ -138,8 +135,11 @@
       variants: +pic
     libyaml:
       version: ['0.2.5']
+    # Newest version of magics needed for oneapi compilers
+    magics:
+      require: "@4.15.3:"
     mapl:
-      version: ['2.40.3']
+      version: ['2.46.2']
       variants: +shared +pflogger ~f2py
     # If making changes here, also check the Discover site configs and the CI workflows
     met:
@@ -195,7 +195,7 @@
     parallel-netcdf:
       version: ['1.12.3']
     pflogger:
-      version: ['1.12.0']
+      #version: ['1.12.0']
       variants: +mpi
     pixman:
       variants: +pic
@@ -291,8 +291,8 @@
       version: ['2.0.8']
     wrf-io:
       version: ['1.2.0']
-    yafyaml:
-      version: ['1.2.0']
+    #yafyaml:
+    #  version: ['1.2.0']
     zstd:
       version: ['1.5.2']
       variants: +programs

configs/containers/specs/jedi-ci.yaml

@@ -9,7 +9,7 @@
     [email protected], [email protected], [email protected], [email protected],
     [email protected], [email protected], [email protected],
     [email protected], py-pip, [email protected], [email protected], [email protected], [email protected],
-    [email protected], [email protected], [email protected], [email protected], [email protected].0, mapl@2.40.3,
+    [email protected], [email protected], [email protected], [email protected], [email protected].1, mapl@2.46.2,
     [email protected], [email protected], [email protected], shumlib@macos_clang_linux_intel_port,
     [email protected], [email protected],
     # Added for new CI system 2024/04/30

configs/sites/nautilus/compilers.yaml

@@ -49,4 +49,23 @@ compilers:
     - scl/gcc-toolset-12
     environment: {}
     extra_rpaths: []
-
+- compiler:
+    spec: [email protected]
+    paths:
+      cc: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2/compiler/2024.1/bin/icx
+      cxx: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2/compiler/2024.1/bin/icpx
+      f77: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2/compiler/2024.1/bin/ifort
+      fc: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2/compiler/2024.1/bin/ifort
+    flags:
+      fflags: -diag-disable=10448
+    operating_system: rhel8
+    target: x86_64
+    modules: []
+    environment:
+      prepend_path:
+        PATH: '/opt/rh/gcc-toolset-11/root/usr/bin'
+        CPATH: '/opt/rh/gcc-toolset-11/root/usr/include'
+        LD_LIBRARY_PATH: '/opt/scyld/slurm/lib64:/opt/scyld/slurm/lib64/slurm:/opt/rh/gcc-toolset-11/root/usr/lib64:/opt/rh/gcc-toolset-11/root/usr/lib'
+      append_path:
+        CPATH: '/p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2/compiler/2024.1/opt/compiler/include/intel64'
+    extra_rpaths: []