diff --git a/.github/workflows/macos-ci-aarch64.yaml b/.github/workflows/macos-ci-aarch64.yaml index e905915f2..cb4c6c219 100644 --- a/.github/workflows/macos-ci-aarch64.yaml +++ b/.github/workflows/macos-ci-aarch64.yaml @@ -35,7 +35,7 @@ jobs: mkdir -p /Users/ec2-user/spack-stack/build-cache mkdir -p /Users/ec2-user/spack-stack/source-cache - - name: create-env + - name: create-buildcache run: | # Get day of week to decide whether to use build caches or not DOW=$(date +%u) @@ -56,15 +56,13 @@ jobs: source ./setup.sh # Important! export SPACK_PYTHON=/usr/bin/python3 - export ENVNAME=ue-apple-clang-14.0.3 - export ENVDIR=$PWD/envs/${ENVNAME} - spack clean -ab spack bootstrap now - + # + export ENVNAME=ue-apple-clang-14.0.3-buildcache + export ENVDIR=$PWD/envs/${ENVNAME} spack stack create env --site macos.default --template unified-dev --name ${ENVNAME} --compiler apple-clang spack env activate ${ENVDIR} - spack add ${{ inputs.specs || '' }} export SPACK_SYSTEM_CONFIG_PATH="${ENVDIR}/site" # Find external packages @@ -86,8 +84,11 @@ jobs: export -n SPACK_SYSTEM_CONFIG_PATH + # For buildcaches + spack config add config:install_tree:padded_length:200 + # Set compiler and MPI specs - spack config add "packages:all:providers:mpi:[openmpi@5.0.3]" + spack config add "packages:all:providers:mpi:[openmpi@5.0.5]" spack config add "packages:all:compiler:[apple-clang@14.0.3]" # Add additional variants for MET packages, different from config/common/packages.yaml @@ -97,8 +98,8 @@ jobs: # *DH # Concretize and check for duplicates - spack concretize 2>&1 | tee log.concretize.apple-clang-14.0.3 - ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.apple-clang-14.0.3 -i fms -i crtm -i esmf -i mapl + spack concretize 2>&1 | tee log.concretize.apple-clang-14.0.3-buildcache + ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.apple-clang-14.0.3-buildcache -i fms -i crtm -i esmf -i mapl # Add and update source cache spack mirror add local-source file:///Users/ec2-user/spack-stack/source-cache/ @@ -121,29 +122,66 @@ jobs: # base-env echo "base-env ..." spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.apple-clang-14.0.3.base-env - spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/ base-env + spack buildcache create -u /Users/ec2-user/spack-stack/build-cache/ base-env # jedi-base-env echo "jedi-base-env ..." spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.apple-clang-14.0.3.jedi-base-env - spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/ jedi-base-env + spack buildcache create -u /Users/ec2-user/spack-stack/build-cache/ jedi-base-env + + # jedi-ufs-env + echo "jedi-ufs-env ..." + spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.apple-clang-14.0.3.jedi-ufs-env + spack buildcache create -u /Users/ec2-user/spack-stack/build-cache/ jedi-ufs-env # the rest echo "unified-env ..." spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.apple-clang-14.0.3.unified-env - spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/ + spack buildcache create -u /Users/ec2-user/spack-stack/build-cache/ # Remove binary cache for next round of concretization if [ "$USE_BINARY_CACHE" = true ] ; then spack mirror rm local-binary fi + # Remove buildcache config settings + spack config remove config:install_tree:padded_length + + # Next steps: synchronize source and build cache to a central/combined mirror? + + # Cleanup + spack clean -a + spack env deactivate + + - name: create-env + run: | + # Set up spack-stack + source ./setup.sh + # Important! + export SPACK_PYTHON=/usr/bin/python3 + export BUILDCACHE_ENVNAME=ue-apple-clang-14.0.3-buildcache + export BUILDCACHE_ENVDIR=$PWD/envs/${BUILDCACHE_ENVNAME} + export ENVNAME=ue-apple-clang-14.0.3 + export ENVDIR=$PWD/envs/${ENVNAME} + rsync -av --exclude='install' --exclude='spack.lock' --exclude='.spack_db' ${BUILDCACHE_ENVDIR}/ ${ENVDIR}/ + spack env activate ${ENVDIR} + + # Concretize and check for duplicates + spack concretize --force 2>&1 | tee log.concretize.apple-clang-14.0.3 + ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.apple-clang-14.0.3 -i fms -i crtm -i esmf -i mapl + + # Add binary cache back in + spack mirror add local-binary file:///Users/ec2-user/spack-stack/build-cache/ + echo "Packages in combined spack build caches:" + spack buildcache list + + # Install from cache + spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.apple-clang-14.0.3.unified-env + # Create modules spack clean -a spack module lmod refresh -y spack stack setup-meta-modules - - # Next steps: synchronize source and build cache to a central/combined mirror? spack env deactivate # Test environment chaining @@ -160,7 +198,6 @@ jobs: - name: test-env run: | - # Set up homebrew and lmod support eval "$(/opt/homebrew/bin/brew shellenv)" source /opt/homebrew/opt/lmod/init/profile @@ -170,7 +207,7 @@ jobs: module use ${ENVDIR}/install/modulefiles/Core module load stack-apple-clang/14.0.3 - module load stack-openmpi/5.0.3 + module load stack-openmpi/5.0.5 module load stack-python/3.11.7 module available diff --git a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml index 3b12d1c11..1a73ad69b 100644 --- a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml @@ -115,17 +115,22 @@ jobs: # base-env echo "base-env ..." spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.base-env - spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ base-env + spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ base-env # jedi-base-env echo "jedi-base-env ..." spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.jedi-base-env - spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env + spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env + + # jedi-ufs-env + echo "jedi-ufs-env ..." + spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.jedi-ufs-env + spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-ufs-env # the rest echo "${TEMPLATE} ..." spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.gnu-11.4.0-buildcache.${TEMPLATE} - spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ + spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ # Remove binary cache for next round of concretization if [ "$USE_BINARY_CACHE" = true ] ; then @@ -145,6 +150,7 @@ jobs: - name: create-env run: | + # Set up spack-stack source ./setup.sh export BUILDCACHE_ENVNAME=ue-gcc-11.4.0-buildcache export BUILDCACHE_ENVDIR=$PWD/envs/${BUILDCACHE_ENVNAME} @@ -153,20 +159,22 @@ jobs: rsync -av --exclude='install' --exclude='spack.lock' --exclude='.spack_db' ${BUILDCACHE_ENVDIR}/ ${ENVDIR}/ spack env activate ${ENVDIR} - # Concretize + # Concretize and check for duplicates spack concretize --force 2>&1 | tee log.concretize.gnu-11.4.0 ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.gnu-11.4.0 -i fms -i crtm -i esmf -i mapl - # Add binary cache back in and reindex it + # Add binary cache back in spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/ echo "Packages in combined spack build caches:" spack buildcache list - # base-env - echo "base-env ..." + # Install from cache spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.gnu-11.4.0.unified-env + # Check shared libraries ${SPACK_STACK_DIR}/util/ldd_check.py $SPACK_ENV 2>&1 | tee log.ldd_check + + # Create modules spack clean -a spack module tcl refresh -y spack stack setup-meta-modules @@ -201,7 +209,7 @@ jobs: module use ${ENVDIR}/install/modulefiles/Core module load stack-gcc/11.4.0 - module load stack-openmpi/5.0.3 + module load stack-openmpi/5.0.5 module load stack-python/3.11.7 module available diff --git a/.github/workflows/ubuntu-ci-x86_64-intel.yaml b/.github/workflows/ubuntu-ci-x86_64-intel.yaml index eed14ce5b..f391ce901 100644 --- a/.github/workflows/ubuntu-ci-x86_64-intel.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-intel.yaml @@ -93,7 +93,7 @@ jobs: echo " intel-oneapi-mpi:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml echo " buildable: false" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml echo " externals:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml + echo " - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0 +classic-names" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml echo " prefix: /opt/intel/oneapi" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml # Add external Intel MKL and oneAPI runtime @@ -155,17 +155,22 @@ jobs: # base-env echo "base-env ..." spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.intel-2021.10.0-buildcache.base-env - spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ base-env + spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ base-env # jedi-base-env echo "jedi-base-env ..." spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.intel-2021.10.0-buildcache.jedi-base-env - spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env + spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env + + # jedi-ufs-env + echo "jedi-ufs-env ..." + spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.intel-2021.10.0-buildcache.jedi-ufs-env + spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-ufs-env # the rest echo "unified-env ..." spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.intel-2021.10.0-buildcache.unified-env - spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ + spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ # Remove binary cache for next round of concretization if [ "$USE_BINARY_CACHE" = true ] ; then @@ -183,6 +188,7 @@ jobs: - name: create-env run: | + # Set up spack-stack source ./setup.sh export BUILDCACHE_ENVNAME=ue-intel-2021.10.0-buildcache export BUILDCACHE_ENVDIR=$PWD/envs/${BUILDCACHE_ENVNAME} @@ -191,20 +197,22 @@ jobs: rsync -av --exclude='install' --exclude='spack.lock' --exclude='.spack_db' ${BUILDCACHE_ENVDIR}/ ${ENVDIR}/ spack env activate ${ENVDIR} - # Concretize + # Concretize and check for duplicates spack concretize --force 2>&1 | tee log.concretize.intel-2021.10.0 ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.intel-2021.10.0 -i fms -i crtm -i esmf -i mapl - # Add binary cache back in and reindex it + # Add binary cache back in spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/ echo "Packages in combined spack build caches:" spack buildcache list - # base-env - echo "base-env ..." + # Install from cache spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.intel-2021.10.0.unified-env + # Check shared libraries ${SPACK_STACK_DIR}/util/ldd_check.py $SPACK_ENV 2>&1 | tee log.ldd_check + + # Create modules spack clean -a spack module tcl refresh -y spack stack setup-meta-modules diff --git a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml b/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml index 0b4f7d337..976813438 100644 --- a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml @@ -161,17 +161,22 @@ jobs: # base-env echo "base-env ..." spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.oneapi-2024.2.0-buildcache.base-env - spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ base-env + spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ base-env # jedi-base-env echo "jedi-base-env ..." spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.oneapi-2024.2.0-buildcache.jedi-base-env - spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env + spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env + + # jedi-ufs-env + echo "jedi-ufs-env ..." + spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.oneapi-2024.2.0-buildcache.jedi-ufs-env + spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-ufs-env # the rest echo "unified-env ..." spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.oneapi-2024.2.0-buildcache.unified-env - spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ + spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ # Remove binary cache for next round of concretization if [ "$USE_BINARY_CACHE" = true ] ; then @@ -189,6 +194,7 @@ jobs: - name: create-env run: | + # Set up spack-stack source ./setup.sh export BUILDCACHE_ENVNAME=ue-oneapi-2024.2.0-buildcache export BUILDCACHE_ENVDIR=$PWD/envs/${BUILDCACHE_ENVNAME} @@ -197,39 +203,27 @@ jobs: rsync -av --exclude='install' --exclude='spack.lock' --exclude='.spack_db' ${BUILDCACHE_ENVDIR}/ ${ENVDIR}/ spack env activate ${ENVDIR} - # Concretize + # Concretize and check for duplicates spack concretize --force 2>&1 | tee log.concretize.oneapi-2024.2.0 ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.oneapi-2024.2.0 -i fms -i crtm -i esmf -i mapl - # Add binary cache back in and reindex it + # Add binary cache back in spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/ echo "Packages in combined spack build caches:" spack buildcache list - # base-env - echo "base-env ..." + # Install from cache spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.oneapi-2024.2.0.unified-env + # Check shared libraries ${SPACK_STACK_DIR}/util/ldd_check.py $SPACK_ENV 2>&1 | tee log.ldd_check + + # Create modules spack clean -a spack module tcl refresh -y spack stack setup-meta-modules spack env deactivate - ### # Test environment chaining - ### echo "Test environment chaining" - ### spack stack create env --name chaintest --template empty --site linux.default --upstream ${ENVDIR}/install --compiler intel - ### # Retain config from upstream so we don't have to rebuild: - ### cp -r ${ENVDIR}/{site,common} $PWD/envs/chaintest/. - ### spack env activate ${PWD}/envs/chaintest - ### # Pin gmake to avoid duplicate packages (excluded gmake from spack external find above) - ### spack config add "packages:gmake:require:'@:4.2'" - ### spack add nccmp@1.9.0.1%oneapi - ### spack concretize | tee envs/chaintest/log.concretize - ### unwanted_duplicates=$(( cat envs/chaintest/log.concretize | grep -E '^ - ' | grep -Fv 'nccmp@1.9.0.1' || true ) | wc -l) - ### if [ ${unwanted_duplicates} -gt 0 ]; then echo "Environment chaining test failed"; exit 1; fi - ### spack env deactivate - - name: test-env run: | source /etc/profile.d/modules.sh diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index ea8049c6d..1c40bd5e8 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -102,6 +102,10 @@ packages: # Note: Uncommenting this entry will break # the container builds. #require: '@2.11.0' + # To avoid duplicate packages being built + gmake: + require: + - one_of: ['@=4.2.1', '@=4.4.1'] grib-util: require: '@1.4.0' gsibec: @@ -138,6 +142,10 @@ packages: mapl: require: '@2.46.3 ~shared ~f2py' variants: '+pflogger' + # To avoid duplicate packages being built + meson: + require: + - '@1.5.1:' # If making changes here, also check the Discover site configs and the CI workflows met: require: '@11.1.1 +python +grib2' @@ -170,7 +178,7 @@ packages: odc: require: '@1.5.2 ~fortran' openblas: - require: '@0.3.24 +noavx512' + require: '@0.3.24' openmpi: require: '~internal-hwloc +two_level_namespace' openssl: diff --git a/configs/containers/docker-ubuntu-gcc-openmpi.yaml b/configs/containers/docker-ubuntu-gcc-openmpi.yaml index f0e495073..526d47589 100644 --- a/configs/containers/docker-ubuntu-gcc-openmpi.yaml +++ b/configs/containers/docker-ubuntu-gcc-openmpi.yaml @@ -28,7 +28,7 @@ spack: require: '%gcc' target: [x86_64] providers: - mpi: [openmpi@5.0.3] + mpi: [openmpi@5.0.5] compiler: [gcc@12.3.0] gcc: buildable: false diff --git a/configs/sites/tier1/narwhal/compilers.yaml b/configs/sites/tier1/narwhal/compilers.yaml index 838a8a1dd..ec9b170c0 100644 --- a/configs/sites/tier1/narwhal/compilers.yaml +++ b/configs/sites/tier1/narwhal/compilers.yaml @@ -9,7 +9,7 @@ compilers:: flags: {} operating_system: sles15 modules: - - PrgEnv-intel/8.3.3 + - PrgEnv-intel/8.4.0 - intel-classic/2023.2.0 - cray-libsci/23.05.1.4 - libfabric/1.12.1.2.2.1 @@ -55,7 +55,7 @@ compilers:: flags: {} operating_system: sles15 modules: - - PrgEnv-gnu/8.3.3 + - PrgEnv-gnu/8.4.0 - gcc/10.3.0 - cray-libsci/23.05.1.4 - libfabric/1.12.1.2.2.1 diff --git a/configs/sites/tier2/blackpearl/config.yaml b/configs/sites/tier2/blackpearl/config.yaml index 77a2e80ad..0eb7669fb 100644 --- a/configs/sites/tier2/blackpearl/config.yaml +++ b/configs/sites/tier2/blackpearl/config.yaml @@ -1,2 +1,2 @@ config: - build_jobs: 2 + build_jobs: 1 diff --git a/configs/sites/tier2/blackpearl/packages_gcc.yaml b/configs/sites/tier2/blackpearl/packages_gcc.yaml index a389fbcfc..730601342 100644 --- a/configs/sites/tier2/blackpearl/packages_gcc.yaml +++ b/configs/sites/tier2/blackpearl/packages_gcc.yaml @@ -2,4 +2,4 @@ packages: all: compiler:: [gcc@13.3.0] providers: - mpi:: [openmpi@5.0.3] + mpi:: [openmpi@5.0.5] diff --git a/configs/sites/tier2/bounty/packages_gcc.yaml b/configs/sites/tier2/bounty/packages_gcc.yaml index a91408366..0ddd36693 100644 --- a/configs/sites/tier2/bounty/packages_gcc.yaml +++ b/configs/sites/tier2/bounty/packages_gcc.yaml @@ -2,4 +2,4 @@ packages: all: compiler:: [gcc@12.3.0] providers: - mpi:: [openmpi@5.0.3] + mpi:: [openmpi@5.0.5] diff --git a/configs/templates/nco/spack.yaml b/configs/templates/nco/spack.yaml index 12787de13..e1263204e 100644 --- a/configs/templates/nco/spack.yaml +++ b/configs/templates/nco/spack.yaml @@ -35,7 +35,7 @@ spack: - g2 - g2c - g2tmpl - - gdal + - gdal@3.8.5 - geos - gfsio - gftl-shared diff --git a/configs/templates/unified-dev/spack.yaml b/configs/templates/unified-dev/spack.yaml index 370a03e7c..744789778 100644 --- a/configs/templates/unified-dev/spack.yaml +++ b/configs/templates/unified-dev/spack.yaml @@ -13,7 +13,7 @@ spack: - geos-gcm-env ^esmf@=8.6.1 - global-workflow-env ^esmf@=8.6.1 - gmao-swell-env - - gsi-env ^esmf@=8.6.1 + - gsi-env - jedi-fv3-env - jedi-geos-env ^esmf@=8.6.1 - jedi-mpas-env diff --git a/spack b/spack index 990f0b84b..b6ef265ec 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 990f0b84b7f6096c0542d04784c291666cdc2b7b +Subproject commit b6ef265ec4582ddca131534a9d314e233919ae69