Merge branch 'develop' of https://github.com/jcsda/spack-stack into f…

…eature/blueback
JCSDA · Nov 14, 2024 · fc82ceb · fc82ceb
2 parents 6c35969 + 9fbd7bb
commit fc82ceb
Show file tree

Hide file tree

Showing 13 changed files with 119 additions and 64 deletions.
diff --git a/.github/workflows/macos-ci-aarch64.yaml b/.github/workflows/macos-ci-aarch64.yaml
@@ -35,7 +35,7 @@ jobs:
           mkdir -p /Users/ec2-user/spack-stack/build-cache
           mkdir -p /Users/ec2-user/spack-stack/source-cache
 
-      - name: create-env
+      - name: create-buildcache
         run: |
           # Get day of week to decide whether to use build caches or not
           DOW=$(date +%u)
@@ -56,15 +56,13 @@ jobs:
           source ./setup.sh
           # Important!
           export SPACK_PYTHON=/usr/bin/python3
-          export ENVNAME=ue-apple-clang-14.0.3
-          export ENVDIR=$PWD/envs/${ENVNAME}
-
           spack clean -ab
           spack bootstrap now
-
+          #
+          export ENVNAME=ue-apple-clang-14.0.3-buildcache
+          export ENVDIR=$PWD/envs/${ENVNAME}
           spack stack create env --site macos.default --template unified-dev --name ${ENVNAME} --compiler apple-clang
           spack env activate ${ENVDIR}
-          spack add ${{ inputs.specs || '' }}
           export SPACK_SYSTEM_CONFIG_PATH="${ENVDIR}/site"
 
           # Find external packages
@@ -86,8 +84,11 @@ jobs:
 
           export -n SPACK_SYSTEM_CONFIG_PATH
 
+          # For buildcaches
+          spack config add config:install_tree:padded_length:200
+
           # Set compiler and MPI specs
-          spack config add "packages:all:providers:mpi:[[email protected].3]"
+          spack config add "packages:all:providers:mpi:[[email protected].5]"
           spack config add "packages:all:compiler:[[email protected]]"
 
           # Add additional variants for MET packages, different from config/common/packages.yaml
@@ -97,8 +98,8 @@ jobs:
           # *DH
 
           # Concretize and check for duplicates
-          spack concretize 2>&1 | tee log.concretize.apple-clang-14.0.3
-          ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.apple-clang-14.0.3 -i fms -i crtm -i esmf -i mapl
+          spack concretize 2>&1 | tee log.concretize.apple-clang-14.0.3-buildcache
+          ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.apple-clang-14.0.3-buildcache -i fms -i crtm -i esmf -i mapl
 
           # Add and update source cache
           spack mirror add local-source file:///Users/ec2-user/spack-stack/source-cache/
@@ -121,29 +122,66 @@ jobs:
           # base-env
           echo "base-env ..."
           spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.apple-clang-14.0.3.base-env
-          spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/ base-env
+          spack buildcache create -u /Users/ec2-user/spack-stack/build-cache/ base-env
 
           # jedi-base-env
           echo "jedi-base-env ..."
           spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.apple-clang-14.0.3.jedi-base-env
-          spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/ jedi-base-env
+          spack buildcache create -u /Users/ec2-user/spack-stack/build-cache/ jedi-base-env
+
+          # jedi-ufs-env
+          echo "jedi-ufs-env ..."
+          spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.apple-clang-14.0.3.jedi-ufs-env
+          spack buildcache create -u /Users/ec2-user/spack-stack/build-cache/ jedi-ufs-env
 
           # the rest
           echo "unified-env ..."
           spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.apple-clang-14.0.3.unified-env
-          spack buildcache create -a -u /Users/ec2-user/spack-stack/build-cache/
+          spack buildcache create -u /Users/ec2-user/spack-stack/build-cache/
 
           # Remove binary cache for next round of concretization
           if [ "$USE_BINARY_CACHE" = true ] ; then
             spack mirror rm local-binary
           fi
 
+          # Remove buildcache config settings
+          spack config remove config:install_tree:padded_length
+
+          # Next steps: synchronize source and build cache to a central/combined mirror?
+
+          # Cleanup
+          spack clean -a
+          spack env deactivate
+
+      - name: create-env
+        run: |
+          # Set up spack-stack
+          source ./setup.sh
+          # Important!
+          export SPACK_PYTHON=/usr/bin/python3
+          export BUILDCACHE_ENVNAME=ue-apple-clang-14.0.3-buildcache
+          export BUILDCACHE_ENVDIR=$PWD/envs/${BUILDCACHE_ENVNAME}
+          export ENVNAME=ue-apple-clang-14.0.3
+          export ENVDIR=$PWD/envs/${ENVNAME}
+          rsync -av --exclude='install' --exclude='spack.lock' --exclude='.spack_db' ${BUILDCACHE_ENVDIR}/ ${ENVDIR}/
+          spack env activate ${ENVDIR}
+
+          # Concretize and check for duplicates
+          spack concretize --force 2>&1 | tee log.concretize.apple-clang-14.0.3
+          ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.apple-clang-14.0.3 -i fms -i crtm -i esmf -i mapl
+
+          # Add binary cache back in
+          spack mirror add local-binary file:///Users/ec2-user/spack-stack/build-cache/
+          echo "Packages in combined spack build caches:"
+          spack buildcache list
+
+          # Install from cache
+          spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.apple-clang-14.0.3.unified-env
+
           # Create modules
           spack clean -a
           spack module lmod refresh -y
           spack stack setup-meta-modules
-
-          # Next steps: synchronize source and build cache to a central/combined mirror?
           spack env deactivate
 
           # Test environment chaining
@@ -160,7 +198,6 @@ jobs:
 
       - name: test-env
         run: |
-          # Set up homebrew and lmod support
           eval "$(/opt/homebrew/bin/brew shellenv)"
           source /opt/homebrew/opt/lmod/init/profile
 
@@ -170,7 +207,7 @@ jobs:
 
           module use ${ENVDIR}/install/modulefiles/Core
           module load stack-apple-clang/14.0.3
-          module load stack-openmpi/5.0.3
+          module load stack-openmpi/5.0.5
           module load stack-python/3.11.7
           module available
 

diff --git a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml
@@ -115,17 +115,22 @@ jobs:
             # base-env
             echo "base-env ..."
             spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.base-env
-            spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ base-env
+            spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ base-env
 
             # jedi-base-env
             echo "jedi-base-env ..."
             spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.jedi-base-env
-            spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env
+            spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env
+
+            # jedi-ufs-env
+            echo "jedi-ufs-env ..."
+            spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.gnu-11.4.0-buildcache.jedi-ufs-env
+            spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-ufs-env
 
             # the rest
             echo "${TEMPLATE} ..."
             spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.gnu-11.4.0-buildcache.${TEMPLATE}
-            spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/
+            spack buildcache create -u /home/ubuntu/spack-stack/build-cache/
 
             # Remove binary cache for next round of concretization
             if [ "$USE_BINARY_CACHE" = true ] ; then
@@ -145,6 +150,7 @@ jobs:
 
       - name: create-env
         run: |
+          # Set up spack-stack
           source ./setup.sh
           export BUILDCACHE_ENVNAME=ue-gcc-11.4.0-buildcache
           export BUILDCACHE_ENVDIR=$PWD/envs/${BUILDCACHE_ENVNAME}
@@ -153,20 +159,22 @@ jobs:
           rsync -av --exclude='install' --exclude='spack.lock' --exclude='.spack_db' ${BUILDCACHE_ENVDIR}/ ${ENVDIR}/
           spack env activate ${ENVDIR}
 
-          # Concretize
+          # Concretize and check for duplicates
           spack concretize --force 2>&1 | tee log.concretize.gnu-11.4.0
           ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.gnu-11.4.0 -i fms -i crtm -i esmf -i mapl
 
-          # Add binary cache back in and reindex it
+          # Add binary cache back in
           spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/
           echo "Packages in combined spack build caches:"
           spack buildcache list
 
-          # base-env
-          echo "base-env ..."
+          # Install from cache
           spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.gnu-11.4.0.unified-env
 
+          # Check shared libraries
           ${SPACK_STACK_DIR}/util/ldd_check.py $SPACK_ENV 2>&1 | tee log.ldd_check
+
+          # Create modules
           spack clean -a
           spack module tcl refresh -y
           spack stack setup-meta-modules
@@ -201,7 +209,7 @@ jobs:
 
           module use ${ENVDIR}/install/modulefiles/Core
           module load stack-gcc/11.4.0
-          module load stack-openmpi/5.0.3
+          module load stack-openmpi/5.0.5
           module load stack-python/3.11.7
           module available
 

diff --git a/.github/workflows/ubuntu-ci-x86_64-intel.yaml b/.github/workflows/ubuntu-ci-x86_64-intel.yaml
@@ -93,7 +93,7 @@ jobs:
           echo "  intel-oneapi-mpi:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
           echo "    buildable: false" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
           echo "    externals:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
-          echo "    - spec: [email protected]%[email protected]" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
+          echo "    - spec: [email protected]%[email protected] +classic-names" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
           echo "      prefix: /opt/intel/oneapi" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml
 
           # Add external Intel MKL and oneAPI runtime
@@ -155,17 +155,22 @@ jobs:
           # base-env
           echo "base-env ..."
           spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.intel-2021.10.0-buildcache.base-env
-          spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ base-env
+          spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ base-env
 
           # jedi-base-env
           echo "jedi-base-env ..."
           spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.intel-2021.10.0-buildcache.jedi-base-env
-          spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env
+          spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env
+
+          # jedi-ufs-env
+          echo "jedi-ufs-env ..."
+          spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.intel-2021.10.0-buildcache.jedi-ufs-env
+          spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-ufs-env
 
           # the rest
           echo "unified-env ..."
           spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.intel-2021.10.0-buildcache.unified-env
-          spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/
+          spack buildcache create -u /home/ubuntu/spack-stack/build-cache/
 
           # Remove binary cache for next round of concretization
           if [ "$USE_BINARY_CACHE" = true ] ; then
@@ -183,6 +188,7 @@ jobs:
 
       - name: create-env
         run: |
+          # Set up spack-stack
           source ./setup.sh
           export BUILDCACHE_ENVNAME=ue-intel-2021.10.0-buildcache
           export BUILDCACHE_ENVDIR=$PWD/envs/${BUILDCACHE_ENVNAME}
@@ -191,20 +197,22 @@ jobs:
           rsync -av --exclude='install' --exclude='spack.lock' --exclude='.spack_db' ${BUILDCACHE_ENVDIR}/ ${ENVDIR}/
           spack env activate ${ENVDIR}
 
-          # Concretize
+          # Concretize and check for duplicates
           spack concretize --force 2>&1 | tee log.concretize.intel-2021.10.0
           ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.intel-2021.10.0 -i fms -i crtm -i esmf -i mapl
 
-          # Add binary cache back in and reindex it
+          # Add binary cache back in
           spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/
           echo "Packages in combined spack build caches:"
           spack buildcache list
 
-          # base-env
-          echo "base-env ..."
+          # Install from cache
           spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.intel-2021.10.0.unified-env
 
+          # Check shared libraries
           ${SPACK_STACK_DIR}/util/ldd_check.py $SPACK_ENV 2>&1 | tee log.ldd_check
+
+          # Create modules
           spack clean -a
           spack module tcl refresh -y
           spack stack setup-meta-modules

diff --git a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml b/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml
@@ -161,17 +161,22 @@ jobs:
           # base-env
           echo "base-env ..."
           spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.oneapi-2024.2.0-buildcache.base-env
-          spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ base-env
+          spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ base-env
 
           # jedi-base-env
           echo "jedi-base-env ..."
           spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.oneapi-2024.2.0-buildcache.jedi-base-env
-          spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env
+          spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env
+
+          # jedi-ufs-env
+          echo "jedi-ufs-env ..."
+          spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.oneapi-2024.2.0-buildcache.jedi-ufs-env
+          spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-ufs-env
 
           # the rest
           echo "unified-env ..."
           spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.oneapi-2024.2.0-buildcache.unified-env
-          spack buildcache create -a -u /home/ubuntu/spack-stack/build-cache/
+          spack buildcache create -u /home/ubuntu/spack-stack/build-cache/
 
           # Remove binary cache for next round of concretization
           if [ "$USE_BINARY_CACHE" = true ] ; then
@@ -189,6 +194,7 @@ jobs:
 
       - name: create-env
         run: |
+          # Set up spack-stack
           source ./setup.sh
           export BUILDCACHE_ENVNAME=ue-oneapi-2024.2.0-buildcache
           export BUILDCACHE_ENVDIR=$PWD/envs/${BUILDCACHE_ENVNAME}
@@ -197,39 +203,27 @@ jobs:
           rsync -av --exclude='install' --exclude='spack.lock' --exclude='.spack_db' ${BUILDCACHE_ENVDIR}/ ${ENVDIR}/
           spack env activate ${ENVDIR}
 
-          # Concretize
+          # Concretize and check for duplicates
           spack concretize --force 2>&1 | tee log.concretize.oneapi-2024.2.0
           ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.oneapi-2024.2.0 -i fms -i crtm -i esmf -i mapl
 
-          # Add binary cache back in and reindex it
+          # Add binary cache back in
           spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/
           echo "Packages in combined spack build caches:"
           spack buildcache list
 
-          # base-env
-          echo "base-env ..."
+          # Install from cache
           spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.oneapi-2024.2.0.unified-env
 
+          # Check shared libraries
           ${SPACK_STACK_DIR}/util/ldd_check.py $SPACK_ENV 2>&1 | tee log.ldd_check
+
+          # Create modules
           spack clean -a
           spack module tcl refresh -y
           spack stack setup-meta-modules
           spack env deactivate
 
-          ### # Test environment chaining
-          ### echo "Test environment chaining"
-          ### spack stack create env --name chaintest --template empty --site linux.default --upstream ${ENVDIR}/install --compiler intel
-          ### # Retain config from upstream so we don't have to rebuild:
-          ### cp -r ${ENVDIR}/{site,common} $PWD/envs/chaintest/.
-          ### spack env activate ${PWD}/envs/chaintest
-          ### # Pin gmake to avoid duplicate packages (excluded gmake from spack external find above)
-          ### spack config add "packages:gmake:require:'@:4.2'"
-          ### spack add [email protected]%oneapi
-          ### spack concretize | tee envs/chaintest/log.concretize
-          ### unwanted_duplicates=$(( cat envs/chaintest/log.concretize | grep -E '^ - ' | grep -Fv '[email protected]' || true ) | wc -l)
-          ### if [ ${unwanted_duplicates} -gt 0 ]; then echo "Environment chaining test failed"; exit 1; fi
-          ### spack env deactivate
-
       - name: test-env
         run: |
           source /etc/profile.d/modules.sh