diff --git a/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml b/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml new file mode 100644 index 000000000..0397b0e51 --- /dev/null +++ b/.github/workflows/ubuntu-ci-x86_64-oneapi-ifx.yaml @@ -0,0 +1,245 @@ +name: ubuntu-ci-c6a-x86_64-oneapi-ifx-build +on: + pull_request: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +defaults: + run: + shell: bash + +jobs: + ubuntu-ci-c6a-x86_64-oneapi-ifx-build: + runs-on: [ubuntu-ci-c6a-x86_64] + + steps: + - name: cleanup + run: | + pwd + ls -lart + set +e + find ./* -type d -exec chmod u+xw {} \; + set -e + rm -fr * + + - name: checkout + uses: actions/checkout@v4 + with: + submodules: true + + - name: prepare-directories + run: | + mkdir -p /home/ubuntu/spack-stack/build-cache/ + mkdir -p /home/ubuntu/spack-stack/source-cache/ + + - name: create-buildcache + run: | + # Get day of week to decide whether to use build caches or not + DOW=$(date +%u) + # Monday is 1 ... Sunday is 7 + if [[ $DOW == 7 ]]; then + export USE_BINARY_CACHE=false + echo "Ignore existing binary cache for creating buildcache environment" + else + export USE_BINARY_CACHE=true + echo "Use existing binary cache for creating buildcache environment" + fi + + # Set up spack-stack + source ./setup.sh + export ENVNAME=ue-oneapi-ifx-2024.2.0-buildcache + export ENVDIR=$PWD/envs/${ENVNAME} + spack stack create env --site linux.default --template unified-dev --name ${ENVNAME} --compiler oneapi + spack env activate ${ENVDIR} + export SPACK_SYSTEM_CONFIG_PATH="${ENVDIR}/site" + + # Find external packages + spack external find --scope system \ + --exclude bison --exclude openssl \ + --exclude curl --exclude python \ + --exclude gmake + spack external find --scope system sed + spack external find --scope system perl + spack external find --scope system wget + spack external find --scope system texlive + spack external find --scope system mysql + + # Find compilers + spack compiler find --scope system + + # For Intel oneAPI + echo "" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml + echo "- compiler:" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml + echo " spec: oneapi@2024.2.0" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml + echo " paths:" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml + echo " cc: /opt/intel/oneapi/compiler/2024.2/bin/icx" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml + echo " cxx: /opt/intel/oneapi/compiler/2024.2/bin/icpx" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml + echo " f77: /opt/intel/oneapi/compiler/2024.2/bin/ifx" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml + echo " fc: /opt/intel/oneapi/compiler/2024.2/bin/ifx" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml + echo " flags: {}" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml + echo " operating_system: ubuntu22.04" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml + echo " target: x86_64" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml + echo " modules: []" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml + echo " environment: {}" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml + echo " extra_rpaths: []" >> ${SPACK_SYSTEM_CONFIG_PATH}/compilers.yaml + + # Need to find external Intel MPI and annotate with the + # correct compiler, no way to do that with spack commands. + echo "" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml + echo " intel-oneapi-mpi:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml + echo " buildable: false" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml + echo " externals:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml + echo " - spec: intel-oneapi-mpi@2021.13%oneapi@2024.2.0" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml + echo " prefix: /opt/intel/oneapi" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml + + # Add external Intel MKL and oneAPI runtime + echo "" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml + echo " intel-oneapi-mkl:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml + echo " externals:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml + echo " - spec: intel-oneapi-mkl@2024.2%oneapi@2024.2.0" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml + echo " prefix: /opt/intel/oneapi" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml + echo " intel-oneapi-runtime:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml + echo " externals:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml + echo " - spec: intel-oneapi-runtime@2024.2.0%oneapi@2024.2.0" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml + echo " prefix: /opt/intel/oneapi" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml + + export -n SPACK_SYSTEM_CONFIG_PATH + + # For buildcaches + spack config add config:install_tree:padded_length:200 + + # Set compiler and MPI specs + spack config add "packages:all:providers:mpi:[intel-oneapi-mpi@2021.13]" + spack config add "packages:all:compiler:[oneapi@2024.2.0,gcc@11.4.0]" + + # Add additional variants for MET packages, different from config/common/packages.yaml + spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis" + + # Switch providers for blas, lapack, fftw-api + spack config add "packages:all:providers:mpi:[intel-oneapi-mpi]" + spack config add "packages:all:providers:blas:[intel-oneapi-mkl]" + spack config add "packages:all:providers:fftw-api:[intel-oneapi-mkl]" + spack config add "packages:all:providers:lapack:[intel-oneapi-mkl]" + spack config add "packages:ectrans:require:'+mkl ~fftw'" + spack config add "packages:gsibec:require:'+mkl'" + spack config add "packages:py-numpy:require:['^intel-oneapi-mkl']" + + # Pin gmake to avoid duplicate packages (excluded gmake from spack external find above) + spack config add "packages:gmake:require:'@:4.2'" + + # Remove wgrib2 from all virtual packages, since it doesn't build with oneAPI + sed -i 's/depends_on("wgrib2")/#depends_on("wgrib2")/g' `grep -lRie wgrib2 spack-ext/` + + # Don't generate ecflow module when using external package + spack config add "modules:default:tcl:exclude:[ecflow]" + + # Concretize and check for duplicates + spack concretize 2>&1 | tee log.concretize.oneapi-ifx-2024.2.0-buildcache + ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.oneapi-ifx-2024.2.0-buildcache -i fms -i crtm -i esmf -i mapl + + # Add and update source cache + spack mirror add local-source file:///home/ubuntu/spack-stack/source-cache/ + spack mirror create -a -d /home/ubuntu/spack-stack/source-cache/ + + # Add binary cache if requested + if [ "$USE_BINARY_CACHE" = true ] ; then + set +e + spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/ + spack buildcache update-index local-binary || (echo "No valid binary cache found, proceed without" && spack mirror rm local-binary) + set +e + echo "Packages in spack binary cache:" + spack buildcache list + fi + + # Break installation up in pieces and create build caches in between + # This allows us to "spin up" builds that altogether take longer than + # six hours, and/or fail later in the build process. + + # base-env + echo "base-env ..." + spack install --fail-fast --source --no-check-signature base-env 2>&1 | tee log.install.oneapi-ifx-2024.2.0-buildcache.base-env + spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ base-env + + # jedi-base-env + echo "jedi-base-env ..." + spack install --fail-fast --source --no-check-signature jedi-base-env 2>&1 | tee log.install.oneapi-ifx-2024.2.0-buildcache.jedi-base-env + spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-base-env + + # jedi-ufs-env + echo "jedi-ufs-env ..." + spack install --fail-fast --source --no-check-signature jedi-ufs-env 2>&1 | tee log.install.oneapi-ifx-2024.2.0-buildcache.jedi-ufs-env + spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ jedi-ufs-env + + # the rest + echo "unified-env ..." + spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.oneapi-ifx-2024.2.0-buildcache.unified-env + spack buildcache create -u /home/ubuntu/spack-stack/build-cache/ + + # Remove binary cache for next round of concretization + if [ "$USE_BINARY_CACHE" = true ] ; then + spack mirror rm local-binary + fi + + # Remove buildcache config settings + spack config remove config:install_tree:padded_length + + # Next steps: synchronize source and build cache to a central/combined mirror? + + # Cleanup + spack clean -a + spack env deactivate + + - name: create-env + run: | + # Set up spack-stack + source ./setup.sh + export BUILDCACHE_ENVNAME=ue-oneapi-ifx-2024.2.0-buildcache + export BUILDCACHE_ENVDIR=$PWD/envs/${BUILDCACHE_ENVNAME} + export ENVNAME=ue-oneapi-ifx-2024.2.0 + export ENVDIR=$PWD/envs/${ENVNAME} + rsync -av --exclude='install' --exclude='spack.lock' --exclude='.spack_db' ${BUILDCACHE_ENVDIR}/ ${ENVDIR}/ + spack env activate ${ENVDIR} + + # Concretize and check for duplicates + spack concretize --force 2>&1 | tee log.concretize.oneapi-ifx-2024.2.0 + ${SPACK_STACK_DIR}/util/show_duplicate_packages.py -d log.concretize.oneapi-ifx-2024.2.0 -i fms -i crtm -i esmf -i mapl + + # Add binary cache back in + spack mirror add local-binary file:///home/ubuntu/spack-stack/build-cache/ + echo "Packages in combined spack build caches:" + spack buildcache list + + # Install from cache + spack install --fail-fast --source --no-check-signature 2>&1 | tee log.install.oneapi-ifx-2024.2.0.unified-env + + # Check shared libraries + ${SPACK_STACK_DIR}/util/ldd_check.py $SPACK_ENV 2>&1 | tee log.ldd_check + + # Create modules + spack clean -a + spack module tcl refresh -y + spack stack setup-meta-modules + spack env deactivate + + - name: test-env + run: | + source /etc/profile.d/modules.sh + module use /home/ubuntu/spack-stack/modulefiles + + export ENVNAME=ue-oneapi-ifx-2024.2.0 + export ENVDIR=$PWD/envs/${ENVNAME} + ls -l ${ENVDIR}/install/modulefiles/Core + + module use ${ENVDIR}/install/modulefiles/Core + module load stack-oneapi/2024.2.0 + module load stack-intel-oneapi-mpi/2021.13 + module load stack-python/3.11.7 + module available + + module load jedi-ufs-env + module load ewok-env + module load soca-env + module list diff --git a/configs/sites/tier1/atlantis/packages_intel.yaml b/configs/sites/tier1/atlantis/packages_intel.yaml index 4271f3f10..e6eb565f3 100644 --- a/configs/sites/tier1/atlantis/packages_intel.yaml +++ b/configs/sites/tier1/atlantis/packages_intel.yaml @@ -7,7 +7,7 @@ packages: buildable: False intel-oneapi-mpi: externals: - - spec: intel-oneapi-mpi@2021.6.0%intel@2021.6.0 + - spec: intel-oneapi-mpi@2021.6.0%intel@2021.6.0 +classic-names prefix: /cm/shared/apps/intel/oneapi modules: - mpi/2021.6.0 diff --git a/configs/sites/tier1/aws-pcluster/packages_intel.yaml b/configs/sites/tier1/aws-pcluster/packages_intel.yaml index 7fb3d8477..e0f255a05 100644 --- a/configs/sites/tier1/aws-pcluster/packages_intel.yaml +++ b/configs/sites/tier1/aws-pcluster/packages_intel.yaml @@ -11,7 +11,7 @@ packages: buildable: False intel-oneapi-mpi: externals: - - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0 + - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0 +classic-names prefix: /opt/intel/oneapi modules: - libfabric-aws/1.19.0amzn4.0 diff --git a/configs/sites/tier1/discover-scu16/packages_intel.yaml b/configs/sites/tier1/discover-scu16/packages_intel.yaml index 8bd48da65..a0f857ac8 100644 --- a/configs/sites/tier1/discover-scu16/packages_intel.yaml +++ b/configs/sites/tier1/discover-scu16/packages_intel.yaml @@ -7,7 +7,7 @@ packages: buildable: False intel-oneapi-mpi: externals: - - spec: intel-oneapi-mpi@2021.6.0%intel@2021.6.0 + - spec: intel-oneapi-mpi@2021.6.0%intel@2021.6.0 +classic-names prefix: /usr/local/intel/oneapi/2021 modules: - mpi/impi/2021.6.0 diff --git a/configs/sites/tier1/discover-scu17/packages_intel.yaml b/configs/sites/tier1/discover-scu17/packages_intel.yaml index d971e2387..e28f13a05 100644 --- a/configs/sites/tier1/discover-scu17/packages_intel.yaml +++ b/configs/sites/tier1/discover-scu17/packages_intel.yaml @@ -7,7 +7,7 @@ packages: buildable: False intel-oneapi-mpi: externals: - - spec: intel-oneapi-mpi@2021.10.0%intel@=2021.10.0 + - spec: intel-oneapi-mpi@2021.10.0%intel@=2021.10.0 +classic-names prefix: /usr/local/intel/oneapi/2021 modules: - mpi/impi/2021.10.0 diff --git a/configs/sites/tier1/hera/packages_intel.yaml b/configs/sites/tier1/hera/packages_intel.yaml index 5ac52dd8b..34b77b76d 100644 --- a/configs/sites/tier1/hera/packages_intel.yaml +++ b/configs/sites/tier1/hera/packages_intel.yaml @@ -11,7 +11,7 @@ packages: buildable: False intel-oneapi-mpi: externals: - - spec: intel-oneapi-mpi@2021.5.1%intel@2021.5.0 + - spec: intel-oneapi-mpi@2021.5.1%intel@2021.5.0 +classic-names modules: - impi/2022.1.2 prefix: /apps/oneapi diff --git a/configs/sites/tier1/hercules/packages_intel.yaml b/configs/sites/tier1/hercules/packages_intel.yaml index eb4a22dc7..b58287ac3 100644 --- a/configs/sites/tier1/hercules/packages_intel.yaml +++ b/configs/sites/tier1/hercules/packages_intel.yaml @@ -11,7 +11,7 @@ packages: buildable: False intel-oneapi-mpi: externals: - - spec: intel-oneapi-mpi@2021.9.0%intel@2021.9.0 + - spec: intel-oneapi-mpi@2021.9.0%intel@2021.9.0 +classic-names modules: - intel-oneapi-mpi/2021.9.0 intel-oneapi-mkl: diff --git a/configs/sites/tier1/jet/packages_intel.yaml b/configs/sites/tier1/jet/packages_intel.yaml index 5e0b82aaf..e04c4fa67 100644 --- a/configs/sites/tier1/jet/packages_intel.yaml +++ b/configs/sites/tier1/jet/packages_intel.yaml @@ -7,7 +7,7 @@ packages: buildable: False intel-oneapi-mpi: externals: - - spec: intel-oneapi-mpi@2021.5.1%intel@2021.5.0 + - spec: intel-oneapi-mpi@2021.5.1%intel@2021.5.0 +classic-names modules: - impi/2022.1.2 prefix: /apps/oneapi diff --git a/configs/sites/tier1/noaa-aws/packages_intel.yaml b/configs/sites/tier1/noaa-aws/packages_intel.yaml index 1ffe7cf2f..5cc0e66d7 100644 --- a/configs/sites/tier1/noaa-aws/packages_intel.yaml +++ b/configs/sites/tier1/noaa-aws/packages_intel.yaml @@ -11,7 +11,7 @@ packages: buildable: False intel-oneapi-mpi: externals: - - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0 + - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0 +classic-names prefix: /apps/oneapi modules: - impi/2023.2.0 diff --git a/configs/sites/tier1/noaa-azure/packages_intel.yaml b/configs/sites/tier1/noaa-azure/packages_intel.yaml index 1ffe7cf2f..5cc0e66d7 100644 --- a/configs/sites/tier1/noaa-azure/packages_intel.yaml +++ b/configs/sites/tier1/noaa-azure/packages_intel.yaml @@ -11,7 +11,7 @@ packages: buildable: False intel-oneapi-mpi: externals: - - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0 + - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0 +classic-names prefix: /apps/oneapi modules: - impi/2023.2.0 diff --git a/configs/sites/tier1/noaa-gcloud/packages_intel.yaml b/configs/sites/tier1/noaa-gcloud/packages_intel.yaml index 1ffe7cf2f..5cc0e66d7 100644 --- a/configs/sites/tier1/noaa-gcloud/packages_intel.yaml +++ b/configs/sites/tier1/noaa-gcloud/packages_intel.yaml @@ -11,7 +11,7 @@ packages: buildable: False intel-oneapi-mpi: externals: - - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0 + - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0 +classic-names prefix: /apps/oneapi modules: - impi/2023.2.0 diff --git a/configs/sites/tier1/orion/packages_intel.yaml b/configs/sites/tier1/orion/packages_intel.yaml index eb4a22dc7..b58287ac3 100644 --- a/configs/sites/tier1/orion/packages_intel.yaml +++ b/configs/sites/tier1/orion/packages_intel.yaml @@ -11,7 +11,7 @@ packages: buildable: False intel-oneapi-mpi: externals: - - spec: intel-oneapi-mpi@2021.9.0%intel@2021.9.0 + - spec: intel-oneapi-mpi@2021.9.0%intel@2021.9.0 +classic-names modules: - intel-oneapi-mpi/2021.9.0 intel-oneapi-mkl: diff --git a/configs/sites/tier1/s4/packages.yaml b/configs/sites/tier1/s4/packages.yaml index a2457222e..ed2478503 100644 --- a/configs/sites/tier1/s4/packages.yaml +++ b/configs/sites/tier1/s4/packages.yaml @@ -13,7 +13,7 @@ packages: buildable: False intel-oneapi-mpi: externals: - - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0 + - spec: intel-oneapi-mpi@2021.10.0%intel@2021.10.0 +classic-names prefix: /opt/intel/oneapi/2023.2 modules: - intel/2023.2 diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py index e2d75b651..a5ce92173 100755 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py @@ -361,7 +361,10 @@ def setup_meta_modules(): # Determine the preferred compiler and sort the flattened list of compilers # such that the preferred compiler comes last. This is so that all other # compilers populate the MODULEPATHS_SAVE list before the preferred compiler - # takes it and adds it to the stack-COMPILER metamodule. + # takes it and adds it to the stack-COMPILER metamodule. Likewise, we need + # to save the list of compiler substitutions from the preferred compiler + # so that we have access to it when we build the MPI meta module. Note that + # by definition, only the preferred compiler can be used for MPI dependencies. try: preferred_compilers = spack.config.get("packages")["all"]["prefer"] except: @@ -386,6 +389,8 @@ def custom_sort_key(entry): compiler_config = spack.config.get("compilers") # Collect and save modulepaths for the preferred compiler MODULEPATHS_SAVE = [] + # Initialize saved substitutes to None (populate for preferred compiler later) + SUBSTITUTES_SAVE = None for compiler_identifier in sorted_flattened_compiler_list: (compiler_name, compiler_version) = compiler_identifier.replace("@=","@").split("@") # Loop through all configured compilers and find the correct match @@ -523,6 +528,10 @@ def custom_sort_key(entry): with open(compiler_module_file, "w") as f: f.write(module_content) logging.info(" ... writing {}".format(compiler_module_file)) + # If this is the last compiler in the list (i.e. the preferred compiler), + # then save the substitutes for later use for building the MPI meta module. + if compiler_identifier == sorted_flattened_compiler_list[-1]: + SUBSTITUTES_SAVE = substitutes del MODULEPATHS_SAVE # Create mpi modules @@ -685,7 +694,18 @@ def custom_sort_key(entry): ) # Compiler wrapper environment variables - if "intel" in mpi_name: + if "intel" in mpi_name and compiler_name == "oneapi": + substitutes["MPICC"] = os.path.join("mpiicx") + substitutes["MPICXX"] = os.path.join("mpiicpx") + if "ifx" in SUBSTITUTES_SAVE["FC"] and not "ifort" in SUBSTITUTES_SAVE["FC"]: + substitutes["MPIF77"] = os.path.join("mpiifx") + substitutes["MPIF90"] = os.path.join("mpiifx") + elif not "ifx" in SUBSTITUTES_SAVE["FC"] and "ifort" in SUBSTITUTES_SAVE["FC"]: + substitutes["MPIF77"] = os.path.join("mpiifort") + substitutes["MPIF90"] = os.path.join("mpiifort") + else: + raise Exception(f"For {mpi_name}, cannot determine MPI wrapper from FC={SUBSTITUTES_SAVE['FC']}") + elif "intel" in mpi_name and compiler_name == "intel": substitutes["MPICC"] = os.path.join("mpiicc") substitutes["MPICXX"] = os.path.join("mpiicpc") substitutes["MPIF77"] = os.path.join("mpiifort") @@ -772,6 +792,7 @@ def custom_sort_key(entry): with open(mpi_module_file, "w") as f: f.write(module_content) logging.info(" ... writing {}".format(mpi_module_file)) + del SUBSTITUTES_SAVE # Create python modules. Need to accommodate both external # Python distributions and spack-built Python distributions.