From 6a692131c4fcf4e6612b8f6a6b170f1424ccd2fd Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Sun, 25 Feb 2024 20:56:25 -0700 Subject: [PATCH 1/7] Bump openmpi on Nautilus for Intel to 4.1.6 --- configs/sites/nautilus/packages.yaml | 14 ++++---------- doc/source/PreConfiguredSites.rst | 2 +- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/configs/sites/nautilus/packages.yaml b/configs/sites/nautilus/packages.yaml index 78d2dfab1..812027634 100644 --- a/configs/sites/nautilus/packages.yaml +++ b/configs/sites/nautilus/packages.yaml @@ -4,8 +4,7 @@ packages: providers: # For now need to enable one or the other; # see https://github.com/JCSDA/spack-stack/issues/659 - mpi:: [openmpi@4.1.5rc2] - #mpi:: [openmpi@4.1.4] + mpi:: [openmpi@4.1.6] blas:: [intel-oneapi-mkl] fftw-api:: [intel-oneapi-mkl] lapack:: [intel-oneapi-mkl] @@ -25,15 +24,10 @@ packages: # prefix: /p/app/compilers/intel/oneapi openmpi: externals: - - spec: openmpi@4.1.5rc2%intel@2021.5.0~cuda~cxx~cxx_exceptions~java~memchecker+pmi~static~wrapper-rpath fabrics=ucx schedulers=slurm - prefix: /p/app/penguin/openmpi/4.1.5rc2/intel + - spec: openmpi@4.1.6%intel@2021.5.0~cuda~cxx~cxx_exceptions~java~memchecker+pmi~static~wrapper-rpath fabrics=ucx schedulers=slurm + prefix: /p/app/penguin/openmpi/4.1.6/intel-classic-2022.0.2 modules: - - penguin/openmpi/4.1.5rc2/intel - - slurm - - spec: openmpi@4.1.4%aocc@4.0.0~cuda~cxx~cxx_exceptions~java~memchecker+pmi~static~wrapper-rpath fabrics=ucx schedulers=slurm - prefix: /p/app/penguin/openmpi/4.1.4/aoc - modules: - - penguin/openmpi/4.1.4/aocc + - penguin/openmpi/4.1.6/intel-classic-2022.0.2 - slurm intel-oneapi-mkl: externals: diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 45849dd72..eb6b76ca0 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -312,7 +312,7 @@ With Intel, the following is required for building new spack environments and fo module load slurm module load intel/compiler/2022.0.2 - module load penguin/openmpi/4.1.5rc2/intel + module load penguin/openmpi/4.1.6/intel-classic-2022.0.2 module use /p/app/projects/NEPTUNE/spack-stack/modulefiles module load ecflow/5.8.4 From cd60be30b73a50a77ee928bc34e1f2c4e583498e Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 27 Feb 2024 13:52:26 -0700 Subject: [PATCH 2/7] Update .gitmodules and submodule pointer for spack for code review and testing --- .gitmodules | 12 +++++++----- spack | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.gitmodules b/.gitmodules index b1f475b64..c99474954 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,10 +1,12 @@ [submodule "spack"] path = spack - #url = https://github.com/spack/spack - #branch = develop - url = https://github.com/jcsda/spack - branch = jcsda_emc_spack_stack -[submodule "doc/CMakeModules"] + ##url = https://github.com/spack/spack + ##branch = develop + #url = https://github.com/jcsda/spack + #branch = jcsda_emc_spack_stack + url = https://github.com/rhoneyager-tomorrow/spack + branch = bugfix/hdf5_fpe + [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules branch = develop diff --git a/spack b/spack index a36fdf2ad..78fa50348 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit a36fdf2ad09c770205b7dc61a0a5ffbf284cb83f +Subproject commit 78fa50348e4d1ba8c65f7b126497ed11ce1c3dbe From 0e358fa11ff97803993501e7d4343881b80a025e Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 27 Feb 2024 13:53:22 -0700 Subject: [PATCH 3/7] Fix CI on Ubuntu R&D cluster --- .github/workflows/ubuntu-rnd-x86_64.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu-rnd-x86_64.yaml b/.github/workflows/ubuntu-rnd-x86_64.yaml index 6b924468c..0b32d4f24 100644 --- a/.github/workflows/ubuntu-rnd-x86_64.yaml +++ b/.github/workflows/ubuntu-rnd-x86_64.yaml @@ -188,7 +188,7 @@ jobs: ls -l /home/ubuntu/spack-stack/CI/unified-env/${TODAY}/modulefiles/Core module use /home/ubuntu/spack-stack/CI/unified-env/${TODAY}/modulefiles/Core - module load stack-intel/2022.1.0 + module load stack-intel/2021.6.0 module load stack-intel-oneapi-mpi/2021.6.0 module load stack-python/3.10.13 module available From ba6b74887c4313270b1cffe89abbae7cebb9cb8f Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 27 Feb 2024 14:38:50 -0700 Subject: [PATCH 4/7] Update documentation for Nautilus: use openmpo@4.1.6 --- configs/sites/nautilus/packages.yaml | 6 ++++++ doc/source/PreConfiguredSites.rst | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/configs/sites/nautilus/packages.yaml b/configs/sites/nautilus/packages.yaml index 812027634..ff4a9ded9 100644 --- a/configs/sites/nautilus/packages.yaml +++ b/configs/sites/nautilus/packages.yaml @@ -29,6 +29,12 @@ packages: modules: - penguin/openmpi/4.1.6/intel-classic-2022.0.2 - slurm + - spec: openmpi@4.1.4%aocc@4.0.0~cuda~cxx~cxx_exceptions~java~memchecker+pmi~static~wrapper-rpath + fabrics=ucx schedulers=slurm + prefix: /p/app/penguin/openmpi/4.1.4/aoc + modules: + - penguin/openmpi/4.1.4/aocc + - slurm intel-oneapi-mkl: externals: - spec: intel-oneapi-mkl@2022.0.2 diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index eb6b76ca0..0a0486d40 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -321,9 +321,9 @@ For ``spack-stack-1.6.0`` with Intel, proceed with loading the following modules .. code-block:: console - module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core + module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-1.6.0/envs/ue-openmpi416/install/modulefiles/Core module load stack-intel/2021.5.0 - module load stack-openmpi/4.1.5rc2 + module load stack-openmpi/4.1.6 module load stack-python/3.10.13 With AMD clang/flang (aocc), the following is required for building new spack environments and for using spack to build and run software. From 18621ffb63589257474c6e42197f5f2baf64a30e Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 27 Feb 2024 14:45:58 -0700 Subject: [PATCH 5/7] Bug fixes --- .gitmodules | 2 +- configs/sites/nautilus/packages.yaml | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index c99474954..2de19dab2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -6,7 +6,7 @@ #branch = jcsda_emc_spack_stack url = https://github.com/rhoneyager-tomorrow/spack branch = bugfix/hdf5_fpe - [submodule "doc/CMakeModules"] +[submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules branch = develop diff --git a/configs/sites/nautilus/packages.yaml b/configs/sites/nautilus/packages.yaml index ff4a9ded9..aac29954e 100644 --- a/configs/sites/nautilus/packages.yaml +++ b/configs/sites/nautilus/packages.yaml @@ -29,8 +29,7 @@ packages: modules: - penguin/openmpi/4.1.6/intel-classic-2022.0.2 - slurm - - spec: openmpi@4.1.4%aocc@4.0.0~cuda~cxx~cxx_exceptions~java~memchecker+pmi~static~wrapper-rpath - fabrics=ucx schedulers=slurm + - spec: openmpi@4.1.4%aocc@4.0.0~cuda~cxx~cxx_exceptions~java~memchecker+pmi~static~wrapper-rpath fabrics=ucx schedulers=slurm prefix: /p/app/penguin/openmpi/4.1.4/aoc modules: - penguin/openmpi/4.1.4/aocc From 67f1665cfa87acd0f1835840e0aa0044348c5441 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 27 Feb 2024 15:12:13 -0700 Subject: [PATCH 6/7] Fix url in .gitmodules --- .gitmodules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitmodules b/.gitmodules index 2de19dab2..548e939af 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,7 +4,7 @@ ##branch = develop #url = https://github.com/jcsda/spack #branch = jcsda_emc_spack_stack - url = https://github.com/rhoneyager-tomorrow/spack + url = https://github.com/rhoneyager-tomorrow/spack-1 branch = bugfix/hdf5_fpe [submodule "doc/CMakeModules"] path = doc/CMakeModules From f6e84faff14b889ecffe749475bf6538b2ca914d Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 28 Feb 2024 11:25:26 -0700 Subject: [PATCH 7/7] Revert .gitmodules and update submodule pointer for spack --- .gitmodules | 10 ++++------ spack | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.gitmodules b/.gitmodules index 548e939af..b1f475b64 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,11 +1,9 @@ [submodule "spack"] path = spack - ##url = https://github.com/spack/spack - ##branch = develop - #url = https://github.com/jcsda/spack - #branch = jcsda_emc_spack_stack - url = https://github.com/rhoneyager-tomorrow/spack-1 - branch = bugfix/hdf5_fpe + #url = https://github.com/spack/spack + #branch = develop + url = https://github.com/jcsda/spack + branch = jcsda_emc_spack_stack [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules diff --git a/spack b/spack index 78fa50348..8e40b08ee 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 78fa50348e4d1ba8c65f7b126497ed11ce1c3dbe +Subproject commit 8e40b08ee6e321181d6dd0c0c926e4f406d7fa85