From 859f66870a62da1c904c992e2c42d8e83409d9b6 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 19 Dec 2023 16:17:53 -0700 Subject: [PATCH 01/13] Update .gitmodules and submodule pointer for spack for code review and testing --- .gitmodules | 10 ++++++---- spack | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.gitmodules b/.gitmodules index 17f5bc5fc..9f4516e45 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,11 @@ [submodule "spack"] path = spack - #url = https://github.com/spack/spack - #branch = develop - url = https://github.com/jcsda/spack - branch = release/1.6.0 + ##url = https://github.com/spack/spack + ##branch = develop + #url = https://github.com/jcsda/spack + #branch = release/1.6.0 + url = https://github.com/climbfuji/spack + branch = feature/rel160_crtm_release [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules diff --git a/spack b/spack index 67633edb8..d71fff9b1 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 67633edb8b36aa88ef488ba175ada8978192d562 +Subproject commit d71fff9b13cd0667e1c5a8d535e583dc0b31f1d1 From 0216729645aba10b8142fee525a2a0068744e9df Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 19 Dec 2023 17:05:30 -0700 Subject: [PATCH 02/13] Generate module for qhull --- configs/common/modules_lmod.yaml | 1 - configs/common/modules_tcl.yaml | 1 - 2 files changed, 2 deletions(-) diff --git a/configs/common/modules_lmod.yaml b/configs/common/modules_lmod.yaml index d8b853121..7d9957bfc 100644 --- a/configs/common/modules_lmod.yaml +++ b/configs/common/modules_lmod.yaml @@ -71,7 +71,6 @@ modules: - openssl - perl - pkgconf - - qhull - qt - randrproto - readline diff --git a/configs/common/modules_tcl.yaml b/configs/common/modules_tcl.yaml index 9a8affb11..afe12a2be 100644 --- a/configs/common/modules_tcl.yaml +++ b/configs/common/modules_tcl.yaml @@ -73,7 +73,6 @@ modules: - openssl - perl - pkgconf - - qhull - qt - randrproto - readline From 397410c0f7c696f9b56a1f9d4bb75d864e4748ef Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 20 Dec 2023 10:54:09 -0700 Subject: [PATCH 03/13] Revert .gitmodules and update submodule pointer for spack --- .gitmodules | 10 ++++------ spack | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.gitmodules b/.gitmodules index 9f4516e45..17f5bc5fc 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,11 +1,9 @@ [submodule "spack"] path = spack - ##url = https://github.com/spack/spack - ##branch = develop - #url = https://github.com/jcsda/spack - #branch = release/1.6.0 - url = https://github.com/climbfuji/spack - branch = feature/rel160_crtm_release + #url = https://github.com/spack/spack + #branch = develop + url = https://github.com/jcsda/spack + branch = release/1.6.0 [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules diff --git a/spack b/spack index d71fff9b1..4c6dcd2a0 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit d71fff9b13cd0667e1c5a8d535e583dc0b31f1d1 +Subproject commit 4c6dcd2a01927d06ccc1005d8403811d0eccdc30 From b32f57190e29a5be1056047700ff335fd8e80c92 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 28 Dec 2023 14:30:34 -0700 Subject: [PATCH 04/13] Update site config for Hercules for release/1.6.0 --- configs/sites/hercules/compilers.yaml | 5 ++++- configs/sites/hercules/packages.yaml | 8 ++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/configs/sites/hercules/compilers.yaml b/configs/sites/hercules/compilers.yaml index a86be4858..36c12beac 100644 --- a/configs/sites/hercules/compilers.yaml +++ b/configs/sites/hercules/compilers.yaml @@ -11,7 +11,10 @@ compilers: target: x86_64 modules: - intel-oneapi-compilers/2023.1.0 - environment: {} + environment: + set: + # https://github.com/ufs-community/ufs-weather-model/issues/2015#issuecomment-1864438186 + I_MPI_EXTRA_FILESYSTEM: 'ON' extra_rpaths: [] - compiler: spec: gcc@12.2.0 diff --git a/configs/sites/hercules/packages.yaml b/configs/sites/hercules/packages.yaml index c71b30cf1..955bedfd7 100644 --- a/configs/sites/hercules/packages.yaml +++ b/configs/sites/hercules/packages.yaml @@ -84,10 +84,6 @@ packages: externals: - spec: groff@1.22.4 prefix: /usr - libtool: - externals: - - spec: libtool@2.4.6 - prefix: /usr m4: externals: - spec: m4@1.4.19 @@ -109,6 +105,10 @@ packages: prefix: /apps/spack-managed/gcc-11.3.1/qt-5.15.8-d47tsna6f5dylcpblkfgw4gpn2cucihd modules: - qt/5.15.8 + sed: + externals: + - spec: sed@4.8 + prefix: /usr subversion: externals: - spec: subversion@1.14.1 From 4599cf40da8706ca87bade9a651019118859d5a3 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 28 Dec 2023 15:58:52 -0700 Subject: [PATCH 05/13] Update documentation for spack-stack-1.6.0 for Hercules --- doc/source/PreConfiguredSites.rst | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 538a72ef8..e534a9b71 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -8,17 +8,17 @@ Directory ``configs/sites`` contains site configurations for several HPC systems Pre-configured sites are split into two categories: Tier 1 with officially supported spack-stack installations (see :numref:`Section %s `), and Tier 2 (sites with configuration files that were tested or contributed by others in the past, but that are not officially supported by the spack-stack team; see :numref:`Section %s `). ============================================================= -Officially supported spack-stack 1.5.1 installations (tier 1) +Officially supported spack-stack 1.6.0 installations (tier 1) ============================================================= -Ready-to-use spack-stack 1.5.1 installations are available on the following, fully supported platforms. This version supports the JEDI Skylab release 5 of June 2023, and the UFS Weather Model of July 2023. It can also be used for testing spack-stack with other UFS applications (e.g. the UFS Short Range Weather Application, and the EMC Global Workflow). Amazon Web Services AMI are available in the US East 1 or 2 regions. +Ready-to-use spack-stack 1.6.0 installations are available on the following, fully supported platforms. This version supports JEDI-Skylab and various UFS Applications (UFS Weather Model, EMC Global Workflow, GSI, UFS Short Range Weather Application). Amazon Web Services AMI are available in the US East 1 or 2 regions. +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | Organization | System | Compilers | Location | Maintainers | +=====================+==================================+=================+=========================================================================================================+===============================+ | **HPC platforms** | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Hercules^* | GCC, Intel | ``/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.1/envs/unified-env`` | Cam Book / Dom Heinzeller | +| | Hercules | GCC, Intel | ``/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/unified-env`` | Cam Book / Dom Heinzeller | | MSU +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Orion | GCC, Intel | ``/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.1/envs/unified-env`` | Cam Book / Dom Heinzeller | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ @@ -61,6 +61,8 @@ Ready-to-use spack-stack 1.5.1 installations are available on the following, ful ^** Uses a slightly newer version than the spack-stack 1.5.1 release, so that `gmao-swell-env` is available in the environment. +For more information about a specific platform, please see the individual sections below. + For questions or problems, please consult the known issues in :numref:`Section %s `, the currently open GitHub `issues `_ and `discussions `_ first. .. _Preconfigured_Sites_Tier1: @@ -118,26 +120,32 @@ The following is required for building new spack environments and for using spac module load ecflow/5.8.4 module load mysql/8.0.31 -For ``spack-stack-1.5.1`` with Intel, load the following modules after loading mysql and ecflow: +For ``spack-stack-1.6.0`` with Intel, load the following modules after loading mysql and ecflow: .. code-block:: console - module use /work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core + module use /work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-intel/2021.9.0 module load stack-intel-oneapi-mpi/2021.9.0 - module load stack-python/3.10.8 + module load stack-python/3.10.13 module available -For ``spack-stack-1.5.1`` with GNU, load the following modules after loading mysql and ecflow: +For ``spack-stack-1.6.0`` with GNU, load the following modules after loading mysql and ecflow: .. code-block:: console - module use /work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core + module use /work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-gcc/12.2.0 module load stack-mvapich2/2.3.7 - module load stack-python/3.10.8 + module load stack-python/3.10.13 module available +.. note:: + spack-stack-1.6.0 on Hercules provides a chained environment `gsi-addon-env` for GSI. To use this environment, replace `unified-env` in the above `module use` statements with `gsi-addon-env`. + +.. note:: + spack-stack-1.6.0 on Hercules has `fms@2023.02.01` installed in the unified environment, in addition to the two standard versions `fms@2023.04` and `fms@release-jcsda`. + .. _Preconfigured_Sites_Discover: ------------------------------ From bb14ba9d5b677742314424b5060905980db91c53 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 29 Dec 2023 09:34:10 -0700 Subject: [PATCH 06/13] Update .gitmodules and submodule pointer for spack for code review and testing --- .gitmodules | 10 ++++++---- spack | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.gitmodules b/.gitmodules index 17f5bc5fc..5127cacd5 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,11 @@ [submodule "spack"] path = spack - #url = https://github.com/spack/spack - #branch = develop - url = https://github.com/jcsda/spack - branch = release/1.6.0 + ##url = https://github.com/spack/spack + ##branch = develop + #url = https://github.com/jcsda/spack + #branch = release/1.6.0 + url = https://github.com/climbfuji/spack + branch = bugfix/py_gevent_intel [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules diff --git a/spack b/spack index 4c6dcd2a0..da5394638 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 4c6dcd2a01927d06ccc1005d8403811d0eccdc30 +Subproject commit da539463828f00e3d2fc5f2cddcd178e2d149bf0 From 096cd94e56b76cae8aca34b91c3cbe11d8e569b2 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 29 Dec 2023 09:45:21 -0700 Subject: [PATCH 07/13] Update site config and documentation for Nautilus --- configs/sites/hercules/packages.yaml | 3 --- configs/sites/nautilus/packages.yaml | 8 ++++---- doc/source/PreConfiguredSites.rst | 21 ++++++++++++--------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/configs/sites/hercules/packages.yaml b/configs/sites/hercules/packages.yaml index 955bedfd7..900b60913 100644 --- a/configs/sites/hercules/packages.yaml +++ b/configs/sites/hercules/packages.yaml @@ -22,9 +22,6 @@ packages: - mvapich2/2.3.7 ### Modifications of common packages - # DH* TRY IF THIS IS STILL TRUE # Version 2.0.8 doesn't compile on Hercules - # DH* TRY IF THIS IS STILL TRUE wgrib2: - # DH* TRY IF THIS IS STILL TRUE version:: ['3.1.1'] ### All other external packages listed alphabetically autoconf: diff --git a/configs/sites/nautilus/packages.yaml b/configs/sites/nautilus/packages.yaml index 233e6d993..78d2dfab1 100644 --- a/configs/sites/nautilus/packages.yaml +++ b/configs/sites/nautilus/packages.yaml @@ -103,10 +103,6 @@ packages: externals: - spec: groff@1.22.3 prefix: /usr - libtool: - externals: - - spec: libtool@2.4.6 - prefix: /usr m4: externals: - spec: m4@1.4.18 @@ -125,6 +121,10 @@ packages: externals: - spec: pkgconf@1.4.2 prefix: /usr + sed: + externals: + - spec: sed@4.5 + prefix: /usr tar: externals: - spec: tar@1.30 diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index e534a9b71..ff203126f 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -40,7 +40,7 @@ Ready-to-use spack-stack 1.6.0 installations are available on the following, ful | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Narwhal | GCC | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.5.1/envs/unified-env-gcc-10.3.0`` | Dom Heinzeller / Sarah King | | U.S. Navy (HPCMP) +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Nautilus | Intel^* | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.5.1/envs/unified-env`` | Dom Heinzeller / Sarah King | +| | Nautilus | Intel | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.6.0/envs/unified-env`` | Dom Heinzeller / Sarah King | | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Nautilus | AOCC | *currently not supported* | Dom Heinzeller / Sarah King | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ @@ -57,8 +57,6 @@ Ready-to-use spack-stack 1.6.0 installations are available on the following, ful | NOAA (RDHPCS) | RDHPCS Cloud (Parallel Works) | Intel | ``/contrib/spack-stack/spack-stack-1.5.1/envs/unified-env`` | Mark Potts / Cam Book / Dom H | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -^* Uses a different ``wgrib2`` version 3.1.1 than the default 2.0.8. - ^** Uses a slightly newer version than the spack-stack 1.5.1 release, so that `gmao-swell-env` is available in the environment. For more information about a specific platform, please see the individual sections below. @@ -119,6 +117,7 @@ The following is required for building new spack environments and for using spac module use /work/noaa/epic/role-epic/spack-stack/hercules/modulefiles module load ecflow/5.8.4 module load mysql/8.0.31 + module load git-lfs/3.1.2 For ``spack-stack-1.6.0`` with Intel, load the following modules after loading mysql and ecflow: @@ -141,10 +140,10 @@ For ``spack-stack-1.6.0`` with GNU, load the following modules after loading mys module available .. note:: - spack-stack-1.6.0 on Hercules provides a chained environment `gsi-addon-env` for GSI. To use this environment, replace `unified-env` in the above `module use` statements with `gsi-addon-env`. + spack-stack-1.6.0 on Hercules provides a chained environment `gsi-addon-env` for GSI with Intel and GNU. To use this environment, replace `unified-env` in the above `module use` statements with `gsi-addon-env`, and load module `stack-python/3.11.6` instead of `stack-python/3.10.13`. .. note:: - spack-stack-1.6.0 on Hercules has `fms@2023.02.01` installed in the unified environment, in addition to the two standard versions `fms@2023.04` and `fms@release-jcsda`. + spack-stack-1.6.0 on Hercules has `fms@2023.02.01` installed in the unified environment, in addition to the two default versions `fms@2023.04` and `fms@release-jcsda`. .. _Preconfigured_Sites_Discover: @@ -267,14 +266,14 @@ With Intel, the following is required for building new spack environments and fo module load ecflow/5.8.4 module load mysql/8.0.31 -For ``spack-stack-1.5.1`` with Intel, load the following modules after loading the above modules. +For ``spack-stack-1.6.0`` with Intel, load the following modules after loading the above modules. .. code-block:: console - module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core + module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-intel/2021.5.0 module load stack-openmpi/4.1.5rc2 - module load stack-python/3.10.8 + module load stack-python/3.10.13 With AMD clang/flang (aocc), the following is required for building new spack environments and for using spack to build and run software. @@ -294,7 +293,11 @@ With AMD clang/flang (aocc), the following is required for building new spack en .. note:: - ``spack-stack-1.5.1`` is not yet supported with the Arm clang/flang compilers. Use Intel instead. + ``spack-stack-1.6.0`` is not yet supported with the Arm clang/flang compilers. Use Intel instead. + +.. note:: + + `wgrib2@2.0.8` does not build on Nautilus, therefore we are using `wgrib2@3.1.1` on this system. .. note:: From f38e52e2b1e4693645b8069cb4a7e5236e5ae390 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Sat, 30 Dec 2023 19:07:05 -0700 Subject: [PATCH 08/13] Update site config for Discover and documentation for Discover and ParallelWorks --- configs/sites/discover/packages.yaml | 4 ---- doc/source/PreConfiguredSites.rst | 24 +++++++++++------------- 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/configs/sites/discover/packages.yaml b/configs/sites/discover/packages.yaml index f5dfcd3d2..6539d8472 100644 --- a/configs/sites/discover/packages.yaml +++ b/configs/sites/discover/packages.yaml @@ -121,10 +121,6 @@ packages: externals: - spec: libfuse@2.9.3 prefix: /usr - libtool: - externals: - - spec: libtool@2.4.2 - prefix: /usr libxpm: externals: - spec: libxpm@4.11.0 diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index ff203126f..e1b1bd990 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -22,7 +22,7 @@ Ready-to-use spack-stack 1.6.0 installations are available on the following, ful | MSU +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Orion | GCC, Intel | ``/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.1/envs/unified-env`` | Cam Book / Dom Heinzeller | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| NASA | Discover^** | GCC, Intel | ``/gpfsm/dswdev/jcsda/spack-stack/spack-stack-dev-20231114/envs/unified-env`` | Dom Heinzeller / ??? | +| NASA | Discover | GCC, Intel | ``/gpfsm/dswdev/jcsda/spack-stack/spack-stack-1.6.0/envs/unified-env`` | Dom Heinzeller / ??? | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Casper | GCC | ``/glade/work/epicufsrt/contrib/spack-stack/casper/spack-stack-1.5.1/envs/unified-env`` | Dom Heinzeller / ??? | | NCAR-Wyoming +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ @@ -54,11 +54,9 @@ Ready-to-use spack-stack 1.6.0 installations are available on the following, ful + Amazon Web Services +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Parallelcluster JCSDA R&D | Intel | ``/mnt/experiments-efs/skylab-v7/spack-stack-1.5.1/envs/unified-env`` | Dom Heinzeller / ??? | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| NOAA (RDHPCS) | RDHPCS Cloud (Parallel Works) | Intel | ``/contrib/spack-stack/spack-stack-1.5.1/envs/unified-env`` | Mark Potts / Cam Book / Dom H | +| NOAA (RDHPCS) | RDHPCS Cloud (Parallel Works) | Intel | ``/contrib/spack-stack/spack-stack-1.6.0/envs/unified-env`` | Mark Potts / Cam Book / Dom H | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -^** Uses a slightly newer version than the spack-stack 1.5.1 release, so that `gmao-swell-env` is available in the environment. - For more information about a specific platform, please see the individual sections below. For questions or problems, please consult the known issues in :numref:`Section %s `, the currently open GitHub `issues `_ and `discussions `_ first. @@ -161,24 +159,24 @@ The following is required for building new spack environments and for using spac module load ecflow/5.8.4 module load mysql/8.0.31 -For ``spack-stack-1.5.1`` with Intel, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.6.0`` with Intel, load the following modules after loading miniconda and ecflow: .. code-block:: console - module use /gpfsm/dswdev/jcsda/spack-stack/spack-stack-dev-20231114/envs/unified-env/install/modulefiles/Core + module use /gpfsm/dswdev/jcsda/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-intel/2022.0.1 module load stack-intel-oneapi-mpi/2021.5.0 - module load stack-python/3.10.8 + module load stack-python/3.10.13 module available -For ``spack-stack-1.5.1`` with GNU, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.6.0`` with GNU, load the following modules after loading miniconda and ecflow: .. code-block:: console - module use /gpfsm/dswdev/jcsda/spack-stack/spack-stack-dev-20231114/envs/unified-env/install/modulefiles/Core + module use /gpfsm/dswdev/jcsda/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-gcc/10.1.0 module load stack-openmpi/4.1.3 - module load stack-python/3.10.8 + module load stack-python/3.10.13 module available .. _Preconfigured_Sites_Narwhal: @@ -413,14 +411,14 @@ The following is required for building new spack environments and for using spac module load mysql/8.0.31 module load git-lfs/2.4.1 -For ``spack-stack-1.5.1`` with Intel, proceed with loading the following modules: +For ``spack-stack-1.6.0`` with Intel, proceed with loading the following modules: .. code-block:: console - module use /contrib/spack-stack/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core + module use /contrib/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-intel/2021.3.0 module load stack-intel-oneapi-mpi/2021.3.0 - module load stack-python/3.10.8 + module load stack-python/3.10.13 module available .. _Preconfigured_Sites_Gaea_C5: From 66dbe7dd5681ab3779db7dfdbd54a627e5d2fc52 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 2 Jan 2024 13:12:49 -0700 Subject: [PATCH 09/13] Revert .gitmodules and update submodule pointer for spack --- .gitmodules | 10 ++++------ spack | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.gitmodules b/.gitmodules index 5127cacd5..17f5bc5fc 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,11 +1,9 @@ [submodule "spack"] path = spack - ##url = https://github.com/spack/spack - ##branch = develop - #url = https://github.com/jcsda/spack - #branch = release/1.6.0 - url = https://github.com/climbfuji/spack - branch = bugfix/py_gevent_intel + #url = https://github.com/spack/spack + #branch = develop + url = https://github.com/jcsda/spack + branch = release/1.6.0 [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules diff --git a/spack b/spack index da5394638..46e4875a0 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit da539463828f00e3d2fc5f2cddcd178e2d149bf0 +Subproject commit 46e4875a0b7021094f0ddabae74e087c64849641 From 1eef392d2582ffb1df1369ca8bfcb32810db8910 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 2 Jan 2024 13:13:40 -0700 Subject: [PATCH 10/13] Update site config for Narwhal --- configs/sites/narwhal/compilers.yaml | 2 ++ configs/sites/narwhal/packages.yaml | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/configs/sites/narwhal/compilers.yaml b/configs/sites/narwhal/compilers.yaml index 4102aa0cd..d279a5213 100644 --- a/configs/sites/narwhal/compilers.yaml +++ b/configs/sites/narwhal/compilers.yaml @@ -11,6 +11,7 @@ compilers:: modules: - PrgEnv-intel/8.3.2 - intel-classic/2021.4.0 + - node.js/20.10.0 environment: prepend_path: PATH: '/opt/cray/pe/gcc/10.3.0/snos/bin' @@ -31,6 +32,7 @@ compilers:: modules: - PrgEnv-gnu/8.3.2 - gcc/10.3.0 + - node.js/20.10.0 environment: prepend_path: LD_LIBRARY_PATH: '/opt/cray/pe/libsci/22.08.1.1/GNU/9.1/x86_64/lib' diff --git a/configs/sites/narwhal/packages.yaml b/configs/sites/narwhal/packages.yaml index 7188d7b4e..cdeebff1e 100644 --- a/configs/sites/narwhal/packages.yaml +++ b/configs/sites/narwhal/packages.yaml @@ -115,6 +115,10 @@ packages: prefix: /p/app/projects/NEPTUNE/spack-stack/qt-5.15.2/5.15.2/gcc_64 modules: - qt/5.15.2 + sed: + externals: + - spec: sed@4.4 + prefix: /usr subversion: externals: - spec: subversion@1.10.6 From 53835dcf7e81af52e7c13a6ffd340ddcdc4a7b94 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 2 Jan 2024 14:46:54 -0700 Subject: [PATCH 11/13] Add doc/modulefile_templates/node.js and update documentation for spack-stack-1.6.0 for Narwhal (Intel) and S4 --- doc/modulefile_templates/node.js | 23 +++++++++++++++++++++++ doc/source/MaintainersSection.rst | 22 +++++++++++++++++++++- doc/source/PreConfiguredSites.rst | 16 +++++++++------- 3 files changed, 53 insertions(+), 8 deletions(-) create mode 100644 doc/modulefile_templates/node.js diff --git a/doc/modulefile_templates/node.js b/doc/modulefile_templates/node.js new file mode 100644 index 000000000..7f5fc113c --- /dev/null +++ b/doc/modulefile_templates/node.js @@ -0,0 +1,23 @@ +#%Module1.0 + +module-whatis "Provides an node.js-20.10.0 installation for use with spack." + +# Only allow one instance of compiler to load +conflict ecflow + +proc ModulesHelp { } { +puts stderr "Provides an node.js-20.10.0 installation for use with spack." +} + +if { [ module-info mode load ] && ![ is-loaded gcc/10.3.0 ] } { + module load gcc/10.3.0 +} + +# Set this value +set NODEJS_PATH "/p/app/projects/NEPTUNE/spack-stack/node-js-20.10.0/gcc-10.3.0" + +prepend-path PATH "${NODEJS_PATH}/bin" +prepend-path LD_LIBRARY_PATH "${NODEJS_PATH}/lib" +prepend-path LIBRARY_PATH "${NODEJS_PATH}/lib" +prepend-path CPATH "${NODEJS_PATH}/include" +prepend-path CMAKE_PREFIX_PATH "${NODEJS_PATH}" diff --git a/doc/source/MaintainersSection.rst b/doc/source/MaintainersSection.rst index 5235641c1..5abfdaca7 100644 --- a/doc/source/MaintainersSection.rst +++ b/doc/source/MaintainersSection.rst @@ -346,7 +346,7 @@ mysql NAVY HPCMP Narwhal ------------------------------ -On Narwhal, ``git-lfs``, ``qt``, ``ecflow``, and ``mysql`` need to be installed as a one-off before spack can be used. +On Narwhal, ``git-lfs``, ``qt``, ``ecflow``, and ``mysql`` need to be installed as a one-off before spack can be used. Also, temporarily it is necessary to install ``node.js`` as an external package to work around build errors for ``py-jupyter-server`` (see https://github.com/JCSDA/spack-stack/issues/928 and https://github.com/spack/spack/issues/41899). git-lfs The following instructions install ``git-lfs`` in ``/p/app/projects/NEPTUNE/spack-stack/git-lfs-2.10.0``. Version 2.10.0 is the default version for Narwhal. First, download the ``git-lfs`` RPM on a system with full internet access (e.g., Derecho) using ``wget https://download.opensuse.org/repositories/openSUSE:/Leap:/15.2/standard/x86_64/git-lfs-2.10.0-lp152.1.2.x86_64.rpm`` and copy this file to ``/p/app/projects/NEPTUNE/spack-stack/git-lfs-2.10.0/src``. Then switch to Narwhal and run the following commands. @@ -397,6 +397,26 @@ ecflow mysql ``mysql`` must be installed separately from ``spack`` using a binary tarball provided by the MySQL community. Follow the instructions in :numref:`Section %s ` to install ``mysql`` in ``/p/app/projects/NEPTUNE/spack-stack/mysql-8.0.31``. +node.js + ``node.js`` is difficult to install via ``spack``, but is needed to install certain Python packages. The complication is that when using a newer ``gcc`` compiler (either directly or as backend for ``icc`` etc.), the OS ``node.js`` errors out with unresolved symbols in the ``libstdc++`` library. Therefore, we need to install ``node.js`` with ``gcc@10.3.0`` loaded, and create modulefile ``node.js/20.10.0`` from template ``modulefiles/node.js``. + +.. code-block:: console + + module unload PrgEnv-cray + module load PrgEnv-gnu/8.3.2 + module unload gcc + module load gcc/10.3.0 + + mkdir -p node-js-20.10.0/src && cd node-js-20.10.0/src + wget https://nodejs.org/dist/v20.10.0/node-v20.10.0.tar.gz + tar -xvzf node-v20.10.0.tar.gz + cd node-v20.10.0/ + ./configure --partly-static \ + --prefix=/p/app/projects/NEPTUNE/spack-stack/node-js-20.10.0/gcc-10.3.0 \ + 2>&1 | tee log.config + make 2>&1 | tee log.make + make install 2>&1 | tee log.install + .. _MaintainersSection_Nautilus: ------------------------------ diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index e1b1bd990..ce1867cae 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -36,7 +36,7 @@ Ready-to-use spack-stack 1.6.0 installations are available on the following, ful | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Jet | GCC, Intel | ``/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.5.1/envs/unified-env`` | Cam Book / Dom Heinzeller | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Narwhal | Intel | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.5.1/envs/unified-env-intel-2021.4.0`` | Dom Heinzeller / Sarah King | +| | Narwhal | Intel | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.6.0/envs/unified-env-intel-2021.4.0`` | Dom Heinzeller / Sarah King | | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Narwhal | GCC | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.5.1/envs/unified-env-gcc-10.3.0`` | Dom Heinzeller / Sarah King | | U.S. Navy (HPCMP) +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ @@ -44,7 +44,7 @@ Ready-to-use spack-stack 1.6.0 installations are available on the following, ful | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Nautilus | AOCC | *currently not supported* | Dom Heinzeller / Sarah King | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | S4 | Intel | ``/data/prod/jedi/spack-stack/spack-stack-1.5.1/envs/unified-env`` | Dom Heinzeller / Mark Potts | +| | S4 | Intel | ``/data/prod/jedi/spack-stack/spack-stack-1.6.0/envs/unified-env`` | Dom Heinzeller / Mark Potts | | Univ. of Wisconsin +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | S4 | GCC | *currently not supported* | Dom Heinzeller / Mark Potts | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ @@ -204,15 +204,16 @@ With Intel, the following is required for building new spack environments and fo module use /p/app/projects/NEPTUNE/spack-stack/modulefiles module load ecflow/5.8.4 module load mysql/8.0.31 + module load node.js/20.10.0 -For ``spack-stack-1.5.1`` with Intel, load the following modules after loading the above modules. +For ``spack-stack-1.6.0`` with Intel, load the following modules after loading the above modules. .. code-block:: console - module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-1.5.1/envs/unified-env-intel-2021.4.0/install/modulefiles/Core + module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-1.6.0/envs/unified-env-intel-2021.4.0/install/modulefiles/Core module load stack-intel/2021.4.0 module load stack-cray-mpich/8.1.14 - module load stack-python/3.10.8 + module load stack-python/3.10.13 With GNU, the following is required for building new spack environments and for using spack to build and run software. Don't use ``module purge`` on Narwhal! @@ -233,6 +234,7 @@ With GNU, the following is required for building new spack environments and for module use /p/app/projects/NEPTUNE/spack-stack/modulefiles module load ecflow/5.8.4 module load mysql/8.0.31 + module load node.js/20.10.0 For ``spack-stack-1.5.1`` with GNU, load the following modules after loading the above modules. @@ -549,11 +551,11 @@ The following is required for building new spack environments and for using spac module load ecflow/5.8.4 module load mysql/8.0.31 -For ``spack-stack-1.5.1`` with Intel, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.6.0`` with Intel, load the following modules after loading miniconda and ecflow: .. code-block:: console - module use /data/prod/jedi/spack-stack/spack-stack-1.5.1/envs/unified-env/install/modulefiles/Core + module use /data/prod/jedi/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-intel/2021.5.0 module load stack-intel-oneapi-mpi/2021.5.0 module load stack-python/3.10.8 From fa309610cc35031739b95a70198371c68320ddd7 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 2 Jan 2024 15:17:31 -0700 Subject: [PATCH 12/13] More updates for Hercules (special installs) --- doc/source/PreConfiguredSites.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index ce1867cae..3e253e3a2 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -141,7 +141,7 @@ For ``spack-stack-1.6.0`` with GNU, load the following modules after loading mys spack-stack-1.6.0 on Hercules provides a chained environment `gsi-addon-env` for GSI with Intel and GNU. To use this environment, replace `unified-env` in the above `module use` statements with `gsi-addon-env`, and load module `stack-python/3.11.6` instead of `stack-python/3.10.13`. .. note:: - spack-stack-1.6.0 on Hercules has `fms@2023.02.01` installed in the unified environment, in addition to the two default versions `fms@2023.04` and `fms@release-jcsda`. + spack-stack-1.6.0 on Hercules has additional packages `fms@2023.02.01`, `sp@2.3.0`, and `crtm@2.4.0` installed in the unified environment, in addition to the two default versions `fms@2023.04` and `fms@release-jcsda`. .. _Preconfigured_Sites_Discover: From 22f2cdd5c28d5834d85762a0413493f807015db9 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 2 Jan 2024 18:22:52 -0700 Subject: [PATCH 13/13] Update doc/source/PreConfiguredSites.rst for gsi-addon-env on S4 --- doc/source/PreConfiguredSites.rst | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 3e253e3a2..e7146740d 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -558,13 +558,16 @@ For ``spack-stack-1.6.0`` with Intel, load the following modules after loading m module use /data/prod/jedi/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core module load stack-intel/2021.5.0 module load stack-intel-oneapi-mpi/2021.5.0 - module load stack-python/3.10.8 + module load stack-python/3.10.13 module unuse /opt/apps/modulefiles/Compiler/intel/non-default/22 module unuse /opt/apps/modulefiles/Compiler/intel/22 module available Note the two `module unuse` commands, that need to be run after the stack metamodules are loaded. Loading the Intel compiler meta module loads the Intel compiler module provided by the sysadmins, which adds those two directories to the module path. These contain duplicate libraries that are not compatible with our stack, such as ``hdf4``. +.. note:: + spack-stack-1.6.0 on S4 provides a chained environment `gsi-addon-env` for GSI with Intel. To use this environment, replace `unified-env` in the above `module use` statements with `gsi-addon-env`, and load module `stack-python/3.11.6` instead of `stack-python/3.10.13`. + .. note:: There is currently no support for GNU on S4, because recent updates to ``hdf5`` require a newer version of ``mpich`` (or other MPI library) than available on the system.