From 0fae4bd887ed976d944ce80305503b73f10cc491 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 11 Mar 2024 11:29:09 -0600 Subject: [PATCH] Update project_charter/documentation, update util/check_package_config.py, remove upp from unified-dev (#1022) 1. Update README.md, project_charter.md and documentation with new maintainers and install responsibilities 2. Remove upp from ufs-srw-app-env virtual package and clean up unified-dev template 3. Fix several bugs in util/check_package_config.py: import yaml from spack _vendored, catch non-existent variants, test on macOS 4. Update util/ldd_check.py and documentation - this tool only works on Linux --- .github/workflows/util-tests.yaml | 3 +- README.md | 2 +- configs/templates/skylab-dev/spack.yaml | 4 ++ configs/templates/unified-dev/spack.yaml | 4 -- doc/source/Overview.rst | 4 +- doc/source/PreConfiguredSites.rst | 40 +++++++++---------- doc/source/Utilities.rst | 4 +- project_charter.md | 6 +-- .../packages/ufs-srw-app-env/package.py | 1 - util/check_package_config.py | 21 +++++++--- util/ldd_check.py | 7 +--- util/util_tests.sh | 3 ++ 12 files changed, 54 insertions(+), 45 deletions(-) diff --git a/.github/workflows/util-tests.yaml b/.github/workflows/util-tests.yaml index 41d494fc1..3bddf71eb 100644 --- a/.github/workflows/util-tests.yaml +++ b/.github/workflows/util-tests.yaml @@ -14,10 +14,11 @@ jobs: - name: checkout uses: actions/checkout@v2 with: - submodules: false + submodules: true - name: run-util-tests run: | export SPACK_STACK_DIR=$PWD + source ./setup.sh cd util/ ./util_tests.sh diff --git a/README.md b/README.md index c7c59e77d..d8bfbd89a 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,7 @@ Full documentation with table of contents can be found at https://spack-stack.re Spack-stack is a collaborative effort between: * [NOAA Environmental Modeling Center (EMC)](https://www.emc.ncep.noaa.gov/emc_new.php): [Alex Richert](https://www.github.com/AlexanderRichert-NOAA), [Hang Lei](https://www.github.com/Hang-Lei-NOAA), [Ed Hartnett](https://www.github.com/edwardhartnett) * [UCAR Joint Center for Satellite Data Assimilation (JCSDA)](https://www.jcsda.org/): [Dom Heinzeller](https://www.github.com/climbfuji), [Steve Herbener](https://github.com/srherbener) -* [Earth Prediction Innovation Center (EPIC)](https://epic.noaa.gov/): [Cam Book](https://github.com/ulmononian), [Natalie Perlin](https://github.com/natalie-perlin) +* [Earth Prediction Innovation Center (EPIC)](https://epic.noaa.gov/): [Cam Book](https://github.com/ulmononian), [Natalie Perlin](https://github.com/natalie-perlin), [Ratko Vasic](https://github.com/ratkovasic-noaa) For more information about the organization of the spack-stack project, see the [Project Charter](project_charter.md). diff --git a/configs/templates/skylab-dev/spack.yaml b/configs/templates/skylab-dev/spack.yaml index 2d9cab3b2..050cbd2d5 100644 --- a/configs/templates/skylab-dev/spack.yaml +++ b/configs/templates/skylab-dev/spack.yaml @@ -9,6 +9,7 @@ spack: - compilers: ['%aocc', '%apple-clang', '%gcc', '%intel'] - packages: - ewok-env +ecflow +cylc + - ai-env - geos-gcm-env - jedi-fv3-env - jedi-geos-env @@ -30,3 +31,6 @@ spack: - matrix: - [$packages] - [$compilers] + exclude: + # py-torch in ai-env doesn't build with Intel + - ai-env%intel diff --git a/configs/templates/unified-dev/spack.yaml b/configs/templates/unified-dev/spack.yaml index 2c10f2f72..8f9b0dfde 100644 --- a/configs/templates/unified-dev/spack.yaml +++ b/configs/templates/unified-dev/spack.yaml @@ -21,13 +21,9 @@ spack: - jedi-tools-env - jedi-ufs-env - jedi-um-env - #- nceplibs-env - soca-env - ufs-srw-app-env - #- ufs-utils-env - ufs-weather-model-env - #- upp-env - #- ww3-env # Various fms tags (list all to avoid duplicate packages) - fms@release-jcsda diff --git a/doc/source/Overview.rst b/doc/source/Overview.rst index 0d3303d06..0ef47d45d 100644 --- a/doc/source/Overview.rst +++ b/doc/source/Overview.rst @@ -15,11 +15,11 @@ spack-stack is mainly a collection of Spack configuration files, but provides a spack-stack is maintained by: -- Alex Richert (@AlexanderRichert-NOAA) and Ed Hartnett (@edwardhartnett), NOAA-EMC +- Alex Richert (@AlexanderRichert-NOAA), Hang Lei (@Hang-Lei-NOAA) and Ed Hartnett (@edwardhartnett), NOAA-EMC - Dom Heinzeller (@climbfuji) and Steve Herbener (@srherbener), JCSDA -- Cameron Book (@ulmononian) and Mark Potts (@mark-a-potts), EPIC +- Cameron Book (@ulmononian), Natalie Perlin (@natalie-perlin) and Ratko Vasic (@ratkovasic-noaa), EPIC =============== Getting started diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 05e4c81b2..ebe307ce8 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -16,49 +16,49 @@ Ready-to-use spack-stack 1.6.0 installations are available on the following, ful On selected systems, developmental versions / release candidates are installed that are newer than spack-stack 1.6.0 (see following table). For information on the spack-stack 1.6.0 releases on this platforms, please revert to version 1.6.0 of the documentation (https://spack-stack.readthedocs.io/en/1.6.0/PreConfiguredSites.html#pre-configured-sites-tier-1). +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| Organization | System | Compilers | Location | Maintainers | +| Organization | System | Compilers | Location | Maintainers (principal/backup)| +=====================+==================================+=================+=========================================================================================================+===============================+ | **HPC platforms** | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| MSU | Hercules GCC+OpenMPI recommended | GCC | ``/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/ue-gcc12-openmpi416`` | Dom Heinzeller | +| MSU | Hercules GCC+OpenMPI recommended | GCC | ``/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/ue-gcc12-openmpi416`` | EPIC / JCSDA | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Hercules | (GCC), Intel | ``/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/unified-env`` | Cam Book / Dom Heinzeller | +| | Hercules | (GCC), Intel | ``/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/unified-env`` | EPIC / JCSDA | | MSU +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Orion | GCC, Intel | ``/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/unified-env`` | Cam Book / Dom Heinzeller | +| | Orion | GCC, Intel | ``/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/unified-env`` | EPIC / JCSDA | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Discover SCU16 | GCC, Intel | ``/gpfsm/dswdev/jcsda/spack-stack/scu16/spack-stack-20240228/envs/unified-env-*`` | Dom Heinzeller / ??? | +| | Discover SCU16 | GCC, Intel | ``/gpfsm/dswdev/jcsda/spack-stack/scu16/spack-stack-20240228/envs/unified-env-*`` | JCSDA | | NASA +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Discover SCU17 | GCC, Intel | ``/gpfsm/dswdev/jcsda/spack-stack/scu17/spack-stack-20240228/envs/unified-env-*`` | Dom Heinzeller / ??? | +| | Discover SCU17 | GCC, Intel | ``/gpfsm/dswdev/jcsda/spack-stack/scu17/spack-stack-20240228/envs/unified-env-*`` | JCSDA | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Casper | GCC | ``/glade/work/epicufsrt/contrib/spack-stack/casper/spack-stack-1.6.0/envs/unified-env`` | Dom Heinzeller / ??? | +| | Casper | GCC | ``/glade/work/epicufsrt/contrib/spack-stack/casper/spack-stack-1.6.0/envs/unified-env`` | JCSDA / EPIC | | NCAR-Wyoming +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Derecho | GCC, Intel | ``/glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.6.0/envs/unified-env`` | Dom Heinzeller / Cam Book | +| | Derecho | GCC, Intel | ``/glade/work/epicufsrt/contrib/spack-stack/derecho/spack-stack-1.6.0/envs/unified-env`` | EPIC / JCSDA | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| NOAA (NCEP) | Acorn | Intel | ``/lfs/h1/emc/nceplibs/noscrub/spack-stack/spack-stack-1.6.0/envs/unified-env-intel{19,2022}`` | Hang Lei / Alex Richert | +| NOAA (NCEP) | Acorn | Intel | ``/lfs/h1/emc/nceplibs/noscrub/spack-stack/spack-stack-1.6.0/envs/unified-env-intel{19,2022}`` | NOAA-EMC | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Gaea C5 | Intel | ``/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env`` | Cam Book / Dom Heinzeller | +| | Gaea C5 | Intel | ``/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env`` | EPIC / NOAA-EMC | | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| NOAA (RDHPCS) | Hera | GCC, Intel | ``/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env`` | Mark Potts / Dom Heinzeller | +| NOAA (RDHPCS) | Hera | GCC, Intel | ``/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env`` | EPIC / NOAA-EMC | | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Jet | GCC, Intel | ``/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env`` | Cam Book / Dom Heinzeller | +| | Jet | GCC, Intel | ``/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env`` | EPIC / NOAA-EMC | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Narwhal | GCC, Intel | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.6.0/envs/unified-env-*`` | Dom Heinzeller / Sarah King | +| | Narwhal | GCC, Intel | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.6.0/envs/unified-env-*`` | JCSDA / NRL | | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| U.S. Navy (HPCMP) | Nautilus | Intel | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.6.0/envs/unified-env`` | Dom Heinzeller / Sarah King | +| U.S. Navy (HPCMP) | Nautilus | Intel | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.6.0/envs/unified-env`` | JCSDA / NRL | | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Nautilus | AOCC | *currently not supported* | Dom Heinzeller / Sarah King | +| | Nautilus | AOCC | *currently not supported* | JCSDA / NRL | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | S4 | Intel | ``/data/prod/jedi/spack-stack/spack-stack-1.6.0/envs/unified-env`` | Dom Heinzeller / Mark Potts | +| | S4 | Intel | ``/data/prod/jedi/spack-stack/spack-stack-1.6.0/envs/unified-env`` | JCSDA | | Univ. of Wisconsin +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | S4 | GCC | *currently not supported* | Dom Heinzeller / Mark Potts | +| | S4 | GCC | *currently not supported* | JCSDA | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | **Cloud platforms** | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | AMI Red Hat 8 | GCC | ``/home/ec2-user/spack-stack/spack-stack-1.6.0/envs/unified-env`` | Dom Heinzeller / ??? | +| | AMI Red Hat 8 | GCC | ``/home/ec2-user/spack-stack/spack-stack-1.6.0/envs/unified-env`` | JCSDA | + Amazon Web Services +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Parallelcluster JCSDA R&D | Intel | ``/mnt/experiments-efs/skylab-v8/spack-stack-20240207/envs/unified-env-*`` | Dom Heinzeller / ??? | +| | Parallelcluster JCSDA R&D | Intel | ``/mnt/experiments-efs/skylab-v8/spack-stack-20240207/envs/unified-env-*`` | JCSDA | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| NOAA (RDHPCS) | RDHPCS Cloud (Parallel Works) | Intel | ``/contrib/spack-stack/spack-stack-1.6.0/envs/unified-env`` | Mark Potts / Cam Book / Dom H | +| NOAA (RDHPCS) | RDHPCS Cloud (Parallel Works) | Intel | ``/contrib/spack-stack/spack-stack-1.6.0/envs/unified-env`` | EPIC / JCSDA | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ For more information about a specific platform, please see the individual sections below. diff --git a/doc/source/Utilities.rst b/doc/source/Utilities.rst index 22b2de344..b5eb0e1f7 100644 --- a/doc/source/Utilities.rst +++ b/doc/source/Utilities.rst @@ -46,10 +46,10 @@ The utility located at util/check_permissions.sh can be run inside any spack-sta .. _LDD_Checker: ------------------------------ -ldd_check.py +ldd_check.py (Linux only) ------------------------------ -The util/ldd_check.py utility should be run for new installations to ensure that no shared library or executable that uses shared libraries is missing a shared library dependency. If the script returns a warning for a given file, this may indicate that Spack's RPATH substitution has not been properly applied. In some instances, missing library dependencies may not indicate a problem, such as a library that is intended to be found through $LD_LIBRARY_PATH after, say, a compiler or MPI environment module is loaded. Though these paths should probably also be RPATH-ified, such instances of harmless missing dependencies may be ignored with ldd_check.py's ``--ignore`` option by specifying a Python regular expression to be excluded from consideration (see example below), or can be permanently whitelisted by modifying the ``whitelist`` variable at the top of the ldd_check.py script itself (in which case please submit a PR). The script searches the 'install/' subdirectory of a given path and runs ``ldd`` on all shared objects. The base path to be search can be specified as a lone positional argument, and by default is the current directory. In practice, this should be ``$SPACK_ENV`` for the environment in question. +The util/ldd_check.py utility should be run for new installations to ensure that no shared library or executable that uses shared libraries is missing a shared library dependency. If the script returns a warning for a given file, this may indicate that Spack's RPATH substitution has not been properly applied. In some instances, missing library dependencies may not indicate a problem, such as a library that is intended to be found through $LD_LIBRARY_PATH after, say, a compiler or MPI environment module is loaded. Though these paths should probably also be RPATH-ified, such instances of harmless missing dependencies may be ignored with ldd_check.py's ``--ignore`` option by specifying a Python regular expression to be excluded from consideration (see example below), or can be permanently whitelisted by modifying the ``whitelist`` variable at the top of the ldd_check.py script itself (in which case please submit a PR). The script searches the 'install/' subdirectory of a given path and runs ``ldd`` on all shared objects. The base path to be search can be specified as a lone positional argument, and by default is the current directory. In practice, this should be ``$SPACK_ENV`` for the environment in question. This utility is available for Linux only. .. code-block:: console diff --git a/project_charter.md b/project_charter.md index 60c77ff2e..ca9bbc235 100644 --- a/project_charter.md +++ b/project_charter.md @@ -34,7 +34,7 @@ Organization | Code Manager(s) -------------|---------------- NOAA EMC | Alex Richert, Hang Lei, Ed Hartnett JCSDA | Dom Heinzeller, Steve Herbener -EPIC | Cam Book, Natalie Perlin +EPIC | Cam Book, Natalie Perlin, Ratko Vasic ## Project Process @@ -116,10 +116,10 @@ which case the work will be shared by the code managers. #### Directory structure of spack-stack installs -/path/to/spack-stack/spack-stack-x.y.z/envs/unified-env/install/compiler-name/compiler-version/package-name-version-hash +/path/to/spack-stack/spack-stack-x.y.z/envs/unified-env-compiler-name-compiler-version/install/compiler-name/compiler-version/package-name-version-hash _Example_ -/Users/heinzell/prod/spack-stack-1.4.0/envs/unified-env/install/apple-clang/13.1.6/netcdf-c-4.9.2-vrrvi2u +/Users/heinzell/prod/spack-stack-1.4.0/envs/unified-env-apple-clang-13.1.6/install/apple-clang/13.1.6/netcdf-c-4.9.2-vrrvi2u #### Auto-generated modules structure (no MPI dependency) diff --git a/spack-ext/repos/spack-stack/packages/ufs-srw-app-env/package.py b/spack-ext/repos/spack-stack/packages/ufs-srw-app-env/package.py index ebeef49a3..ae5501f1d 100644 --- a/spack-ext/repos/spack-stack/packages/ufs-srw-app-env/package.py +++ b/spack-ext/repos/spack-stack/packages/ufs-srw-app-env/package.py @@ -33,7 +33,6 @@ class UfsSrwAppEnv(BundlePackage): depends_on("ip") depends_on("sp") depends_on("w3nco") - depends_on("upp") depends_on("gfsio") depends_on("landsfcutil") depends_on("nemsio") diff --git a/util/check_package_config.py b/util/check_package_config.py index 16a6eb2bc..aa457e549 100755 --- a/util/check_package_config.py +++ b/util/check_package_config.py @@ -19,7 +19,11 @@ import os import sys -import yaml +SPACK_ROOT = os.getenv("SPACK_ROOT") +assert SPACK_ROOT, "$SPACK_ROOT must be set but is not!" + +sys.path.append(os.path.join(SPACK_ROOT, "lib/spack/external/_vendoring")) +from ruamel import yaml SPACK_ENV = os.getenv("SPACK_ENV") assert SPACK_ENV, "$SPACK_ENV must be set but is not!" @@ -74,16 +78,21 @@ # Boolean variant if config_variant[0] in ("+", "~"): config_value = config_variant[0] == "+" - if concrete_spec["parameters"][config_variant[1:]] != config_value: + if not 'config_variant[1:]' in concrete_spec["parameters"].keys(): + variant_mismatch = True + elif concrete_spec["parameters"][config_variant[1:]] != config_value: variant_mismatch = True # Named variant elif "=" in config_variant: config_variant, config_value = config_variant.split("=") - concrete_values = concrete_spec["parameters"][config_variant] - if type(concrete_values) is str: - concrete_values = [concrete_values] - if set(config_value.split(",")) != set(concrete_values): + if not config_variant in concrete_spec["parameters"].keys(): variant_mismatch = True + else: + concrete_values = concrete_spec["parameters"][config_variant] + if type(concrete_values) is str: + concrete_values = [concrete_values] + if set(config_value.split(",")) != set(concrete_values): + variant_mismatch = True if variant_mismatch: iret = 1 print( diff --git a/util/ldd_check.py b/util/ldd_check.py index af6d3fa77..9cee8a838 100755 --- a/util/ldd_check.py +++ b/util/ldd_check.py @@ -49,17 +49,14 @@ ldd_cmd = "ldd" error_pattern = " => not found" getlibname = lambda line: re.findall("^[^ ]+", line)[0] -elif platform=="Darwin": - print("macOS not yet supported", file=sys.stderr) - sys.exit(1) + libext = "so" else: print(f"Platform '{platform}' not supported", file=sys.stderr) sys.exit(1) searchpath = os.path.join(args.path, "install") - bin_list = glob.glob(os.path.join(searchpath, "*/*/*/bin/*")) -dlib_list = glob.glob(os.path.join(searchpath, "*/*/*/lib*/*.{so,dylib}")) +dlib_list = glob.glob(os.path.join(searchpath, f"*/*/*/lib*/*.{libext}")) master_list = bin_list + dlib_list diff --git a/util/util_tests.sh b/util/util_tests.sh index 609a1fcdf..079c20f92 100755 --- a/util/util_tests.sh +++ b/util/util_tests.sh @@ -80,6 +80,9 @@ fi ## Check check_package_config.py export SPACK_ENV=${SPACK_STACK_DIR}/util/test_env +${SPACK_STACK_DIR}/util/check_package_config.py +${SPACK_STACK_DIR}/util/check_package_config.py | sort +${SPACK_STACK_DIR}/util/check_package_config.py | sort | md5sum output_checksum=$(${SPACK_STACK_DIR}/util/check_package_config.py | sort | md5sum) reference_checksum=$(cat ${SPACK_STACK_DIR}/util/test_env/package_check_baseline.txt | md5sum) if [[ "$output_checksum" != "$reference_checksum" ]]; then