From a1400fe21b934c9f6e51734c1edefbee7c1aff13 Mon Sep 17 00:00:00 2001 From: Alex Richert Date: Tue, 11 Jun 2024 18:33:01 -0700 Subject: [PATCH] remove check_package_config.py --- doc/source/Utilities.rst | 16 -- util/check_package_config.py | 116 --------- util/test_env/README | 1 - util/test_env/common/packages.yaml | 286 ----------------------- util/test_env/package_check_baseline.txt | 6 - util/test_env/site/packages.yaml | 10 - util/test_env/spack.lock | 1 - util/util_tests.sh | 18 -- 8 files changed, 454 deletions(-) delete mode 100755 util/check_package_config.py delete mode 100644 util/test_env/README delete mode 100644 util/test_env/common/packages.yaml delete mode 100644 util/test_env/package_check_baseline.txt delete mode 100644 util/test_env/site/packages.yaml delete mode 100644 util/test_env/spack.lock diff --git a/doc/source/Utilities.rst b/doc/source/Utilities.rst index c2e11c9e4..0796a69db 100644 --- a/doc/source/Utilities.rst +++ b/doc/source/Utilities.rst @@ -20,22 +20,6 @@ The utility located at util/show_duplicate_packages.py parses the output of ``sp The ``-d`` option shows only a list of the duplicates, as opposed to the default behavior, which is to show a print-out of all packages with colorized duplicates. In any case, the identification of any duplicates will yield a return code of 1. The ``-i`` option can be invoked multiple times to skip specific package names. The ``-c`` option can be used to ignore duplicates associated with different compilers; in an environment with, say, GCC and Intel copies of any given package, those two copies of a package will not be reported as duplicates. -.. _Package_Config_Checker: - ------------------------------- -check_package_config.py ------------------------------- - -The utility at util/check_package_config.py is run after concretization in an active spack-stack environment (i.e., `$SPACK_ENV` is set) to confirm that the packages versions and variants in common/packages.yaml are respected in the concretization, as well as that any externals specified in site/packages.yaml are not being omitted. It does this by reading common/packages.yaml (for the version and variant settings), site/packages.yaml (for the external settings), and spack.lock. Usage is as follows: - -.. code-block:: console - - spack env active envs/unified-env/ - # To verify versions, variants, and externals: - ${SPACK_STACK_DIR}/util/check_package_config.py - # To ignore a known mismatch in version, variant, or external status for package 'esmf', use -i/--ignore option: - ${SPACK_STACK_DIR}/util/check_package_config.py -i esmf - .. _Permissions_Checker: ------------------------------ diff --git a/util/check_package_config.py b/util/check_package_config.py deleted file mode 100755 index 89e70900a..000000000 --- a/util/check_package_config.py +++ /dev/null @@ -1,116 +0,0 @@ -#!/usr/bin/env python3 -# This utility checks whether the package versions and variants set in -# common/packages.yaml are being respected in the concretization, and -# whether the externals in site/packages.yaml are being used. -# -# To use this script, run it in a loaded spack-stack environment. -# Package names to be ignored can be provided as optional arguments -# using -i/--ignore. -# -# Usage: -# $ spack env active myenv -# $ ${SPACK_STACK_DIR}/util/check_package_config.py -# Ignore packages foo and bar: -# $ ${SPACK_STACK_DIR}/util/check_package_config.py -i foo -i bar -# -# Alex Richert, Jan 2024 - -import json -import os -import sys - -SPACK_ROOT = os.getenv("SPACK_ROOT") -assert SPACK_ROOT, "$SPACK_ROOT must be set but is not!" - -sys.path.append(os.path.join(SPACK_ROOT, "lib/spack/external/_vendoring")) -from ruamel import yaml - -SPACK_ENV = os.getenv("SPACK_ENV") -assert SPACK_ENV, "$SPACK_ENV must be set but is not!" - -# Load common/packages.yaml and site/packages.yaml for versions and externals, respectively -packages_versions_path = os.path.join(SPACK_ENV, "common", "packages.yaml") -with open(packages_versions_path, "r") as f: - packages_versions = yaml.safe_load(f) - -packages_externals_path = os.path.join(SPACK_ENV, "site", "packages.yaml") -if os.path.isfile(packages_externals_path): - with open(packages_externals_path, "r") as f: - packages_externals = yaml.safe_load(f) - -# Load spack.lock -spack_lock_path = os.path.join(SPACK_ENV, "spack.lock") -with open(spack_lock_path, "r") as f: - spack_lock = json.load(f) - -iret = 0 - -# Set up list of packages to ignore -args = sys.argv[1:] -if args: - assert args[-1] not in ("-i", "--ignore"), "-i/--ignore option requires package name" - ignore_list = [args[iarg+1] for iarg in range(len(args)) if args[iarg] in ("-i", "--ignore")] - if ignore_list: - print("Ignoring the following packages: %s" % ", ".join(ignore_list), file=sys.stderr) -else: - ignore_list = [] - -# Iterate over concretized packages -for concrete_spec in spack_lock["concrete_specs"].values(): - concrete_name = concrete_spec["name"] - # Ignore user-specified packages: - if concrete_name in ignore_list: - continue - concrete_version = concrete_spec["version"] - if concrete_name in packages_versions["packages"].keys(): - # Check whether concretized package has specified version from common/packages.yaml - if "version" in packages_versions["packages"][concrete_name].keys(): - config_version = packages_versions["packages"][concrete_name]["version"][0] - if concrete_version != config_version: - iret = 1 - print( - f"WARNING: '{concrete_name}' concretized version {concrete_version} does not match {config_version} specified in $SPACK_ENV/common/packages.yaml" - ) - # Check whether concretized variants match settings from common/packages.yaml - if "variants" in packages_versions["packages"][concrete_name].keys(): - config_variants = packages_versions["packages"][concrete_name]["variants"].replace("+"," +").replace("~"," ~").split() - for config_variant in config_variants: - variant_mismatch = False - # Boolean variant - if config_variant[0] in ("+", "~"): - config_value = config_variant[0] == "+" - if not config_variant[1:] in concrete_spec["parameters"].keys(): - variant_mismatch = True - elif concrete_spec["parameters"][config_variant[1:]] != config_value: - variant_mismatch = True - # Named variant - elif "=" in config_variant: - config_variant, config_value = config_variant.split("=") - if not config_variant in concrete_spec["parameters"].keys(): - variant_mismatch = True - else: - concrete_values = concrete_spec["parameters"][config_variant] - if type(concrete_values) is str: - concrete_values = [concrete_values] - if set(config_value.split(",")) != set(concrete_values): - variant_mismatch = True - if variant_mismatch: - iret = 1 - print( - f"WARNING: '{concrete_name}' concretized variant '{config_variant}' does not match configured value in $SPACK_ENV/common/packages.yaml" - ) - # Check whether concretized package is an external based on site/packages.yaml - if os.path.isfile(packages_externals_path): - if concrete_name in packages_externals["packages"].keys(): - is_external_config = "externals" in packages_externals["packages"][concrete_name].keys() - else: - is_external_config = False - is_external_concrete = "external" in concrete_spec.keys() - if is_external_config != is_external_concrete: - iret = 1 - print( - f"WARNING: '{concrete_name}' is %sconfigured as external in $SPACK_ENV/site/packages.yaml but was %sconcretized as external" - % ((not is_external_config) * "not ", (not is_external_concrete) * "not ") - ) - -sys.exit(iret) diff --git a/util/test_env/README b/util/test_env/README deleted file mode 100644 index 68ad4ca39..000000000 --- a/util/test_env/README +++ /dev/null @@ -1 +0,0 @@ -This directory provides test files to verify the check_package_config.py utility. diff --git a/util/test_env/common/packages.yaml b/util/test_env/common/packages.yaml deleted file mode 100644 index 979803db7..000000000 --- a/util/test_env/common/packages.yaml +++ /dev/null @@ -1,286 +0,0 @@ -# Pin versions and specs when building packages - packages: - # - all: - providers: - blas: [openblas] - fftw-api: [fftw] - gl: [opengl] - glu: [openglu] - jpeg: [libjpeg-turbo] - lapack: [openblas] - yacc: [bison] - zlib-api: [zlib] - # - # This version of awscli goes with py-pyyaml@5.4.1 - awscli: - version: ['1.27.84'] - bacio: - version: ['2.4.1'] - bison: - version: ['3.8.2'] - boost: - version: ['1.83.0'] - variants: ~atomic +chrono +date_time +exception +filesystem ~graph ~iostreams ~locale ~log ~math ~mpi ~numpy +pic +program_options +python ~random +regex +serialization ~signals +system +test +thread +timer ~wave cxxstd=17 visibility=hidden - bufr: - version: ['12.0.1'] - variants: +python - cairo: - variants: +pic - cdo: - version: ['2.2.0'] - variants: ~openmp - cmake: - version: ['3.23.1'] - variants: +ownlibs - # Attention - when updating also check the various jcsda-emc-bundles env packages - crtm: - version: ['2.4.0.1'] - variants: +fix - ecbuild: - version: ['3.7.2'] - eccodes: - version: ['2.32.0'] - variants: +png - ecflow: - version: ['5.11.4'] - variants: +ui - eckit: - version: ['1.24.5'] - variants: linalg=eigen,lapack compression=lz4,bzip2 - ecmwf-atlas: - version: ['0.35.1'] - variants: +fckit +trans +tesselation +fftw - ectrans: - version: ['1.2.0'] - variants: ~mkl +fftw - eigen: - version: ['3.4.0'] - # Attention - when updating the version also check the common modules.yaml - # config and update the projections for lmod/tcl. - # Also, check the acorn and derecho site configs which have esmf modifications. - esmf: - version: ['8.6.0'] - variants: ~xerces ~pnetcdf snapshot=none +shared +external-parallelio - require: - - any_of: ['fflags="-fp-model precise" cxxflags="-fp-model precise"'] - when: "%intel" - message: "Extra ESMF compile options for Intel" - - any_of: [''] - when: "%gcc" - message: "Extra ESMF compile options for GCC" - fckit: - version: ['0.11.0'] - variants: +eckit - fftw: - version: ['3.3.10'] - fiat: - version: ['1.2.0'] - fms: - version: ['2023.04'] - variants: precision=32,64 +quad_precision +gfs_phys +openmp +pic constants=GFS build_type=Release +deprecated_io - fontconfig: - variants: +pic - freetype: - variants: +pic - g2: - version: ['3.4.5'] - g2c: - version: ['1.6.4'] - g2tmpl: - version: ['1.10.2'] - gettext: - version: ['0.21.1'] - gfsio: - version: ['1.4.1'] - gftl-shared: - version: ['1.6.1'] - #git-lfs: - # Assume git-lfs is provided, hard to install - # because of dependencies on go/go-bootstrap. - # Note: Uncommenting this entry will break - # the container builds. - #version: ['2.11.0'] - grib-util: - version: ['1.3.0'] - gsibec: - version: ['1.1.3'] - gsi-ncdiag: - version: ['1.1.2'] - gsl-lite: - version: ['0.37.0'] - hdf: - version: ['4.2.15'] - variants: +external-xdr ~fortran ~netcdf - hdf5: - version: ['1.14.0'] - variants: +hl +fortran +mpi ~threadsafe +szip - ip: - version: ['4.3.0'] - variants: precision=4,d,8 - ip2: - version: ['1.1.2'] - jasper: - version: ['2.0.32'] - jedi-cmake: - version: ['1.4.0'] - jpeg: - version: ['9.1.0'] - landsfcutil: - version: ['2.4.1'] - libjpeg-turbo: - version: ['2.1.0'] - libpng: - version: ['1.6.37'] - variants: +pic - libyaml: - version: ['0.2.5'] - mapl: - version: ['2.40.3'] - variants: +shared +pflogger ~f2py - # If making changes here, also check the Discover site config and the CI workflows - met: - version: ['11.1.0'] - variants: +python +grib2 - metplus: - version: ['5.1.0'] - metis: - require: "+int64 +real64" - mpich: - variants: ~hwloc +two_level_namespace - mysql: - variants: +download_boost - nco: - version: ['5.0.6'] - variants: ~doc - # ncview - when adding information here, also check Orion - # and Discover site configs - nemsio: - version: ['2.5.4'] - nemsiogfs: - version: ['2.5.3'] - nccmp: - version: ['1.9.0.1'] - ncio: - version: ['1.1.2'] - netcdf-c: - version: ['4.9.2'] - # If using 4.9.1, turn off byterange variant to fix compile error: ~byterange - variants: +dap +mpi ~parallel-netcdf - netcdf-cxx4: - version: ['4.3.1'] - netcdf-fortran: - version: ['4.6.1'] - # ninja - when adding information here, also check Discover site config - nlohmann-json: - version: ['3.10.5'] - nlohmann-json-schema-validator: - version: ['2.1.0'] - odc: - version: ['1.4.6'] - variants: ~fortran - openblas: - version: ['0.3.24'] - variants: +noavx512 - openmpi: - variants: +internal-hwloc +two_level_namespace - # Pin openssl to avoid duplicate packages being built - openssl: - variants: +shared - p4est: - version: ['2.8'] - parallelio: - version: ['2.5.10'] - variants: +pnetcdf - parallel-netcdf: - version: ['1.12.2'] - pflogger: - version: ['1.12.0'] - variants: +mpi - pixman: - variants: +pic - # Do not build pkgconf - https://github.com/jcsda/spack-stack/issues/123 - pkgconf: - buildable: False - prod-util: - version: ['2.1.1'] - proj: - version: ['8.1.0'] - variants: ~tiff - python: - require: "@3.10.13" - py-attrs: - # https://github.com/JCSDA/spack-stack/issues/740 - version: ['21.4.0'] - py-cartopy: - variants: +plotting - require: "@0.21.1" - py-cryptography: - variants: +rust_bootstrap - # Introduced in https://github.com/JCSDA/spack-stack/pull/894, pin py-cython - # to avoid duplicate packages being built (cylc dependencies soft-want @3:) - py-cython: - require: "@0.29.36" - py-h5py: - version: ['3.7.0'] - variants: ~mpi - # Comment out for now until build problems are solved - # https://github.com/jcsda/spack-stack/issues/522 - # see also ewok-env virtual package and container - # README.md - #py-mysql-connector-python: - # version: ['8.0.32'] - py-netcdf4: - version: ['1.5.8'] - variants: ~mpi - py-numpy: - require: ['@1.22.3'] - py-pandas: - variants: +excel - # To avoid pip._vendor.pep517.wrappers.BackendInvalid errors with newer - # versions of py-poetry-core when using external/homebrew Python as - # we do at the moment in spack-stack. - # Pin the py-setuptools version to avoid duplicate Python packages - py-setuptools: - require: ['@63.4.3'] - py-setuptools-rust: - variants: +rust_bootstrap - py-shapely: - require: ['@1.8.0'] - qt: - version: ['5.15.3'] - scotch: - version: ['7.0.4'] - variants: +mpi+metis~shared~threads~mpi_thread+noarch - sfcio: - version: ['1.4.1'] - shumlib: - version: ['macos_clang_linux_intel_port'] - sigio: - version: ['2.3.2'] - sp: - version: ['2.5.0'] - variants: precision=4,d,8 derp=auto - udunits: - version: ['2.2.28'] - upp: - version: ['10.0.10'] - w3emc: - version: ['2.10.0'] - variants: precision=4,d,8 - w3nco: - version: ['2.4.1'] - wget: - version: ['1.21.2'] - # When changing wgrib2, also check Hercules and Nautilus site configs - wgrib2: - version: ['2.0.8'] - wrf-io: - version: ['1.2.0'] - yafyaml: - version: ['0.5.1'] - zlib: - version: ['1.2.13'] - zstd: - version: ['1.5.2'] - variants: +programs diff --git a/util/test_env/package_check_baseline.txt b/util/test_env/package_check_baseline.txt deleted file mode 100644 index aa83c411a..000000000 --- a/util/test_env/package_check_baseline.txt +++ /dev/null @@ -1,6 +0,0 @@ -WARNING: 'cmake' concretized variant '+ownlibs' does not match configured value in $SPACK_ENV/common/packages.yaml -WARNING: 'cmake' concretized version 3.20.2 does not match 3.23.1 specified in $SPACK_ENV/common/packages.yaml -WARNING: 'cmake' is not configured as external in $SPACK_ENV/site/packages.yaml but was concretized as external -WARNING: 'sp' concretized variant 'derp' does not match configured value in $SPACK_ENV/common/packages.yaml -WARNING: 'sp' concretized variant 'precision' does not match configured value in $SPACK_ENV/common/packages.yaml -WARNING: 'sp' is configured as external in $SPACK_ENV/site/packages.yaml but was not concretized as external diff --git a/util/test_env/site/packages.yaml b/util/test_env/site/packages.yaml deleted file mode 100644 index 9edf85972..000000000 --- a/util/test_env/site/packages.yaml +++ /dev/null @@ -1,10 +0,0 @@ -packages: - gmake: - buildable: false - externals: - - spec: gmake@4.2.1 - prefix: /usr - sp: - externals: - - spec: sp@2.3.3 - prefix: /dev/null diff --git a/util/test_env/spack.lock b/util/test_env/spack.lock deleted file mode 100644 index 0be5d3c36..000000000 --- a/util/test_env/spack.lock +++ /dev/null @@ -1 +0,0 @@ -{"_meta":{"file-type":"spack-lockfile","lockfile-version":5,"specfile-version":4},"spack":{"version":"0.21.1.dev0","type":"git","commit":"5a06f19dd176f84159ced4ca75d8cbee3a6b9e56"},"roots":[{"hash":"5dqa22jiadepasivmtbkvplu6ucry3qh","spec":"sp@2.5.0"}],"concrete_specs":{"5dqa22jiadepasivmtbkvplu6ucry3qh":{"name":"sp","version":"2.5.0","arch":{"platform":"linux","platform_os":"centos8","target":{"name":"zen","vendor":"AuthenticAMD","features":["abm","aes","avx","avx2","bmi1","bmi2","clflushopt","clzero","cx16","f16c","fma","fsgsbase","mmx","movbe","pclmulqdq","popcnt","rdseed","sse","sse2","sse4_1","sse4_2","sse4a","ssse3","xsavec","xsaveopt"],"generation":0,"parents":["x86_64_v3"]}},"compiler":{"name":"gcc","version":"8.5.0"},"namespace":"builtin","parameters":{"build_system":"cmake","build_type":"Release","generator":"make","ipo":false,"openmp":false,"pic":true,"precision":["8","d"],"derp":"none","shared":false,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"wgwpmenbjmbqyiar5djncippvybyxbw3zwgofoqsixx6kf5n737a====","dependencies":[{"name":"cmake","hash":"qjf2medg2vff5gn4htfd43rgdex7sv4d","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"d2a25i7irwfiy3librk4xaqpiv3qzkkg","parameters":{"deptypes":["build"],"virtuals":[]}}],"hash":"5dqa22jiadepasivmtbkvplu6ucry3qh"},"qjf2medg2vff5gn4htfd43rgdex7sv4d":{"name":"cmake","version":"3.20.2","arch":{"platform":"linux","platform_os":"centos8","target":{"name":"zen","vendor":"AuthenticAMD","features":["abm","aes","avx","avx2","bmi1","bmi2","clflushopt","clzero","cx16","f16c","fma","fsgsbase","mmx","movbe","pclmulqdq","popcnt","rdseed","sse","sse2","sse4_1","sse4_2","sse4a","ssse3","xsavec","xsaveopt"],"generation":0,"parents":["x86_64_v3"]}},"compiler":{"name":"gcc","version":"8.5.0"},"namespace":"builtin","parameters":{"build_system":"generic","build_type":"Release","doc":false,"ncurses":true,"owncurl":false,"ownlibs":false,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"external":{"path":"/usr","module":null,"extra_attributes":{}},"package_hash":"usp3csiedkkf6skym4g5erzxxcbck4d7is3ilzzx7fv6vkuk452q====","hash":"qjf2medg2vff5gn4htfd43rgdex7sv4d"},"d2a25i7irwfiy3librk4xaqpiv3qzkkg":{"name":"gmake","version":"4.2.1","arch":{"platform":"linux","platform_os":"centos8","target":{"name":"zen","vendor":"AuthenticAMD","features":["abm","aes","avx","avx2","bmi1","bmi2","clflushopt","clzero","cx16","f16c","fma","fsgsbase","mmx","movbe","pclmulqdq","popcnt","rdseed","sse","sse2","sse4_1","sse4_2","sse4a","ssse3","xsavec","xsaveopt"],"generation":0,"parents":["x86_64_v3"]}},"compiler":{"name":"gcc","version":"8.5.0"},"namespace":"builtin","parameters":{"build_system":"generic","guile":false,"patches":["ca60bd9c1a1b35bc0dc58b6a4a19d5c2651f7a94a4b22b2c5ea001a1ca7a8a7f","fe5b60d091c33f169740df8cb718bf4259f84528b42435194ffe0dd5b79cd125"],"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"external":{"path":"/usr","module":null,"extra_attributes":{}},"patches":["fe5b60d091c33f169740df8cb718bf4259f84528b42435194ffe0dd5b79cd125","ca60bd9c1a1b35bc0dc58b6a4a19d5c2651f7a94a4b22b2c5ea001a1ca7a8a7f"],"package_hash":"duirzgwy5733by27cjxcobmm4wsnw63uvriq5y7x45f4din4m4qa====","hash":"d2a25i7irwfiy3librk4xaqpiv3qzkkg"}}} diff --git a/util/util_tests.sh b/util/util_tests.sh index 079c20f92..27ae0d88f 100755 --- a/util/util_tests.sh +++ b/util/util_tests.sh @@ -78,22 +78,4 @@ if [ $(eval "$cmd") -ne 2 ] ; then fail=1 fi -## Check check_package_config.py -export SPACK_ENV=${SPACK_STACK_DIR}/util/test_env -${SPACK_STACK_DIR}/util/check_package_config.py -${SPACK_STACK_DIR}/util/check_package_config.py | sort -${SPACK_STACK_DIR}/util/check_package_config.py | sort | md5sum -output_checksum=$(${SPACK_STACK_DIR}/util/check_package_config.py | sort | md5sum) -reference_checksum=$(cat ${SPACK_STACK_DIR}/util/test_env/package_check_baseline.txt | md5sum) -if [[ "$output_checksum" != "$reference_checksum" ]]; then - echo "check_package_config.py check A failed!" - fail=1 -fi -# Test ignoring packages -count=$(${SPACK_STACK_DIR}/util/check_package_config.py -i sp --ignore cmake | wc -l) -if [ "$count" -ne 0 ]; then - echo "check_package_config.py check B failed!" - fail=1 -fi - exit $fail