diff --git a/doc/source/Utilities.rst b/doc/source/Utilities.rst index 09cfad693..22b2de344 100644 --- a/doc/source/Utilities.rst +++ b/doc/source/Utilities.rst @@ -20,6 +20,21 @@ The utility located at util/show_duplicate_packages.py parses the output of ``sp The ``-d`` option shows only a list of the duplicates, as opposed to the default behavior, which is to show a print-out of all packages with colorized duplicates. In any case, the identification of any duplicates will yield a return code of 1. The ``-i`` option can be invoked multiple times to skip specific package names. The ``-c`` option can be used to ignore duplicates associated with different compilers; in an environment with, say, GCC and Intel copies of any given package, those two copies of a package will not be reported as duplicates. +.. _Package_Config_Checker: + +------------------------------ +check_package_config.py +------------------------------ + +The utility at util/check_package_config.py is run after concretization in an active spack-stack environment (i.e., `$SPACK_ENV` is set) to confirm that the packages versions and variants in common/packages.yaml are respected in the concretization, as well as that any externals specified in site/packages.yaml are not being omitted. It does this by reading common/packages.yaml (for the version and variant settings), site/packages.yaml (for the external settings), and spack.lock. Usage is as follows: + +.. code-block:: console + spack env active envs/unified-env/ + # To verify versions, variants, and externals: + ${SPACK_STACK_DIR}/util/check_package_config.py + # To ignore a known mismatch in version, variant, or external status for package 'esmf', use -i/--ignore option: + ${SPACK_STACK_DIR}/util/check_package_config.py -i esmf + .. _Permissions_Checker: ------------------------------ diff --git a/util/check_package_config.py b/util/check_package_config.py new file mode 100755 index 000000000..16a6eb2bc --- /dev/null +++ b/util/check_package_config.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +# This utility checks whether the package versions and variants set in +# common/packages.yaml are being respected in the concretization, and +# whether the externals in site/packages.yaml are being used. +# +# To use this script, run it in a loaded spack-stack environment. +# Package names to be ignored can be provided as optional arguments +# using -i/--ignore. +# +# Usage: +# $ spack env active myenv +# $ ${SPACK_STACK_DIR}/util/check_package_config.py +# Ignore packages foo and bar: +# $ ${SPACK_STACK_DIR}/util/check_package_config.py -i foo -i bar +# +# Alex Richert, Jan 2024 + +import json +import os +import sys + +import yaml + +SPACK_ENV = os.getenv("SPACK_ENV") +assert SPACK_ENV, "$SPACK_ENV must be set but is not!" + +# Load common/packages.yaml and site/packages.yaml for versions and externals, respectively +packages_versions_path = os.path.join(SPACK_ENV, "common", "packages.yaml") +with open(packages_versions_path, "r") as f: + packages_versions = yaml.safe_load(f) + +packages_externals_path = os.path.join(SPACK_ENV, "site", "packages.yaml") +with open(packages_externals_path, "r") as f: + packages_externals = yaml.safe_load(f) + +# Load spack.lock +spack_lock_path = os.path.join(SPACK_ENV, "spack.lock") +with open(spack_lock_path, "r") as f: + spack_lock = json.load(f) + +iret = 0 + +# Set up list of packages to ignore +args = sys.argv[1:] +if args: + assert args[-1] not in ("-i", "--ignore"), "-i/--ignore option requires package name" + ignore_list = [args[iarg+1] for iarg in range(len(args)) if args[iarg] in ("-i", "--ignore")] + if ignore_list: + print("Ignoring the following packages: %s" % ", ".join(ignore_list), file=sys.stderr) +else: + ignore_list = [] + +# Iterate over concretized packages +for concrete_spec in spack_lock["concrete_specs"].values(): + concrete_name = concrete_spec["name"] + # Ignore user-specified packages: + if concrete_name in ignore_list: + continue + concrete_version = concrete_spec["version"] + if concrete_name in packages_versions["packages"].keys(): + # Check whether concretized package has specified version from common/packages.yaml + if "version" in packages_versions["packages"][concrete_name].keys(): + config_version = packages_versions["packages"][concrete_name]["version"][0] + if concrete_version != config_version: + iret = 1 + print( + f"WARNING: '{concrete_name}' concretized version {concrete_version} does not match {config_version} specified in $SPACK_ENV/common/packages.yaml" + ) + # Check whether concretized variants match settings from common/packages.yaml + if "variants" in packages_versions["packages"][concrete_name].keys(): + config_variants = packages_versions["packages"][concrete_name]["variants"].split() + for config_variant in config_variants: + variant_mismatch = False + # Boolean variant + if config_variant[0] in ("+", "~"): + config_value = config_variant[0] == "+" + if concrete_spec["parameters"][config_variant[1:]] != config_value: + variant_mismatch = True + # Named variant + elif "=" in config_variant: + config_variant, config_value = config_variant.split("=") + concrete_values = concrete_spec["parameters"][config_variant] + if type(concrete_values) is str: + concrete_values = [concrete_values] + if set(config_value.split(",")) != set(concrete_values): + variant_mismatch = True + if variant_mismatch: + iret = 1 + print( + f"WARNING: '{concrete_name}' concretized variant '{config_variant}' does not match configured value in $SPACK_ENV/common/packages.yaml" + ) + # Check whether concretized package is an external based on site/packages.yaml + if concrete_name in packages_externals["packages"].keys(): + is_external_config = "externals" in packages_externals["packages"][concrete_name].keys() + else: + is_external_config = False + is_external_concrete = "external" in concrete_spec.keys() + if is_external_config != is_external_concrete: + iret = 1 + print( + f"WARNING: '{concrete_name}' is %sconfigured as external in $SPACK_ENV/site/packages.yaml but was %sconcretized as external" + % ((not is_external_config) * "not ", (not is_external_concrete) * "not ") + ) + +sys.exit(iret) diff --git a/util/test_env/README b/util/test_env/README new file mode 100644 index 000000000..68ad4ca39 --- /dev/null +++ b/util/test_env/README @@ -0,0 +1 @@ +This directory provides test files to verify the check_package_config.py utility. diff --git a/util/test_env/common/packages.yaml b/util/test_env/common/packages.yaml new file mode 100644 index 000000000..979803db7 --- /dev/null +++ b/util/test_env/common/packages.yaml @@ -0,0 +1,286 @@ +# Pin versions and specs when building packages + packages: + # + all: + providers: + blas: [openblas] + fftw-api: [fftw] + gl: [opengl] + glu: [openglu] + jpeg: [libjpeg-turbo] + lapack: [openblas] + yacc: [bison] + zlib-api: [zlib] + # + # This version of awscli goes with py-pyyaml@5.4.1 + awscli: + version: ['1.27.84'] + bacio: + version: ['2.4.1'] + bison: + version: ['3.8.2'] + boost: + version: ['1.83.0'] + variants: ~atomic +chrono +date_time +exception +filesystem ~graph ~iostreams ~locale ~log ~math ~mpi ~numpy +pic +program_options +python ~random +regex +serialization ~signals +system +test +thread +timer ~wave cxxstd=17 visibility=hidden + bufr: + version: ['12.0.1'] + variants: +python + cairo: + variants: +pic + cdo: + version: ['2.2.0'] + variants: ~openmp + cmake: + version: ['3.23.1'] + variants: +ownlibs + # Attention - when updating also check the various jcsda-emc-bundles env packages + crtm: + version: ['2.4.0.1'] + variants: +fix + ecbuild: + version: ['3.7.2'] + eccodes: + version: ['2.32.0'] + variants: +png + ecflow: + version: ['5.11.4'] + variants: +ui + eckit: + version: ['1.24.5'] + variants: linalg=eigen,lapack compression=lz4,bzip2 + ecmwf-atlas: + version: ['0.35.1'] + variants: +fckit +trans +tesselation +fftw + ectrans: + version: ['1.2.0'] + variants: ~mkl +fftw + eigen: + version: ['3.4.0'] + # Attention - when updating the version also check the common modules.yaml + # config and update the projections for lmod/tcl. + # Also, check the acorn and derecho site configs which have esmf modifications. + esmf: + version: ['8.6.0'] + variants: ~xerces ~pnetcdf snapshot=none +shared +external-parallelio + require: + - any_of: ['fflags="-fp-model precise" cxxflags="-fp-model precise"'] + when: "%intel" + message: "Extra ESMF compile options for Intel" + - any_of: [''] + when: "%gcc" + message: "Extra ESMF compile options for GCC" + fckit: + version: ['0.11.0'] + variants: +eckit + fftw: + version: ['3.3.10'] + fiat: + version: ['1.2.0'] + fms: + version: ['2023.04'] + variants: precision=32,64 +quad_precision +gfs_phys +openmp +pic constants=GFS build_type=Release +deprecated_io + fontconfig: + variants: +pic + freetype: + variants: +pic + g2: + version: ['3.4.5'] + g2c: + version: ['1.6.4'] + g2tmpl: + version: ['1.10.2'] + gettext: + version: ['0.21.1'] + gfsio: + version: ['1.4.1'] + gftl-shared: + version: ['1.6.1'] + #git-lfs: + # Assume git-lfs is provided, hard to install + # because of dependencies on go/go-bootstrap. + # Note: Uncommenting this entry will break + # the container builds. + #version: ['2.11.0'] + grib-util: + version: ['1.3.0'] + gsibec: + version: ['1.1.3'] + gsi-ncdiag: + version: ['1.1.2'] + gsl-lite: + version: ['0.37.0'] + hdf: + version: ['4.2.15'] + variants: +external-xdr ~fortran ~netcdf + hdf5: + version: ['1.14.0'] + variants: +hl +fortran +mpi ~threadsafe +szip + ip: + version: ['4.3.0'] + variants: precision=4,d,8 + ip2: + version: ['1.1.2'] + jasper: + version: ['2.0.32'] + jedi-cmake: + version: ['1.4.0'] + jpeg: + version: ['9.1.0'] + landsfcutil: + version: ['2.4.1'] + libjpeg-turbo: + version: ['2.1.0'] + libpng: + version: ['1.6.37'] + variants: +pic + libyaml: + version: ['0.2.5'] + mapl: + version: ['2.40.3'] + variants: +shared +pflogger ~f2py + # If making changes here, also check the Discover site config and the CI workflows + met: + version: ['11.1.0'] + variants: +python +grib2 + metplus: + version: ['5.1.0'] + metis: + require: "+int64 +real64" + mpich: + variants: ~hwloc +two_level_namespace + mysql: + variants: +download_boost + nco: + version: ['5.0.6'] + variants: ~doc + # ncview - when adding information here, also check Orion + # and Discover site configs + nemsio: + version: ['2.5.4'] + nemsiogfs: + version: ['2.5.3'] + nccmp: + version: ['1.9.0.1'] + ncio: + version: ['1.1.2'] + netcdf-c: + version: ['4.9.2'] + # If using 4.9.1, turn off byterange variant to fix compile error: ~byterange + variants: +dap +mpi ~parallel-netcdf + netcdf-cxx4: + version: ['4.3.1'] + netcdf-fortran: + version: ['4.6.1'] + # ninja - when adding information here, also check Discover site config + nlohmann-json: + version: ['3.10.5'] + nlohmann-json-schema-validator: + version: ['2.1.0'] + odc: + version: ['1.4.6'] + variants: ~fortran + openblas: + version: ['0.3.24'] + variants: +noavx512 + openmpi: + variants: +internal-hwloc +two_level_namespace + # Pin openssl to avoid duplicate packages being built + openssl: + variants: +shared + p4est: + version: ['2.8'] + parallelio: + version: ['2.5.10'] + variants: +pnetcdf + parallel-netcdf: + version: ['1.12.2'] + pflogger: + version: ['1.12.0'] + variants: +mpi + pixman: + variants: +pic + # Do not build pkgconf - https://github.com/jcsda/spack-stack/issues/123 + pkgconf: + buildable: False + prod-util: + version: ['2.1.1'] + proj: + version: ['8.1.0'] + variants: ~tiff + python: + require: "@3.10.13" + py-attrs: + # https://github.com/JCSDA/spack-stack/issues/740 + version: ['21.4.0'] + py-cartopy: + variants: +plotting + require: "@0.21.1" + py-cryptography: + variants: +rust_bootstrap + # Introduced in https://github.com/JCSDA/spack-stack/pull/894, pin py-cython + # to avoid duplicate packages being built (cylc dependencies soft-want @3:) + py-cython: + require: "@0.29.36" + py-h5py: + version: ['3.7.0'] + variants: ~mpi + # Comment out for now until build problems are solved + # https://github.com/jcsda/spack-stack/issues/522 + # see also ewok-env virtual package and container + # README.md + #py-mysql-connector-python: + # version: ['8.0.32'] + py-netcdf4: + version: ['1.5.8'] + variants: ~mpi + py-numpy: + require: ['@1.22.3'] + py-pandas: + variants: +excel + # To avoid pip._vendor.pep517.wrappers.BackendInvalid errors with newer + # versions of py-poetry-core when using external/homebrew Python as + # we do at the moment in spack-stack. + # Pin the py-setuptools version to avoid duplicate Python packages + py-setuptools: + require: ['@63.4.3'] + py-setuptools-rust: + variants: +rust_bootstrap + py-shapely: + require: ['@1.8.0'] + qt: + version: ['5.15.3'] + scotch: + version: ['7.0.4'] + variants: +mpi+metis~shared~threads~mpi_thread+noarch + sfcio: + version: ['1.4.1'] + shumlib: + version: ['macos_clang_linux_intel_port'] + sigio: + version: ['2.3.2'] + sp: + version: ['2.5.0'] + variants: precision=4,d,8 derp=auto + udunits: + version: ['2.2.28'] + upp: + version: ['10.0.10'] + w3emc: + version: ['2.10.0'] + variants: precision=4,d,8 + w3nco: + version: ['2.4.1'] + wget: + version: ['1.21.2'] + # When changing wgrib2, also check Hercules and Nautilus site configs + wgrib2: + version: ['2.0.8'] + wrf-io: + version: ['1.2.0'] + yafyaml: + version: ['0.5.1'] + zlib: + version: ['1.2.13'] + zstd: + version: ['1.5.2'] + variants: +programs diff --git a/util/test_env/package_check_baseline.txt b/util/test_env/package_check_baseline.txt new file mode 100644 index 000000000..aa83c411a --- /dev/null +++ b/util/test_env/package_check_baseline.txt @@ -0,0 +1,6 @@ +WARNING: 'cmake' concretized variant '+ownlibs' does not match configured value in $SPACK_ENV/common/packages.yaml +WARNING: 'cmake' concretized version 3.20.2 does not match 3.23.1 specified in $SPACK_ENV/common/packages.yaml +WARNING: 'cmake' is not configured as external in $SPACK_ENV/site/packages.yaml but was concretized as external +WARNING: 'sp' concretized variant 'derp' does not match configured value in $SPACK_ENV/common/packages.yaml +WARNING: 'sp' concretized variant 'precision' does not match configured value in $SPACK_ENV/common/packages.yaml +WARNING: 'sp' is configured as external in $SPACK_ENV/site/packages.yaml but was not concretized as external diff --git a/util/test_env/site/packages.yaml b/util/test_env/site/packages.yaml new file mode 100644 index 000000000..9edf85972 --- /dev/null +++ b/util/test_env/site/packages.yaml @@ -0,0 +1,10 @@ +packages: + gmake: + buildable: false + externals: + - spec: gmake@4.2.1 + prefix: /usr + sp: + externals: + - spec: sp@2.3.3 + prefix: /dev/null diff --git a/util/test_env/spack.lock b/util/test_env/spack.lock new file mode 100644 index 000000000..0be5d3c36 --- /dev/null +++ b/util/test_env/spack.lock @@ -0,0 +1 @@ +{"_meta":{"file-type":"spack-lockfile","lockfile-version":5,"specfile-version":4},"spack":{"version":"0.21.1.dev0","type":"git","commit":"5a06f19dd176f84159ced4ca75d8cbee3a6b9e56"},"roots":[{"hash":"5dqa22jiadepasivmtbkvplu6ucry3qh","spec":"sp@2.5.0"}],"concrete_specs":{"5dqa22jiadepasivmtbkvplu6ucry3qh":{"name":"sp","version":"2.5.0","arch":{"platform":"linux","platform_os":"centos8","target":{"name":"zen","vendor":"AuthenticAMD","features":["abm","aes","avx","avx2","bmi1","bmi2","clflushopt","clzero","cx16","f16c","fma","fsgsbase","mmx","movbe","pclmulqdq","popcnt","rdseed","sse","sse2","sse4_1","sse4_2","sse4a","ssse3","xsavec","xsaveopt"],"generation":0,"parents":["x86_64_v3"]}},"compiler":{"name":"gcc","version":"8.5.0"},"namespace":"builtin","parameters":{"build_system":"cmake","build_type":"Release","generator":"make","ipo":false,"openmp":false,"pic":true,"precision":["8","d"],"derp":"none","shared":false,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"wgwpmenbjmbqyiar5djncippvybyxbw3zwgofoqsixx6kf5n737a====","dependencies":[{"name":"cmake","hash":"qjf2medg2vff5gn4htfd43rgdex7sv4d","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"gmake","hash":"d2a25i7irwfiy3librk4xaqpiv3qzkkg","parameters":{"deptypes":["build"],"virtuals":[]}}],"hash":"5dqa22jiadepasivmtbkvplu6ucry3qh"},"qjf2medg2vff5gn4htfd43rgdex7sv4d":{"name":"cmake","version":"3.20.2","arch":{"platform":"linux","platform_os":"centos8","target":{"name":"zen","vendor":"AuthenticAMD","features":["abm","aes","avx","avx2","bmi1","bmi2","clflushopt","clzero","cx16","f16c","fma","fsgsbase","mmx","movbe","pclmulqdq","popcnt","rdseed","sse","sse2","sse4_1","sse4_2","sse4a","ssse3","xsavec","xsaveopt"],"generation":0,"parents":["x86_64_v3"]}},"compiler":{"name":"gcc","version":"8.5.0"},"namespace":"builtin","parameters":{"build_system":"generic","build_type":"Release","doc":false,"ncurses":true,"owncurl":false,"ownlibs":false,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"external":{"path":"/usr","module":null,"extra_attributes":{}},"package_hash":"usp3csiedkkf6skym4g5erzxxcbck4d7is3ilzzx7fv6vkuk452q====","hash":"qjf2medg2vff5gn4htfd43rgdex7sv4d"},"d2a25i7irwfiy3librk4xaqpiv3qzkkg":{"name":"gmake","version":"4.2.1","arch":{"platform":"linux","platform_os":"centos8","target":{"name":"zen","vendor":"AuthenticAMD","features":["abm","aes","avx","avx2","bmi1","bmi2","clflushopt","clzero","cx16","f16c","fma","fsgsbase","mmx","movbe","pclmulqdq","popcnt","rdseed","sse","sse2","sse4_1","sse4_2","sse4a","ssse3","xsavec","xsaveopt"],"generation":0,"parents":["x86_64_v3"]}},"compiler":{"name":"gcc","version":"8.5.0"},"namespace":"builtin","parameters":{"build_system":"generic","guile":false,"patches":["ca60bd9c1a1b35bc0dc58b6a4a19d5c2651f7a94a4b22b2c5ea001a1ca7a8a7f","fe5b60d091c33f169740df8cb718bf4259f84528b42435194ffe0dd5b79cd125"],"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"external":{"path":"/usr","module":null,"extra_attributes":{}},"patches":["fe5b60d091c33f169740df8cb718bf4259f84528b42435194ffe0dd5b79cd125","ca60bd9c1a1b35bc0dc58b6a4a19d5c2651f7a94a4b22b2c5ea001a1ca7a8a7f"],"package_hash":"duirzgwy5733by27cjxcobmm4wsnw63uvriq5y7x45f4din4m4qa====","hash":"d2a25i7irwfiy3librk4xaqpiv3qzkkg"}}} diff --git a/util/util_tests.sh b/util/util_tests.sh index 8d6bcfbd3..609a1fcdf 100755 --- a/util/util_tests.sh +++ b/util/util_tests.sh @@ -22,7 +22,7 @@ chmod o+rX $HOME mkdir -p ${SPACK_STACK_DIR}/util/checks cd ${SPACK_STACK_DIR}/util/checks -# Check check_permissions.sh +## Check check_permissions.sh mkdir -p perm_check1/perm_check2/perm_check3 cd perm_check1/perm_check2 chmod 777 ../../perm_check1 @@ -43,7 +43,7 @@ run_and_check 1 "check_permissions F" ${SPACK_STACK_DIR}/util/check_permissions. chmod 770 perm_check3 run_and_check 1 "check_permissions G" ${SPACK_STACK_DIR}/util/check_permissions.sh -# Check show_duplicate_packages.py +## Check show_duplicate_packages.py cd ${SPACK_STACK_DIR}/util/checks echo -e " - abcdefg hdf6@1.2.3%intel\n - tuvwxyz hdf6@1.2.3%gcc" > fakeconcrete.A run_and_check 1 "show_duplicate_packages.py A1" ${SPACK_STACK_DIR}/util/show_duplicate_packages.py fakeconcrete.A @@ -78,4 +78,19 @@ if [ $(eval "$cmd") -ne 2 ] ; then fail=1 fi +## Check check_package_config.py +export SPACK_ENV=${SPACK_STACK_DIR}/util/test_env +output_checksum=$(${SPACK_STACK_DIR}/util/check_package_config.py | sort | md5sum) +reference_checksum=$(cat ${SPACK_STACK_DIR}/util/test_env/package_check_baseline.txt | md5sum) +if [[ "$output_checksum" != "$reference_checksum" ]]; then + echo "check_package_config.py check A failed!" + fail=1 +fi +# Test ignoring packages +count=$(${SPACK_STACK_DIR}/util/check_package_config.py -i sp --ignore cmake | wc -l) +if [ "$count" -ne 0 ]; then + echo "check_package_config.py check B failed!" + fail=1 +fi + exit $fail