Merge branch 'develop' into gributil_wgrib

.github/workflows/macos-ci-aarch64.yaml

@@ -60,6 +60,7 @@ jobs:
               --exclude bison --exclude openssl \
               --exclude python
           spack external find --scope system perl
+          spack external find --scope system libiconv
           spack external find --scope system wget
           PATH="/opt/homebrew/opt/curl/bin:$PATH" \
               spack external find --scope system curl
@@ -79,12 +80,15 @@ jobs:
           # *DH
 
           # Set compiler and MPI
-          spack config add "packages:all:providers:mpi:[[email protected].1]"
+          spack config add "packages:all:providers:mpi:[[email protected].3]"
           spack config add "packages:all:compiler:[[email protected]]"
           sed -i '' "s/\['\%aocc', '\%apple-clang', '\%gcc', '\%intel'\]/\['\%apple-clang'\]/g" $ENVDIR/spack.yaml
 
           # Add additional variants for MET packages, different from config/common/packages.yaml
-          spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis"
+          # DH* 20240513 - avoid hdf-eos2 until https://github.com/spack/spack/issues/44168 is resolved
+          #spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis"
+          spack config add "packages:met:variants:+python +grib2 +graphics"
+          # *DH
 
           # Concretize and check for duplicates
           spack concretize 2>&1 | tee log.concretize.apple-clang-14.0.3
@@ -138,7 +142,7 @@ jobs:
 
           module use ${ENVDIR}/install/modulefiles/Core
           module load stack-apple-clang/14.0.3
-          module load stack-openmpi/5.0.1
+          module load stack-openmpi/5.0.3
           module load stack-python/3.10.13
           module available
 

.github/workflows/ubuntu-ci-x86_64-gnu.yaml

@@ -72,7 +72,7 @@ jobs:
           spack config add config:install_tree:padded_length:200
 
           # Set compiler and MPI
-          spack config add "packages:all:providers:mpi:[[email protected].1]"
+          spack config add "packages:all:providers:mpi:[[email protected].3]"
           spack config add "packages:all:compiler:[[email protected]]"
           sed -i "s/\['\%aocc', '\%apple-clang', '\%gcc', '\%intel'\]/\['\%gcc'\]/g" $ENVDIR/spack.yaml
 
@@ -180,7 +180,7 @@ jobs:
 
           module use ${ENVDIR}/install/modulefiles/Core
           module load stack-gcc/11.4.0
-          module load stack-openmpi/5.0.1
+          module load stack-openmpi/5.0.3
           module load stack-python/3.10.13
           module available
 

.gitmodules

@@ -1,7 +1,9 @@
 [submodule "spack"]
   path = spack
-  url = https://github.com/jcsda/spack
-  branch = spack-stack-dev
+  #url = https://github.com/jcsda/spack
+  #branch = spack-stack-dev
+  url = https://github.com/climbfuji/spack
+  branch = feature/merge_spack_develop_into_spack_stack_dev_20240509
 [submodule "doc/CMakeModules"]
   path = doc/CMakeModules
   url = https://github.com/noaa-emc/cmakemodules

configs/common/packages.yaml

@@ -18,8 +18,9 @@
       version: ['2.4.1']
     bison:
       version: ['3.8.2']
+    # 1.85 incompatible with [email protected] - use latest "good version"
     boost:
-      require: "@1.83: ~atomic +chrono +date_time +exception +filesystem ~graph ~iostreams ~locale ~log ~math ~mpi ~numpy +pic +program_options +python ~random +regex +serialization ~signals +system +test +thread +timer ~wave cxxstd=17 visibility=hidden"
+      require: "@1.84 ~atomic +chrono +date_time +exception +filesystem ~graph ~iostreams ~locale ~log ~math ~mpi ~numpy +pic +program_options +python ~random +regex +serialization ~signals +system +test +thread +timer ~wave cxxstd=17 visibility=hidden"
     bufr:
       version: ['12.0.1']
       variants: +python
@@ -29,7 +30,7 @@
       version: ['2.2.0']
       variants: ~openmp
     cmake:
-      version: ['3.23.1']
+      version: ['3.27.9']
       variants: +ownlibs
     # Attention - when updating also check the various jcsda-emc-bundles env packages
     crtm:
@@ -115,8 +116,10 @@
       version: ['4.2.15']
       variants: +external-xdr ~fortran ~netcdf
     hdf5:
-      version: ['1.14.3']
-      variants: +hl +fortran +mpi ~threadsafe ~szip
+      require: '@1.14.3 +hl +fortran +mpi +threadsafe ~szip'
+    # Newer versions of hdf-eos2 require manual downloading, avoid
+    hdf-eos2:
+      require: "@2.20v1.00"
     ip:
       version: ['5.0.0']
       variants: precision=4,d,8
@@ -162,9 +165,8 @@
     ncio:
       version: ['1.1.2']
     netcdf-c:
-      version: ['4.9.2']
       # If using 4.9.1, turn off byterange variant to fix compile error: ~byterange
-      variants: +dap +mpi ~parallel-netcdf ~szip
+      require: "@4.9.2 +dap +mpi ~parallel-netcdf ~szip build_system=autotools"
     netcdf-cxx4:
       version: ['4.3.1']
     netcdf-fortran:
@@ -225,6 +227,12 @@
     py-h5py:
       version: ['3.7.0']
       variants: ~mpi
+    # To avoid duplicate packages
+    py-flit-core:
+      require: "@3.8.0"
+    # To avoid duplicate packages
+    py-jinja2:
+      require: "@3.0.3"
     # Comment out for now until build problems are solved
     # https://github.com/jcsda/spack-stack/issues/522
     # see also ewok-env virtual package and container
@@ -246,7 +254,10 @@
     py-shapely:
       require: ['@1.8.0']
     py-torch:
-      require: "+internal-protobuf ~mkldnn"
+      require: "+custom-protobuf ~mkldnn"
+    # To avoid duplicate packages
+    py-urllib3:
+      require: "@1.26.12"
     qt:
       version: ['5.15.3']
     scotch:

configs/containers/docker-ubuntu-clang-mpich.yaml

@@ -30,7 +30,7 @@ spack:
       require: '%clang'
       target: [x86_64]
       providers:
-        mpi: [[email protected].0]
+        mpi: [[email protected].1]
       compiler: [[email protected]]
     gcc:
       buildable: false
@@ -71,9 +71,9 @@ spack:
     mpich:
       buildable: false
       externals:
-      - spec: [email protected].0
-        prefix: /opt/mpich-4.2.0
-      version: [4.2.0]
+      - spec: [email protected].1
+        prefix: /opt/mpich-4.2.1
+      version: [4.2.1]
     mysql:
       buildable: false
       externals:
@@ -188,7 +188,7 @@ spack:
         ln -svf libc++abi.so.1.0 libc++abi.so
         #
         # Build mpich outside of spack-stack
-        ENV MPICH_VERSION=4.2.0
+        ENV MPICH_VERSION=4.2.1
         ENV CC=clang
         ENV CXX=clang++
         ENV FC=gfortran
@@ -214,7 +214,7 @@ spack:
         # Set environment variables for installing tzdata
         ENV DEBIAN_FRONTEND=noninteractive
         ENV TZ=Etc/UTC
-        ENV MPICH_VERSION=4.2.0
+        ENV MPICH_VERSION=4.2.1
         ENV PATH=/opt/mpich-${MPICH_VERSION}/bin:${PATH}
         ENV CPATH=/opt/mpich-${MPICH_VERSION}/include:${CPATH}
         ENV LD_LIBRARY_PATH=/opt/mpich-${MPICH_VERSION}/lib:${LD_LIBRARY_PATH}
@@ -228,7 +228,7 @@ spack:
         ln -svf libc++abi.so.1.0 libc++abi.so
         # Copy spack find output from builder
         COPY --from=builder /root/spack_find.out /root/spack_find.out
-        ENV MPICH_VERSION=4.2.0
+        ENV MPICH_VERSION=4.2.1
         # Copy mpich-${MPICH_VERSION} installation from builder
         COPY --from=builder /opt/mpich-${MPICH_VERSION} /opt/mpich-${MPICH_VERSION}
         # Make a non-root user:nonroot / group:nonroot for running MPI

configs/containers/docker-ubuntu-gcc-openmpi.yaml

@@ -28,7 +28,7 @@ spack:
       require: '%gcc'
       target: [x86_64]
       providers:
-        mpi: [openmpi@5.1.6]
+        mpi: [openmpi@5.0.3]
       compiler: [[email protected]]
     gcc:
       buildable: false

doc/modulefile_templates/ecflow

@@ -17,6 +17,7 @@ if { [ module-info mode load ] && ![ is-loaded qt/5.15.2 ] } {
 # Set this value
 set ECFLOW_PATH "/discover/swdev/jcsda/spack-stack/ecflow-5.8.4"
 
+setenv       ecflow_ROOT "${ECFLOW_PATH}"
 prepend-path PATH "${ECFLOW_PATH}/bin"
 prepend-path LD_LIBRARY_PATH "${ECFLOW_PATH}/lib"
 prepend-path LD_LIBRARY_PATH "${ECFLOW_PATH}/lib64"

doc/source/NewSiteConfigs.rst

@@ -299,7 +299,7 @@ Remember to activate the ``lua`` module environment and have MacTeX in your sear
    # Check your clang version then add it to your site compiler config.
    clang --version
    spack config add "packages:all:compiler:[apple-clang@YOUR-VERSION]"
-   spack config add "packages:all:providers:mpi:[[email protected].1]"
+   spack config add "packages:all:providers:mpi:[[email protected].3]"
 
 8. If the environment will be used to run JCSDA's JEDI-Skylab experiments using R2D2 with a local MySQL server, run the following command:
 
@@ -550,10 +550,10 @@ It is recommended to increase the stacksize limit by using ``ulimit -S -s unlimi
    spack config add "packages:all:compiler:[gcc@YOUR-VERSION]"
 
    # Example for Red Hat 8 following the above instructions
-   spack config add "packages:all:providers:mpi:[[email protected].1]"
+   spack config add "packages:all:providers:mpi:[[email protected].3]"
 
    # Example for Ubuntu 20.04 or 22.04 following the above instructions
-   spack config add "packages:all:providers:mpi:[mpich@4.1.2]"
+   spack config add "packages:all:providers:mpi:[mpich@4.2.1]"
 
 .. warning::
    On some systems, the default compiler (e.g., ``gcc`` on Ubuntu 20) may not get used by spack if a newer version is found. Compare your entry to the output of the concretization step later and adjust the entry, if necessary.

spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py

@@ -718,15 +718,13 @@ def setup_meta_modules():
                 or not compiler_version in python_dict[compiler_name].keys()
             ):
                 logging.warn(
-                    "No Python version found for compiler {compiler_name}@{compiler_version}, skipping Python modules"
+                    f"No Python version found for compiler {compiler_name}@{compiler_version}, skipping Python modules"
                 )
                 continue
             spec = python_dict[compiler_name][compiler_version]
             python_version = str(spec.version)
             logging.info(
-                "  ... configuring stack python interpreter {}@{} for compiler {}@{}".format(
-                    python_name, python_version, compiler_name, compiler_version
-                )
+                f"  ... configuring stack python interpreter {python_name}@{python_version} for compiler {compiler_name}@{compiler_version}"
             )
 
             if spec.external:

spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py

@@ -298,7 +298,7 @@ def write(self):
             original = original_sections.get(section, {})
             existing = spack.config.get(section, scope=env_scope)
             new = spack.config.merge_yaml(existing, original)
-            if section in existing:
+            if existing and section in existing:
                 spack.config.set(section, new[section], env_scope)
 
         with env.write_transaction():