From 5a65ac54f71a9de0a3cb0eba0d0d93425909b3b0 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Fri, 22 Sep 2023 18:01:27 +0000 Subject: [PATCH 01/33] Determine IC structure based on start type #1783 --- workflow/setup_expt.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/workflow/setup_expt.py b/workflow/setup_expt.py index 7f9651f79a..646d1f9dd5 100755 --- a/workflow/setup_expt.py +++ b/workflow/setup_expt.py @@ -71,7 +71,15 @@ def fill_COMROT_cycled(host, inputs): rdatestr = datetime_to_YMDH(inputs.idate - to_timedelta('T06H')) idatestr = datetime_to_YMDH(inputs.idate) - if os.path.isdir(os.path.join(inputs.icsdir, f'{inputs.cdump}.{rdatestr[:8]}', rdatestr[8:], 'model_data', 'atmos')): + # Test if we are using the new COM structure or the old flat one for ICs + if(inputs.start in ['warm']): + pathstr = os.path.join(inputs.icsdir, f'{inputs.cdump}.{rdatestr[:8]}', + rdatestr[8:], 'model_data', 'atmos') + else: + pathstr = os.path.join(inputs.icsdir, f'{inputs.cdump}.{idatestr[:8]}', + idatestr[8:], 'model_data', 'atmos') + + if os.path.isdir(pathstr): flat_structure = False else: flat_structure = True From 5622b797b35f63b0fedcecded80e53066dc6e0b1 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Fri, 22 Sep 2023 18:16:21 +0000 Subject: [PATCH 02/33] Fix spacing for pycodestyle tests #1783 --- workflow/setup_expt.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/workflow/setup_expt.py b/workflow/setup_expt.py index 646d1f9dd5..ccf8203610 100755 --- a/workflow/setup_expt.py +++ b/workflow/setup_expt.py @@ -73,11 +73,11 @@ def fill_COMROT_cycled(host, inputs): # Test if we are using the new COM structure or the old flat one for ICs if(inputs.start in ['warm']): - pathstr = os.path.join(inputs.icsdir, f'{inputs.cdump}.{rdatestr[:8]}', - rdatestr[8:], 'model_data', 'atmos') + pathstr = os.path.join(inputs.icsdir, f'{inputs.cdump}.{rdatestr[:8]}', + rdatestr[8:], 'model_data', 'atmos') else: - pathstr = os.path.join(inputs.icsdir, f'{inputs.cdump}.{idatestr[:8]}', - idatestr[8:], 'model_data', 'atmos') + pathstr = os.path.join(inputs.icsdir, f'{inputs.cdump}.{idatestr[:8]}', + idatestr[8:], 'model_data', 'atmos') if os.path.isdir(pathstr): flat_structure = False From 9761692f094c22531e20d0164df9496451e1c789 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Fri, 22 Sep 2023 18:22:18 +0000 Subject: [PATCH 03/33] Added a space after 'if' --- workflow/setup_expt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/workflow/setup_expt.py b/workflow/setup_expt.py index ccf8203610..c9aa9188f6 100755 --- a/workflow/setup_expt.py +++ b/workflow/setup_expt.py @@ -72,7 +72,7 @@ def fill_COMROT_cycled(host, inputs): idatestr = datetime_to_YMDH(inputs.idate) # Test if we are using the new COM structure or the old flat one for ICs - if(inputs.start in ['warm']): + if inputs.start in ['warm']: pathstr = os.path.join(inputs.icsdir, f'{inputs.cdump}.{rdatestr[:8]}', rdatestr[8:], 'model_data', 'atmos') else: From 90efb17891728a49d10d56440a90294650c7f9ac Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Tue, 17 Oct 2023 19:22:16 +0000 Subject: [PATCH 04/33] Point to new gw spack-stack. --- jobs/rocoto/anal.sh | 40 ++------------------- jobs/rocoto/efcs.sh | 21 ++--------- jobs/rocoto/eobs.sh | 40 ++------------------- jobs/rocoto/eupd.sh | 40 ++------------------- jobs/rocoto/fcst.sh | 54 ++-------------------------- jobs/rocoto/prep.sh | 1 - modulefiles/module_base.hera.lua | 41 +++++++++++---------- modulefiles/module_gwci.hera.lua | 13 +++---- modulefiles/module_gwsetup.hera.lua | 16 ++++++--- parm/config/gefs/config.base.emc.dyn | 4 +-- parm/config/gfs/config.base.emc.dyn | 2 +- scripts/exglobal_diag.sh | 2 +- sorc/checkout.sh | 6 ++-- ush/getncdimlen | 2 +- ush/gsi_utils.py | 2 +- versions/build.hera.ver | 28 +++++++-------- versions/run.hera.ver | 27 +++++++------- 17 files changed, 93 insertions(+), 246 deletions(-) mode change 100755 => 100644 sorc/checkout.sh diff --git a/jobs/rocoto/anal.sh b/jobs/rocoto/anal.sh index cb07749b9e..d99152ef19 100755 --- a/jobs/rocoto/anal.sh +++ b/jobs/rocoto/anal.sh @@ -4,43 +4,9 @@ source "${HOMEgfs}/ush/preamble.sh" ############################################################### # Source FV3GFS workflow modules -#. ${HOMEgfs}/ush/load_fv3gfs_modules.sh -#status=$? -#[[ ${status} -ne 0 ]] && exit ${status} - -# TODO: clean this up -#Source appropriate modulefiles based on machine - -source "${HOMEgfs}/ush/detect_machine.sh" -if [[ "${MACHINE_ID}" = "wcoss2" ]]; then - # Source FV3GFS workflow modules - . "${HOMEgfs}"/ush/load_fv3gfs_modules.sh - status=$? - [[ ${status} -ne 0 ]] && exit "${status}" -else - # Append compiler (only on machines that have multiple compilers) - COMPILER=${COMPILER:-"intel"} - if [[ "${MACHINE_ID}" = "hera" ]] || [[ "${MACHINE_ID}" = "cheyenne" ]]; then - MACHINE_ID=${MACHINE_ID}.${COMPILER} - fi - - # Source machine specific GSI-EnKF modules - set +x - source "${HOMEgfs}/ush/module-setup.sh" - module use "${HOMEgfs}/sorc/gsi_enkf.fd/modulefiles" - module load gsi_"${MACHINE_ID}" - - if [[ "${MACHINE_ID}" = "orion" ]]; then - module load miniconda3/4.12.0 - set +u - conda activate regional_workflow_cmaq - set_strict - fi - - module list - unset MACHINE_ID - set_trace -fi +. ${HOMEgfs}/ush/load_fv3gfs_modules.sh +status=$? +[[ ${status} -ne 0 ]] && exit ${status} export job="anal" export jobid="${job}.$$" diff --git a/jobs/rocoto/efcs.sh b/jobs/rocoto/efcs.sh index 46a25ac759..b202015149 100755 --- a/jobs/rocoto/efcs.sh +++ b/jobs/rocoto/efcs.sh @@ -4,24 +4,9 @@ source "${HOMEgfs}/ush/preamble.sh" ############################################################### # Source FV3GFS workflow modules -#. ${HOMEgfs}/ush/load_fv3gfs_modules.sh -#status=$? -#[[ ${status} -ne 0 ]] && exit ${status} - -# TODO: clean this up -source "${HOMEgfs}/ush/detect_machine.sh" -set +x -source "${HOMEgfs}/ush/module-setup.sh" -module use "${HOMEgfs}/sorc/ufs_model.fd/tests" -module load modules.ufs_model.lua -# Workflow needs utilities from prod_util (setPDY.sh, ndate, etc.) -module load prod_util -if [[ "${MACHINE_ID}" = "wcoss2" ]]; then - module load cray-pals -fi -module list -unset MACHINE_ID -set_trace +. ${HOMEgfs}/ush/load_fv3gfs_modules.sh +status=$? +[[ ${status} -ne 0 ]] && exit ${status} export job="efcs" export jobid="${job}.$$" diff --git a/jobs/rocoto/eobs.sh b/jobs/rocoto/eobs.sh index 4f813a8d84..95fa42cb08 100755 --- a/jobs/rocoto/eobs.sh +++ b/jobs/rocoto/eobs.sh @@ -4,43 +4,9 @@ source "${HOMEgfs}/ush/preamble.sh" ############################################################### # Source FV3GFS workflow modules -#. ${HOMEgfs}/ush/load_fv3gfs_modules.sh -#status=$? -#[[ ${status} -ne 0 ]] && exit ${status} - -# TODO: clean this up -#Source appropriate modulefiles based on machine - -source "${HOMEgfs}/ush/detect_machine.sh" -if [[ "${MACHINE_ID}" = "wcoss2" ]]; then - # Source FV3GFS workflow modules - . "${HOMEgfs}"/ush/load_fv3gfs_modules.sh - status=$? - [[ ${status} -ne 0 ]] && exit "${status}" -else - # Append compiler (only on machines that have multiple compilers) - COMPILER=${COMPILER:-"intel"} - if [[ "${MACHINE_ID}" = "hera" ]] || [[ "${MACHINE_ID}" = "cheyenne" ]]; then - MACHINE_ID=${MACHINE_ID}.${COMPILER} - fi - - # Source machine specific GSI-EnKF modules - set +x - source "${HOMEgfs}/ush/module-setup.sh" - module use "${HOMEgfs}/sorc/gsi_enkf.fd/modulefiles" - module load gsi_"${MACHINE_ID}" - - if [[ "${MACHINE_ID}" = "orion" ]]; then - module load miniconda3/4.12.0 - set +u - conda activate regional_workflow_cmaq - set_strict - fi - - module list - unset MACHINE_ID - set_trace -fi +. ${HOMEgfs}/ush/load_fv3gfs_modules.sh +status=$? +[[ ${status} -ne 0 ]] && exit ${status} export job="eobs" export jobid="${job}.$$" diff --git a/jobs/rocoto/eupd.sh b/jobs/rocoto/eupd.sh index 9ddde92d36..3ed028f87a 100755 --- a/jobs/rocoto/eupd.sh +++ b/jobs/rocoto/eupd.sh @@ -4,43 +4,9 @@ source "${HOMEgfs}/ush/preamble.sh" ############################################################### # Source FV3GFS workflow modules -#. ${HOMEgfs}/ush/load_fv3gfs_modules.sh -#status=$? -#[[ ${status} -ne 0 ]] && exit ${status} - -# TODO: clean this up -#Source appropriate modulefiles based on machine - -source "${HOMEgfs}/ush/detect_machine.sh" -if [[ "${MACHINE_ID}" = "wcoss2" ]]; then - # Source FV3GFS workflow modules - . "${HOMEgfs}"/ush/load_fv3gfs_modules.sh - status=$? - [[ ${status} -ne 0 ]] && exit "${status}" -else - # Append compiler (only on machines that have multiple compilers) - COMPILER=${COMPILER:-"intel"} - if [[ "${MACHINE_ID}" = "hera" ]] || [[ "${MACHINE_ID}" = "cheyenne" ]]; then - MACHINE_ID=${MACHINE_ID}.${COMPILER} - fi - - # Source machine specific GSI-EnKF modules - set +x - source "${HOMEgfs}/ush/module-setup.sh" - module use "${HOMEgfs}/sorc/gsi_enkf.fd/modulefiles" - module load gsi_"${MACHINE_ID}" - - if [[ "${MACHINE_ID}" = "orion" ]]; then - module load miniconda3/4.12.0 - set +u - conda activate regional_workflow_cmaq - set_strict - fi - - module list - unset MACHINE_ID - set_trace -fi +. ${HOMEgfs}/ush/load_fv3gfs_modules.sh +status=$? +[[ ${status} -ne 0 ]] && exit ${status} export job="eupd" export jobid="${job}.$$" diff --git a/jobs/rocoto/fcst.sh b/jobs/rocoto/fcst.sh index 9d59f70bd8..daefd6c449 100755 --- a/jobs/rocoto/fcst.sh +++ b/jobs/rocoto/fcst.sh @@ -4,57 +4,9 @@ source "${HOMEgfs}/ush/preamble.sh" ############################################################### # Source FV3GFS workflow modules -#. ${HOMEgfs}/ush/load_fv3gfs_modules.sh -#status=$? -#[[ ${status} -ne 0 ]] && exit ${status} - -# TODO: clean this up -source "${HOMEgfs}/ush/detect_machine.sh" -set +x -source "${HOMEgfs}/ush/module-setup.sh" -if [[ "${MACHINE_ID}" != "noaacloud" ]]; then - module use "${HOMEgfs}/sorc/ufs_model.fd/tests" - module load modules.ufs_model.lua - module load prod_util -fi - -if [[ "${MACHINE_ID}" = "wcoss2" ]]; then - module load cray-pals -fi -if [[ "${MACHINE_ID}" = "hera" ]]; then - module use "/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/modulefiles/core" - module load "miniconda3/4.6.14" - module load "gfs_workflow/1.0.0" -# TODO: orion and wcoss2 will be uncommented when they are ready. This comment block will be removed in the next PR -#elif [[ "${MACHINE_ID}" = "orion" ]]; then -# module use "/home/rmahajan/opt/global-workflow/modulefiles/core" -# module load "python/3.7.5" -# module load "gfs_workflow/1.0.0" -#elif [[ "${MACHINE_ID}" = "wcoss2" ]]; then -# module load "python/3.7.5" -fi -if [[ "${MACHINE_ID}" == "noaacloud" ]]; then - if [[ "${PW_CSP:-}" = "aws" ]]; then - - # TODO: This can be cleaned-up; most of this is a hack for now. - module use "/contrib/spack-stack/envs/ufswm/install/modulefiles/Core" - module load "stack-intel" - module load "stack-intel-oneapi-mpi" - module use -a "/contrib/spack-stack/miniconda/modulefiles/miniconda/" - module load "py39_4.12.0" - module load "ufs-weather-model-env/1.0.0" - export NETCDF="/contrib/spack-stack/miniconda/apps/miniconda/py39_4.12.0" - # TODO: Are there plans for EPIC to maintain this package or should GW provide support? - export UTILROOT="/contrib/global-workflow/NCEPLIBS-prod_util" - export PATH="${PATH}:/contrib/global-workflow/bin" - ndate_path="$(command -v ndate)" - export NDATE="${ndate_path}" - fi -fi - -module list -unset MACHINE_ID -set_trace +. ${HOMEgfs}/ush/load_fv3gfs_modules.sh +status=$? +[[ ${status} -ne 0 ]] && exit ${status} ############################################################### # exglobal_forecast.py requires the following in PYTHONPATH diff --git a/jobs/rocoto/prep.sh b/jobs/rocoto/prep.sh index f90be861d9..dfb541abb6 100755 --- a/jobs/rocoto/prep.sh +++ b/jobs/rocoto/prep.sh @@ -94,7 +94,6 @@ if [[ ${MAKE_PREPBUFR} = "YES" ]]; then fi export job="j${CDUMP}_prep_${cyc}" - export DATAROOT="${RUNDIR}/${CDATE}/${CDUMP}/prepbufr" export COMIN=${COM_OBS} export COMOUT=${COM_OBS} RUN="gdas" YMD=${PDY} HH=${cyc} generate_com -rx COMINgdas:COM_ATMOS_HISTORY_TMPL diff --git a/modulefiles/module_base.hera.lua b/modulefiles/module_base.hera.lua index 7464ac7bec..92cbce9ba0 100644 --- a/modulefiles/module_base.hera.lua +++ b/modulefiles/module_base.hera.lua @@ -2,46 +2,49 @@ help([[ Load environment to run GFS on Hera ]]) -prepend_path("MODULEPATH", "/scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack") +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/Alexander.Richert/spack-stack-1.4.1-gw/envs/gw/install/modulefiles/Core") +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/jcsda/jedipara/spack-stack/modulefiles") -load(pathJoin("hpc", os.getenv("hpc_ver"))) -load(pathJoin("hpc-intel", os.getenv("hpc_intel_ver"))) -load(pathJoin("hpc-impi", os.getenv("hpc_impi_ver"))) +load(pathJoin("stack-intel", os.getenv("stack_intel_ver"))) +load(pathJoin("stack-intel-oneapi-mpi", os.getenv("stack_impi_ver"))) +load(pathJoin("stack-python", os.getenv("stack_python_ver"))) +load(pathJoin("miniconda", os.getenv("miniconda_ver"))) load(pathJoin("hpss", os.getenv("hpss_ver"))) load(pathJoin("gempak", os.getenv("gempak_ver"))) load(pathJoin("ncl", os.getenv("ncl_ver"))) load(pathJoin("jasper", os.getenv("jasper_ver"))) -load(pathJoin("png", os.getenv("libpng_ver"))) +load(pathJoin("libpng", os.getenv("libpng_ver"))) load(pathJoin("cdo", os.getenv("cdo_ver"))) load(pathJoin("R", os.getenv("R_ver"))) load(pathJoin("hdf5", os.getenv("hdf5_ver"))) -load(pathJoin("netcdf", os.getenv("netcdf_ver"))) +load(pathJoin("netcdf-c", os.getenv("netcdf_c_ver"))) +load(pathJoin("netcdf-fortran", os.getenv("netcdf_fortran_ver"))) load(pathJoin("nco", os.getenv("nco_ver"))) -load(pathJoin("prod_util", os.getenv("prod_util_ver"))) -load(pathJoin("grib_util", os.getenv("grib_util_ver"))) +load(pathJoin("prod-util", os.getenv("prod_util_ver"))) +load(pathJoin("grib-util", os.getenv("grib_util_ver"))) load(pathJoin("g2tmpl", os.getenv("g2tmpl_ver"))) -load(pathJoin("ncdiag", os.getenv("ncdiag_ver"))) +load(pathJoin("gsi-ncdiag", os.getenv("gsi_ncdiag_ver"))) load(pathJoin("crtm", os.getenv("crtm_ver"))) +load(pathJoin("bufr", os.getenv("bufr_ver"))) load(pathJoin("wgrib2", os.getenv("wgrib2_ver"))) +load(pathJoin("py-netcdf4", os.getenv("py_netcdf4_ver"))) +load(pathJoin("py-pyyaml", os.getenv("py_pyyaml_ver"))) +load(pathJoin("py-jinja2", os.getenv("py_jinja2_ver"))) + +load(pathJoin("met", os.getenv("met_ver"))) +load(pathJoin("metplus", os.getenv("metplus_ver"))) + setenv("WGRIB2","wgrib2") +setenv("UTILROOT",os.getenv("prod_util_ROOT")) --prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/prepobs/v" .. os.getenv("prepobs_run_ver"), "modulefiles")) prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/prepobs/feature-GFSv17_com_reorg_log_update/modulefiles")) load(pathJoin("prepobs", os.getenv("prepobs_run_ver"))) -prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/Fit2Obs/v" .. os.getenv("fit2obs_ver"), "modulefiles")) +prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) load(pathJoin("fit2obs", os.getenv("fit2obs_ver"))) --- Temporary until official hpc-stack is updated -prepend_path("MODULEPATH", "/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/hpc-stack/modulefiles/stack") -load(pathJoin("hpc", "1.2.0")) -load(pathJoin("hpc-intel", "18.0.5.274")) -load(pathJoin("hpc-miniconda3", "4.6.14")) -load(pathJoin("gfs_workflow", "1.0.0")) -load(pathJoin("met", "9.1")) -load(pathJoin("metplus", "3.1")) - whatis("Description: GFS run environment") diff --git a/modulefiles/module_gwci.hera.lua b/modulefiles/module_gwci.hera.lua index f4b62a5fd2..cce41f34da 100644 --- a/modulefiles/module_gwci.hera.lua +++ b/modulefiles/module_gwci.hera.lua @@ -2,14 +2,15 @@ help([[ Load environment to run GFS workflow setup scripts on Hera ]]) -prepend_path("MODULEPATH", "/scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack") +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/Alexander.Richert/spack-stack-1.4.1-gw/envs/gw/install/modulefiles/Core") +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/jcsda/jedipara/spack-stack/modulefiles") -load(pathJoin("hpc", "1.1.0")) -load(pathJoin("hpc-intel", "18.0.5.274")) -load(pathJoin("hpc-impi", "2018.0.4")) +load(pathJoin("stack-intel", os.getenv("2021.5.0"))) +load(pathJoin("stack-intel-oneapi-mpi", os.getenv("2021.5.1"))) -load(pathJoin("netcdf","4.7.4")) -load(pathJoin("nccmp","1.8.7.0")) +load(pathJoin("netcdf-c", os.getenv("4.9.2"))) +load(pathJoin("netcdf-fortran", os.getenv("4.6.0"))) +load(pathJoin("nccmp","1.9.0.1")) load(pathJoin("wgrib2", "2.0.8")) whatis("Description: GFS run setup CI environment") diff --git a/modulefiles/module_gwsetup.hera.lua b/modulefiles/module_gwsetup.hera.lua index a07b32b6a6..0858353669 100644 --- a/modulefiles/module_gwsetup.hera.lua +++ b/modulefiles/module_gwsetup.hera.lua @@ -4,10 +4,16 @@ Load environment to run GFS workflow setup scripts on Hera load(pathJoin("rocoto")) --- Temporary until official hpc-stack is updated -prepend_path("MODULEPATH", "/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/hpc-stack/modulefiles/stack") -load(pathJoin("hpc", "1.2.0")) -load(pathJoin("hpc-miniconda3", "4.6.14")) -load(pathJoin("gfs_workflow", "1.0.0")) +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/Alexander.Richert/spack-stack-1.4.1-gw/envs/gw/install/modulefiles/Core") +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/jcsda/jedipara/spack-stack/modulefiles") + +local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" +local stack_python_ver=os.getenv("stack_python_ver") or "3.9.12" + +load(pathJoin("stack-intel", stack_intel_ver)) +load(pathJoin("stack-python", stack_python_ver)) +load("py-jinja2") +load("py-pyyaml") +load("py-numpy") whatis("Description: GFS run setup environment") diff --git a/parm/config/gefs/config.base.emc.dyn b/parm/config/gefs/config.base.emc.dyn index 7140ef477b..b6cf086fbb 100644 --- a/parm/config/gefs/config.base.emc.dyn +++ b/parm/config/gefs/config.base.emc.dyn @@ -82,7 +82,7 @@ export VERBOSE="YES" export KEEPDATA="NO" export CHGRP_RSTPROD="@CHGRP_RSTPROD@" export CHGRP_CMD="@CHGRP_CMD@" -export NCDUMP="${NETCDF}/bin/ncdump" +export NCDUMP="${NETCDF:-${netcdf_c_ROOT:-}}/bin/ncdump" export NCLEN="${HOMEgfs}/ush/getncdimlen" # Machine environment, jobs, and other utility scripts @@ -126,7 +126,7 @@ export SENDCOM=${SENDCOM:-"YES"} export SENDSDM=${SENDSDM:-"NO"} export SENDDBN_NTC=${SENDDBN_NTC:-"NO"} export SENDDBN=${SENDDBN:-"NO"} -export DBNROOT=${DBNROOT:-${UTILROOT}/fakedbn} +export DBNROOT=${DBNROOT:-${UTILROOT:-}/fakedbn} # APP settings export APP=@APP@ diff --git a/parm/config/gfs/config.base.emc.dyn b/parm/config/gfs/config.base.emc.dyn index db4cd22e7f..2c38fd20db 100644 --- a/parm/config/gfs/config.base.emc.dyn +++ b/parm/config/gfs/config.base.emc.dyn @@ -86,7 +86,7 @@ export VERBOSE="YES" export KEEPDATA="NO" export CHGRP_RSTPROD="@CHGRP_RSTPROD@" export CHGRP_CMD="@CHGRP_CMD@" -export NCDUMP="${NETCDF:-}/bin/ncdump" +export NCDUMP="${NETCDF:-${netcdf_c_ROOT:-}}/bin/ncdump" export NCLEN="${HOMEgfs}/ush/getncdimlen" # Machine environment, jobs, and other utility scripts diff --git a/scripts/exglobal_diag.sh b/scripts/exglobal_diag.sh index 3aa1093fad..2322f8c93f 100755 --- a/scripts/exglobal_diag.sh +++ b/scripts/exglobal_diag.sh @@ -35,7 +35,7 @@ export NMV=${NMV:-"/bin/mv"} export NLN=${NLN:-"/bin/ln -sf"} export CHGRP_CMD=${CHGRP_CMD:-"chgrp ${group_name:-rstprod}"} export NCLEN=${NCLEN:-$HOMEgfs/ush/getncdimlen} -export CATEXEC=${CATEXEC:-$ncdiag_ROOT/bin/ncdiag_cat_serial.x} +export CATEXEC=${CATEXEC:-${ncdiag_ROOT:-${gsi_ncdiag_ROOT}}/bin/ncdiag_cat_serial.x} COMPRESS=${COMPRESS:-gzip} UNCOMPRESS=${UNCOMPRESS:-gunzip} APRUNCFP=${APRUNCFP:-""} diff --git a/sorc/checkout.sh b/sorc/checkout.sh old mode 100755 new mode 100644 index 9c9addad1d..eb195a2b1a --- a/sorc/checkout.sh +++ b/sorc/checkout.sh @@ -150,7 +150,7 @@ source "${topdir}/../workflow/gw_setup.sh" # The checkout version should always be a speciifc commit (hash or tag), not a branch errs=0 checkout "wxflow" "https://github.com/NOAA-EMC/wxflow" "528f5ab" ; errs=$((errs + $?)) -checkout "gfs_utils.fd" "https://github.com/NOAA-EMC/gfs-utils" "a283262" ; errs=$((errs + $?)) +checkout "gfs_utils.fd" "https://github.com/davidhuber-noaa/gfs-utils" "feature/spack-stack" ; errs=$((errs + $?)) checkout "ufs_utils.fd" "https://github.com/ufs-community/UFS_UTILS.git" "72a0471" ; errs=$((errs + $?)) checkout "ufs_model.fd" "https://github.com/ufs-community/ufs-weather-model" "${ufs_model_hash:-4d05445}" ; errs=$((errs + $?)) checkout "verif-global.fd" "https://github.com/NOAA-EMC/EMC_verif-global.git" "c267780" ; errs=$((errs + $?)) @@ -164,8 +164,8 @@ if [[ ${checkout_gdas} == "YES" ]]; then fi if [[ ${checkout_gsi} == "YES" || ${checkout_gdas} == "YES" ]]; then - checkout "gsi_utils.fd" "https://github.com/NOAA-EMC/GSI-Utils.git" "322cc7b"; errs=$((errs + $?)) - checkout "gsi_monitor.fd" "https://github.com/NOAA-EMC/GSI-Monitor.git" "45783e3"; errs=$((errs + $?)) + checkout "gsi_utils.fd" "https://github.com/davidhuber-noaa/GSI-Utils.git" "feature/spack-stack"; errs=$((errs + $?)) + checkout "gsi_monitor.fd" "https://github.com/davidhuber-noaa/GSI-Monitor.git" "spack-stack"; errs=$((errs + $?)) fi if (( errs > 0 )); then diff --git a/ush/getncdimlen b/ush/getncdimlen index 5d230f6cc3..fcf231947b 100755 --- a/ush/getncdimlen +++ b/ush/getncdimlen @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # getncdimlen # cory.r.martin@noaa.gov # 2019-10-17 diff --git a/ush/gsi_utils.py b/ush/gsi_utils.py index 94a2ae1348..8088cb7233 100644 --- a/ush/gsi_utils.py +++ b/ush/gsi_utils.py @@ -78,7 +78,7 @@ def get_ncdims(ncfile): try: import netCDF4 as nc except ImportError as err: - raise ImportError(f"Unable to import netCDF4 module\n{err}") + raise ImportError(f"Unable to import netCDF4 module") ncf = nc.Dataset(ncfile) ncdims = {} for d in ncf.dimensions.keys(): diff --git a/versions/build.hera.ver b/versions/build.hera.ver index dcc30aee76..0f7f7bb053 100644 --- a/versions/build.hera.ver +++ b/versions/build.hera.ver @@ -1,19 +1,19 @@ -export hpc_ver=1.1.0 -export hpc_intel_ver=18.0.5.274 -export hpc_impi_ver=2018.0.4 +export stack_intel_ver=2021.5.0 +export stack_impi_ver=2021.5.1 -export cmake_ver=3.20.1 +export cmake_ver=3.23.1 export gempak_ver=7.4.2 -export jasper_ver=2.0.25 -export libpng_ver=1.6.35 +export jasper_ver=2.0.32 +export libpng_ver=1.6.37 export zlib_ver=1.2.11 export esmf_ver=8.3.0b09 export pio_ver=2.5.2 export fms_ver=2022.04 -export hdf5_ver=1.10.6 -export netcdf_ver=4.7.4 +export hdf5_ver=1.14.0 +export netcdf_c_ver=4.9.2 +export netcdf_fortran_ver=4.6.0 export bacio_ver=2.4.1 export w3nco_ver=2.4.1 @@ -24,11 +24,11 @@ export bufr_ver=11.7.0 export g2_ver=3.4.5 export sp_ver=2.3.3 export ip_ver=3.3.3 -export wrf_io_ver=1.2.0 -export ncio_ver=1.1.2 -export ncdiag_ver=1.0.0 -export g2tmpl_ver=1.10.0 +export gsi_ncdiag_ver=1.1.2 +export g2tmpl_ver=1.10.2 export crtm_ver=2.4.0 export wgrib2_ver=2.0.8 - -export upp_ver=10.0.8 +export py_netcdf4_ver=1.5.3 +export py_pyyaml_ver=6.0 +export py_jinja2_ver=3.1.2 +export fit2obs_ver=1.0.0 diff --git a/versions/run.hera.ver b/versions/run.hera.ver index 471e019dcd..5b730013af 100644 --- a/versions/run.hera.ver +++ b/versions/run.hera.ver @@ -1,24 +1,27 @@ -export hpc_ver=1.1.0 -export hpc_intel_ver=18.0.5.274 -export hpc_impi_ver=2018.0.4 +export stack_intel_ver=2021.5.0 +export stack_impi_ver=2021.5.1 +export stack_python_ver=3.9.12 +export miniconda_ver=3.9.12 export hpss_ver=hpss export gempak_ver=7.4.2 export ncl_ver=6.6.2 -export jasper_ver=2.0.25 -export libpng_ver=1.6.35 -export cdo_ver=1.9.5 +export jasper_ver=2.0.32 +export libpng_ver=1.6.37 +export cdo_ver=2.0.5 export R_ver=3.5.0 -export hdf5_ver=1.10.6 -export netcdf_ver=4.7.4 +export hdf5_ver=1.14.0 +export netcdf_c_ver=4.9.2 +export netcdf_fortran_ver=4.6.0 -export nco_ver=4.9.1 +export nco_ver=5.0.6 export prod_util_ver=1.2.2 -export grib_util_ver=1.2.2 -export g2tmpl_ver=1.10.0 -export ncdiag_ver=1.0.0 +export grib_util_ver=1.2.3 +export g2tmpl_ver=1.10.2 +export gsi_ncdiag_ver=1.1.2 export crtm_ver=2.4.0 +export bufr_ver=11.7.0 export wgrib2_ver=2.0.8 export obsproc_run_ver=1.1.2 From 678ce48fd87a58f7d01a33551bb811b601d5936f Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 16 Nov 2023 14:36:50 +0000 Subject: [PATCH 05/33] Upgrade to spack-stack/1.5.1 on Hera #1868 --- modulefiles/module_base.hera.lua | 10 ++++---- modulefiles/module_gwci.hera.lua | 3 +-- modulefiles/module_gwsetup.hera.lua | 7 +++--- sorc/checkout.sh | 4 +-- versions/build.hera.ver | 27 +++++++++++++------- versions/run.hera.ver | 39 ++++++++++++++++++----------- 6 files changed, 54 insertions(+), 36 deletions(-) diff --git a/modulefiles/module_base.hera.lua b/modulefiles/module_base.hera.lua index 92cbce9ba0..08c546c31c 100644 --- a/modulefiles/module_base.hera.lua +++ b/modulefiles/module_base.hera.lua @@ -2,13 +2,13 @@ help([[ Load environment to run GFS on Hera ]]) -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/Alexander.Richert/spack-stack-1.4.1-gw/envs/gw/install/modulefiles/Core") -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/jcsda/jedipara/spack-stack/modulefiles") +spack_stack_ver=(os.getenv("spack_stack_ver") or "None") +spack_env=(os.getenv("spack_env") or "None") +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-" .. spack_stack_ver .. "/envs/" .. spack_env .. "/install/modulefiles/Core") load(pathJoin("stack-intel", os.getenv("stack_intel_ver"))) load(pathJoin("stack-intel-oneapi-mpi", os.getenv("stack_impi_ver"))) -load(pathJoin("stack-python", os.getenv("stack_python_ver"))) -load(pathJoin("miniconda", os.getenv("miniconda_ver"))) +load(pathJoin("python", os.getenv("python_ver"))) load(pathJoin("hpss", os.getenv("hpss_ver"))) load(pathJoin("gempak", os.getenv("gempak_ver"))) @@ -23,7 +23,7 @@ load(pathJoin("netcdf-c", os.getenv("netcdf_c_ver"))) load(pathJoin("netcdf-fortran", os.getenv("netcdf_fortran_ver"))) load(pathJoin("nco", os.getenv("nco_ver"))) -load(pathJoin("prod-util", os.getenv("prod_util_ver"))) +load(pathJoin("prod_util", os.getenv("prod_util_ver"))) load(pathJoin("grib-util", os.getenv("grib_util_ver"))) load(pathJoin("g2tmpl", os.getenv("g2tmpl_ver"))) load(pathJoin("gsi-ncdiag", os.getenv("gsi_ncdiag_ver"))) diff --git a/modulefiles/module_gwci.hera.lua b/modulefiles/module_gwci.hera.lua index cce41f34da..1ee7a1cde1 100644 --- a/modulefiles/module_gwci.hera.lua +++ b/modulefiles/module_gwci.hera.lua @@ -2,8 +2,7 @@ help([[ Load environment to run GFS workflow setup scripts on Hera ]]) -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/Alexander.Richert/spack-stack-1.4.1-gw/envs/gw/install/modulefiles/Core") -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/jcsda/jedipara/spack-stack/modulefiles") +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") load(pathJoin("stack-intel", os.getenv("2021.5.0"))) load(pathJoin("stack-intel-oneapi-mpi", os.getenv("2021.5.1"))) diff --git a/modulefiles/module_gwsetup.hera.lua b/modulefiles/module_gwsetup.hera.lua index 0858353669..c86cac7b02 100644 --- a/modulefiles/module_gwsetup.hera.lua +++ b/modulefiles/module_gwsetup.hera.lua @@ -4,14 +4,13 @@ Load environment to run GFS workflow setup scripts on Hera load(pathJoin("rocoto")) -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/Alexander.Richert/spack-stack-1.4.1-gw/envs/gw/install/modulefiles/Core") -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/jcsda/jedipara/spack-stack/modulefiles") +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" -local stack_python_ver=os.getenv("stack_python_ver") or "3.9.12" +local python_ver=os.getenv("python_ver") or "3.10.8" load(pathJoin("stack-intel", stack_intel_ver)) -load(pathJoin("stack-python", stack_python_ver)) +load(pathJoin("python", python_ver)) load("py-jinja2") load("py-pyyaml") load("py-numpy") diff --git a/sorc/checkout.sh b/sorc/checkout.sh index 83b0866eb3..ce242ca4c9 100755 --- a/sorc/checkout.sh +++ b/sorc/checkout.sh @@ -156,11 +156,11 @@ checkout "ufs_model.fd" "https://github.com/ufs-community/ufs-weather-model" # Run all other checkouts simultaneously with just 1 core each to handle submodules. checkout "wxflow" "https://github.com/NOAA-EMC/wxflow" "528f5ab" & checkout "gfs_utils.fd" "https://github.com/davidhuber-noaa/gfs-utils" "feature/spack-stack" & -checkout "ufs_utils.fd" "https://github.com/ufs-community/UFS_UTILS.git" "72a0471" & +checkout "ufs_utils.fd" "https://github.com/ufs-community/UFS_UTILS.git" "892b693" & checkout "verif-global.fd" "https://github.com/NOAA-EMC/EMC_verif-global.git" "c267780" & if [[ ${checkout_gsi} == "YES" ]]; then - checkout "gsi_enkf.fd" "https://github.com/NOAA-EMC/GSI.git" "ca19008" "1" "NO" & + checkout "gsi_enkf.fd" "https://github.com/DavidHuber-NOAA/GSI.git" "spack-stack" "1" "NO" & fi if [[ ${checkout_gdas} == "YES" ]]; then diff --git a/versions/build.hera.ver b/versions/build.hera.ver index 0f7f7bb053..64a4e57dd4 100644 --- a/versions/build.hera.ver +++ b/versions/build.hera.ver @@ -1,34 +1,43 @@ +export spack_stack_ver=1.5.1 +export spack_env=gsi-addon export stack_intel_ver=2021.5.0 export stack_impi_ver=2021.5.1 +export python_ver=3.10.8 +export hpss_ver=hpss export cmake_ver=3.23.1 export gempak_ver=7.4.2 export jasper_ver=2.0.32 export libpng_ver=1.6.37 export zlib_ver=1.2.11 -export esmf_ver=8.3.0b09 -export pio_ver=2.5.2 -export fms_ver=2022.04 +export esmf_ver=8.5.0 +export fms_ver=2023.02.01 +export cdo_ver=2.0.5 +export nco_ver=5.0.6 +export ncl_ver=6.6.2 +export prod_util_ver=1.2.2 export hdf5_ver=1.14.0 export netcdf_c_ver=4.9.2 export netcdf_fortran_ver=4.6.0 export bacio_ver=2.4.1 -export w3nco_ver=2.4.1 -export nemsio_ver=2.5.2 +export nemsio_ver=2.5.4 export sigio_ver=2.3.2 -export w3emc_ver=2.9.2 +export w3emc_ver=2.10.0 export bufr_ver=11.7.0 export g2_ver=3.4.5 export sp_ver=2.3.3 -export ip_ver=3.3.3 +export ip_ver=4.3.0 export gsi_ncdiag_ver=1.1.2 export g2tmpl_ver=1.10.2 export crtm_ver=2.4.0 export wgrib2_ver=2.0.8 -export py_netcdf4_ver=1.5.3 -export py_pyyaml_ver=6.0 +export grib_util_ver=1.3.0 +export py_netcdf4_ver=1.5.8 +export py_pyyaml_ver=5.4.1 export py_jinja2_ver=3.1.2 export fit2obs_ver=1.0.0 +export met_ver=9.1.3 +export metplus_ver=3.1.1 diff --git a/versions/run.hera.ver b/versions/run.hera.ver index 5b730013af..8083d7e892 100644 --- a/versions/run.hera.ver +++ b/versions/run.hera.ver @@ -1,31 +1,42 @@ +export spack_stack_ver=1.5.1 +export spack_env=gsi-addon export stack_intel_ver=2021.5.0 export stack_impi_ver=2021.5.1 -export stack_python_ver=3.9.12 -export miniconda_ver=3.9.12 +export python_ver=3.10.8 export hpss_ver=hpss +export cmake_ver=3.23.1 + export gempak_ver=7.4.2 -export ncl_ver=6.6.2 export jasper_ver=2.0.32 export libpng_ver=1.6.37 +export zlib_ver=1.2.11 +export esmf_ver=8.5.0 +export fms_ver=2023.02.01 export cdo_ver=2.0.5 -export R_ver=3.5.0 +export nco_ver=5.0.6 +export ncl_ver=6.6.2 export hdf5_ver=1.14.0 export netcdf_c_ver=4.9.2 export netcdf_fortran_ver=4.6.0 -export nco_ver=5.0.6 -export prod_util_ver=1.2.2 -export grib_util_ver=1.2.3 -export g2tmpl_ver=1.10.2 +export bacio_ver=2.4.1 +export nemsio_ver=2.5.4 +export sigio_ver=2.3.2 +export w3emc_ver=2.10.0 +export bufr_ver=11.7.0 +export g2_ver=3.4.5 +export sp_ver=2.3.3 +export ip_ver=4.3.0 export gsi_ncdiag_ver=1.1.2 +export g2tmpl_ver=1.10.2 export crtm_ver=2.4.0 -export bufr_ver=11.7.0 export wgrib2_ver=2.0.8 - -export obsproc_run_ver=1.1.2 -export prepobs_run_ver=1.0.1 - -export tracker_ver=feature-GFSv17_com_reorg +export py_netcdf4_ver=1.5.8 +export py_pyyaml_ver=5.4.1 +export py_jinja2_ver=3.1.2 export fit2obs_ver=1.0.0 +export met_ver=9.1.3 +export metplus_ver=3.1.1 +export tracker_ver=feature-GFSv17_com_reorg From 7039de69bfa1d8c4b66bec086704e5ca080c0651 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 16 Nov 2023 20:40:04 +0000 Subject: [PATCH 06/33] Update version files for all systems on spack-stack. #1868 --- versions/build.jet.ver | 51 ++++++++++++++++++++++--------------- versions/build.orion.ver | 53 ++++++++++++++++++++++---------------- versions/build.s4.ver | 52 ++++++++++++++++++++++--------------- versions/run.jet.ver | 54 ++++++++++++++++++++++++--------------- versions/run.orion.ver | 55 +++++++++++++++++++++++++--------------- versions/run.s4.ver | 55 +++++++++++++++++++++++++--------------- 6 files changed, 195 insertions(+), 125 deletions(-) diff --git a/versions/build.jet.ver b/versions/build.jet.ver index e92a2dc3d8..64a4e57dd4 100644 --- a/versions/build.jet.ver +++ b/versions/build.jet.ver @@ -1,34 +1,43 @@ -export hpc_ver=1.2.0 -export hpc_intel_ver=18.0.5.274 -export hpc_impi_ver=2018.4.274 +export spack_stack_ver=1.5.1 +export spack_env=gsi-addon +export stack_intel_ver=2021.5.0 +export stack_impi_ver=2021.5.1 +export python_ver=3.10.8 -export cmake_ver=3.20.1 +export hpss_ver=hpss +export cmake_ver=3.23.1 export gempak_ver=7.4.2 -export jasper_ver=2.0.25 -export libpng_ver=1.6.35 +export jasper_ver=2.0.32 +export libpng_ver=1.6.37 export zlib_ver=1.2.11 -export esmf_ver=8.3.0b09 -export pio_ver=2.5.2 -export fms_ver=2022.04 +export esmf_ver=8.5.0 +export fms_ver=2023.02.01 +export cdo_ver=2.0.5 +export nco_ver=5.0.6 +export ncl_ver=6.6.2 +export prod_util_ver=1.2.2 -export hdf5_ver=1.10.6 -export netcdf_ver=4.7.4 +export hdf5_ver=1.14.0 +export netcdf_c_ver=4.9.2 +export netcdf_fortran_ver=4.6.0 export bacio_ver=2.4.1 -export w3nco_ver=2.4.1 -export nemsio_ver=2.5.2 +export nemsio_ver=2.5.4 export sigio_ver=2.3.2 -export w3emc_ver=2.9.2 +export w3emc_ver=2.10.0 export bufr_ver=11.7.0 export g2_ver=3.4.5 export sp_ver=2.3.3 -export ip_ver=3.3.3 -export wrf_io_ver=1.2.0 -export ncio_ver=1.1.2 -export ncdiag_ver=1.0.0 -export g2tmpl_ver=1.10.0 +export ip_ver=4.3.0 +export gsi_ncdiag_ver=1.1.2 +export g2tmpl_ver=1.10.2 export crtm_ver=2.4.0 export wgrib2_ver=2.0.8 - -export upp_ver=10.0.8 +export grib_util_ver=1.3.0 +export py_netcdf4_ver=1.5.8 +export py_pyyaml_ver=5.4.1 +export py_jinja2_ver=3.1.2 +export fit2obs_ver=1.0.0 +export met_ver=9.1.3 +export metplus_ver=3.1.1 diff --git a/versions/build.orion.ver b/versions/build.orion.ver index 0084e9e506..64a4e57dd4 100644 --- a/versions/build.orion.ver +++ b/versions/build.orion.ver @@ -1,34 +1,43 @@ -export hpc_ver=1.1.0 -export hpc_intel_ver=2018.4 -export hpc_impi_ver=2018.4 +export spack_stack_ver=1.5.1 +export spack_env=gsi-addon +export stack_intel_ver=2021.5.0 +export stack_impi_ver=2021.5.1 +export python_ver=3.10.8 -export cmake_ver=3.22.1 +export hpss_ver=hpss +export cmake_ver=3.23.1 -export gempak_ver=7.5.1 -export jasper_ver=2.0.25 -export libpng_ver=1.6.35 +export gempak_ver=7.4.2 +export jasper_ver=2.0.32 +export libpng_ver=1.6.37 export zlib_ver=1.2.11 -export esmf_ver=8.3.0b09 -export pio_ver=2.5.2 -export fms_ver=2022.04 +export esmf_ver=8.5.0 +export fms_ver=2023.02.01 +export cdo_ver=2.0.5 +export nco_ver=5.0.6 +export ncl_ver=6.6.2 +export prod_util_ver=1.2.2 -export hdf5_ver=1.10.6 -export netcdf_ver=4.7.4 +export hdf5_ver=1.14.0 +export netcdf_c_ver=4.9.2 +export netcdf_fortran_ver=4.6.0 export bacio_ver=2.4.1 -export w3nco_ver=2.4.1 -export nemsio_ver=2.5.2 +export nemsio_ver=2.5.4 export sigio_ver=2.3.2 -export w3emc_ver=2.9.2 +export w3emc_ver=2.10.0 export bufr_ver=11.7.0 export g2_ver=3.4.5 export sp_ver=2.3.3 -export ip_ver=3.3.3 -export wrf_io_ver=1.2.0 -export ncio_ver=1.1.2 -export ncdiag_ver=1.0.0 -export g2tmpl_ver=1.10.0 +export ip_ver=4.3.0 +export gsi_ncdiag_ver=1.1.2 +export g2tmpl_ver=1.10.2 export crtm_ver=2.4.0 export wgrib2_ver=2.0.8 - -export upp_ver=10.0.8 +export grib_util_ver=1.3.0 +export py_netcdf4_ver=1.5.8 +export py_pyyaml_ver=5.4.1 +export py_jinja2_ver=3.1.2 +export fit2obs_ver=1.0.0 +export met_ver=9.1.3 +export metplus_ver=3.1.1 diff --git a/versions/build.s4.ver b/versions/build.s4.ver index d08ee7a74b..64a4e57dd4 100644 --- a/versions/build.s4.ver +++ b/versions/build.s4.ver @@ -1,33 +1,43 @@ -export hpc_ver=1.2.0 -export hpc_intel_ver=18.0.4 -export hpc_impi_ver=18.0.4 +export spack_stack_ver=1.5.1 +export spack_env=gsi-addon +export stack_intel_ver=2021.5.0 +export stack_impi_ver=2021.5.1 +export python_ver=3.10.8 -export cmake_ver=3.20.5 +export hpss_ver=hpss +export cmake_ver=3.23.1 -export jasper_ver=2.0.25 -export libpng_ver=1.6.35 +export gempak_ver=7.4.2 +export jasper_ver=2.0.32 +export libpng_ver=1.6.37 export zlib_ver=1.2.11 -export esmf_ver=8.3.0b09 -export pio_ver=2.5.2 -export fms_ver=2022.04 +export esmf_ver=8.5.0 +export fms_ver=2023.02.01 +export cdo_ver=2.0.5 +export nco_ver=5.0.6 +export ncl_ver=6.6.2 +export prod_util_ver=1.2.2 -export hdf5_ver=1.10.6 -export netcdf_ver=4.7.4 +export hdf5_ver=1.14.0 +export netcdf_c_ver=4.9.2 +export netcdf_fortran_ver=4.6.0 export bacio_ver=2.4.1 -export w3nco_ver=2.4.1 -export nemsio_ver=2.5.2 +export nemsio_ver=2.5.4 export sigio_ver=2.3.2 -export w3emc_ver=2.9.2 +export w3emc_ver=2.10.0 export bufr_ver=11.7.0 export g2_ver=3.4.5 export sp_ver=2.3.3 -export ip_ver=3.3.3 -export wrf_io_ver=1.2.0 -export ncio_ver=1.1.2 -export ncdiag_ver=1.0.0 -export g2tmpl_ver=1.10.0 +export ip_ver=4.3.0 +export gsi_ncdiag_ver=1.1.2 +export g2tmpl_ver=1.10.2 export crtm_ver=2.4.0 export wgrib2_ver=2.0.8 - -export upp_ver=10.0.8 +export grib_util_ver=1.3.0 +export py_netcdf4_ver=1.5.8 +export py_pyyaml_ver=5.4.1 +export py_jinja2_ver=3.1.2 +export fit2obs_ver=1.0.0 +export met_ver=9.1.3 +export metplus_ver=3.1.1 diff --git a/versions/run.jet.ver b/versions/run.jet.ver index 3cab138a1a..8083d7e892 100644 --- a/versions/run.jet.ver +++ b/versions/run.jet.ver @@ -1,30 +1,42 @@ -export hpc_ver=1.2.0 -export hpc_intel_ver=18.0.5.274 -export hpc_impi_ver=2018.4.274 +export spack_stack_ver=1.5.1 +export spack_env=gsi-addon +export stack_intel_ver=2021.5.0 +export stack_impi_ver=2021.5.1 +export python_ver=3.10.8 export hpss_ver=hpss +export cmake_ver=3.23.1 + export gempak_ver=7.4.2 +export jasper_ver=2.0.32 +export libpng_ver=1.6.37 +export zlib_ver=1.2.11 +export esmf_ver=8.5.0 +export fms_ver=2023.02.01 +export cdo_ver=2.0.5 +export nco_ver=5.0.6 export ncl_ver=6.6.2 -export jasper_ver=2.0.25 -export libpng_ver=1.6.35 -export cdo_ver=1.9.5 -export R_ver=4.0.2 -export hdf5_ver=1.10.6 -export netcdf_ver=4.7.4 +export hdf5_ver=1.14.0 +export netcdf_c_ver=4.9.2 +export netcdf_fortran_ver=4.6.0 -export nco_ver=4.9.1 -export prod_util_ver=1.2.2 -export grib_util_ver=1.2.2 -export g2tmpl_ver=1.10.0 -export ncdiag_ver=1.0.0 +export bacio_ver=2.4.1 +export nemsio_ver=2.5.4 +export sigio_ver=2.3.2 +export w3emc_ver=2.10.0 +export bufr_ver=11.7.0 +export g2_ver=3.4.5 +export sp_ver=2.3.3 +export ip_ver=4.3.0 +export gsi_ncdiag_ver=1.1.2 +export g2tmpl_ver=1.10.2 export crtm_ver=2.4.0 export wgrib2_ver=2.0.8 - -export anaconda_ver=5.3.1 - -export obsproc_run_ver=1.1.2 -export prepobs_run_ver=1.0.1 - -export tracker_ver=feature-GFSv17_com_reorg +export py_netcdf4_ver=1.5.8 +export py_pyyaml_ver=5.4.1 +export py_jinja2_ver=3.1.2 export fit2obs_ver=1.0.0 +export met_ver=9.1.3 +export metplus_ver=3.1.1 +export tracker_ver=feature-GFSv17_com_reorg diff --git a/versions/run.orion.ver b/versions/run.orion.ver index 80a05580cf..8083d7e892 100644 --- a/versions/run.orion.ver +++ b/versions/run.orion.ver @@ -1,27 +1,42 @@ -export hpc_ver=1.1.0 -export hpc_intel_ver=2018.4 -export hpc_impi_ver=2018.4 +export spack_stack_ver=1.5.1 +export spack_env=gsi-addon +export stack_intel_ver=2021.5.0 +export stack_impi_ver=2021.5.1 +export python_ver=3.10.8 -export gempak_ver=7.5.1 -export ncl_ver=6.6.2 -export jasper_ver=2.0.25 +export hpss_ver=hpss +export cmake_ver=3.23.1 + +export gempak_ver=7.4.2 +export jasper_ver=2.0.32 +export libpng_ver=1.6.37 export zlib_ver=1.2.11 -export libpng_ver=1.6.35 -export cdo_ver=1.9.5 +export esmf_ver=8.5.0 +export fms_ver=2023.02.01 +export cdo_ver=2.0.5 +export nco_ver=5.0.6 +export ncl_ver=6.6.2 -export hdf5_ver=1.10.6 -export netcdf_ver=4.7.4 +export hdf5_ver=1.14.0 +export netcdf_c_ver=4.9.2 +export netcdf_fortran_ver=4.6.0 -export nco_ver=4.8.1 -export prod_util_ver=1.2.2 -export grib_util_ver=1.2.2 -export g2tmpl_ver=1.10.0 -export ncdiag_ver=1.0.0 +export bacio_ver=2.4.1 +export nemsio_ver=2.5.4 +export sigio_ver=2.3.2 +export w3emc_ver=2.10.0 +export bufr_ver=11.7.0 +export g2_ver=3.4.5 +export sp_ver=2.3.3 +export ip_ver=4.3.0 +export gsi_ncdiag_ver=1.1.2 +export g2tmpl_ver=1.10.2 export crtm_ver=2.4.0 export wgrib2_ver=2.0.8 - -export obsproc_run_ver=1.1.2 -export prepobs_run_ver=1.0.1 - -export tracker_ver=feature-GFSv17_com_reorg +export py_netcdf4_ver=1.5.8 +export py_pyyaml_ver=5.4.1 +export py_jinja2_ver=3.1.2 export fit2obs_ver=1.0.0 +export met_ver=9.1.3 +export metplus_ver=3.1.1 +export tracker_ver=feature-GFSv17_com_reorg diff --git a/versions/run.s4.ver b/versions/run.s4.ver index 966c86fe01..8083d7e892 100644 --- a/versions/run.s4.ver +++ b/versions/run.s4.ver @@ -1,27 +1,42 @@ -export hpc_ver=1.1.0 -export hpc_intel_ver=18.0.4 -export hpc_impi_ver=18.0.4 +export spack_stack_ver=1.5.1 +export spack_env=gsi-addon +export stack_intel_ver=2021.5.0 +export stack_impi_ver=2021.5.1 +export python_ver=3.10.8 -export miniconda_ver=3.8-s4 -export ncl_ver=6.4.0-precompiled -export cdo_ver=1.9.8 -export jasper_ver=2.0.25 +export hpss_ver=hpss +export cmake_ver=3.23.1 + +export gempak_ver=7.4.2 +export jasper_ver=2.0.32 +export libpng_ver=1.6.37 export zlib_ver=1.2.11 -export libpng_ver=1.6.35 +export esmf_ver=8.5.0 +export fms_ver=2023.02.01 +export cdo_ver=2.0.5 +export nco_ver=5.0.6 +export ncl_ver=6.6.2 -export hdf5_ver=1.10.6 -export netcdf_ver=4.7.4 +export hdf5_ver=1.14.0 +export netcdf_c_ver=4.9.2 +export netcdf_fortran_ver=4.6.0 -export nco_ver=4.9.3 -export prod_util_ver=1.2.2 -export grib_util_ver=1.2.2 -export g2tmpl_ver=1.10.0 -export ncdiag_ver=1.0.0 +export bacio_ver=2.4.1 +export nemsio_ver=2.5.4 +export sigio_ver=2.3.2 +export w3emc_ver=2.10.0 +export bufr_ver=11.7.0 +export g2_ver=3.4.5 +export sp_ver=2.3.3 +export ip_ver=4.3.0 +export gsi_ncdiag_ver=1.1.2 +export g2tmpl_ver=1.10.2 export crtm_ver=2.4.0 export wgrib2_ver=2.0.8 - -export obsproc_run_ver=1.1.2 -export prepobs_run_ver=1.0.1 - -export tracker_ver=feature-GFSv17_com_reorg +export py_netcdf4_ver=1.5.8 +export py_pyyaml_ver=5.4.1 +export py_jinja2_ver=3.1.2 export fit2obs_ver=1.0.0 +export met_ver=9.1.3 +export metplus_ver=3.1.1 +export tracker_ver=feature-GFSv17_com_reorg From 94981b64ee4499320e11838fc3d4fc5bd495fc6c Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 16 Nov 2023 20:41:11 +0000 Subject: [PATCH 07/33] Update Orion and Jet modulefiles. #1868 --- modulefiles/module_base.jet.lua | 13 ++++++------- modulefiles/module_base.orion.lua | 21 +++++++++------------ 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/modulefiles/module_base.jet.lua b/modulefiles/module_base.jet.lua index c8cb9a9abf..a032cd6273 100644 --- a/modulefiles/module_base.jet.lua +++ b/modulefiles/module_base.jet.lua @@ -2,11 +2,13 @@ help([[ Load environment to run GFS on Jet ]]) -prepend_path("MODULEPATH", "/lfs4/HFIP/hfv3gfs/role.epic/hpc-stack/libs/intel-18.0.5.274/modulefiles/stack") +prepend_path("MODULEPATH", "/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") -load(pathJoin("hpc", os.getenv("hpc_ver"))) -load(pathJoin("hpc-intel", os.getenv("hpc_intel_ver"))) -load(pathJoin("hpc-impi", os.getenv("hpc_impi_ver"))) +local stack_python_ver=os.getenv("stack_python_ver") or "3.10.8" +local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" +local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1" +local cmake_ver=os.getenv("cmake_ver") or "3.23.1" +local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" load("hpss") load(pathJoin("gempak", os.getenv("gempak_ver"))) @@ -27,9 +29,6 @@ load(pathJoin("ncdiag", os.getenv("ncdiag_ver"))) load(pathJoin("crtm", os.getenv("crtm_ver"))) load(pathJoin("wgrib2", os.getenv("wgrib2_ver"))) -prepend_path("MODULEPATH", "/contrib/anaconda/modulefiles") -load(pathJoin("anaconda", os.getenv("anaconda_ver"))) - --prepend_path("MODULEPATH", pathJoin("/lfs4/HFIP/hfv3gfs/glopara/git/prepobs/v" .. os.getenv("prepobs_run_ver"), "modulefiles")) prepend_path("MODULEPATH", pathJoin("/lfs4/HFIP/hfv3gfs/glopara/git/prepobs/feature-GFSv17_com_reorg_log_update/modulefiles")) load(pathJoin("prepobs", os.getenv("prepobs_run_ver"))) diff --git a/modulefiles/module_base.orion.lua b/modulefiles/module_base.orion.lua index bb00594cce..5a88d65d69 100644 --- a/modulefiles/module_base.orion.lua +++ b/modulefiles/module_base.orion.lua @@ -2,11 +2,13 @@ help([[ Load environment to run GFS on Orion ]]) -prepend_path("MODULEPATH", "/apps/contrib/NCEP/hpc-stack/libs/hpc-stack/modulefiles/stack") +prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") -load(pathJoin("hpc", os.getenv("hpc_ver"))) -load(pathJoin("hpc-intel", os.getenv("hpc_intel_ver"))) -load(pathJoin("hpc-impi", os.getenv("hpc_impi_ver"))) +local stack_python_ver=os.getenv("stack_python_ver") or "3.10.8" +local stack_intel_ver=os.getenv("stack_intel_ver") or "2022.0.2" +local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1" +local cmake_ver=os.getenv("cmake_ver") or "3.23.1" +local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" load(pathJoin("gempak", os.getenv("gempak_ver"))) load(pathJoin("ncl", os.getenv("ncl_ver"))) @@ -34,13 +36,8 @@ load(pathJoin("prepobs", os.getenv("prepobs_run_ver"))) prepend_path("MODULEPATH", pathJoin("/work/noaa/global/glopara/git/Fit2Obs/v" .. os.getenv("fit2obs_ver"), "modulefiles")) load(pathJoin("fit2obs", os.getenv("fit2obs_ver"))) --- Temporary until official hpc-stack is updated -prepend_path("MODULEPATH", "/work2/noaa/global/wkolczyn/save/hpc-stack/modulefiles/stack") -load(pathJoin("hpc", "1.2.0")) -load(pathJoin("hpc-intel", "2018.4")) -load(pathJoin("hpc-miniconda3", "4.6.14")) -load(pathJoin("gfs_workflow", "1.0.0")) -load(pathJoin("met", "9.1")) -load(pathJoin("metplus", "3.1")) +-- Disabled verif-global until met/metplus are included in spack-stack +--load(pathJoin("met", "9.1")) +--load(pathJoin("metplus", "3.1")) whatis("Description: GFS run environment") From 8c0b50ee3390617b38213dfffb2e26c18f5f9cd2 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 16 Nov 2023 20:41:48 +0000 Subject: [PATCH 08/33] Fix AWIPS variable names. --- scripts/exgfs_atmos_awips_20km_1p0deg.sh | 2 +- scripts/exgfs_atmos_grib_awips.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/exgfs_atmos_awips_20km_1p0deg.sh b/scripts/exgfs_atmos_awips_20km_1p0deg.sh index caf7f9cc58..7546f3cabe 100755 --- a/scripts/exgfs_atmos_awips_20km_1p0deg.sh +++ b/scripts/exgfs_atmos_awips_20km_1p0deg.sh @@ -174,7 +174,7 @@ for GRID in conus ak prico pac 003; do export pgm; prep_step startmsg - if [[ ${GRID} = "003" && $(( fcsthrs % 6 )) == 0 ]]; then + if [[ ${GRID} = "003" && $(( 10#${fcsthrs} % 6 )) == 0 ]]; then export FORT11="awps_file_f${fcsthrs}_${GRID}" export FORT31="awps_file_fi${fcsthrs}_${GRID}" export FORT51="grib2.awpgfs${fcsthrs}.${GRID}" diff --git a/scripts/exgfs_atmos_grib_awips.sh b/scripts/exgfs_atmos_grib_awips.sh index efdd6beb81..037b4ce191 100755 --- a/scripts/exgfs_atmos_grib_awips.sh +++ b/scripts/exgfs_atmos_grib_awips.sh @@ -122,7 +122,7 @@ if [[ "${SENDDBN}" == 'YES' || "${SENDAWIP}" == 'YES' ]] ; then "${DBNROOT}/bin/dbn_alert" "${DBNALERT_TYPE}" "${NET}" "${job}" \ "${COM_ATMOS_WMO}/xtrn.awpgfs${fcsthrs}.${GRID}.${job_name}" else - echo "File ${output_grb}.${job_name} not posted to db_net." + echo "File xtrn.awpgfs${fcsthrs}.${GRID}.${job_name} not posted to db_net." fi if [[ -e "${pgmout}" ]] ; then From 58c917a14fca215d2fec9805fc948b676d7162b1 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Fri, 17 Nov 2023 14:26:09 +0000 Subject: [PATCH 09/33] Disable METplus jobs. #1868 --- parm/config/gfs/config.base.emc.dyn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/gfs/config.base.emc.dyn b/parm/config/gfs/config.base.emc.dyn index 6398daac35..ddbd2bffaf 100644 --- a/parm/config/gfs/config.base.emc.dyn +++ b/parm/config/gfs/config.base.emc.dyn @@ -386,7 +386,7 @@ export netcdf_diag=".true." export binary_diag=".false." # Verification options -export DO_METP="YES" # Run METPLUS jobs - set METPLUS settings in config.metp +export DO_METP="NO" # Run METPLUS jobs - set METPLUS settings in config.metp export DO_FIT2OBS="YES" # Run fit to observations package # Archiving options From 5b614d1e6a399fe1f4ee21b26ef78cd72bed7d35 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Fri, 17 Nov 2023 14:38:25 +0000 Subject: [PATCH 10/33] Implement single spack version build/run files, update modulefiles for spack #1868 --- modulefiles/module_base.orion.lua | 1 - modulefiles/module_base.s4.lua | 3 +- sorc/link_workflow.sh | 9 +++- versions/build.jet.ver | 43 ------------------ versions/build.orion.ver | 43 ------------------ versions/build.s4.ver | 43 ------------------ versions/{build.hera.ver => build.spack.ver} | 7 +-- versions/run.jet.ver | 46 -------------------- versions/run.orion.ver | 46 -------------------- versions/run.s4.ver | 46 -------------------- versions/{run.hera.ver => run.spack.ver} | 11 ----- 11 files changed, 9 insertions(+), 289 deletions(-) delete mode 100644 versions/build.jet.ver delete mode 100644 versions/build.orion.ver delete mode 100644 versions/build.s4.ver rename versions/{build.hera.ver => build.spack.ver} (85%) delete mode 100644 versions/run.jet.ver delete mode 100644 versions/run.orion.ver delete mode 100644 versions/run.s4.ver rename versions/{run.hera.ver => run.spack.ver} (75%) diff --git a/modulefiles/module_base.orion.lua b/modulefiles/module_base.orion.lua index 5a88d65d69..9a34ba7b38 100644 --- a/modulefiles/module_base.orion.lua +++ b/modulefiles/module_base.orion.lua @@ -13,7 +13,6 @@ local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" load(pathJoin("gempak", os.getenv("gempak_ver"))) load(pathJoin("ncl", os.getenv("ncl_ver"))) load(pathJoin("jasper", os.getenv("jasper_ver"))) -load(pathJoin("zlib", os.getenv("zlib_ver"))) load(pathJoin("png", os.getenv("libpng_ver"))) load(pathJoin("cdo", os.getenv("cdo_ver"))) diff --git a/modulefiles/module_base.s4.lua b/modulefiles/module_base.s4.lua index 21b574e763..ce42012417 100644 --- a/modulefiles/module_base.s4.lua +++ b/modulefiles/module_base.s4.lua @@ -13,8 +13,7 @@ load(pathJoin("miniconda", os.getenv("miniconda_ver"))) load(pathJoin("ncl", os.getenv("ncl_ver"))) load(pathJoin("cdo", os.getenv("cdo_ver"))) load(pathJoin("jasper", os.getenv("jasper_ver"))) -load(pathJoin("zlib", os.getenv("zlib_ver"))) -load(pathJoin("png", os.getenv("libpng_ver"))) +load(pathJoin("libpng", os.getenv("libpng_ver"))) load(pathJoin("hdf5", os.getenv("hdf5_ver"))) load(pathJoin("netcdf", os.getenv("netcdf_ver"))) diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index d2328caba9..f274c714ba 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -49,8 +49,13 @@ machine=$(echo "${MACHINE_ID}" | cut -d. -f1) #------------------------------ #--Set up build.ver and run.ver #------------------------------ -cp "${HOMEgfs}/versions/build.${machine}.ver" "${HOMEgfs}/versions/build.ver" -cp "${HOMEgfs}/versions/run.${machine}.ver" "${HOMEgfs}/versions/run.ver" +if [[ "${machine}" == "wcoss2" ]]; then + cp "${HOMEgfs}/versions/build.${machine}.ver" "${HOMEgfs}/versions/build.ver" + cp "${HOMEgfs}/versions/run.${machine}.ver" "${HOMEgfs}/versions/run.ver" +else + cp "${HOMEgfs}/versions/build.spack.ver" "${HOMEgfs}/versions/build.ver" + cp "${HOMEgfs}/versions/run.spack.ver" "${HOMEgfs}/versions/run.ver" +fi #------------------------------ #--model fix fields diff --git a/versions/build.jet.ver b/versions/build.jet.ver deleted file mode 100644 index 64a4e57dd4..0000000000 --- a/versions/build.jet.ver +++ /dev/null @@ -1,43 +0,0 @@ -export spack_stack_ver=1.5.1 -export spack_env=gsi-addon -export stack_intel_ver=2021.5.0 -export stack_impi_ver=2021.5.1 -export python_ver=3.10.8 - -export hpss_ver=hpss -export cmake_ver=3.23.1 - -export gempak_ver=7.4.2 -export jasper_ver=2.0.32 -export libpng_ver=1.6.37 -export zlib_ver=1.2.11 -export esmf_ver=8.5.0 -export fms_ver=2023.02.01 -export cdo_ver=2.0.5 -export nco_ver=5.0.6 -export ncl_ver=6.6.2 -export prod_util_ver=1.2.2 - -export hdf5_ver=1.14.0 -export netcdf_c_ver=4.9.2 -export netcdf_fortran_ver=4.6.0 - -export bacio_ver=2.4.1 -export nemsio_ver=2.5.4 -export sigio_ver=2.3.2 -export w3emc_ver=2.10.0 -export bufr_ver=11.7.0 -export g2_ver=3.4.5 -export sp_ver=2.3.3 -export ip_ver=4.3.0 -export gsi_ncdiag_ver=1.1.2 -export g2tmpl_ver=1.10.2 -export crtm_ver=2.4.0 -export wgrib2_ver=2.0.8 -export grib_util_ver=1.3.0 -export py_netcdf4_ver=1.5.8 -export py_pyyaml_ver=5.4.1 -export py_jinja2_ver=3.1.2 -export fit2obs_ver=1.0.0 -export met_ver=9.1.3 -export metplus_ver=3.1.1 diff --git a/versions/build.orion.ver b/versions/build.orion.ver deleted file mode 100644 index 64a4e57dd4..0000000000 --- a/versions/build.orion.ver +++ /dev/null @@ -1,43 +0,0 @@ -export spack_stack_ver=1.5.1 -export spack_env=gsi-addon -export stack_intel_ver=2021.5.0 -export stack_impi_ver=2021.5.1 -export python_ver=3.10.8 - -export hpss_ver=hpss -export cmake_ver=3.23.1 - -export gempak_ver=7.4.2 -export jasper_ver=2.0.32 -export libpng_ver=1.6.37 -export zlib_ver=1.2.11 -export esmf_ver=8.5.0 -export fms_ver=2023.02.01 -export cdo_ver=2.0.5 -export nco_ver=5.0.6 -export ncl_ver=6.6.2 -export prod_util_ver=1.2.2 - -export hdf5_ver=1.14.0 -export netcdf_c_ver=4.9.2 -export netcdf_fortran_ver=4.6.0 - -export bacio_ver=2.4.1 -export nemsio_ver=2.5.4 -export sigio_ver=2.3.2 -export w3emc_ver=2.10.0 -export bufr_ver=11.7.0 -export g2_ver=3.4.5 -export sp_ver=2.3.3 -export ip_ver=4.3.0 -export gsi_ncdiag_ver=1.1.2 -export g2tmpl_ver=1.10.2 -export crtm_ver=2.4.0 -export wgrib2_ver=2.0.8 -export grib_util_ver=1.3.0 -export py_netcdf4_ver=1.5.8 -export py_pyyaml_ver=5.4.1 -export py_jinja2_ver=3.1.2 -export fit2obs_ver=1.0.0 -export met_ver=9.1.3 -export metplus_ver=3.1.1 diff --git a/versions/build.s4.ver b/versions/build.s4.ver deleted file mode 100644 index 64a4e57dd4..0000000000 --- a/versions/build.s4.ver +++ /dev/null @@ -1,43 +0,0 @@ -export spack_stack_ver=1.5.1 -export spack_env=gsi-addon -export stack_intel_ver=2021.5.0 -export stack_impi_ver=2021.5.1 -export python_ver=3.10.8 - -export hpss_ver=hpss -export cmake_ver=3.23.1 - -export gempak_ver=7.4.2 -export jasper_ver=2.0.32 -export libpng_ver=1.6.37 -export zlib_ver=1.2.11 -export esmf_ver=8.5.0 -export fms_ver=2023.02.01 -export cdo_ver=2.0.5 -export nco_ver=5.0.6 -export ncl_ver=6.6.2 -export prod_util_ver=1.2.2 - -export hdf5_ver=1.14.0 -export netcdf_c_ver=4.9.2 -export netcdf_fortran_ver=4.6.0 - -export bacio_ver=2.4.1 -export nemsio_ver=2.5.4 -export sigio_ver=2.3.2 -export w3emc_ver=2.10.0 -export bufr_ver=11.7.0 -export g2_ver=3.4.5 -export sp_ver=2.3.3 -export ip_ver=4.3.0 -export gsi_ncdiag_ver=1.1.2 -export g2tmpl_ver=1.10.2 -export crtm_ver=2.4.0 -export wgrib2_ver=2.0.8 -export grib_util_ver=1.3.0 -export py_netcdf4_ver=1.5.8 -export py_pyyaml_ver=5.4.1 -export py_jinja2_ver=3.1.2 -export fit2obs_ver=1.0.0 -export met_ver=9.1.3 -export metplus_ver=3.1.1 diff --git a/versions/build.hera.ver b/versions/build.spack.ver similarity index 85% rename from versions/build.hera.ver rename to versions/build.spack.ver index 64a4e57dd4..679da91116 100644 --- a/versions/build.hera.ver +++ b/versions/build.spack.ver @@ -4,19 +4,17 @@ export stack_intel_ver=2021.5.0 export stack_impi_ver=2021.5.1 export python_ver=3.10.8 -export hpss_ver=hpss export cmake_ver=3.23.1 export gempak_ver=7.4.2 export jasper_ver=2.0.32 export libpng_ver=1.6.37 -export zlib_ver=1.2.11 +export zlib_ver=1.2.13 export esmf_ver=8.5.0 export fms_ver=2023.02.01 export cdo_ver=2.0.5 export nco_ver=5.0.6 export ncl_ver=6.6.2 -export prod_util_ver=1.2.2 export hdf5_ver=1.14.0 export netcdf_c_ver=4.9.2 @@ -38,6 +36,3 @@ export grib_util_ver=1.3.0 export py_netcdf4_ver=1.5.8 export py_pyyaml_ver=5.4.1 export py_jinja2_ver=3.1.2 -export fit2obs_ver=1.0.0 -export met_ver=9.1.3 -export metplus_ver=3.1.1 diff --git a/versions/run.jet.ver b/versions/run.jet.ver deleted file mode 100644 index dcb8badb3e..0000000000 --- a/versions/run.jet.ver +++ /dev/null @@ -1,46 +0,0 @@ -export spack_stack_ver=1.5.1 -export spack_env=gsi-addon -export stack_intel_ver=2021.5.0 -export stack_impi_ver=2021.5.1 -export python_ver=3.10.8 - -export hpss_ver=hpss -export cmake_ver=3.23.1 - -export gempak_ver=7.4.2 -export jasper_ver=2.0.32 -export libpng_ver=1.6.37 -export zlib_ver=1.2.11 -export esmf_ver=8.5.0 -export fms_ver=2023.02.01 -export cdo_ver=2.0.5 -export nco_ver=5.0.6 -export ncl_ver=6.6.2 - -export hdf5_ver=1.14.0 -export netcdf_c_ver=4.9.2 -export netcdf_fortran_ver=4.6.0 - -export bacio_ver=2.4.1 -export nemsio_ver=2.5.4 -export sigio_ver=2.3.2 -export w3emc_ver=2.10.0 -export bufr_ver=11.7.0 -export g2_ver=3.4.5 -export sp_ver=2.3.3 -export ip_ver=4.3.0 -export gsi_ncdiag_ver=1.1.2 -export g2tmpl_ver=1.10.2 -export crtm_ver=2.4.0 -export wgrib2_ver=2.0.8 -export py_netcdf4_ver=1.5.8 -export py_pyyaml_ver=5.4.1 -export py_jinja2_ver=3.1.2 - -export obsproc_run_ver=1.1.2 -export prepobs_run_ver=1.0.1 - -export ens_tracker_ver=feature-GFSv17_com_reorg -export fit2obs_ver=1.0.0 -export met_ver=9.1.3 -export metplus_ver=3.1.1 diff --git a/versions/run.orion.ver b/versions/run.orion.ver deleted file mode 100644 index dcb8badb3e..0000000000 --- a/versions/run.orion.ver +++ /dev/null @@ -1,46 +0,0 @@ -export spack_stack_ver=1.5.1 -export spack_env=gsi-addon -export stack_intel_ver=2021.5.0 -export stack_impi_ver=2021.5.1 -export python_ver=3.10.8 - -export hpss_ver=hpss -export cmake_ver=3.23.1 - -export gempak_ver=7.4.2 -export jasper_ver=2.0.32 -export libpng_ver=1.6.37 -export zlib_ver=1.2.11 -export esmf_ver=8.5.0 -export fms_ver=2023.02.01 -export cdo_ver=2.0.5 -export nco_ver=5.0.6 -export ncl_ver=6.6.2 - -export hdf5_ver=1.14.0 -export netcdf_c_ver=4.9.2 -export netcdf_fortran_ver=4.6.0 - -export bacio_ver=2.4.1 -export nemsio_ver=2.5.4 -export sigio_ver=2.3.2 -export w3emc_ver=2.10.0 -export bufr_ver=11.7.0 -export g2_ver=3.4.5 -export sp_ver=2.3.3 -export ip_ver=4.3.0 -export gsi_ncdiag_ver=1.1.2 -export g2tmpl_ver=1.10.2 -export crtm_ver=2.4.0 -export wgrib2_ver=2.0.8 -export py_netcdf4_ver=1.5.8 -export py_pyyaml_ver=5.4.1 -export py_jinja2_ver=3.1.2 - -export obsproc_run_ver=1.1.2 -export prepobs_run_ver=1.0.1 - -export ens_tracker_ver=feature-GFSv17_com_reorg -export fit2obs_ver=1.0.0 -export met_ver=9.1.3 -export metplus_ver=3.1.1 diff --git a/versions/run.s4.ver b/versions/run.s4.ver deleted file mode 100644 index dcb8badb3e..0000000000 --- a/versions/run.s4.ver +++ /dev/null @@ -1,46 +0,0 @@ -export spack_stack_ver=1.5.1 -export spack_env=gsi-addon -export stack_intel_ver=2021.5.0 -export stack_impi_ver=2021.5.1 -export python_ver=3.10.8 - -export hpss_ver=hpss -export cmake_ver=3.23.1 - -export gempak_ver=7.4.2 -export jasper_ver=2.0.32 -export libpng_ver=1.6.37 -export zlib_ver=1.2.11 -export esmf_ver=8.5.0 -export fms_ver=2023.02.01 -export cdo_ver=2.0.5 -export nco_ver=5.0.6 -export ncl_ver=6.6.2 - -export hdf5_ver=1.14.0 -export netcdf_c_ver=4.9.2 -export netcdf_fortran_ver=4.6.0 - -export bacio_ver=2.4.1 -export nemsio_ver=2.5.4 -export sigio_ver=2.3.2 -export w3emc_ver=2.10.0 -export bufr_ver=11.7.0 -export g2_ver=3.4.5 -export sp_ver=2.3.3 -export ip_ver=4.3.0 -export gsi_ncdiag_ver=1.1.2 -export g2tmpl_ver=1.10.2 -export crtm_ver=2.4.0 -export wgrib2_ver=2.0.8 -export py_netcdf4_ver=1.5.8 -export py_pyyaml_ver=5.4.1 -export py_jinja2_ver=3.1.2 - -export obsproc_run_ver=1.1.2 -export prepobs_run_ver=1.0.1 - -export ens_tracker_ver=feature-GFSv17_com_reorg -export fit2obs_ver=1.0.0 -export met_ver=9.1.3 -export metplus_ver=3.1.1 diff --git a/versions/run.hera.ver b/versions/run.spack.ver similarity index 75% rename from versions/run.hera.ver rename to versions/run.spack.ver index dcb8badb3e..d5e64f8be5 100644 --- a/versions/run.hera.ver +++ b/versions/run.spack.ver @@ -5,14 +5,10 @@ export stack_impi_ver=2021.5.1 export python_ver=3.10.8 export hpss_ver=hpss -export cmake_ver=3.23.1 export gempak_ver=7.4.2 export jasper_ver=2.0.32 export libpng_ver=1.6.37 -export zlib_ver=1.2.11 -export esmf_ver=8.5.0 -export fms_ver=2023.02.01 export cdo_ver=2.0.5 export nco_ver=5.0.6 export ncl_ver=6.6.2 @@ -21,14 +17,7 @@ export hdf5_ver=1.14.0 export netcdf_c_ver=4.9.2 export netcdf_fortran_ver=4.6.0 -export bacio_ver=2.4.1 -export nemsio_ver=2.5.4 -export sigio_ver=2.3.2 -export w3emc_ver=2.10.0 export bufr_ver=11.7.0 -export g2_ver=3.4.5 -export sp_ver=2.3.3 -export ip_ver=4.3.0 export gsi_ncdiag_ver=1.1.2 export g2tmpl_ver=1.10.2 export crtm_ver=2.4.0 From 14931d72ff45dd4b07b9a3431c53e31adcc7c480 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Mon, 20 Nov 2023 19:23:25 +0000 Subject: [PATCH 11/33] Update modulefiles, reinstate (pared down) machine version files. --- modulefiles/module_base.jet.lua | 28 ++++++++++++++++++---------- modulefiles/module_base.orion.lua | 4 ---- modulefiles/module_base.s4.lua | 23 +++++++++++++++-------- modulefiles/module_gwsetup.jet.lua | 22 +++++++++------------- modulefiles/module_gwsetup.s4.lua | 12 +++++++++++- sorc/link_workflow.sh | 2 ++ versions/build.hera.ver | 2 ++ versions/build.jet.ver | 2 ++ versions/build.orion.ver | 2 ++ versions/build.s4.ver | 2 ++ versions/run.hera.ver | 11 +++++++++++ versions/run.jet.ver | 7 +++++++ versions/run.orion.ver | 5 +++++ versions/run.s4.ver | 4 ++++ versions/run.spack.ver | 8 -------- 15 files changed, 90 insertions(+), 44 deletions(-) create mode 100644 versions/build.hera.ver create mode 100644 versions/build.jet.ver create mode 100644 versions/build.orion.ver create mode 100644 versions/build.s4.ver create mode 100644 versions/run.hera.ver create mode 100644 versions/run.jet.ver create mode 100644 versions/run.orion.ver create mode 100644 versions/run.s4.ver diff --git a/modulefiles/module_base.jet.lua b/modulefiles/module_base.jet.lua index a032cd6273..2e13191797 100644 --- a/modulefiles/module_base.jet.lua +++ b/modulefiles/module_base.jet.lua @@ -2,15 +2,15 @@ help([[ Load environment to run GFS on Jet ]]) -prepend_path("MODULEPATH", "/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") +spack_stack_ver=(os.getenv("spack_stack_ver") or "None") +spack_env=(os.getenv("spack_env") or "None") +prepend_path("MODULEPATH", "/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-" .. spack_stack_ver .. "/envs/" .. spack_env .. "/install/modulefiles/Core") -local stack_python_ver=os.getenv("stack_python_ver") or "3.10.8" -local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" -local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1" -local cmake_ver=os.getenv("cmake_ver") or "3.23.1" -local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" +load(pathJoin("stack-intel", os.getenv("stack_intel_ver"))) +load(pathJoin("stack-intel-oneapi-mpi", os.getenv("stack_impi_ver"))) +load(pathJoin("python", os.getenv("python_ver"))) -load("hpss") +load(pathJoin("hpss", os.getenv("hpss_ver"))) load(pathJoin("gempak", os.getenv("gempak_ver"))) load(pathJoin("ncl", os.getenv("ncl_ver"))) load(pathJoin("jasper", os.getenv("jasper_ver"))) @@ -19,15 +19,23 @@ load(pathJoin("cdo", os.getenv("cdo_ver"))) load(pathJoin("R", os.getenv("R_ver"))) load(pathJoin("hdf5", os.getenv("hdf5_ver"))) -load(pathJoin("netcdf", os.getenv("netcdf_ver"))) +load(pathJoin("netcdf-c", os.getenv("netcdf_c_ver"))) +load(pathJoin("netcdf-fortran", os.getenv("netcdf_fortran_ver"))) load(pathJoin("nco", os.getenv("nco_ver"))) load(pathJoin("prod_util", os.getenv("prod_util_ver"))) -load(pathJoin("grib_util", os.getenv("grib_util_ver"))) +load(pathJoin("grib-util", os.getenv("grib_util_ver"))) load(pathJoin("g2tmpl", os.getenv("g2tmpl_ver"))) -load(pathJoin("ncdiag", os.getenv("ncdiag_ver"))) +load(pathJoin("gsi-ncdiag", os.getenv("gsi_ncdiag_ver"))) load(pathJoin("crtm", os.getenv("crtm_ver"))) +load(pathJoin("bufr", os.getenv("bufr_ver"))) load(pathJoin("wgrib2", os.getenv("wgrib2_ver"))) +load(pathJoin("py-netcdf4", os.getenv("py_netcdf4_ver"))) +load(pathJoin("py-pyyaml", os.getenv("py_pyyaml_ver"))) +load(pathJoin("py-jinja2", os.getenv("py_jinja2_ver"))) + +setenv("WGRIB2","wgrib2") +setenv("UTILROOT",os.getenv("prod_util_ROOT")) --prepend_path("MODULEPATH", pathJoin("/lfs4/HFIP/hfv3gfs/glopara/git/prepobs/v" .. os.getenv("prepobs_run_ver"), "modulefiles")) prepend_path("MODULEPATH", pathJoin("/lfs4/HFIP/hfv3gfs/glopara/git/prepobs/feature-GFSv17_com_reorg_log_update/modulefiles")) diff --git a/modulefiles/module_base.orion.lua b/modulefiles/module_base.orion.lua index 9a34ba7b38..1c683157b5 100644 --- a/modulefiles/module_base.orion.lua +++ b/modulefiles/module_base.orion.lua @@ -35,8 +35,4 @@ load(pathJoin("prepobs", os.getenv("prepobs_run_ver"))) prepend_path("MODULEPATH", pathJoin("/work/noaa/global/glopara/git/Fit2Obs/v" .. os.getenv("fit2obs_ver"), "modulefiles")) load(pathJoin("fit2obs", os.getenv("fit2obs_ver"))) --- Disabled verif-global until met/metplus are included in spack-stack ---load(pathJoin("met", "9.1")) ---load(pathJoin("metplus", "3.1")) - whatis("Description: GFS run environment") diff --git a/modulefiles/module_base.s4.lua b/modulefiles/module_base.s4.lua index ce42012417..23e61ef9a7 100644 --- a/modulefiles/module_base.s4.lua +++ b/modulefiles/module_base.s4.lua @@ -2,30 +2,37 @@ help([[ Load environment to run GFS on S4 ]]) -load("license_intel") -prepend_path("MODULEPATH", "/data/prod/hpc-stack/modulefiles/stack") +spack_stack_ver=(os.getenv("spack_stack_ver") or "None") +spack_env=(os.getenv("spack_env") or "None") +prepend_path("MODULEPATH", "/data/prod/jedi/spack-stack/spack-stack-" .. spack_stack_ver .. "/envs/" .. spack_env .. "/install/modulefiles/Core") -load(pathJoin("hpc", os.getenv("hpc_ver"))) -load(pathJoin("hpc-intel", os.getenv("hpc_intel_ver"))) -load(pathJoin("hpc-impi", os.getenv("hpc_impi_ver"))) +load(pathJoin("stack-intel", os.getenv("stack_intel_ver"))) +load(pathJoin("stack-intel-oneapi-mpi", os.getenv("stack_impi_ver"))) +load(pathJoin("python", os.getenv("python_ver"))) -load(pathJoin("miniconda", os.getenv("miniconda_ver"))) load(pathJoin("ncl", os.getenv("ncl_ver"))) load(pathJoin("cdo", os.getenv("cdo_ver"))) load(pathJoin("jasper", os.getenv("jasper_ver"))) load(pathJoin("libpng", os.getenv("libpng_ver"))) load(pathJoin("hdf5", os.getenv("hdf5_ver"))) -load(pathJoin("netcdf", os.getenv("netcdf_ver"))) +load(pathJoin("netcdf-c", os.getenv("netcdf_c_ver"))) +load(pathJoin("netcdf-fortran", os.getenv("netcdf_fortran_ver"))) load(pathJoin("nco", os.getenv("nco_ver"))) load(pathJoin("prod_util", os.getenv("prod_util_ver"))) load(pathJoin("grib_util", os.getenv("grib_util_ver"))) load(pathJoin("g2tmpl", os.getenv("g2tmpl_ver"))) -load(pathJoin("ncdiag", os.getenv("ncdiag_ver"))) +load(pathJoin("gsi-ncdiag", os.getenv("gsi_ncdiag_ver"))) load(pathJoin("crtm", os.getenv("crtm_ver"))) load(pathJoin("wgrib2", os.getenv("wgrib2_ver"))) +load(pathJoin("bufr", os.getenv("bufr_ver"))) +load(pathJoin("py-netcdf4", os.getenv("py_netcdf4_ver"))) +load(pathJoin("py-pyyaml", os.getenv("py_pyyaml_ver"))) +load(pathJoin("py-jinja2", os.getenv("py_jinja2_ver"))) + setenv("WGRIB2","wgrib2") +setenv("UTILROOT",os.getenv("prod_util_ROOT")) --prepend_path("MODULEPATH", pathJoin("/data/prod/glopara/git/prepobs/v" .. os.getenv("prepobs_run_ver"), "modulefiles")) prepend_path("MODULEPATH", pathJoin("/data/prod/glopara/git/prepobs/feature-GFSv17_com_reorg/modulefiles")) diff --git a/modulefiles/module_gwsetup.jet.lua b/modulefiles/module_gwsetup.jet.lua index 10fc39ce8a..d08389c711 100644 --- a/modulefiles/module_gwsetup.jet.lua +++ b/modulefiles/module_gwsetup.jet.lua @@ -4,19 +4,15 @@ Load environment to run GFS workflow setup scripts on Jet load(pathJoin("rocoto", "1.3.3")) -if (mode() == "unload") then - -- `execute` delays commands until last, but we need conda deactivated - -- before unloading miniconda. `print` (bizarrely) still executes the - -- command, but does it immediately. The semicolon is necessary - -- because otherwise other commands get tacked onto the same line. - print("conda deactivate;") -end +prepend_path("MODULEPATH", "/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") --- Temporary until official hpc-stack is updated -prepend_path("MODULEPATH", "/mnt/lfs4/HFIP/hfv3gfs/role.epic/miniconda3/modulefiles") -load(pathJoin("miniconda3", "4.12.0")) -if (mode() == "load") then - execute{cmd="conda activate ufswm", modeA={"load"}} -end +local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" +local python_ver=os.getenv("python_ver") or "3.10.8" + +load(pathJoin("stack-intel", stack_intel_ver)) +load(pathJoin("python", python_ver)) +load("py-jinja2") +load("py-pyyaml") +load("py-numpy") whatis("Description: GFS run setup environment") diff --git a/modulefiles/module_gwsetup.s4.lua b/modulefiles/module_gwsetup.s4.lua index 8324a148d8..291c654bb3 100644 --- a/modulefiles/module_gwsetup.s4.lua +++ b/modulefiles/module_gwsetup.s4.lua @@ -2,8 +2,18 @@ help([[ Load environment to run GFS workflow setup scripts on S4 ]]) -load(pathJoin("miniconda", "3.8-s4")) load(pathJoin("rocoto","1.3.5")) load(pathJoin("git","2.30.0")) +prepend_path("MODULEPATH", "/data/prod/jedi/spack-stack/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") + +local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" +local python_ver=os.getenv("python_ver") or "3.10.8" + +load(pathJoin("stack-intel", stack_intel_ver)) +load(pathJoin("python", python_ver)) +load("py-jinja2") +load("py-pyyaml") +load("py-numpy") + whatis("Description: GFS run setup environment") diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index f274c714ba..5b039030ca 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -55,6 +55,8 @@ if [[ "${machine}" == "wcoss2" ]]; then else cp "${HOMEgfs}/versions/build.spack.ver" "${HOMEgfs}/versions/build.ver" cp "${HOMEgfs}/versions/run.spack.ver" "${HOMEgfs}/versions/run.ver" + cat "${HOMEgfs}/versions/build.${machine}.ver" >> "${HOMEgfs}/versions/build.ver" + cat "${HOMEgfs}/versions/run.${machine}.ver" >> "${HOMEgfs}/versions/run.ver" fi #------------------------------ diff --git a/versions/build.hera.ver b/versions/build.hera.ver new file mode 100644 index 0000000000..cba2bb5a76 --- /dev/null +++ b/versions/build.hera.ver @@ -0,0 +1,2 @@ +export stack_intel_ver=2021.5.0 +export stack_impi_ver=2021.5.1 diff --git a/versions/build.jet.ver b/versions/build.jet.ver new file mode 100644 index 0000000000..cba2bb5a76 --- /dev/null +++ b/versions/build.jet.ver @@ -0,0 +1,2 @@ +export stack_intel_ver=2021.5.0 +export stack_impi_ver=2021.5.1 diff --git a/versions/build.orion.ver b/versions/build.orion.ver new file mode 100644 index 0000000000..cba2bb5a76 --- /dev/null +++ b/versions/build.orion.ver @@ -0,0 +1,2 @@ +export stack_intel_ver=2021.5.0 +export stack_impi_ver=2021.5.1 diff --git a/versions/build.s4.ver b/versions/build.s4.ver new file mode 100644 index 0000000000..cba2bb5a76 --- /dev/null +++ b/versions/build.s4.ver @@ -0,0 +1,2 @@ +export stack_intel_ver=2021.5.0 +export stack_impi_ver=2021.5.1 diff --git a/versions/run.hera.ver b/versions/run.hera.ver new file mode 100644 index 0000000000..d82e9fba62 --- /dev/null +++ b/versions/run.hera.ver @@ -0,0 +1,11 @@ +export stack_intel_ver=2021.5.0 +export stack_impi_ver=2021.5.1 + +export hpss_ver=hpss +export ncl_ver=6.6.2 +export R_ver=3.5.0 +export gempak_ver=7.4.2 + +#For metplus jobs, not currently working with spack-stack +#export met_ver=9.1.3 +#export metplus_ver=3.1.1 diff --git a/versions/run.jet.ver b/versions/run.jet.ver new file mode 100644 index 0000000000..ab350da770 --- /dev/null +++ b/versions/run.jet.ver @@ -0,0 +1,7 @@ +export stack_intel_ver=2021.5.0 +export stack_impi_ver=2021.5.1 + +export hpss_ver= +export ncl_ver=6.6.2 +export R_ver=4.0.2 +export gempak_ver=7.4.2 diff --git a/versions/run.orion.ver b/versions/run.orion.ver new file mode 100644 index 0000000000..f016816a15 --- /dev/null +++ b/versions/run.orion.ver @@ -0,0 +1,5 @@ +export stack_intel_ver=2022.0.2 +export stack_impi_ver=2021.5.1 + +export ncl_ver=6.6.2 +export gempak_ver=7.4.1 diff --git a/versions/run.s4.ver b/versions/run.s4.ver new file mode 100644 index 0000000000..1125787b09 --- /dev/null +++ b/versions/run.s4.ver @@ -0,0 +1,4 @@ +export stack_intel_ver=2021.5.0 +export stack_impi_ver=2021.5.1 + +export ncl_ver=6.6.2 diff --git a/versions/run.spack.ver b/versions/run.spack.ver index d5e64f8be5..7045f2ed01 100644 --- a/versions/run.spack.ver +++ b/versions/run.spack.ver @@ -1,17 +1,11 @@ export spack_stack_ver=1.5.1 export spack_env=gsi-addon -export stack_intel_ver=2021.5.0 -export stack_impi_ver=2021.5.1 export python_ver=3.10.8 -export hpss_ver=hpss - -export gempak_ver=7.4.2 export jasper_ver=2.0.32 export libpng_ver=1.6.37 export cdo_ver=2.0.5 export nco_ver=5.0.6 -export ncl_ver=6.6.2 export hdf5_ver=1.14.0 export netcdf_c_ver=4.9.2 @@ -31,5 +25,3 @@ export prepobs_run_ver=1.0.1 export ens_tracker_ver=feature-GFSv17_com_reorg export fit2obs_ver=1.0.0 -export met_ver=9.1.3 -export metplus_ver=3.1.1 From 123c8ffc4fd67cbfc10c6c8da807d91b52526313 Mon Sep 17 00:00:00 2001 From: David Huber Date: Mon, 20 Nov 2023 13:46:53 -0600 Subject: [PATCH 12/33] Fix Orion modules. #1868 --- modulefiles/module_base.orion.lua | 26 ++++++++++++++++---------- versions/run.orion.ver | 2 +- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/modulefiles/module_base.orion.lua b/modulefiles/module_base.orion.lua index 1c683157b5..71cab99f74 100644 --- a/modulefiles/module_base.orion.lua +++ b/modulefiles/module_base.orion.lua @@ -2,30 +2,36 @@ help([[ Load environment to run GFS on Orion ]]) -prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") +spack_stack_ver=(os.getenv("spack_stack_ver") or "None") +spack_env=(os.getenv("spack_env") or "None") +prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-" .. spack_stack_ver .. "/envs/" .. spack_env .. "/install/modulefiles/Core") -local stack_python_ver=os.getenv("stack_python_ver") or "3.10.8" -local stack_intel_ver=os.getenv("stack_intel_ver") or "2022.0.2" -local stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1" -local cmake_ver=os.getenv("cmake_ver") or "3.23.1" -local prod_util_ver=os.getenv("prod_util_ver") or "1.2.2" +load(pathJoin("stack-intel", os.getenv("stack_intel_ver"))) +load(pathJoin("stack-intel-oneapi-mpi", os.getenv("stack_impi_ver"))) +load(pathJoin("python", os.getenv("python_ver"))) load(pathJoin("gempak", os.getenv("gempak_ver"))) load(pathJoin("ncl", os.getenv("ncl_ver"))) load(pathJoin("jasper", os.getenv("jasper_ver"))) -load(pathJoin("png", os.getenv("libpng_ver"))) +load(pathJoin("libpng", os.getenv("libpng_ver"))) load(pathJoin("cdo", os.getenv("cdo_ver"))) load(pathJoin("hdf5", os.getenv("hdf5_ver"))) -load(pathJoin("netcdf", os.getenv("netcdf_ver"))) +load(pathJoin("netcdf-c", os.getenv("netcdf_c_ver"))) +load(pathJoin("netcdf-fortran", os.getenv("netcdf_fortran_ver"))) load(pathJoin("nco", os.getenv("nco_ver"))) load(pathJoin("prod_util", os.getenv("prod_util_ver"))) -load(pathJoin("grib_util", os.getenv("grib_util_ver"))) +load(pathJoin("grib-util", os.getenv("grib_util_ver"))) load(pathJoin("g2tmpl", os.getenv("g2tmpl_ver"))) -load(pathJoin("ncdiag", os.getenv("ncdiag_ver"))) +load(pathJoin("gsi-ncdiag", os.getenv("gsi_ncdiag_ver"))) load(pathJoin("crtm", os.getenv("crtm_ver"))) +load(pathJoin("bufr", os.getenv("bufr_ver"))) load(pathJoin("wgrib2", os.getenv("wgrib2_ver"))) +load(pathJoin("py-netcdf4", os.getenv("py_netcdf4_ver"))) +load(pathJoin("py-pyyaml", os.getenv("py_pyyaml_ver"))) +load(pathJoin("py-jinja2", os.getenv("py_jinja2_ver"))) + setenv("WGRIB2","wgrib2") --prepend_path("MODULEPATH", pathJoin("/work/noaa/global/glopara/git/prepobs/v" .. os.getenv("prepobs_run_ver"), "modulefiles")) diff --git a/versions/run.orion.ver b/versions/run.orion.ver index f016816a15..ca3258ba8e 100644 --- a/versions/run.orion.ver +++ b/versions/run.orion.ver @@ -2,4 +2,4 @@ export stack_intel_ver=2022.0.2 export stack_impi_ver=2021.5.1 export ncl_ver=6.6.2 -export gempak_ver=7.4.1 +export gempak_ver=7.5.1 From 856f7cc050095310529c1c0deb502c0a6abbc5f3 Mon Sep 17 00:00:00 2001 From: David Huber Date: Mon, 20 Nov 2023 21:16:31 +0000 Subject: [PATCH 13/33] Fix S4 modules. #1868 --- modulefiles/module_base.hera.lua | 4 ++-- modulefiles/module_base.jet.lua | 6 +++--- modulefiles/module_base.orion.lua | 5 +++-- modulefiles/module_base.s4.lua | 14 +++++++------- versions/build.s4.ver | 2 +- versions/run.s4.ver | 4 ++-- 6 files changed, 18 insertions(+), 17 deletions(-) diff --git a/modulefiles/module_base.hera.lua b/modulefiles/module_base.hera.lua index 08c546c31c..cf77df520a 100644 --- a/modulefiles/module_base.hera.lua +++ b/modulefiles/module_base.hera.lua @@ -38,9 +38,9 @@ load(pathJoin("met", os.getenv("met_ver"))) load(pathJoin("metplus", os.getenv("metplus_ver"))) setenv("WGRIB2","wgrib2") -setenv("UTILROOT",os.getenv("prod_util_ROOT")) +setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None")) ---prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/prepobs/v" .. os.getenv("prepobs_run_ver"), "modulefiles")) +--prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/prepobs/feature-GFSv17_com_reorg_log_update/modulefiles")) load(pathJoin("prepobs", os.getenv("prepobs_run_ver"))) diff --git a/modulefiles/module_base.jet.lua b/modulefiles/module_base.jet.lua index 2e13191797..7a0faaaeb6 100644 --- a/modulefiles/module_base.jet.lua +++ b/modulefiles/module_base.jet.lua @@ -35,13 +35,13 @@ load(pathJoin("py-pyyaml", os.getenv("py_pyyaml_ver"))) load(pathJoin("py-jinja2", os.getenv("py_jinja2_ver"))) setenv("WGRIB2","wgrib2") -setenv("UTILROOT",os.getenv("prod_util_ROOT")) +setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None")) ---prepend_path("MODULEPATH", pathJoin("/lfs4/HFIP/hfv3gfs/glopara/git/prepobs/v" .. os.getenv("prepobs_run_ver"), "modulefiles")) +--prepend_path("MODULEPATH", pathJoin("/lfs4/HFIP/hfv3gfs/glopara/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) prepend_path("MODULEPATH", pathJoin("/lfs4/HFIP/hfv3gfs/glopara/git/prepobs/feature-GFSv17_com_reorg_log_update/modulefiles")) load(pathJoin("prepobs", os.getenv("prepobs_run_ver"))) -prepend_path("MODULEPATH", pathJoin("/lfs4/HFIP/hfv3gfs/glopara/git/Fit2Obs/v" .. os.getenv("fit2obs_ver"), "modulefiles")) +prepend_path("MODULEPATH", pathJoin("/lfs4/HFIP/hfv3gfs/glopara/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) load(pathJoin("fit2obs", os.getenv("fit2obs_ver"))) whatis("Description: GFS run environment") diff --git a/modulefiles/module_base.orion.lua b/modulefiles/module_base.orion.lua index 71cab99f74..5ae43d5d34 100644 --- a/modulefiles/module_base.orion.lua +++ b/modulefiles/module_base.orion.lua @@ -33,12 +33,13 @@ load(pathJoin("py-pyyaml", os.getenv("py_pyyaml_ver"))) load(pathJoin("py-jinja2", os.getenv("py_jinja2_ver"))) setenv("WGRIB2","wgrib2") +setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None")) ---prepend_path("MODULEPATH", pathJoin("/work/noaa/global/glopara/git/prepobs/v" .. os.getenv("prepobs_run_ver"), "modulefiles")) +--prepend_path("MODULEPATH", pathJoin("/work/noaa/global/glopara/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) prepend_path("MODULEPATH", pathJoin("/work/noaa/global/glopara/git/prepobs/feature-GFSv17_com_reorg_log_update/modulefiles")) load(pathJoin("prepobs", os.getenv("prepobs_run_ver"))) -prepend_path("MODULEPATH", pathJoin("/work/noaa/global/glopara/git/Fit2Obs/v" .. os.getenv("fit2obs_ver"), "modulefiles")) +prepend_path("MODULEPATH", pathJoin("/work/noaa/global/glopara/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) load(pathJoin("fit2obs", os.getenv("fit2obs_ver"))) whatis("Description: GFS run environment") diff --git a/modulefiles/module_base.s4.lua b/modulefiles/module_base.s4.lua index 23e61ef9a7..51eeee0f10 100644 --- a/modulefiles/module_base.s4.lua +++ b/modulefiles/module_base.s4.lua @@ -11,9 +11,9 @@ load(pathJoin("stack-intel-oneapi-mpi", os.getenv("stack_impi_ver"))) load(pathJoin("python", os.getenv("python_ver"))) load(pathJoin("ncl", os.getenv("ncl_ver"))) -load(pathJoin("cdo", os.getenv("cdo_ver"))) load(pathJoin("jasper", os.getenv("jasper_ver"))) load(pathJoin("libpng", os.getenv("libpng_ver"))) +load(pathJoin("cdo", os.getenv("cdo_ver"))) load(pathJoin("hdf5", os.getenv("hdf5_ver"))) load(pathJoin("netcdf-c", os.getenv("netcdf_c_ver"))) @@ -21,24 +21,24 @@ load(pathJoin("netcdf-fortran", os.getenv("netcdf_fortran_ver"))) load(pathJoin("nco", os.getenv("nco_ver"))) load(pathJoin("prod_util", os.getenv("prod_util_ver"))) -load(pathJoin("grib_util", os.getenv("grib_util_ver"))) +load(pathJoin("grib-util", os.getenv("grib_util_ver"))) load(pathJoin("g2tmpl", os.getenv("g2tmpl_ver"))) load(pathJoin("gsi-ncdiag", os.getenv("gsi_ncdiag_ver"))) load(pathJoin("crtm", os.getenv("crtm_ver"))) -load(pathJoin("wgrib2", os.getenv("wgrib2_ver"))) load(pathJoin("bufr", os.getenv("bufr_ver"))) +load(pathJoin("wgrib2", os.getenv("wgrib2_ver"))) load(pathJoin("py-netcdf4", os.getenv("py_netcdf4_ver"))) load(pathJoin("py-pyyaml", os.getenv("py_pyyaml_ver"))) load(pathJoin("py-jinja2", os.getenv("py_jinja2_ver"))) setenv("WGRIB2","wgrib2") -setenv("UTILROOT",os.getenv("prod_util_ROOT")) +setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None")) ---prepend_path("MODULEPATH", pathJoin("/data/prod/glopara/git/prepobs/v" .. os.getenv("prepobs_run_ver"), "modulefiles")) -prepend_path("MODULEPATH", pathJoin("/data/prod/glopara/git/prepobs/feature-GFSv17_com_reorg/modulefiles")) +--prepend_path("MODULEPATH", pathJoin("/data/prod/glopara/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) +prepend_path("MODULEPATH", pathJoin("/data/prod/glopara/git/prepobs/feature-GFSv17_com_reorg_log_update/modulefiles")) load(pathJoin("prepobs", os.getenv("prepobs_run_ver"))) -prepend_path("MODULEPATH", pathJoin("/data/prod/glopara/git/Fit2Obs/v" .. os.getenv("fit2obs_ver"), "modulefiles")) +prepend_path("MODULEPATH", pathJoin("/data/prod/glopara/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) load(pathJoin("fit2obs", os.getenv("fit2obs_ver"))) whatis("Description: GFS run environment") diff --git a/versions/build.s4.ver b/versions/build.s4.ver index cba2bb5a76..a0aed0ee24 100644 --- a/versions/build.s4.ver +++ b/versions/build.s4.ver @@ -1,2 +1,2 @@ export stack_intel_ver=2021.5.0 -export stack_impi_ver=2021.5.1 +export stack_impi_ver=2021.5.0 diff --git a/versions/run.s4.ver b/versions/run.s4.ver index 1125787b09..72206b21d6 100644 --- a/versions/run.s4.ver +++ b/versions/run.s4.ver @@ -1,4 +1,4 @@ export stack_intel_ver=2021.5.0 -export stack_impi_ver=2021.5.1 +export stack_impi_ver=2021.5.0 -export ncl_ver=6.6.2 +export ncl_ver=6.4.0-precompiled From 755a5e85aed75d74b3ccfca32ca963b56f9dc603 Mon Sep 17 00:00:00 2001 From: David Huber Date: Mon, 20 Nov 2023 15:30:49 -0600 Subject: [PATCH 14/33] Fix gwci module files. --- modulefiles/module_gwci.hera.lua | 2 +- modulefiles/module_gwci.orion.lua | 21 +++++++-------------- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/modulefiles/module_gwci.hera.lua b/modulefiles/module_gwci.hera.lua index 1ee7a1cde1..1aecddf549 100644 --- a/modulefiles/module_gwci.hera.lua +++ b/modulefiles/module_gwci.hera.lua @@ -2,7 +2,7 @@ help([[ Load environment to run GFS workflow setup scripts on Hera ]]) -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") load(pathJoin("stack-intel", os.getenv("2021.5.0"))) load(pathJoin("stack-intel-oneapi-mpi", os.getenv("2021.5.1"))) diff --git a/modulefiles/module_gwci.orion.lua b/modulefiles/module_gwci.orion.lua index 971ba01c65..18851ba7d4 100644 --- a/modulefiles/module_gwci.orion.lua +++ b/modulefiles/module_gwci.orion.lua @@ -2,21 +2,14 @@ help([[ Load environment to run GFS workflow ci scripts on Orion ]]) -prepend_path("MODULEPATH", "/apps/contrib/NCEP/hpc-stack/libs/hpc-stack/modulefiles/stack") +prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") -load(pathJoin("hpc", "1.1.0")) -load(pathJoin("hpc-intel", "2018.4")) -load(pathJoin("hpc-impi", "2018.4")) -load(pathJoin("netcdf","4.7.4")) -load(pathJoin("netcdf","4.7.4-parallel")) -load(pathJoin("nccmp"," 1.8.7.0")) -load(pathJoin("contrib","0.1")) -load(pathJoin("wgrib2","3.0.2")) +load(pathJoin("stack-intel", os.getenv("2022.0.2"))) +load(pathJoin("stack-intel-oneapi-mpi", os.getenv("2021.5.1"))) -prepend_path("MODULEPATH", "/work2/noaa/global/wkolczyn/save/hpc-stack/modulefiles/stack") -load(pathJoin("hpc", "1.2.0")) -load(pathJoin("hpc-intel", "2018.4")) -load(pathJoin("hpc-miniconda3", "4.6.14")) -load(pathJoin("gfs_workflow", "1.0.0")) +load(pathJoin("netcdf-c", os.getenv("4.9.2"))) +load(pathJoin("netcdf-fortran", os.getenv("4.6.0"))) +load(pathJoin("nccmp","1.9.0.1")) +load(pathJoin("wgrib2", "2.0.8")) whatis("Description: GFS run ci top-level sripts environment") From e817c85618a62819492876fa931bf11e24b673f4 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Tue, 21 Nov 2023 16:08:44 +0000 Subject: [PATCH 15/33] Add comment about metplus support. #1868 --- parm/config/gfs/config.base.emc.dyn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/gfs/config.base.emc.dyn b/parm/config/gfs/config.base.emc.dyn index 4ff418370c..78cff9bfb8 100644 --- a/parm/config/gfs/config.base.emc.dyn +++ b/parm/config/gfs/config.base.emc.dyn @@ -386,7 +386,7 @@ export netcdf_diag=".true." export binary_diag=".false." # Verification options -export DO_METP="NO" # Run METPLUS jobs - set METPLUS settings in config.metp +export DO_METP="NO" # Run METPLUS jobs - set METPLUS settings in config.metp; not supported with spack-stack export DO_FIT2OBS="YES" # Run fit to observations package # Archiving options From 6faca55fd4e630ee1faa83eecc8e58b2ea1305cf Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Tue, 21 Nov 2023 16:14:16 +0000 Subject: [PATCH 16/33] Address shellcheck warnings. --- jobs/rocoto/anal.sh | 4 ++-- jobs/rocoto/eobs.sh | 4 ++-- jobs/rocoto/eupd.sh | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/jobs/rocoto/anal.sh b/jobs/rocoto/anal.sh index d99152ef19..00f03e3832 100755 --- a/jobs/rocoto/anal.sh +++ b/jobs/rocoto/anal.sh @@ -13,8 +13,8 @@ export jobid="${job}.$$" ############################################################### # Execute the JJOB -${HOMEgfs}/jobs/JGLOBAL_ATMOS_ANALYSIS +"${HOMEgfs}/jobs/JGLOBAL_ATMOS_ANALYSIS" status=$? -exit ${status} +exit "${status}" diff --git a/jobs/rocoto/eobs.sh b/jobs/rocoto/eobs.sh index 95fa42cb08..1139c3e623 100755 --- a/jobs/rocoto/eobs.sh +++ b/jobs/rocoto/eobs.sh @@ -13,8 +13,8 @@ export jobid="${job}.$$" ############################################################### # Execute the JJOB -${HOMEgfs}/jobs/JGDAS_ENKF_SELECT_OBS +"${HOMEgfs}/jobs/JGDAS_ENKF_SELECT_OBS" status=$? -exit ${status} +exit "${status}" diff --git a/jobs/rocoto/eupd.sh b/jobs/rocoto/eupd.sh index 3ed028f87a..9134ca5f14 100755 --- a/jobs/rocoto/eupd.sh +++ b/jobs/rocoto/eupd.sh @@ -13,8 +13,8 @@ export jobid="${job}.$$" ############################################################### # Execute the JJOB -${HOMEgfs}/jobs/JGDAS_ENKF_UPDATE +"${HOMEgfs}/jobs/JGDAS_ENKF_UPDATE" status=$? -exit ${status} +exit "${status}" From 1cdde6915c1183ec048c7e5c0b40f43b747bf3c1 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Tue, 21 Nov 2023 17:03:02 +0000 Subject: [PATCH 17/33] Update gfs-utils hash. #1868 This points to the head of develop on NOAA-EMC which supports spack-stack. --- sorc/checkout.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/checkout.sh b/sorc/checkout.sh index 4d9ae35735..57b83a346a 100755 --- a/sorc/checkout.sh +++ b/sorc/checkout.sh @@ -155,7 +155,7 @@ checkout "ufs_model.fd" "https://github.com/ufs-community/ufs-weather-model" # Run all other checkouts simultaneously with just 1 core each to handle submodules. checkout "wxflow" "https://github.com/NOAA-EMC/wxflow" "528f5ab" & -checkout "gfs_utils.fd" "https://github.com/davidhuber-noaa/gfs-utils" "feature/spack-stack" & +checkout "gfs_utils.fd" "https://github.com/NOAA-EMC/gfs-utils" "427d467" & checkout "ufs_utils.fd" "https://github.com/ufs-community/UFS_UTILS.git" "892b693" & checkout "verif-global.fd" "https://github.com/NOAA-EMC/EMC_verif-global.git" "c267780" & From 55e898595e51b71f45d095dde29759b4f2af8e6c Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Tue, 21 Nov 2023 20:54:06 +0000 Subject: [PATCH 18/33] Remove post hack. #1868 --- jobs/rocoto/post.sh | 23 +++-------------------- 1 file changed, 3 insertions(+), 20 deletions(-) diff --git a/jobs/rocoto/post.sh b/jobs/rocoto/post.sh index ca8bd7aa24..c91b9d4679 100755 --- a/jobs/rocoto/post.sh +++ b/jobs/rocoto/post.sh @@ -8,26 +8,9 @@ source "${HOMEgfs}/ush/preamble.sh" ############################################################### # Source FV3GFS workflow modules -# . ${HOMEgfs}/ush/load_fv3gfs_modules.sh -# status=$? -# [[ ${status} -ne 0 ]] && exit ${status} -# Temporarily load modules from UPP -source "${HOMEgfs}/ush/detect_machine.sh" -source "${HOMEgfs}/ush/module-setup.sh" -module use "${HOMEgfs}/sorc/ufs_model.fd/FV3/upp/modulefiles" -module load "${MACHINE_ID}" -module load prod_util -if [[ "${MACHINE_ID}" = "wcoss2" ]]; then - module load cray-pals - module load cfp -else - # shellcheck disable=SC2154 - export UTILROOT="${prod_util_ROOT}" -fi -module load grib-util -module load wgrib2 -export WGRIB2=wgrib2 -# End hack +. ${HOMEgfs}/ush/load_fv3gfs_modules.sh +status=$? +[[ ${status} -ne 0 ]] && exit ${status} export job="post" export jobid="${job}.$$" From 906a9fcb26ba95ca1f09e00bb7ffa6e7355e8491 Mon Sep 17 00:00:00 2001 From: David Huber Date: Thu, 30 Nov 2023 08:10:06 -0600 Subject: [PATCH 19/33] Update GSI-utils, GSI-mon hashes #1868 --- sorc/checkout.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sorc/checkout.sh b/sorc/checkout.sh index 57b83a346a..b7a871cc7e 100755 --- a/sorc/checkout.sh +++ b/sorc/checkout.sh @@ -168,8 +168,8 @@ if [[ ${checkout_gdas} == "YES" ]]; then fi if [[ ${checkout_gsi} == "YES" || ${checkout_gdas} == "YES" ]]; then - checkout "gsi_utils.fd" "https://github.com/davidhuber-noaa/GSI-Utils.git" "feature/spack-stack" & - checkout "gsi_monitor.fd" "https://github.com/davidhuber-noaa/GSI-Monitor.git" "spack-stack" & + checkout "gsi_utils.fd" "https://github.com/NOAA-EMC/GSI-Utils.git" "f371890" & + checkout "gsi_monitor.fd" "https://github.com/NOAA-EMC/GSI-Monitor.git" "ae256c0" & fi # Go through each PID and verify no errors were reported. From bb2fc24d7381818014b6f4f3282e53fc0c0f49a0 Mon Sep 17 00:00:00 2001 From: David Huber Date: Thu, 30 Nov 2023 08:31:34 -0600 Subject: [PATCH 20/33] Address lint warnings. --- jobs/rocoto/post.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jobs/rocoto/post.sh b/jobs/rocoto/post.sh index c91b9d4679..4c696c2e7f 100755 --- a/jobs/rocoto/post.sh +++ b/jobs/rocoto/post.sh @@ -20,9 +20,9 @@ fhrlst=$(echo ${FHRLST} | sed -e 's/_/ /g; s/f/ /g; s/,/ /g') #--------------------------------------------------------------- for fhr in ${fhrlst}; do export post_times=${fhr} - ${HOMEgfs}/jobs/JGLOBAL_ATMOS_POST + "${HOMEgfs}/jobs/JGLOBAL_ATMOS_POST" status=$? - [[ ${status} -ne 0 ]] && exit ${status} + [[ ${status} -ne 0 ]] && exit "${status}" done exit 0 From 87fd614fc6e7cec9324d926d48c94fb01e09ddd3 Mon Sep 17 00:00:00 2001 From: David Huber Date: Thu, 30 Nov 2023 16:18:51 +0000 Subject: [PATCH 21/33] Reinstated hacks for WCOSS2 in efcs, fcst, and post jobs. #1868 --- jobs/rocoto/efcs.sh | 8 +++++++- jobs/rocoto/fcst.sh | 8 +++++++- jobs/rocoto/post.sh | 24 ++++++++++++++++++++---- 3 files changed, 34 insertions(+), 6 deletions(-) diff --git a/jobs/rocoto/efcs.sh b/jobs/rocoto/efcs.sh index b202015149..27612c40b6 100755 --- a/jobs/rocoto/efcs.sh +++ b/jobs/rocoto/efcs.sh @@ -4,7 +4,13 @@ source "${HOMEgfs}/ush/preamble.sh" ############################################################### # Source FV3GFS workflow modules -. ${HOMEgfs}/ush/load_fv3gfs_modules.sh +# TODO clean this up once ncdiag/1.1.2 is installed on WCOSS2 +source "${HOMEgfs}/ush/detect_machine.sh" +if [[ "${MACHINE_ID}" == "wcoss2" ]]; then + . ${HOMEgfs}/ush/load_ufswm_modules.sh +else + . ${HOMEgfs}/ush/load_fv3gfs_modules.sh +fi status=$? [[ ${status} -ne 0 ]] && exit ${status} diff --git a/jobs/rocoto/fcst.sh b/jobs/rocoto/fcst.sh index 75dd58929f..9d09f869d7 100755 --- a/jobs/rocoto/fcst.sh +++ b/jobs/rocoto/fcst.sh @@ -4,7 +4,13 @@ source "${HOMEgfs}/ush/preamble.sh" ############################################################### # Source FV3GFS workflow modules -. ${HOMEgfs}/ush/load_fv3gfs_modules.sh +# TODO clean this up once ncdiag/1.1.2 is installed on WCOSS2 +source "${HOMEgfs}/ush/detect_machine.sh" +if [[ "${MACHINE_ID}" == "wcoss2" ]]; then + . ${HOMEgfs}/ush/load_ufswm_modules.sh +else + . ${HOMEgfs}/ush/load_fv3gfs_modules.sh +fi status=$? [[ ${status} -ne 0 ]] && exit ${status} diff --git a/jobs/rocoto/post.sh b/jobs/rocoto/post.sh index 4c696c2e7f..92abef8767 100755 --- a/jobs/rocoto/post.sh +++ b/jobs/rocoto/post.sh @@ -7,10 +7,26 @@ source "${HOMEgfs}/ush/preamble.sh" ## FHRLST : forecast hourlist to be post-process (e.g. anl, f000, f000_f001_f002, ...) ############################################################### -# Source FV3GFS workflow modules -. ${HOMEgfs}/ush/load_fv3gfs_modules.sh -status=$? -[[ ${status} -ne 0 ]] && exit ${status} +# TODO clean this up once ncdiag/1.1.2 is installed on WCOSS2 +source "${HOMEgfs}/ush/detect_machine.sh" +if [[ "${MACHINE_ID}" = "wcoss2" ]]; then + # Temporarily load modules from UPP + source "${HOMEgfs}/ush/module-setup.sh" + module use "${HOMEgfs}/sorc/ufs_model.fd/FV3/upp/modulefiles" + module load "${MACHINE_ID}" + module load prod_util + module load cray-pals + module load cfp + module load libjpeg + module load grib_util + module load wgrib2 + export WGRIB2=wgrib2 + # End hack +else + . ${HOMEgfs}/ush/load_fv3gfs_modules.sh + status=$? + [[ ${status} -ne 0 ]] && exit ${status} +fi export job="post" export jobid="${job}.$$" From a3cb8ef51afaf72535678106fb79f6d4d2752cc6 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 30 Nov 2023 16:33:10 +0000 Subject: [PATCH 22/33] Address lint warnings. --- jobs/rocoto/efcs.sh | 2 +- jobs/rocoto/fcst.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/jobs/rocoto/efcs.sh b/jobs/rocoto/efcs.sh index 27612c40b6..c5667cb970 100755 --- a/jobs/rocoto/efcs.sh +++ b/jobs/rocoto/efcs.sh @@ -19,7 +19,7 @@ export jobid="${job}.$$" ############################################################### # Execute the JJOB -${HOMEgfs}/jobs/JGDAS_ENKF_FCST +"${HOMEgfs}/jobs/JGDAS_ENKF_FCST" status=$? exit ${status} diff --git a/jobs/rocoto/fcst.sh b/jobs/rocoto/fcst.sh index 9d09f869d7..9138b4eb43 100755 --- a/jobs/rocoto/fcst.sh +++ b/jobs/rocoto/fcst.sh @@ -18,7 +18,7 @@ export job="fcst" export jobid="${job}.$$" # Execute the JJOB -${HOMEgfs}/jobs/JGLOBAL_FORECAST +"${HOMEgfs}/jobs/JGLOBAL_FORECAST" status=$? exit ${status} From 7181fb2d9eaf625f7e7cea07f68335ed7254cdf2 Mon Sep 17 00:00:00 2001 From: DavidHuber Date: Thu, 30 Nov 2023 17:03:01 +0000 Subject: [PATCH 23/33] Update GSI hash #1868 --- sorc/checkout.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sorc/checkout.sh b/sorc/checkout.sh index b7a871cc7e..fc3842d968 100755 --- a/sorc/checkout.sh +++ b/sorc/checkout.sh @@ -160,16 +160,16 @@ checkout "ufs_utils.fd" "https://github.com/ufs-community/UFS_UTILS.git" checkout "verif-global.fd" "https://github.com/NOAA-EMC/EMC_verif-global.git" "c267780" & if [[ ${checkout_gsi} == "YES" ]]; then - checkout "gsi_enkf.fd" "https://github.com/DavidHuber-NOAA/GSI.git" "spack-stack" "1" "NO" & + checkout "gsi_enkf.fd" "https://github.com/NOAA-EMC/GSI.git" "c94bc72" "1" "NO" & fi if [[ ${checkout_gdas} == "YES" ]]; then - checkout "gdas.cd" "https://github.com/NOAA-EMC/GDASApp.git" "7659c10" & + checkout "gdas.cd" "https://github.com/NOAA-EMC/GDASApp.git" "7659c10" & fi if [[ ${checkout_gsi} == "YES" || ${checkout_gdas} == "YES" ]]; then - checkout "gsi_utils.fd" "https://github.com/NOAA-EMC/GSI-Utils.git" "f371890" & - checkout "gsi_monitor.fd" "https://github.com/NOAA-EMC/GSI-Monitor.git" "ae256c0" & + checkout "gsi_utils.fd" "https://github.com/NOAA-EMC/GSI-Utils.git" "f371890" & + checkout "gsi_monitor.fd" "https://github.com/NOAA-EMC/GSI-Monitor.git" "ae256c0" & fi # Go through each PID and verify no errors were reported. From e041132e6588170f5a8658fcc9f697cf2155b706 Mon Sep 17 00:00:00 2001 From: David Huber Date: Thu, 30 Nov 2023 12:02:49 -0600 Subject: [PATCH 24/33] Cleanup Orion module files. #1868 --- modulefiles/module_base.orion.lua | 3 +++ modulefiles/module_gwsetup.orion.lua | 17 +++++++++++------ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/modulefiles/module_base.orion.lua b/modulefiles/module_base.orion.lua index 5ae43d5d34..3464accc89 100644 --- a/modulefiles/module_base.orion.lua +++ b/modulefiles/module_base.orion.lua @@ -32,6 +32,9 @@ load(pathJoin("py-netcdf4", os.getenv("py_netcdf4_ver"))) load(pathJoin("py-pyyaml", os.getenv("py_pyyaml_ver"))) load(pathJoin("py-jinja2", os.getenv("py_jinja2_ver"))) +load(pathJoin("met", os.getenv("met_ver"))) +load(pathJoin("metplus", os.getenv("metplus_ver"))) + setenv("WGRIB2","wgrib2") setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None")) diff --git a/modulefiles/module_gwsetup.orion.lua b/modulefiles/module_gwsetup.orion.lua index 477ceb6251..93a59c8e50 100644 --- a/modulefiles/module_gwsetup.orion.lua +++ b/modulefiles/module_gwsetup.orion.lua @@ -7,10 +7,15 @@ load(pathJoin("contrib","0.1")) load(pathJoin("rocoto","1.3.3")) load(pathJoin("git","2.28.0")) --- Temporary until official hpc-stack is updated -prepend_path("MODULEPATH", "/work2/noaa/global/wkolczyn/save/hpc-stack/modulefiles/stack") -load(pathJoin("hpc", "1.2.0")) -load(pathJoin("hpc-miniconda3", "4.6.14")) -load(pathJoin("gfs_workflow", "1.0.0")) +prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") -whatis("Description: GFS run ci top-level sripts environment") +local stack_intel_ver=os.getenv("stack_intel_ver") or "2022.0.2" +local python_ver=os.getenv("python_ver") or "3.10.8" + +load(pathJoin("stack-intel", stack_intel_ver)) +load(pathJoin("python", python_ver)) +load("py-jinja2") +load("py-pyyaml") +load("py-numpy") + +whatis("Description: GFS run setup environment") From 6e5ec635b6d684556267cf0c00f308b2f724e502 Mon Sep 17 00:00:00 2001 From: David Huber Date: Thu, 30 Nov 2023 12:03:16 -0600 Subject: [PATCH 25/33] Correct WCOSS2 modulefile comment. --- modulefiles/module_gwsetup.wcoss2.lua | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modulefiles/module_gwsetup.wcoss2.lua b/modulefiles/module_gwsetup.wcoss2.lua index 31f644090b..d4e64548b0 100644 --- a/modulefiles/module_gwsetup.wcoss2.lua +++ b/modulefiles/module_gwsetup.wcoss2.lua @@ -7,4 +7,4 @@ load(pathJoin("git","2.29.0")) prepend_path("MODULEPATH", "/apps/ops/test/nco/modulefiles/core") load(pathJoin("rocoto","1.3.5")) -whatis("Description: GFS run ci top-level sripts environment") +whatis("Description: GFS run setup environment") From 06f52bdf7f8de13d96d2a9fdb9fa75397cabb7c9 Mon Sep 17 00:00:00 2001 From: David Huber Date: Wed, 6 Dec 2023 07:58:22 -0600 Subject: [PATCH 26/33] Initial Hercules port. #1588 --- README.md | 1 + docs/note_fixfield.txt | 2 +- docs/source/clone.rst | 4 +- docs/source/components.rst | 2 +- docs/source/hpc.rst | 19 +++--- docs/source/init.rst | 12 ++-- docs/source/start.rst | 2 +- env/HERCULES.env | 78 +++++++++++++++++++++++++ modulefiles/module-setup.csh.inc | 17 ++++-- modulefiles/module-setup.sh.inc | 11 +++- modulefiles/module_base.hercules.lua | 48 +++++++++++++++ modulefiles/module_gwci.hercules.lua | 15 +++++ modulefiles/module_gwsetup.hercules.lua | 19 ++++++ parm/config/gefs/config.resources | 2 +- parm/config/gefs/config.ufs | 2 +- parm/config/gfs/config.aero | 2 +- parm/config/gfs/config.resources | 2 + parm/config/gfs/config.ufs | 2 +- sorc/link_workflow.sh | 1 + ush/detect_machine.sh | 12 +++- ush/load_fv3gfs_modules.sh | 10 +++- ush/module-setup.sh | 12 +++- versions/build.hercules.ver | 2 + versions/run.hercules.ver | 5 ++ workflow/hosts.py | 8 ++- workflow/hosts/hercules.yaml | 25 ++++++++ 26 files changed, 278 insertions(+), 37 deletions(-) create mode 100755 env/HERCULES.env create mode 100644 modulefiles/module_base.hercules.lua create mode 100644 modulefiles/module_gwci.hercules.lua create mode 100644 modulefiles/module_gwsetup.hercules.lua create mode 100644 versions/build.hercules.ver create mode 100644 versions/run.hercules.ver create mode 100644 workflow/hosts/hercules.yaml diff --git a/README.md b/README.md index 6b8fbf589f..289e74933b 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ The `global-workflow` current supports the following tier-1 machines: * NOAA RDHPCS - Hera * MSU HPC - Orion +* MSU HPC - Hercules * NOAA's operational HPC - WCOSS2 Additionally, the following tier-2 machine is supported: diff --git a/docs/note_fixfield.txt b/docs/note_fixfield.txt index af2539e48a..a7a4001561 100644 --- a/docs/note_fixfield.txt +++ b/docs/note_fixfield.txt @@ -3,7 +3,7 @@ For EMC, the fix fields for running the model are not included in git repository They are saved locally on all platforms Hera: /scratch1/NCEPDEV/global/glopara/fix -Orion: /work/noaa/global/glopara/fix +Orion/Hercules: /work/noaa/global/glopara/fix Jet: /mnt/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix S4: /data/prod/glopara/fix diff --git a/docs/source/clone.rst b/docs/source/clone.rst index 3b2d555520..c098a34f7e 100644 --- a/docs/source/clone.rst +++ b/docs/source/clone.rst @@ -32,7 +32,7 @@ For cycled (w/ data assimilation) use the `-g` option during checkout: For coupled cycling (include new UFSDA) use the `-gu` options during checkout: -[Currently only available on Hera and Orion] +[Currently only available on Hera, Orion, and Hercules] :: @@ -110,7 +110,7 @@ Or with the ``-g`` switch to include data assimilation (GSI) for cycling: ./checkout.sh -g Or also with the ``-u`` swtich to include coupled DA (via UFSDA): -[Currently only available on Hera and Orion] +[Currently only available on Hera, Orion, and Hercules] :: diff --git a/docs/source/components.rst b/docs/source/components.rst index 59385c45d0..4d2619e44e 100644 --- a/docs/source/components.rst +++ b/docs/source/components.rst @@ -57,7 +57,7 @@ Data Observation data, also known as dump data, is prepared in production and then archived in a global dump archive (GDA) for use by users when running cycled experiments. The GDA (identified as ``$DMPDIR`` in the workflow) is available on supported platforms and the workflow system knows where to find the data. * Hera: /scratch1/NCEPDEV/global/glopara/dump -* Orion: /work/noaa/rstprod/dump +* Orion/Hercules: /work/noaa/rstprod/dump * Jet: /mnt/lfs4/HFIP/hfv3gfs/glopara/dump * WCOSS2: /lfs/h2/emc/global/noscrub/emc.global/dump * S4: /data/prod/glopara/dump diff --git a/docs/source/hpc.rst b/docs/source/hpc.rst index 7442b0bd69..3ce6a889d9 100644 --- a/docs/source/hpc.rst +++ b/docs/source/hpc.rst @@ -19,6 +19,7 @@ HPC helpdesks * WCOSS2: hpc.wcoss2-help@noaa.gov * Hera: rdhpcs.hera.help@noaa.gov * Orion: rdhpcs.orion.help@noaa.gov +* Hercules: rdhpcs.hercules.help@noaa.gov * HPSS: rdhpcs.hpss.help@noaa.gov * Gaea: oar.gfdl.help@noaa.gov * S4: david.huber@noaa.gov @@ -72,19 +73,21 @@ Version It is advised to use Git v2+ when available. At the time of writing this documentation the default Git clients on the different machines were as noted in the table below. It is recommended that you check the default modules before loading recommended ones: +---------+----------+---------------------------------------+ -| Machine | Default | Recommended | +| Machine | Default | Recommended | +---------+----------+---------------------------------------+ -| Hera | v2.18.0 | default | +| Hera | v2.18.0 | default | +---------+----------+---------------------------------------+ -| Orion | v1.8.3.1 | **module load git/2.28.0** | +| Hercules | v2.31.1 | default | +---------+----------+---------------------------------------+ -| Jet | v2.18.0 | default | +| Orion | v1.8.3.1 | **module load git/2.28.0** | +---------+----------+---------------------------------------+ -| WCOSS2 | v2.26.2 | default or **module load git/2.29.0** | +| Jet | v2.18.0 | default | +---------+----------+---------------------------------------+ -| S4 | v1.8.3.1 | **module load git/2.30.0** | +| WCOSS2 | v2.26.2 | default or **module load git/2.29.0** | +---------+----------+---------------------------------------+ -| AWS PW | v1.8.3.1 | default +| S4 | v1.8.3.1 | **module load git/2.30.0** | ++---------+----------+---------------------------------------+ +| AWS PW | v1.8.3.1 | default +---------+----------+---------------------------------------+ ^^^^^^^^^^^^^ @@ -103,7 +106,7 @@ For the manage_externals utility functioning:: Fix: git config --global ssh.variant ssh ======================================== -Stacksize on R&Ds (Hera, Orion, Jet, S4) +Stacksize on R&Ds (Hera, Orion, Hercules, Jet, S4) ======================================== Some GFS components, like the UPP, need an unlimited stacksize. Add the following setting into your appropriate .*rc file to support these components: diff --git a/docs/source/init.rst b/docs/source/init.rst index 5c9c811052..f9562a3a7d 100644 --- a/docs/source/init.rst +++ b/docs/source/init.rst @@ -49,7 +49,7 @@ Cold-start atmosphere-only cycled C96 deterministic C48 enkf (80 members) ICs ar :: Hera: /scratch1/NCEPDEV/global/glopara/data/ICSDIR/C96C48 - Orion: /work/noaa/global/glopara/data/ICSDIR/C96C48 + Orion/Hercules: /work/noaa/global/glopara/data/ICSDIR/C96C48 WCOSS2: /lfs/h2/emc/global/noscrub/emc.global/data/ICSDIR/C96C48 Start date = 2021122018 @@ -108,7 +108,7 @@ Warm-start cycled w/ coupled (S2S) model C48 atmosphere C48 enkf (80 members) 5 :: Hera: /scratch1/NCEPDEV/global/glopara/data/ICSDIR/C48C48mx500 - Orion: /work/noaa/global/glopara/data/ICSDIR/C48C48mx500 + Orion/Hercules: /work/noaa/global/glopara/data/ICSDIR/C48C48mx500 WCOSS2: /lfs/h2/emc/global/noscrub/emc.global/data/ICSDIR/C48C48mx500 Jet: /lfs4/HFIP/hfv3gfs/glopara/data/ICSDIR/C48C48mx500 @@ -224,7 +224,7 @@ Forecast-only P8 prototype initial conditions are made available to users on sup WCOSS2: /lfs/h2/emc/global/noscrub/emc.global/IC/COUPLED HERA: /scratch1/NCEPDEV/climate/role.ufscpara/IC - ORION: /work/noaa/global/glopara/data/ICSDIR/prototype_ICs + ORION/Hercules: /work/noaa/global/glopara/data/ICSDIR/prototype_ICs JET: /mnt/lfs4/HFIP/hfv3gfs/glopara/data/ICSDIR/prototype_ICs S4: /data/prod/glopara/coupled_ICs @@ -253,7 +253,7 @@ Not yet supported. See :ref:`Manual Generation` section below --------------------- Forecast-only coupled --------------------- -Coupled initial conditions are currently only generated offline and copied prior to the forecast run. Prototype initial conditions will automatically be used when setting up an experiment as an S2SW app, there is no need to do anything additional. Copies of initial conditions from the prototype runs are currently maintained on Hera, Orion, Jet, and WCOSS2. The locations used are determined by ``parm/config/config.coupled_ic``. If you need prototype ICs on another machine, please contact Walter (Walter.Kolczynski@noaa.gov). +Coupled initial conditions are currently only generated offline and copied prior to the forecast run. Prototype initial conditions will automatically be used when setting up an experiment as an S2SW app, there is no need to do anything additional. Copies of initial conditions from the prototype runs are currently maintained on Hera, Orion/Hercules, Jet, and WCOSS2. The locations used are determined by ``parm/config/config.coupled_ic``. If you need prototype ICs on another machine, please contact Walter (Walter.Kolczynski@noaa.gov). .. _forecastonly-atmonly: @@ -354,7 +354,7 @@ Then switch to a different tag or use the default branch (develop). where ``$MACHINE`` is ``wcoss2``, ``hera``, or ``jet``. .. note:: - UFS-UTILS builds on Orion but due to the lack of HPSS access on Orion the ``gdas_init`` utility is not supported there. + UFS-UTILS builds on Orion/Hercules but due to the lack of HPSS access on Orion/Hercules the ``gdas_init`` utility is not supported there. 3. Configure your conversion: @@ -380,7 +380,7 @@ Most users will want to adjust the following ``config`` settings for the current where ``$MACHINE`` is currently ``wcoss2``, ``hera`` or ``jet``. Additional options will be available as support for other machines expands. .. note:: - UFS-UTILS builds on Orion but due to lack of HPSS access there is no ``gdas_init`` driver for Orion nor support to pull initial conditions from HPSS for the ``gdas_init`` utility. + UFS-UTILS builds on Orion/Hercules but due to lack of HPSS access there is no ``gdas_init`` driver for Orion/Hercules nor support to pull initial conditions from HPSS for the ``gdas_init`` utility. Several small jobs will be submitted: diff --git a/docs/source/start.rst b/docs/source/start.rst index 957971e637..7dc093e8a4 100644 --- a/docs/source/start.rst +++ b/docs/source/start.rst @@ -23,7 +23,7 @@ Set up your experiment cron ^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. note:: - Orion currently only supports cron on Orion-login-1. Cron support for other login nodes is coming in the future. + Orion and Hercules currently only support cron on Orion-login-1 and Hercules-login-1, respectively. Cron support for other login nodes is coming in the future. :: diff --git a/env/HERCULES.env b/env/HERCULES.env new file mode 100755 index 0000000000..798549eed8 --- /dev/null +++ b/env/HERCULES.env @@ -0,0 +1,78 @@ +#! /usr/bin/env bash + +if [[ $# -ne 1 ]]; then + + echo "Must specify an input argument to set runtime environment variables!" + echo "argument can be any one of the following:" + echo "fcst post" + echo "Note: Hercules is only set up to run in forecast-only mode" + exit 1 + +fi + +step=$1 + +export npe_node_max=40 +export launcher="srun -l --export=ALL" +export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out" + +# Configure MPI environment +export MPI_BUFS_PER_PROC=2048 +export MPI_BUFS_PER_HOST=2048 +export MPI_GROUP_MAX=256 +export MPI_MEMMAP_OFF=1 +export MP_STDOUTMODE="ORDERED" +export KMP_AFFINITY=scatter +export OMP_STACKSIZE=2048000 +export NTHSTACK=1024000000 +#export LD_BIND_NOW=1 + +ulimit -s unlimited +ulimit -a + +if [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || \ + [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostpnt" ]] || [[ "${step}" == "wavepostbndpntbll" ]]; then + + export CFP_MP="YES" + if [[ "${step}" = "waveprep" ]]; then export MP_PULSE=0 ; fi + export wavempexec=${launcher} + export wave_mpmd=${mpmd_opt} + +elif [[ "${step}" = "fcst" ]]; then + + export OMP_STACKSIZE=512M + if [[ "${CDUMP}" =~ "gfs" ]]; then + nprocs="npe_${step}_gfs" + ppn="npe_node_${step}_gfs" || ppn="npe_node_${step}" + else + nprocs="npe_${step}" + ppn="npe_node_${step}" + fi + (( nnodes = (${!nprocs}+${!ppn}-1)/${!ppn} )) + (( ntasks = nnodes*${!ppn} )) + # With ESMF threading, the model wants to use the full node + export APRUN_UFS="${launcher} -n ${ntasks}" + unset nprocs ppn nnodes ntasks + +elif [[ "${step}" = "upp" ]]; then + + nth_max=$((npe_node_max / npe_node_upp)) + + export NTHREADS_UPP=${nth_upp:-1} + [[ ${NTHREADS_UPP} -gt ${nth_max} ]] && export NTHREADS_UPP=${nth_max} + export APRUN_UPP="${launcher} -n ${npe_upp} --cpus-per-task=${NTHREADS_UPP}" + +elif [[ "${step}" = "post" ]]; then + + nth_max=$((npe_node_max / npe_node_post)) + + export NTHREADS_NP=${nth_np:-1} + [[ ${NTHREADS_NP} -gt ${nth_max} ]] && export NTHREADS_NP=${nth_max} + export APRUN_NP="${launcher} -n ${npe_post} --cpus-per-task=${NTHREADS_NP}" + + export USE_CFP="YES" # Use MPMD for downstream product generation on Hercules + export NTHREADS_DWN=${nth_dwn:-1} + [[ ${NTHREADS_DWN} -gt ${nth_max} ]] && export NTHREADS_DWN=${nth_max} + export APRUN_DWN="${launcher} -n ${npe_dwn} --cpus-per-task=${NTHREADS_DWN}" + +fi diff --git a/modulefiles/module-setup.csh.inc b/modulefiles/module-setup.csh.inc index e8219424f6..7086326627 100644 --- a/modulefiles/module-setup.csh.inc +++ b/modulefiles/module-setup.csh.inc @@ -19,11 +19,18 @@ else if ( { test -d /scratch1 } ) then source /apps/lmod/lmod/init/$__ms_shell endif module purge -else if ( { test -d /work } ) then - # We are on MSU Orion - if ( ! { module help >& /dev/null } ) then - source /apps/lmod/init/$__ms_shell - endif +elif [[ -d /work ]] ; then + # We are on MSU Orion or Hercules + if [[ -d /apps/other ]] ; then + # Hercules + init_path="/apps/other/lmod/lmod/init/$__ms_shell" + else + # Orion + init_path="/apps/lmod/lmod/init/$__ms_shell" + fi + if ( ! eval module help > /dev/null 2>&1 ) ; then + source "${init_path}" + fi module purge else if ( { test -d /data/prod } ) then # We are on SSEC S4 diff --git a/modulefiles/module-setup.sh.inc b/modulefiles/module-setup.sh.inc index e5322cbb2c..db9dabffe1 100644 --- a/modulefiles/module-setup.sh.inc +++ b/modulefiles/module-setup.sh.inc @@ -35,9 +35,16 @@ elif [[ -d /scratch1 ]] ; then fi module purge elif [[ -d /work ]] ; then - # We are on MSU Orion + # We are on MSU Orion or Hercules + if [[ -d /apps/other ]] ; then + # Hercules + init_path="/apps/other/lmod/lmod/init/$__ms_shell" + else + # Orion + init_path="/apps/lmod/lmod/init/$__ms_shell" + fi if ( ! eval module help > /dev/null 2>&1 ) ; then - source /apps/lmod/lmod/init/$__ms_shell + source "${init_path}" fi module purge elif [[ -d /glade ]] ; then diff --git a/modulefiles/module_base.hercules.lua b/modulefiles/module_base.hercules.lua new file mode 100644 index 0000000000..a16b7e5741 --- /dev/null +++ b/modulefiles/module_base.hercules.lua @@ -0,0 +1,48 @@ +help([[ +Load environment to run GFS on Hercules +]]) + +spack_stack_ver=(os.getenv("spack_stack_ver") or "None") +spack_env=(os.getenv("spack_env") or "None") +prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-" .. spack_stack_ver .. "/envs/" .. spack_env .. "/install/modulefiles/Core") + +load(pathJoin("stack-intel", os.getenv("stack_intel_ver"))) +load(pathJoin("stack-intel-oneapi-mpi", os.getenv("stack_impi_ver"))) +load(pathJoin("python", os.getenv("python_ver"))) + +load(pathJoin("gempak", os.getenv("gempak_ver"))) +load(pathJoin("ncl", os.getenv("ncl_ver"))) +load(pathJoin("jasper", os.getenv("jasper_ver"))) +load(pathJoin("libpng", os.getenv("libpng_ver"))) +load(pathJoin("cdo", os.getenv("cdo_ver"))) + +load(pathJoin("hdf5", os.getenv("hdf5_ver"))) +load(pathJoin("netcdf-c", os.getenv("netcdf_c_ver"))) +load(pathJoin("netcdf-fortran", os.getenv("netcdf_fortran_ver"))) + +load(pathJoin("nco", os.getenv("nco_ver"))) +load(pathJoin("prod_util", os.getenv("prod_util_ver"))) +load(pathJoin("grib-util", os.getenv("grib_util_ver"))) +load(pathJoin("g2tmpl", os.getenv("g2tmpl_ver"))) +load(pathJoin("gsi-ncdiag", os.getenv("gsi_ncdiag_ver"))) +load(pathJoin("crtm", os.getenv("crtm_ver"))) +load(pathJoin("bufr", os.getenv("bufr_ver"))) +load(pathJoin("wgrib2", os.getenv("wgrib2_ver"))) +load(pathJoin("py-netcdf4", os.getenv("py_netcdf4_ver"))) +load(pathJoin("py-pyyaml", os.getenv("py_pyyaml_ver"))) +load(pathJoin("py-jinja2", os.getenv("py_jinja2_ver"))) + +load(pathJoin("met", os.getenv("met_ver"))) +load(pathJoin("metplus", os.getenv("metplus_ver"))) + +setenv("WGRIB2","wgrib2") +setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None")) + +--prepend_path("MODULEPATH", pathJoin("/work/noaa/global/glopara/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) +prepend_path("MODULEPATH", pathJoin("/work/noaa/global/glopara/git/prepobs/feature-GFSv17_com_reorg_log_update/modulefiles")) +load(pathJoin("prepobs", os.getenv("prepobs_run_ver"))) + +prepend_path("MODULEPATH", pathJoin("/work/noaa/global/glopara/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles")) +load(pathJoin("fit2obs", os.getenv("fit2obs_ver"))) + +whatis("Description: GFS run environment") diff --git a/modulefiles/module_gwci.hercules.lua b/modulefiles/module_gwci.hercules.lua new file mode 100644 index 0000000000..9c60aed467 --- /dev/null +++ b/modulefiles/module_gwci.hercules.lua @@ -0,0 +1,15 @@ +help([[ +Load environment to run GFS workflow ci scripts on Hercules +]]) + +prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") + +load(pathJoin("stack-intel", os.getenv("2021.9.0"))) +load(pathJoin("stack-intel-oneapi-mpi", os.getenv("2021.9.0"))) + +load(pathJoin("netcdf-c", os.getenv("4.9.2"))) +load(pathJoin("netcdf-fortran", os.getenv("4.6.0"))) +load(pathJoin("nccmp","1.9.0.1")) +load(pathJoin("wgrib2", "3.1.1")) + +whatis("Description: GFS run ci top-level sripts environment") diff --git a/modulefiles/module_gwsetup.hercules.lua b/modulefiles/module_gwsetup.hercules.lua new file mode 100644 index 0000000000..673928605c --- /dev/null +++ b/modulefiles/module_gwsetup.hercules.lua @@ -0,0 +1,19 @@ +help([[ +Load environment to run GFS workflow ci scripts on Hercules +]]) + +load(pathJoin("contrib","0.1")) +load(pathJoin("rocoto","1.3.5")) + +prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core") + +local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.9.0" +local python_ver=os.getenv("python_ver") or "3.10.8" + +load(pathJoin("stack-intel", stack_intel_ver)) +load(pathJoin("python", python_ver)) +load("py-jinja2") +load("py-pyyaml") +load("py-numpy") + +whatis("Description: GFS run setup environment") diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources index d033ec8d42..dbfcf3bd79 100644 --- a/parm/config/gefs/config.resources +++ b/parm/config/gefs/config.resources @@ -41,7 +41,7 @@ elif [[ ${machine} = "S4" ]]; then elif [[ ${PARTITION_BATCH} = "ivy" ]]; then export npe_node_max=20 fi -elif [[ ${machine} = "ORION" ]]; then +elif [[ ${machine} = "ORION" | ${machine} = "HERCULES" ]]; then export npe_node_max=40 fi diff --git a/parm/config/gefs/config.ufs b/parm/config/gefs/config.ufs index db0589ab6a..68b364529e 100644 --- a/parm/config/gefs/config.ufs +++ b/parm/config/gefs/config.ufs @@ -72,7 +72,7 @@ case "${machine}" in "WCOSS2") npe_node_max=128 ;; - "HERA" | "ORION") + "HERA" | "ORION" | "HERCULES" ) npe_node_max=40 ;; "JET") diff --git a/parm/config/gfs/config.aero b/parm/config/gfs/config.aero index 23d738c642..70283099ca 100644 --- a/parm/config/gfs/config.aero +++ b/parm/config/gfs/config.aero @@ -11,7 +11,7 @@ case ${machine} in "HERA") AERO_INPUTS_DIR="/scratch1/NCEPDEV/global/glopara/data/gocart_emissions" ;; - "ORION") + "ORION"|"HERCULES") AERO_INPUTS_DIR="/work2/noaa/global/wkolczyn/noscrub/global-workflow/gocart_emissions" ;; "S4") diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 4d975c6003..625104500c 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -54,6 +54,8 @@ elif [[ "${machine}" = "AWSPW" ]]; then export npe_node_max=40 elif [[ ${machine} = "ORION" ]]; then export npe_node_max=40 +elif [[ ${machine} = "HERCULES" ]]; then + export npe_node_max=40 fi if [[ ${step} = "prep" ]]; then diff --git a/parm/config/gfs/config.ufs b/parm/config/gfs/config.ufs index 33264ba0aa..000c8b1e99 100644 --- a/parm/config/gfs/config.ufs +++ b/parm/config/gfs/config.ufs @@ -72,7 +72,7 @@ case "${machine}" in "WCOSS2") npe_node_max=128 ;; - "HERA" | "ORION") + "HERA" | "ORION" | "HERCULES") npe_node_max=40 ;; "JET") diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index cdb9256479..378612dffa 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -66,6 +66,7 @@ case "${machine}" in "wcoss2") FIX_DIR="/lfs/h2/emc/global/noscrub/emc.global/FIX/fix" ;; "hera") FIX_DIR="/scratch1/NCEPDEV/global/glopara/fix" ;; "orion") FIX_DIR="/work/noaa/global/glopara/fix" ;; + "hercules") FIX_DIR="/work/noaa/global/glopara/fix" ;; "jet") FIX_DIR="/lfs4/HFIP/hfv3gfs/glopara/git/fv3gfs/fix" ;; "s4") FIX_DIR="/data/prod/glopara/fix" ;; *) diff --git a/ush/detect_machine.sh b/ush/detect_machine.sh index f7e4a9d4f3..f2a394b166 100755 --- a/ush/detect_machine.sh +++ b/ush/detect_machine.sh @@ -26,6 +26,9 @@ case $(hostname -f) in Orion-login-[1-4].HPC.MsState.Edu) MACHINE_ID=orion ;; ### orion1-4 + hercules-login-[1-4].hpc.msstate.edu) MACHINE_ID=hercules ;; ### hercules1-4 + Hercules-login-[1-4].HPC.MsState.Edu) MACHINE_ID=hercules ;; ### hercules1-4 + cheyenne[1-6].cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1-6 cheyenne[1-6].ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1-6 chadmin[1-6].ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1-6 @@ -64,8 +67,13 @@ elif [[ -d /scratch1 ]] ; then # We are on NOAA Hera MACHINE_ID=hera elif [[ -d /work ]] ; then - # We are on MSU Orion - MACHINE_ID=orion + # We are on MSU Orion or Hercules + if [[ -d /apps/other ]] ; then + # We are on Hercules + MACHINE_ID=hercules + else + MACHINE_ID=orion + fi elif [[ -d /glade ]] ; then # We are on NCAR Yellowstone MACHINE_ID=cheyenne diff --git a/ush/load_fv3gfs_modules.sh b/ush/load_fv3gfs_modules.sh index 6ceb5c63ee..b4f23fa331 100755 --- a/ush/load_fv3gfs_modules.sh +++ b/ush/load_fv3gfs_modules.sh @@ -28,8 +28,14 @@ elif [[ -d /scratch1 ]] ; then # We are on NOAA Hera module load module_base.hera elif [[ -d /work ]] ; then - # We are on MSU Orion - module load module_base.orion + # We are on MSU Orion or Hercules + if [[ -d /apps/other ]] ; then + # Hercules + module load module_base.hercules + else + # Orion + module load module_base.orion + fi elif [[ -d /glade ]] ; then # We are on NCAR Yellowstone module load module_base.cheyenne diff --git a/ush/module-setup.sh b/ush/module-setup.sh index e01e379de4..fd656966bf 100755 --- a/ush/module-setup.sh +++ b/ush/module-setup.sh @@ -19,10 +19,20 @@ elif [[ ${MACHINE_ID} = hera* ]] ; then module reset set -u +elif [[ ${MACHINE_ID} = hercules* ]] ; then + # We are on Hercules + if ( ! eval module help > /dev/null 2>&1 ) ; then + source /apps/other/lmod/lmod/init/bash + fi + export LMOD_SYSTEM_DEFAULT_MODULES=contrib + set +u + module reset + set -u + elif [[ ${MACHINE_ID} = orion* ]] ; then # We are on Orion if ( ! eval module help > /dev/null 2>&1 ) ; then - source /apps/lmod/init/bash + source /apps/lmod/lmod/init/bash fi export LMOD_SYSTEM_DEFAULT_MODULES=contrib set +u diff --git a/versions/build.hercules.ver b/versions/build.hercules.ver new file mode 100644 index 0000000000..600827d87d --- /dev/null +++ b/versions/build.hercules.ver @@ -0,0 +1,2 @@ +export stack_intel_ver=2021.9.0 +export stack_impi_ver=2021.9.0 diff --git a/versions/run.hercules.ver b/versions/run.hercules.ver new file mode 100644 index 0000000000..fd8bff6d63 --- /dev/null +++ b/versions/run.hercules.ver @@ -0,0 +1,5 @@ +export stack_intel_ver=2021.9.0 +export stack_impi_ver=2021.9.0 + +export ncl_ver=6.6.2 +export wgrib2_ver=3.1.1 diff --git a/workflow/hosts.py b/workflow/hosts.py index 0b199ba382..5accc4764c 100644 --- a/workflow/hosts.py +++ b/workflow/hosts.py @@ -14,7 +14,7 @@ class Host: Gather Host specific information. """ - SUPPORTED_HOSTS = ['HERA', 'ORION', 'JET', + SUPPORTED_HOSTS = ['HERA', 'ORION', 'JET', 'HERCULES', 'WCOSS2', 'S4', 'CONTAINER', 'AWSPW'] def __init__(self, host=None): @@ -39,7 +39,11 @@ def detect(cls): if os.path.exists('/scratch1/NCEPDEV'): machine = 'HERA' elif os.path.exists('/work/noaa'): - machine = 'ORION' + if os.path.exists('/apps/other'): + machine = 'HERCULES' + print("Hercules") + else: + machine = 'ORION' elif os.path.exists('/lfs4/HFIP'): machine = 'JET' elif os.path.exists('/lfs/f1'): diff --git a/workflow/hosts/hercules.yaml b/workflow/hosts/hercules.yaml new file mode 100644 index 0000000000..e977091ba6 --- /dev/null +++ b/workflow/hosts/hercules.yaml @@ -0,0 +1,25 @@ +BASE_GIT: '/work/noaa/global/glopara/git' +DMPDIR: '/work/noaa/rstprod/dump' +BASE_CPLIC: '/work/noaa/global/glopara/data/ICSDIR/prototype_ICs' +PACKAGEROOT: '/work/noaa/global/glopara/nwpara' +COMROOT: '/work/noaa/global/glopara/com' +COMINsyn: '${COMROOT}/gfs/prod/syndat' +HOMEDIR: '/work/noaa/global/${USER}' +STMP: '/work/noaa/stmp/${USER}' +PTMP: '/work/noaa/stmp/${USER}' +NOSCRUB: $HOMEDIR +SCHEDULER: slurm +ACCOUNT: fv3-cpu +QUEUE: batch +QUEUE_SERVICE: batch +PARTITION_BATCH: hercules +PARTITION_SERVICE: service +CHGRP_RSTPROD: 'YES' +CHGRP_CMD: 'chgrp rstprod' +HPSSARCH: 'NO' +HPSS_PROJECT: emc-global +LOCALARCH: 'NO' +ATARDIR: '${NOSCRUB}/archive_rotdir/${PSLOT}' +MAKE_NSSTBUFR: 'NO' +MAKE_ACFTBUFR: 'NO' +SUPPORTED_RESOLUTIONS: ['C1152', 'C768', 'C384', 'C192', 'C96', 'C48'] From e181fc3e4fb23b108df17ac7682a1d2f7ab6f77d Mon Sep 17 00:00:00 2001 From: David Huber Date: Wed, 6 Dec 2023 13:15:55 -0600 Subject: [PATCH 27/33] Remove debug print. --- workflow/hosts.py | 1 - 1 file changed, 1 deletion(-) diff --git a/workflow/hosts.py b/workflow/hosts.py index 5accc4764c..3e27478dbe 100644 --- a/workflow/hosts.py +++ b/workflow/hosts.py @@ -41,7 +41,6 @@ def detect(cls): elif os.path.exists('/work/noaa'): if os.path.exists('/apps/other'): machine = 'HERCULES' - print("Hercules") else: machine = 'ORION' elif os.path.exists('/lfs4/HFIP'): From 6b1c0afad244759d15175f2ee993d4ec58457253 Mon Sep 17 00:00:00 2001 From: David Huber Date: Wed, 6 Dec 2023 15:20:34 -0600 Subject: [PATCH 28/33] Correct Hercules modules. #1588 --- modulefiles/module_base.hercules.lua | 10 ++++------ versions/run.hercules.ver | 3 ++- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/modulefiles/module_base.hercules.lua b/modulefiles/module_base.hercules.lua index a16b7e5741..d587b90c4f 100644 --- a/modulefiles/module_base.hercules.lua +++ b/modulefiles/module_base.hercules.lua @@ -10,8 +10,10 @@ load(pathJoin("stack-intel", os.getenv("stack_intel_ver"))) load(pathJoin("stack-intel-oneapi-mpi", os.getenv("stack_impi_ver"))) load(pathJoin("python", os.getenv("python_ver"))) -load(pathJoin("gempak", os.getenv("gempak_ver"))) -load(pathJoin("ncl", os.getenv("ncl_ver"))) +-- TODO load NCL once the SAs remove the 'depends_on' statements within it +-- NCL is a static installation and does not depend on any libraries +-- but as is will load, among others, the system netcdf-c/4.9.0 module +--load(pathJoin("ncl", os.getenv("ncl_ver"))) load(pathJoin("jasper", os.getenv("jasper_ver"))) load(pathJoin("libpng", os.getenv("libpng_ver"))) load(pathJoin("cdo", os.getenv("cdo_ver"))) @@ -32,13 +34,9 @@ load(pathJoin("py-netcdf4", os.getenv("py_netcdf4_ver"))) load(pathJoin("py-pyyaml", os.getenv("py_pyyaml_ver"))) load(pathJoin("py-jinja2", os.getenv("py_jinja2_ver"))) -load(pathJoin("met", os.getenv("met_ver"))) -load(pathJoin("metplus", os.getenv("metplus_ver"))) - setenv("WGRIB2","wgrib2") setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None")) ---prepend_path("MODULEPATH", pathJoin("/work/noaa/global/glopara/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles")) prepend_path("MODULEPATH", pathJoin("/work/noaa/global/glopara/git/prepobs/feature-GFSv17_com_reorg_log_update/modulefiles")) load(pathJoin("prepobs", os.getenv("prepobs_run_ver"))) diff --git a/versions/run.hercules.ver b/versions/run.hercules.ver index c9b94e8e98..4bedeb1e96 100644 --- a/versions/run.hercules.ver +++ b/versions/run.hercules.ver @@ -5,5 +5,6 @@ export ncl_ver=6.6.2 source "${HOMEgfs:-}/versions/run.spack.ver" -# wgrib2 is different on Hercules from all the other systems +# wgrib2 and cdo are different on Hercules from all the other systems export wgrib2_ver=3.1.1 +export cdo_ver=2.2.0 From 779627225e1f3d1e4de203587a9efb9bced175d9 Mon Sep 17 00:00:00 2001 From: David Huber Date: Wed, 6 Dec 2023 15:21:13 -0600 Subject: [PATCH 29/33] Enable MPMD for atmos_products on Hercules. #1588 --- env/HERCULES.env | 13 ++----------- scripts/exglobal_atmos_products.sh | 2 +- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/env/HERCULES.env b/env/HERCULES.env index 798549eed8..3721be2b66 100755 --- a/env/HERCULES.env +++ b/env/HERCULES.env @@ -62,17 +62,8 @@ elif [[ "${step}" = "upp" ]]; then [[ ${NTHREADS_UPP} -gt ${nth_max} ]] && export NTHREADS_UPP=${nth_max} export APRUN_UPP="${launcher} -n ${npe_upp} --cpus-per-task=${NTHREADS_UPP}" -elif [[ "${step}" = "post" ]]; then +elif [[ "${step}" = "atmos_products" ]]; then - nth_max=$((npe_node_max / npe_node_post)) - - export NTHREADS_NP=${nth_np:-1} - [[ ${NTHREADS_NP} -gt ${nth_max} ]] && export NTHREADS_NP=${nth_max} - export APRUN_NP="${launcher} -n ${npe_post} --cpus-per-task=${NTHREADS_NP}" - - export USE_CFP="YES" # Use MPMD for downstream product generation on Hercules - export NTHREADS_DWN=${nth_dwn:-1} - [[ ${NTHREADS_DWN} -gt ${nth_max} ]] && export NTHREADS_DWN=${nth_max} - export APRUN_DWN="${launcher} -n ${npe_dwn} --cpus-per-task=${NTHREADS_DWN}" + export USE_CFP="YES" # Use MPMD for downstream product generation fi diff --git a/scripts/exglobal_atmos_products.sh b/scripts/exglobal_atmos_products.sh index beeea99161..d2c0ed7466 100755 --- a/scripts/exglobal_atmos_products.sh +++ b/scripts/exglobal_atmos_products.sh @@ -133,7 +133,7 @@ for (( nset=1 ; nset <= downset ; nset++ )); do export err=$? else chmod 755 "${DATA}/poescript" - bash +x "${DATA}/poescript" 2>&1 mpmd.out + bash +x "${DATA}/poescript" > mpmd.out 2>&1 export err=$? fi err_chk From b7c179d293ad1bcc14858365b456a40b9db62e35 Mon Sep 17 00:00:00 2001 From: David Huber Date: Wed, 6 Dec 2023 15:29:48 -0600 Subject: [PATCH 30/33] Fix tabbing. --- workflow/hosts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/workflow/hosts.py b/workflow/hosts.py index 3e27478dbe..a17cd3f4a8 100644 --- a/workflow/hosts.py +++ b/workflow/hosts.py @@ -40,9 +40,9 @@ def detect(cls): machine = 'HERA' elif os.path.exists('/work/noaa'): if os.path.exists('/apps/other'): - machine = 'HERCULES' + machine = 'HERCULES' else: - machine = 'ORION' + machine = 'ORION' elif os.path.exists('/lfs4/HFIP'): machine = 'JET' elif os.path.exists('/lfs/f1'): From 301a76d48729036f350106e5feaa4d7d15538a4c Mon Sep 17 00:00:00 2001 From: David Huber Date: Wed, 6 Dec 2023 15:37:17 -0600 Subject: [PATCH 31/33] Separate Orion and Hercules in gefs config to match gfs. --- parm/config/gefs/config.resources | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources index ae1ef03be6..a50418d23a 100644 --- a/parm/config/gefs/config.resources +++ b/parm/config/gefs/config.resources @@ -41,7 +41,9 @@ elif [[ ${machine} = "S4" ]]; then elif [[ ${PARTITION_BATCH} = "ivy" ]]; then export npe_node_max=20 fi -elif [[ ${machine} = "ORION" | ${machine} = "HERCULES" ]]; then +elif [[ ${machine} = "ORION" ]]; then + export npe_node_max=40 +elif [[ ${machine} = "HERCULES" ]]; then export npe_node_max=40 fi From bd6ce5fe07200fb732294ade0e9f27db4b100032 Mon Sep 17 00:00:00 2001 From: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> Date: Thu, 7 Dec 2023 08:00:23 -0500 Subject: [PATCH 32/33] Improve logic in config.aero Co-authored-by: Walter Kolczynski - NOAA --- parm/config/gfs/config.aero | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/config/gfs/config.aero b/parm/config/gfs/config.aero index 70283099ca..32993554b4 100644 --- a/parm/config/gfs/config.aero +++ b/parm/config/gfs/config.aero @@ -11,7 +11,7 @@ case ${machine} in "HERA") AERO_INPUTS_DIR="/scratch1/NCEPDEV/global/glopara/data/gocart_emissions" ;; - "ORION"|"HERCULES") + "ORION" | "HERCULES") AERO_INPUTS_DIR="/work2/noaa/global/wkolczyn/noscrub/global-workflow/gocart_emissions" ;; "S4") From 1c042ce16d261bfedd333e2c8dd49c1522c2c767 Mon Sep 17 00:00:00 2001 From: David Huber Date: Thu, 7 Dec 2023 09:15:19 -0600 Subject: [PATCH 33/33] Use current hercules hostname in detect_machine.sh. --- ush/detect_machine.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ush/detect_machine.sh b/ush/detect_machine.sh index f2a394b166..afeb01830b 100755 --- a/ush/detect_machine.sh +++ b/ush/detect_machine.sh @@ -27,7 +27,6 @@ case $(hostname -f) in Orion-login-[1-4].HPC.MsState.Edu) MACHINE_ID=orion ;; ### orion1-4 hercules-login-[1-4].hpc.msstate.edu) MACHINE_ID=hercules ;; ### hercules1-4 - Hercules-login-[1-4].HPC.MsState.Edu) MACHINE_ID=hercules ;; ### hercules1-4 cheyenne[1-6].cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1-6 cheyenne[1-6].ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1-6