From 0b3304eac1cbbe27e54978681da143820fbb11c0 Mon Sep 17 00:00:00 2001 From: AndrewEichmann-NOAA <58948505+AndrewEichmann-NOAA@users.noreply.github.com> Date: Wed, 30 Oct 2024 18:25:12 -0400 Subject: [PATCH 1/4] Add run and finalize methods to marine LETKF task (#2944) Adds run and finalize methods to marine LETKF task, experiment yaml for gw-ci in GDASApp, workflow additions, removes bugs found on the way, and completes the bulk of the work on LETKF task. Conversion of fields to increments pending. Partially resolves NOAA-EMC/GDASApp#1091 and NOAA-EMC/GDASApp#1251 Mutual dependency with GDASApp PR NOAA-EMC/GDASApp#1287 and IC fix file issue https://github.com/NOAA-EMC/global-workflow/pull/2944#issue-2537157488 --------- Co-authored-by: Walter Kolczynski - NOAA Co-authored-by: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> --- ci/cases/pr/C48mx500_hybAOWCDA.yaml | 26 +++++++ env/HERA.env | 6 +- env/ORION.env | 6 +- env/WCOSS2.env | 6 +- jobs/JGLOBAL_MARINE_ANALYSIS_LETKF | 25 ++++-- .../{marineanalletkf.sh => marineanlletkf.sh} | 2 +- parm/config/gfs/config.com | 2 + parm/config/gfs/config.marineanalletkf | 18 ----- parm/config/gfs/config.marineanlletkf | 20 +++++ parm/config/gfs/config.resources | 2 +- parm/stage/ocean_ens_perturbations.yaml.j2 | 2 +- ...f.py => exglobal_marine_analysis_letkf.py} | 0 ush/forecast_postdet.sh | 13 ++-- ush/python/pygfs/task/marine_bmat.py | 2 +- ush/python/pygfs/task/marine_letkf.py | 78 +++++++++++++++---- workflow/applications/gfs_cycled.py | 4 +- workflow/rocoto/gfs_tasks.py | 26 +++++++ workflow/rocoto/tasks.py | 2 +- 18 files changed, 177 insertions(+), 63 deletions(-) create mode 100644 ci/cases/pr/C48mx500_hybAOWCDA.yaml rename jobs/rocoto/{marineanalletkf.sh => marineanlletkf.sh} (95%) delete mode 100644 parm/config/gfs/config.marineanalletkf create mode 100644 parm/config/gfs/config.marineanlletkf rename scripts/{exgdas_global_marine_analysis_letkf.py => exglobal_marine_analysis_letkf.py} (100%) diff --git a/ci/cases/pr/C48mx500_hybAOWCDA.yaml b/ci/cases/pr/C48mx500_hybAOWCDA.yaml new file mode 100644 index 0000000000..036aa8ca60 --- /dev/null +++ b/ci/cases/pr/C48mx500_hybAOWCDA.yaml @@ -0,0 +1,26 @@ +experiment: + system: gfs + mode: cycled + +arguments: + pslot: {{ 'pslot' | getenv }} + app: S2S + resdetatmos: 48 + resdetocean: 5.0 + resensatmos: 48 + comroot: {{ 'RUNTESTS' | getenv }}/COMROOT + expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR + icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C48mx500/20240610 + idate: 2021032412 + edate: 2021032418 + nens: 3 + interval: 0 + start: warm + yaml: {{ HOMEgfs }}/ci/cases/yamls/soca_gfs_defaults_ci.yaml + +skip_ci_on_hosts: + - wcoss2 + - orion + - hercules + - hera + - gaea diff --git a/env/HERA.env b/env/HERA.env index 80cd7cddaf..259461b1ac 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -153,10 +153,10 @@ elif [[ "${step}" = "ocnanalecen" ]]; then export NTHREADS_OCNANALECEN=${NTHREADSmax} export APRUN_OCNANALECEN="${APRUN_default} --cpus-per-task=${NTHREADS_OCNANALECEN}" -elif [[ "${step}" = "marineanalletkf" ]]; then +elif [[ "${step}" = "marineanlletkf" ]]; then - export NTHREADS_MARINEANALLETKF=${NTHREADSmax} - export APRUN_MARINEANALLETKF="${APRUN_default} --cpus-per-task=${NTHREADS_MARINEANALLETKF}" + export NTHREADS_MARINEANLLETKF=${NTHREADSmax} + export APRUN_MARINEANLLETKF=${APRUN_default} elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then diff --git a/env/ORION.env b/env/ORION.env index 45fd607aa5..06ae2c1a63 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -149,10 +149,10 @@ elif [[ "${step}" = "ocnanalecen" ]]; then export NTHREADS_OCNANALECEN=${NTHREADSmax} export APRUN_OCNANALECEN="${APRUN_default} --cpus-per-task=${NTHREADS_OCNANALECEN}" -elif [[ "${step}" = "marineanalletkf" ]]; then +elif [[ "${step}" = "marineanlletkf" ]]; then - export NTHREADS_MARINEANALLETKF=${NTHREADSmax} - export APRUN_MARINEANALLETKF="${APRUN_default} --cpus-per-task=${NTHREADS_MARINEANALLETKF}" + export NTHREADS_MARINEANLLETKF=${NTHREADSmax} + export APRUN_MARINEANLLETKF="${APRUN_default}" elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then diff --git a/env/WCOSS2.env b/env/WCOSS2.env index cea24fb26b..c67c16f929 100755 --- a/env/WCOSS2.env +++ b/env/WCOSS2.env @@ -126,10 +126,10 @@ elif [[ "${step}" = "ocnanalecen" ]]; then export NTHREADS_OCNANALECEN=${NTHREADSmax} export APRUN_OCNANALECEN="${APRUN_default} --cpus-per-task=${NTHREADS_OCNANALECEN}" -elif [[ "${step}" = "marineanalletkf" ]]; then +elif [[ "${step}" = "marineanlletkf" ]]; then - export NTHREADS_MARINEANALLETKF=${NTHREADSmax} - export APRUN_MARINEANALLETKF="${APRUN_default} --cpus-per-task=${NTHREADS_MARINEANALLETKF}" + export NTHREADS_MARINEANLLETKF=${NTHREADSmax} + export APRUN_MARINEANLLETKF="${APRUN_default}" elif [[ "${step}" = "atmanlfv3inc" ]]; then diff --git a/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF b/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF index 38dc3049f9..2a88f89eab 100755 --- a/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF +++ b/jobs/JGLOBAL_MARINE_ANALYSIS_LETKF @@ -1,6 +1,13 @@ #!/bin/bash source "${HOMEgfs}/ush/preamble.sh" -source "${HOMEgfs}/ush/jjob_header.sh" -e "marineanalletkf" -c "base ocnanal marineanalletkf" + +export DATAjob="${DATAROOT}/${RUN}marineanalysis.${PDY:-}${cyc}" +export DATA="${DATAjob}/${jobid}" +# Create the directory to hold ensemble perturbations +export DATAens="${DATAjob}/ensdata" +if [[ ! -d "${DATAens}" ]]; then mkdir -p "${DATAens}"; fi + +source "${HOMEgfs}/ush/jjob_header.sh" -e "marineanlletkf" -c "base marineanl marineanlletkf" ############################################## # Set variables used in the script @@ -11,12 +18,18 @@ GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") gPDY=${GDATE:0:8} gcyc=${GDATE:8:2} +export GDUMP="gdas" +export GDUMP_ENS="enkf${GDUMP}" +export OPREFIX="${RUN}.t${cyc}z." -YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ - COMIN_OCEAN_HISTORY_PREV:COM_OCEAN_HISTORY_TMPL \ - COMIN_ICE_HISTORY_PREV:COM_ICE_HISTORY_TMPL +RUN="${GDUMP}" YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ + COMIN_OCEAN_HISTORY_PREV:COM_OCEAN_HISTORY_TMPL \ + COMIN_ICE_HISTORY_PREV:COM_ICE_HISTORY_TMPL -YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COMIN_OBS:COM_OBS_TMPL +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ + COMIN_OBS:COM_OBS_TMPL \ + COMOUT_OCEAN_LETKF:COM_OCEAN_LETKF_TMPL \ + COMOUT_ICE_LETKF:COM_ICE_LETKF_TMPL ############################################## # Begin JOB SPECIFIC work @@ -25,7 +38,7 @@ YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COMIN_OBS:COM_OBS_TMPL ############################################################### # Run relevant script -EXSCRIPT=${GDASOCNLETKFPY:-${HOMEgfs}/scripts/exgdas_global_marine_analysis_letkf.py} +EXSCRIPT=${GDASOCNLETKFPY:-${HOMEgfs}/scripts/exglobal_marine_analysis_letkf.py} ${EXSCRIPT} status=$? [[ ${status} -ne 0 ]] && exit "${status}" diff --git a/jobs/rocoto/marineanalletkf.sh b/jobs/rocoto/marineanlletkf.sh similarity index 95% rename from jobs/rocoto/marineanalletkf.sh rename to jobs/rocoto/marineanlletkf.sh index f2bfb9f70c..d4333461f3 100755 --- a/jobs/rocoto/marineanalletkf.sh +++ b/jobs/rocoto/marineanlletkf.sh @@ -8,7 +8,7 @@ source "${HOMEgfs}/ush/preamble.sh" status=$? [[ ${status} -ne 0 ]] && exit "${status}" -export job="marineanalletkf" +export job="marineanlletkf" export jobid="${job}.$$" ############################################################### diff --git a/parm/config/gfs/config.com b/parm/config/gfs/config.com index 61d592561d..d949edb33a 100644 --- a/parm/config/gfs/config.com +++ b/parm/config/gfs/config.com @@ -82,12 +82,14 @@ declare -rx COM_OCEAN_HISTORY_TMPL=${COM_BASE}'/model/ocean/history' declare -rx COM_OCEAN_RESTART_TMPL=${COM_BASE}'/model/ocean/restart' declare -rx COM_OCEAN_INPUT_TMPL=${COM_BASE}'/model/ocean/input' declare -rx COM_OCEAN_ANALYSIS_TMPL=${COM_BASE}'/analysis/ocean' +declare -rx COM_OCEAN_LETKF_TMPL=${COM_BASE}'/analysis/ocean/letkf' declare -rx COM_OCEAN_BMATRIX_TMPL=${COM_BASE}'/bmatrix/ocean' declare -rx COM_OCEAN_NETCDF_TMPL=${COM_BASE}'/products/ocean/netcdf' declare -rx COM_OCEAN_GRIB_TMPL=${COM_BASE}'/products/ocean/grib2' declare -rx COM_OCEAN_GRIB_GRID_TMPL=${COM_OCEAN_GRIB_TMPL}'/${GRID}' declare -rx COM_ICE_ANALYSIS_TMPL=${COM_BASE}'/analysis/ice' +declare -rx COM_ICE_LETKF_TMPL=${COM_BASE}'/analysis/ice/letkf' declare -rx COM_ICE_BMATRIX_TMPL=${COM_BASE}'/bmatrix/ice' declare -rx COM_ICE_INPUT_TMPL=${COM_BASE}'/model/ice/input' declare -rx COM_ICE_HISTORY_TMPL=${COM_BASE}'/model/ice/history' diff --git a/parm/config/gfs/config.marineanalletkf b/parm/config/gfs/config.marineanalletkf deleted file mode 100644 index fde3433a13..0000000000 --- a/parm/config/gfs/config.marineanalletkf +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -########## config.marineanalletkf ########## -# Ocn Analysis specific - -echo "BEGIN: config.marineanalletkf" - -# Get task specific resources -. "${EXPDIR}/config.resources" marineanalletkf - -export MARINE_LETKF_EXEC="${JEDI_BIN}/gdas.x" -export MARINE_LETKF_YAML_TMPL="${PARMgfs}/gdas/soca/letkf/letkf.yaml.j2" -export MARINE_LETKF_STAGE_YAML_TMPL="${PARMgfs}/gdas/soca/letkf/letkf_stage.yaml.j2" - -export GRIDGEN_EXEC="${JEDI_BIN}/gdas_soca_gridgen.x" -export GRIDGEN_YAML="${PARMgfs}/gdas/soca/gridgen/gridgen.yaml" - -echo "END: config.marineanalletkf" diff --git a/parm/config/gfs/config.marineanlletkf b/parm/config/gfs/config.marineanlletkf new file mode 100644 index 0000000000..8b84af4eaa --- /dev/null +++ b/parm/config/gfs/config.marineanlletkf @@ -0,0 +1,20 @@ +#!/bin/bash + +########## config.marineanlletkf ########## +# Ocn Analysis specific + +echo "BEGIN: config.marineanlletkf" + +# Get task specific resources +. "${EXPDIR}/config.resources" marineanlletkf + +export MARINE_LETKF_EXEC="${EXECgfs}/gdas.x" +export MARINE_LETKF_YAML_TMPL="${PARMgfs}/gdas/soca/letkf/letkf.yaml.j2" +export MARINE_LETKF_STAGE_YAML_TMPL="${PARMgfs}/gdas/soca/letkf/letkf_stage.yaml.j2" +export MARINE_LETKF_SAVE_YAML_TMPL="${PARMgfs}/gdas/soca/letkf/letkf_save.yaml.j2" + +export GRIDGEN_EXEC="${EXECgfs}/gdas_soca_gridgen.x" +export GRIDGEN_YAML="${PARMgfs}/gdas/soca/gridgen/gridgen.yaml" +export DIST_HALO_SIZE=500000 + +echo "END: config.marineanlletkf" diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 79dbb487db..14e6f0d7fb 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -601,7 +601,7 @@ case ${step} in tasks_per_node=$(( max_tasks_per_node / threads_per_task )) ;; - "marineanalletkf") + "marineanlletkf") ntasks=16 case ${OCNRES} in "025") diff --git a/parm/stage/ocean_ens_perturbations.yaml.j2 b/parm/stage/ocean_ens_perturbations.yaml.j2 index fede3816a7..586b9f66cb 100644 --- a/parm/stage/ocean_ens_perturbations.yaml.j2 +++ b/parm/stage/ocean_ens_perturbations.yaml.j2 @@ -9,5 +9,5 @@ ocean_ens_perturbation: {% for mem in range(first_mem + 1, last_mem + 1) %} {% set imem = mem - first_mem %} {% set COMOUT_OCEAN_ANALYSIS_MEM = COMOUT_OCEAN_ANALYSIS_MEM_list[imem] %} - - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.mom6_perturbation.nc", "{{ COMOUT_OCEAN_ANALYSIS_MEM }}/mom6_increment.nc"] + - ["{{ ICSDIR }}/{{ COMOUT_OCEAN_ANALYSIS_MEM | relpath(ROTDIR) }}/{{ m_prefix }}.mom6_perturbation.nc", "{{ COMOUT_OCEAN_ANALYSIS_MEM }}/{{ RUN }}.t{{ current_cycle_HH }}z.ocninc.nc"] {% endfor %} # mem loop diff --git a/scripts/exgdas_global_marine_analysis_letkf.py b/scripts/exglobal_marine_analysis_letkf.py similarity index 100% rename from scripts/exgdas_global_marine_analysis_letkf.py rename to scripts/exglobal_marine_analysis_letkf.py diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh index 288b251aa8..25b2e28d75 100755 --- a/ush/forecast_postdet.sh +++ b/ush/forecast_postdet.sh @@ -466,12 +466,13 @@ MOM6_postdet() { fi # GEFS perturbations - # TODO if [[ $RUN} == "gefs" ]] block maybe be needed - # to ensure it does not interfere with the GFS when ensemble is updated in the GFS - if (( MEMBER > 0 )) && [[ "${ODA_INCUPD:-False}" == "True" ]]; then - ${NCP} "${COMIN_OCEAN_ANALYSIS}/mom6_increment.nc" "${DATA}/INPUT/mom6_increment.nc" \ - || ( echo "FATAL ERROR: Unable to copy ensemble MOM6 increment, ABORT!"; exit 1 ) - fi + if [[ "${RUN}" == "gefs" ]]; then + # to ensure it does not interfere with the GFS + if (( MEMBER > 0 )) && [[ "${ODA_INCUPD:-False}" == "True" ]]; then + ${NCP} "${COMIN_OCEAN_ANALYSIS}/${RUN}.t${cyc}z.ocninc.nc" "${DATA}/INPUT/mom6_increment.nc" \ + || ( echo "FATAL ERROR: Unable to copy ensemble MOM6 increment, ABORT!"; exit 1 ) + fi + fi # if [[ "${RUN}" == "gefs" ]]; then fi # if [[ "${RERUN}" == "NO" ]]; then # Link output files diff --git a/ush/python/pygfs/task/marine_bmat.py b/ush/python/pygfs/task/marine_bmat.py index 93329f05ac..a4a5b4f144 100644 --- a/ush/python/pygfs/task/marine_bmat.py +++ b/ush/python/pygfs/task/marine_bmat.py @@ -318,7 +318,7 @@ def finalize(self: Task) -> None: FileHandler({'copy': diagb_list}).sync() # Copy the ensemble perturbation diagnostics to the ROTDIR - if self.task_config.DOHYBVAR == "YES" or self.task_config.NMEM_ENS > 3: + if self.task_config.DOHYBVAR == "YES" or self.task_config.NMEM_ENS > 2: window_middle_iso = self.task_config.MARINE_WINDOW_MIDDLE.strftime('%Y-%m-%dT%H:%M:%SZ') weight_list = [] src = os.path.join(self.task_config.DATA, f"ocn.ens_weights.incr.{window_middle_iso}.nc") diff --git a/ush/python/pygfs/task/marine_letkf.py b/ush/python/pygfs/task/marine_letkf.py index 36c26d594b..54d40f8d66 100644 --- a/ush/python/pygfs/task/marine_letkf.py +++ b/ush/python/pygfs/task/marine_letkf.py @@ -1,11 +1,13 @@ #!/usr/bin/env python3 import f90nml +import pygfs.utils.marine_da_utils as mdau from logging import getLogger import os from pygfs.task.analysis import Analysis from typing import Dict from wxflow import (AttrDict, + Executable, FileHandler, logit, parse_j2yaml, @@ -41,6 +43,8 @@ def __init__(self, config: Dict) -> None: 'soca', 'localensembleda', _letkf_yaml_file] + # compute the relative path from self.task_config.DATA to self.task_config.DATAenspert + _enspert_relpath = os.path.relpath(self.task_config.DATAens, self.task_config.DATA) self.task_config.WINDOW_MIDDLE = self.task_config.current_cycle self.task_config.WINDOW_BEGIN = self.task_config.current_cycle - _half_assim_freq @@ -49,6 +53,7 @@ def __init__(self, config: Dict) -> None: self.task_config.mom_input_nml_tmpl = os.path.join(self.task_config.DATA, 'mom_input.nml.tmpl') self.task_config.mom_input_nml = os.path.join(self.task_config.DATA, 'mom_input.nml') self.task_config.obs_dir = os.path.join(self.task_config.DATA, 'obs') + self.task_config.ENSPERT_RELPATH = _enspert_relpath @logit(logger) def initialize(self): @@ -64,26 +69,50 @@ def initialize(self): logger.info("initialize") # make directories and stage ensemble background files - ensbkgconf = AttrDict() - keys = ['previous_cycle', 'current_cycle', 'DATA', 'NMEM_ENS', - 'PARMgfs', 'ROTDIR', 'COM_OCEAN_HISTORY_TMPL', 'COM_ICE_HISTORY_TMPL'] - for key in keys: - ensbkgconf[key] = self.task_config[key] - ensbkgconf.RUN = 'enkfgdas' - soca_ens_bkg_stage_list = parse_j2yaml(self.task_config.SOCA_ENS_BKG_STAGE_YAML_TMPL, ensbkgconf) - FileHandler(soca_ens_bkg_stage_list).sync() soca_fix_stage_list = parse_j2yaml(self.task_config.SOCA_FIX_YAML_TMPL, self.task_config) FileHandler(soca_fix_stage_list).sync() - letkf_stage_list = parse_j2yaml(self.task_config.MARINE_LETKF_STAGE_YAML_TMPL, self.task_config) + stageconf = AttrDict() + keys = ['current_cycle', + 'previous_cycle', + 'COM_ICE_LETKF_TMPL', + 'COM_OCEAN_LETKF_TMPL', + 'COM_ICE_HISTORY_TMPL', + 'COM_OCEAN_HISTORY_TMPL', + 'COMIN_OCEAN_HISTORY_PREV', + 'COMIN_ICE_HISTORY_PREV', + 'COMOUT_ICE_LETKF', + 'COMOUT_OCEAN_LETKF', + 'DATA', + 'ENSPERT_RELPATH', + 'GDUMP_ENS', + 'NMEM_ENS', + 'OPREFIX', + 'PARMgfs', + 'ROTDIR', + 'RUN', + 'WINDOW_BEGIN', + 'WINDOW_MIDDLE'] + for key in keys: + stageconf[key] = self.task_config[key] + + # stage ensemble background files + soca_ens_bkg_stage_list = parse_j2yaml(self.task_config.MARINE_ENSDA_STAGE_BKG_YAML_TMPL, stageconf) + FileHandler(soca_ens_bkg_stage_list).sync() + + # stage letkf-specific files + letkf_stage_list = parse_j2yaml(self.task_config.MARINE_LETKF_STAGE_YAML_TMPL, stageconf) FileHandler(letkf_stage_list).sync() - obs_list = parse_j2yaml(self.task_config.OBS_YAML, self.task_config) + obs_list = parse_j2yaml(self.task_config.MARINE_OBS_LIST_YAML, self.task_config) # get the list of observations obs_files = [] for ob in obs_list['observers']: obs_name = ob['obs space']['name'].lower() - obs_filename = f"{self.task_config.RUN}.t{self.task_config.cyc}z.{obs_name}.{to_YMDH(self.task_config.current_cycle)}.nc" + # TODO(AFE) - this should be removed when the obs config yamls are jinjafied + if 'distribution' not in ob['obs space']: + ob['obs space']['distribution'] = {'name': 'Halo', 'halo size': self.task_config['DIST_HALO_SIZE']} + obs_filename = f"{self.task_config.RUN}.t{self.task_config.cyc}z.{obs_name}.{to_YMDH(self.task_config.current_cycle)}.nc4" obs_files.append((obs_filename, ob)) obs_files_to_copy = [] @@ -102,12 +131,7 @@ def initialize(self): FileHandler({'copy': obs_files_to_copy}).sync() # make the letkf.yaml - letkfconf = AttrDict() - keys = ['WINDOW_BEGIN', 'WINDOW_MIDDLE', 'RUN', 'gcyc', 'NMEM_ENS'] - for key in keys: - letkfconf[key] = self.task_config[key] - letkfconf.RUN = 'enkfgdas' - letkf_yaml = parse_j2yaml(self.task_config.MARINE_LETKF_YAML_TMPL, letkfconf) + letkf_yaml = parse_j2yaml(self.task_config.MARINE_LETKF_YAML_TMPL, stageconf) letkf_yaml.observations.observers = obs_to_use letkf_yaml.save(self.task_config.letkf_yaml_file) @@ -133,6 +157,18 @@ def run(self): logger.info("run") + exec_cmd_gridgen = Executable(self.task_config.APRUN_MARINEANLLETKF) + exec_cmd_gridgen.add_default_arg(self.task_config.GRIDGEN_EXEC) + exec_cmd_gridgen.add_default_arg(self.task_config.GRIDGEN_YAML) + + mdau.run(exec_cmd_gridgen) + + exec_cmd_letkf = Executable(self.task_config.APRUN_MARINEANLLETKF) + for letkf_exec_arg in self.task_config.letkf_exec_args: + exec_cmd_letkf.add_default_arg(letkf_exec_arg) + + mdau.run(exec_cmd_letkf) + @logit(logger) def finalize(self): """Method finalize for ocean and sea ice LETKF task @@ -145,3 +181,11 @@ def finalize(self): """ logger.info("finalize") + + letkfsaveconf = AttrDict() + keys = ['current_cycle', 'DATA', 'NMEM_ENS', 'WINDOW_BEGIN', 'GDUMP_ENS', + 'PARMgfs', 'ROTDIR', 'COM_OCEAN_LETKF_TMPL', 'COM_ICE_LETKF_TMPL'] + for key in keys: + letkfsaveconf[key] = self.task_config[key] + letkf_save_list = parse_j2yaml(self.task_config.MARINE_LETKF_SAVE_YAML_TMPL, letkfsaveconf) + FileHandler(letkf_save_list).sync() diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index da78166ede..f92bf95fba 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -46,7 +46,7 @@ def _get_app_configs(self): if self.do_jediocnvar: configs += ['prepoceanobs', 'marineanlinit', 'marinebmat', 'marineanlvar'] if self.do_hybvar: - configs += ['ocnanalecen'] + configs += ['marineanlletkf', 'ocnanalecen'] configs += ['marineanlchkpt', 'marineanlfinal'] if self.do_vrfy_oceanda: configs += ['ocnanalvrfy'] @@ -148,7 +148,7 @@ def get_task_names(self): if self.do_jediocnvar: gdas_gfs_common_tasks_before_fcst += ['prepoceanobs', 'marineanlinit', 'marinebmat', 'marineanlvar'] if self.do_hybvar: - gdas_gfs_common_tasks_before_fcst += ['ocnanalecen'] + gdas_gfs_common_tasks_before_fcst += ['marineanlletkf', 'ocnanalecen'] gdas_gfs_common_tasks_before_fcst += ['marineanlchkpt', 'marineanlfinal'] if self.do_vrfy_oceanda: gdas_gfs_common_tasks_before_fcst += ['ocnanalvrfy'] diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 82dfb9f1d4..d3bb68a6b8 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -664,6 +664,32 @@ def prepoceanobs(self): return task + def marineanlletkf(self): + + deps = [] + dep_dict = {'type': 'metatask', 'name': f'enkfgdas_fcst', 'offset': f"-{timedelta_to_HMS(self._base['interval_gdas'])}"} + deps.append(rocoto.add_dependency(dep_dict)) + dep_dict = {'type': 'task', 'name': f'{self.run}_prepoceanobs'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) + + resources = self.get_resource('marineanlletkf') + task_name = f'{self.run}_marineanlletkf' + task_dict = {'task_name': task_name, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars, + 'cycledef': self.run.replace('enkf', ''), + 'command': f'{self.HOMEgfs}/jobs/rocoto/marineanlletkf.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + + task = rocoto.create_task(task_dict) + + return task + def marinebmat(self): ocean_hist_path = self._template_to_rocoto_cycstring(self._base["COM_OCEAN_HISTORY_TMPL"], {'RUN': 'gdas'}) diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index 92ceea73aa..b989def13f 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -15,7 +15,7 @@ class Tasks: 'prep', 'anal', 'sfcanl', 'analcalc', 'analdiag', 'arch', "cleanup", 'prepatmiodaobs', 'atmanlinit', 'atmanlvar', 'atmanlfv3inc', 'atmanlfinal', 'prepoceanobs', - 'marineanlinit', 'marinebmat', 'marineanlvar', 'ocnanalecen', 'marineanlchkpt', 'marineanlfinal', 'ocnanalvrfy', + 'marineanlinit', 'marineanlletkf', 'marinebmat', 'marineanlvar', 'ocnanalecen', 'marineanlchkpt', 'marineanlfinal', 'ocnanalvrfy', 'earc', 'ecen', 'echgres', 'ediag', 'efcs', 'eobs', 'eomg', 'epos', 'esfc', 'eupd', 'atmensanlinit', 'atmensanlobs', 'atmensanlsol', 'atmensanlletkf', 'atmensanlfv3inc', 'atmensanlfinal', From f4e380ac4c024d6778c333babbeba73601360d07 Mon Sep 17 00:00:00 2001 From: mingshichen-noaa <48537176+mingshichen-noaa@users.noreply.github.com> Date: Wed, 30 Oct 2024 18:25:45 -0400 Subject: [PATCH 2/4] Update global jdas enkf diag job with COMIN/COMOUT for COM prefix (#2959) NCO has requested that each COM variable specify whether it is an input or an output. This completes that process for the global jdas enkf diagnostics job. Refs https://github.com/NOAA-EMC/global-workflow/issues/2451 --- jobs/JGDAS_ATMOS_ANALYSIS_DIAG | 5 ++-- jobs/JGDAS_ENKF_DIAG | 48 ++++++++++++++++++---------------- scripts/exglobal_diag.sh | 10 +++---- 3 files changed, 33 insertions(+), 30 deletions(-) diff --git a/jobs/JGDAS_ATMOS_ANALYSIS_DIAG b/jobs/JGDAS_ATMOS_ANALYSIS_DIAG index a1e0c9f1d5..c47bd4a47b 100755 --- a/jobs/JGDAS_ATMOS_ANALYSIS_DIAG +++ b/jobs/JGDAS_ATMOS_ANALYSIS_DIAG @@ -27,8 +27,9 @@ export OPREFIX="${RUN/enkf}.t${cyc}z." export GPREFIX="${GDUMP}.t${gcyc}z." export APREFIX="${RUN}.t${cyc}z." -YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_ATMOS_ANALYSIS -mkdir -m 775 -p "${COM_ATMOS_ANALYSIS}" +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ + COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL +mkdir -m 775 -p "${COMOUT_ATMOS_ANALYSIS}" ############################################################### # Run relevant script diff --git a/jobs/JGDAS_ENKF_DIAG b/jobs/JGDAS_ENKF_DIAG index cc8c933cc8..3daa8bfb73 100755 --- a/jobs/JGDAS_ENKF_DIAG +++ b/jobs/JGDAS_ENKF_DIAG @@ -30,56 +30,58 @@ export APREFIX="${RUN}.t${cyc}z." export GPREFIX="${GDUMP_ENS}.t${gcyc}z." GPREFIX_DET="${GDUMP}.t${gcyc}z." -RUN=${RUN/enkf} YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_OBS -MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_ATMOS_ANALYSIS +RUN=${RUN/enkf} YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ + COMIN_OBS:COM_OBS_TMPL +MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ + COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ - COM_OBS_PREV:COM_OBS_TMPL \ - COM_ATMOS_ANALYSIS_DET_PREV:COM_ATMOS_ANALYSIS_TMPL + COMIN_OBS_PREV:COM_OBS_TMPL \ + COMIN_ATMOS_ANALYSIS_DET_PREV:COM_ATMOS_ANALYSIS_TMPL MEMDIR="ensstat" RUN=${GDUMP_ENS} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ - COM_ATMOS_HISTORY_PREV:COM_ATMOS_HISTORY_TMPL + COMIN_ATMOS_HISTORY_PREV:COM_ATMOS_HISTORY_TMPL -export ATMGES_ENSMEAN="${COM_ATMOS_HISTORY_PREV}/${GPREFIX}atmf006.ensmean.nc" +export ATMGES_ENSMEAN="${COMIN_ATMOS_HISTORY_PREV}/${GPREFIX}atmf006.ensmean.nc" if [ ! -f ${ATMGES_ENSMEAN} ]; then echo "FATAL ERROR: FILE MISSING: ATMGES_ENSMEAN = ${ATMGES_ENSMEAN}" exit 1 fi # Link observational data -export PREPQC="${COM_OBS}/${OPREFIX}prepbufr" +export PREPQC="${COMIN_OBS}/${OPREFIX}prepbufr" if [[ ! -f ${PREPQC} ]]; then echo "WARNING: Global PREPBUFR FILE ${PREPQC} MISSING" fi -export TCVITL="${COM_OBS}/${OPREFIX}syndata.tcvitals.tm00" +export TCVITL="${COMIN_OBS}/${OPREFIX}syndata.tcvitals.tm00" if [[ ${DONST} = "YES" ]]; then - export NSSTBF="${COM_OBS}/${OPREFIX}nsstbufr" + export NSSTBF="${COMIN_OBS}/${OPREFIX}nsstbufr" fi -export PREPQCPF="${COM_OBS}/${OPREFIX}prepbufr.acft_profiles" +export PREPQCPF="${COMIN_OBS}/${OPREFIX}prepbufr.acft_profiles" # Guess Bias correction coefficients related to control -export GBIAS=${COM_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}abias -export GBIASPC=${COM_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}abias_pc -export GBIASAIR=${COM_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}abias_air -export GRADSTAT=${COM_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}radstat +export GBIAS=${COMIN_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}abias +export GBIASPC=${COMIN_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}abias_pc +export GBIASAIR=${COMIN_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}abias_air +export GRADSTAT=${COMIN_ATMOS_ANALYSIS_DET_PREV}/${GPREFIX_DET}radstat # Bias correction coefficients related to ensemble mean -export ABIAS="${COM_ATMOS_ANALYSIS}/${APREFIX}abias.ensmean" -export ABIASPC="${COM_ATMOS_ANALYSIS}/${APREFIX}abias_pc.ensmean" -export ABIASAIR="${COM_ATMOS_ANALYSIS}/${APREFIX}abias_air.ensmean" -export ABIASe="${COM_ATMOS_ANALYSIS}/${APREFIX}abias_int.ensmean" +export ABIAS="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias.ensmean" +export ABIASPC="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias_pc.ensmean" +export ABIASAIR="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias_air.ensmean" +export ABIASe="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}abias_int.ensmean" # Diagnostics related to ensemble mean -export GSISTAT="${COM_ATMOS_ANALYSIS}/${APREFIX}gsistat.ensmean" -export CNVSTAT="${COM_ATMOS_ANALYSIS}/${APREFIX}cnvstat.ensmean" -export OZNSTAT="${COM_ATMOS_ANALYSIS}/${APREFIX}oznstat.ensmean" -export RADSTAT="${COM_ATMOS_ANALYSIS}/${APREFIX}radstat.ensmean" +export GSISTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}gsistat.ensmean" +export CNVSTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}cnvstat.ensmean" +export OZNSTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}oznstat.ensmean" +export RADSTAT="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}radstat.ensmean" # Select observations based on ensemble mean export RUN_SELECT="YES" export USE_SELECT="NO" -export SELECT_OBS="${COM_ATMOS_ANALYSIS}/${APREFIX}obsinput.ensmean" +export SELECT_OBS="${COMOUT_ATMOS_ANALYSIS}/${APREFIX}obsinput.ensmean" export DIAG_SUFFIX="_ensmean" export DIAG_COMPRESS="NO" diff --git a/scripts/exglobal_diag.sh b/scripts/exglobal_diag.sh index ed9bef05df..46a6e9863c 100755 --- a/scripts/exglobal_diag.sh +++ b/scripts/exglobal_diag.sh @@ -49,10 +49,10 @@ SENDDBN=${SENDDBN:-"NO"} # Analysis files export APREFIX=${APREFIX:-""} -RADSTAT=${RADSTAT:-${COM_ATMOS_ANALYSIS}/${APREFIX}radstat} -PCPSTAT=${PCPSTAT:-${COM_ATMOS_ANALYSIS}/${APREFIX}pcpstat} -CNVSTAT=${CNVSTAT:-${COM_ATMOS_ANALYSIS}/${APREFIX}cnvstat} -OZNSTAT=${OZNSTAT:-${COM_ATMOS_ANALYSIS}/${APREFIX}oznstat} +RADSTAT=${RADSTAT:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}radstat} +PCPSTAT=${PCPSTAT:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}pcpstat} +CNVSTAT=${CNVSTAT:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}cnvstat} +OZNSTAT=${OZNSTAT:-${COMOUT_ATMOS_ANALYSIS}/${APREFIX}oznstat} # Remove stat file if file already exists [[ -s $RADSTAT ]] && rm -f $RADSTAT @@ -74,7 +74,7 @@ nm="" if [ $CFP_MP = "YES" ]; then nm=0 fi -DIAG_DIR=${DIAG_DIR:-${COM_ATMOS_ANALYSIS}/gsidiags} +DIAG_DIR=${DIAG_DIR:-${COMOUT_ATMOS_ANALYSIS}/gsidiags} REMOVE_DIAG_DIR=${REMOVE_DIAG_DIR:-"NO"} # Set script / GSI control parameters From 1cc407805b48cc2f0fbcc2e65cc960d3688b6161 Mon Sep 17 00:00:00 2001 From: Walter Kolczynski - NOAA Date: Thu, 31 Oct 2024 04:11:23 -0400 Subject: [PATCH 3/4] Make wxflow links static instead of from link_workflow (#3008) Commits symlinks to the repo for wxflow instead of relying on link_workflow to create them. This will allow testing in the ci or workflow directory without needing to run an otherwise unnecessary link_workflow first. --- .gitignore | 5 ----- ci/scripts/wxflow | 1 + sorc/link_workflow.sh | 18 ------------------ ush/python/wxflow | 1 + workflow/wxflow | 1 + 5 files changed, 3 insertions(+), 23 deletions(-) create mode 120000 ci/scripts/wxflow create mode 120000 ush/python/wxflow create mode 120000 workflow/wxflow diff --git a/.gitignore b/.gitignore index 8fc6d0b20b..4ec62993d3 100644 --- a/.gitignore +++ b/.gitignore @@ -171,11 +171,6 @@ ush/bufr2ioda_insitu* versions/build.ver versions/run.ver -# wxflow checkout and symlinks -ush/python/wxflow -workflow/wxflow -ci/scripts/wxflow - # jcb checkout and symlinks ush/python/jcb workflow/jcb diff --git a/ci/scripts/wxflow b/ci/scripts/wxflow new file mode 120000 index 0000000000..9dbee42bc8 --- /dev/null +++ b/ci/scripts/wxflow @@ -0,0 +1 @@ +../../sorc/wxflow/src/wxflow \ No newline at end of file diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 870ddc5eba..3d81f7b7d4 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -86,15 +86,6 @@ esac # Source fix version file source "${HOMEgfs}/versions/fix.ver" -# Link python pacakges in ush/python -# TODO: This will be unnecessary when these are part of the virtualenv -packages=("wxflow") -for package in "${packages[@]}"; do - cd "${HOMEgfs}/ush/python" || exit 1 - [[ -s "${package}" ]] && rm -f "${package}" - ${LINK} "${HOMEgfs}/sorc/${package}/src/${package}" . -done - # Link GDASapp python packages in ush/python packages=("jcb") for package in "${packages[@]}"; do @@ -103,15 +94,6 @@ for package in "${packages[@]}"; do ${LINK} "${HOMEgfs}/sorc/gdas.cd/sorc/${package}/src/${package}" . done -# Link wxflow in workflow and ci/scripts -# TODO: This will be unnecessary when wxflow is part of the virtualenv -cd "${HOMEgfs}/workflow" || exit 1 -[[ -s "wxflow" ]] && rm -f wxflow -${LINK} "${HOMEgfs}/sorc/wxflow/src/wxflow" . -cd "${HOMEgfs}/ci/scripts" || exit 1 -[[ -s "wxflow" ]] && rm -f wxflow -${LINK} "${HOMEgfs}/sorc/wxflow/src/wxflow" . - # Link fix directories if [[ -n "${FIX_DIR}" ]]; then if [[ ! -d "${HOMEgfs}/fix" ]]; then mkdir "${HOMEgfs}/fix" || exit 1; fi diff --git a/ush/python/wxflow b/ush/python/wxflow new file mode 120000 index 0000000000..9dbee42bc8 --- /dev/null +++ b/ush/python/wxflow @@ -0,0 +1 @@ +../../sorc/wxflow/src/wxflow \ No newline at end of file diff --git a/workflow/wxflow b/workflow/wxflow new file mode 120000 index 0000000000..7ea96a12bf --- /dev/null +++ b/workflow/wxflow @@ -0,0 +1 @@ +../sorc/wxflow/src/wxflow \ No newline at end of file From 5a8a5aa13b0143c871dc466e0ed062c55c7cd573 Mon Sep 17 00:00:00 2001 From: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> Date: Thu, 31 Oct 2024 12:10:07 -0400 Subject: [PATCH 4/4] Fix the name of the TC tracker filenames in archive.py (#3030) This corrects the names of the product files created by the `tracker` job when attempting to rename the experiment and push the file to the `ARCDIR` within the `arch` job. --- ush/python/pygfs/task/archive.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/ush/python/pygfs/task/archive.py b/ush/python/pygfs/task/archive.py index d138474e9a..108cd2ed27 100644 --- a/ush/python/pygfs/task/archive.py +++ b/ush/python/pygfs/task/archive.py @@ -88,11 +88,6 @@ def configure(self, arch_dict: Dict[str, Any]) -> (Dict[str, Any], List[Dict[str if not os.path.isdir(arch_dict.ROTDIR): raise FileNotFoundError(f"FATAL ERROR: The ROTDIR ({arch_dict.ROTDIR}) does not exist!") - if arch_dict.RUN in ["gdas", "gfs"]: - - # Copy the cyclone track files and rename the experiments - Archive._rename_cyclone_expt(arch_dict) - # Collect datasets that need to be archived # Each dataset represents one tarball @@ -371,14 +366,14 @@ def _rename_cyclone_expt(arch_dict) -> None: if run == "gfs": in_track_file = (track_dir_in + "/avno.t" + - cycle_HH + "z.cycle.trackatcfunix") + cycle_HH + "z.cyclone.trackatcfunix") in_track_p_file = (track_dir_in + "/avnop.t" + - cycle_HH + "z.cycle.trackatcfunixp") + cycle_HH + "z.cyclone.trackatcfunix") elif run == "gdas": in_track_file = (track_dir_in + "/gdas.t" + - cycle_HH + "z.cycle.trackatcfunix") + cycle_HH + "z.cyclone.trackatcfunix") in_track_p_file = (track_dir_in + "/gdasp.t" + - cycle_HH + "z.cycle.trackatcfunixp") + cycle_HH + "z.cyclone.trackatcfunix") if not os.path.isfile(in_track_file): # Do not attempt to archive the outputs @@ -416,7 +411,7 @@ def replace_string_from_to_file(filename_in, filename_out, search_str, replace_s with open("/tmp/track_file", "w") as new_file: new_file.writelines(out_lines) - shutil.move("tmp/track_file", filename_out) + shutil.move("/tmp/track_file", filename_out) replace_string_from_to_file(in_track_file, out_track_file, "AVNO", pslot4) replace_string_from_to_file(in_track_p_file, out_track_p_file, "AVNO", pslot4)