From 65a7ab75dc0e4baba06a02e11ed0455787056a68 Mon Sep 17 00:00:00 2001 From: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> Date: Tue, 23 Jul 2024 08:35:48 -0400 Subject: [PATCH] Replace Jinja namespaces with replace_tmpl filter and disable ACCOUNT_SERVICE option (#2775) Removes the namespace-based construction of EnKF member COM directories in the enkf archive template. --- .github/workflows/ci_unit_tests.yaml | 2 +- parm/archive/master_enkf.yaml.j2 | 40 +++++++++------------------- workflow/hosts/awspw.yaml | 1 - workflow/hosts/container.yaml | 1 - workflow/hosts/gaea.yaml | 1 - workflow/hosts/hera.yaml | 1 - workflow/hosts/hercules.yaml | 1 - workflow/hosts/jet.yaml | 1 - workflow/hosts/orion.yaml | 1 - workflow/hosts/s4.yaml | 1 - workflow/hosts/wcoss2.yaml | 1 - workflow/rocoto/tasks.py | 2 +- 12 files changed, 15 insertions(+), 38 deletions(-) diff --git a/.github/workflows/ci_unit_tests.yaml b/.github/workflows/ci_unit_tests.yaml index e22f63bf56..6dbc7ee52c 100644 --- a/.github/workflows/ci_unit_tests.yaml +++ b/.github/workflows/ci_unit_tests.yaml @@ -47,7 +47,7 @@ jobs: run: | sudo mkdir -p /scratch1/NCEPDEV cd $GITHUB_WORKSPACE/sorc - git submodule update --init --recursive + git submodule update --init ./link_workflow.sh cd $GITHUB_WORKSPACE/ci/scripts/tests ln -s ../wxflow diff --git a/parm/archive/master_enkf.yaml.j2 b/parm/archive/master_enkf.yaml.j2 index 70f8a2ad89..3ebd52dbad 100644 --- a/parm/archive/master_enkf.yaml.j2 +++ b/parm/archive/master_enkf.yaml.j2 @@ -52,35 +52,21 @@ datasets: {% for mem in range(first_group_mem, last_group_mem + 1) %} # Declare a dict of search and replace terms to run on each template - {% set tmpl_dict = {'ROTDIR':ROTDIR, - 'RUN':RUN, - 'YMD':cycle_YMD, - 'HH':cycle_HH, - 'MEMDIR':"mem" + '%03d' % mem} %} - - # Replace template variables with tmpl_dict, one key at a time - # This must be done in a namespace to overcome jinja scoping - # Variables set inside of a for loop are lost at the end of the loop - # unless they are part of a namespace - {% set com_ns = namespace(COMIN_ATMOS_ANALYSIS_MEM = COM_ATMOS_ANALYSIS_TMPL, - COMIN_ATMOS_HISTORY_MEM = COM_ATMOS_HISTORY_TMPL, - COMIN_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_TMPL) %} - - {% for key in tmpl_dict.keys() %} - {% set search_term = '${' + key + '}' %} - {% set replace_term = tmpl_dict[key] %} - {% set com_ns.COMIN_ATMOS_ANALYSIS_MEM = - com_ns.COMIN_ATMOS_ANALYSIS_MEM.replace(search_term, replace_term) %} - {% set com_ns.COMIN_ATMOS_HISTORY_MEM = - com_ns.COMIN_ATMOS_HISTORY_MEM.replace(search_term, replace_term) %} - {% set com_ns.COMIN_ATMOS_RESTART_MEM = - com_ns.COMIN_ATMOS_RESTART_MEM.replace(search_term, replace_term) %} - {% endfor %} + {% set mem_char = 'mem%03d' | format(mem) %} + {% set tmpl_dict = ({ '${ROTDIR}':ROTDIR, + '${RUN}':RUN, + '${YMD}':cycle_YMD, + '${HH}':cycle_HH, + '${MEMDIR}': mem_char }) %} + + {% set COMIN_ATMOS_ANALYSIS_MEM = COM_ATMOS_ANALYSIS_TMPL | replace_tmpl(tmpl_dict) %} + {% set COMIN_ATMOS_HISTORY_MEM = COM_ATMOS_HISTORY_TMPL | replace_tmpl(tmpl_dict) %} + {% set COMIN_ATMOS_RESTART_MEM = COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) %} # Append the member COM directories - {% do COMIN_ATMOS_ANALYSIS_MEM_list.append(com_ns.COMIN_ATMOS_ANALYSIS_MEM)%} - {% do COMIN_ATMOS_HISTORY_MEM_list.append(com_ns.COMIN_ATMOS_HISTORY_MEM)%} - {% do COMIN_ATMOS_RESTART_MEM_list.append(com_ns.COMIN_ATMOS_RESTART_MEM)%} + {% do COMIN_ATMOS_ANALYSIS_MEM_list.append(COMIN_ATMOS_ANALYSIS_MEM)%} + {% do COMIN_ATMOS_HISTORY_MEM_list.append(COMIN_ATMOS_HISTORY_MEM)%} + {% do COMIN_ATMOS_RESTART_MEM_list.append(COMIN_ATMOS_RESTART_MEM)%} {% endfor %} diff --git a/workflow/hosts/awspw.yaml b/workflow/hosts/awspw.yaml index b7021a6e3f..046dafcfa7 100644 --- a/workflow/hosts/awspw.yaml +++ b/workflow/hosts/awspw.yaml @@ -7,7 +7,6 @@ STMP: '/lustre/${USER}/stmp2/' PTMP: '/lustre/${USER}/stmp4/' NOSCRUB: ${HOMEDIR} ACCOUNT: hwufscpldcld -ACCOUNT_SERVICE: hwufscpldcld SCHEDULER: slurm QUEUE: batch QUEUE_SERVICE: batch diff --git a/workflow/hosts/container.yaml b/workflow/hosts/container.yaml index 907f69754e..d7924724ae 100644 --- a/workflow/hosts/container.yaml +++ b/workflow/hosts/container.yaml @@ -8,7 +8,6 @@ PTMP: '/home/${USER}' NOSCRUB: $HOMEDIR SCHEDULER: none ACCOUNT: '' -ACCOUNT_SERVICE: '' QUEUE: '' QUEUE_SERVICE: '' PARTITION_BATCH: '' diff --git a/workflow/hosts/gaea.yaml b/workflow/hosts/gaea.yaml index ff9877e77b..619a86f2e5 100644 --- a/workflow/hosts/gaea.yaml +++ b/workflow/hosts/gaea.yaml @@ -9,7 +9,6 @@ STMP: '/gpfs/f5/ufs-ard/scratch/${USER}' PTMP: '/gpfs/f5/ufs-ard/scratch/${USER}' NOSCRUB: $HOMEDIR ACCOUNT: ufs-ard -ACCOUNT_SERVICE: ufs-ard SCHEDULER: slurm QUEUE: normal QUEUE_SERVICE: normal diff --git a/workflow/hosts/hera.yaml b/workflow/hosts/hera.yaml index 76a7158f43..731e583961 100644 --- a/workflow/hosts/hera.yaml +++ b/workflow/hosts/hera.yaml @@ -8,7 +8,6 @@ STMP: '/scratch1/NCEPDEV/stmp2/${USER}' PTMP: '/scratch1/NCEPDEV/stmp4/${USER}' NOSCRUB: $HOMEDIR ACCOUNT: fv3-cpu -ACCOUNT_SERVICE: fv3-cpu SCHEDULER: slurm QUEUE: batch QUEUE_SERVICE: batch diff --git a/workflow/hosts/hercules.yaml b/workflow/hosts/hercules.yaml index 975558160f..b513bfd57a 100644 --- a/workflow/hosts/hercules.yaml +++ b/workflow/hosts/hercules.yaml @@ -9,7 +9,6 @@ PTMP: '/work/noaa/stmp/${USER}/HERCULES' NOSCRUB: $HOMEDIR SCHEDULER: slurm ACCOUNT: fv3-cpu -ACCOUNT_SERVICE: fv3-cpu QUEUE: batch QUEUE_SERVICE: batch PARTITION_BATCH: hercules diff --git a/workflow/hosts/jet.yaml b/workflow/hosts/jet.yaml index b526e073c3..ae7267d687 100644 --- a/workflow/hosts/jet.yaml +++ b/workflow/hosts/jet.yaml @@ -8,7 +8,6 @@ STMP: '/lfs4/HFIP/hfv3gfs/${USER}/stmp' PTMP: '/lfs4/HFIP/hfv3gfs/${USER}/ptmp' NOSCRUB: $HOMEDIR ACCOUNT: hfv3gfs -ACCOUNT_SERVICE: hfv3gfs SCHEDULER: slurm QUEUE: batch QUEUE_SERVICE: batch diff --git a/workflow/hosts/orion.yaml b/workflow/hosts/orion.yaml index fe36c8e7ce..f0f807aacf 100644 --- a/workflow/hosts/orion.yaml +++ b/workflow/hosts/orion.yaml @@ -9,7 +9,6 @@ PTMP: '/work/noaa/stmp/${USER}/ORION' NOSCRUB: $HOMEDIR SCHEDULER: slurm ACCOUNT: fv3-cpu -ACCOUNT_SERVICE: fv3-cpu QUEUE: batch QUEUE_SERVICE: batch PARTITION_BATCH: orion diff --git a/workflow/hosts/s4.yaml b/workflow/hosts/s4.yaml index 37479fa13c..aea807da63 100644 --- a/workflow/hosts/s4.yaml +++ b/workflow/hosts/s4.yaml @@ -8,7 +8,6 @@ STMP: '/scratch/users/${USER}' PTMP: '/scratch/users/${USER}' NOSCRUB: ${HOMEDIR} ACCOUNT: star -ACCOUNT_SERVICE: star SCHEDULER: slurm QUEUE: s4 QUEUE_SERVICE: serial diff --git a/workflow/hosts/wcoss2.yaml b/workflow/hosts/wcoss2.yaml index e3650e4710..7ae2be1424 100644 --- a/workflow/hosts/wcoss2.yaml +++ b/workflow/hosts/wcoss2.yaml @@ -8,7 +8,6 @@ STMP: '/lfs/h2/emc/stmp/${USER}' PTMP: '/lfs/h2/emc/ptmp/${USER}' NOSCRUB: $HOMEDIR ACCOUNT: 'GFS-DEV' -ACCOUNT_SERVICE: 'GFS-DEV' SCHEDULER: pbspro QUEUE: 'dev' QUEUE_SERVICE: 'dev_transfer' diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index 097d1adef5..404203f02d 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -176,7 +176,7 @@ def get_resource(self, task_name): task_config = self._configs[task_name] - account = task_config['ACCOUNT_SERVICE'] if task_name in Tasks.SERVICE_TASKS else task_config['ACCOUNT'] + account = task_config['ACCOUNT'] if f'wtime_{task_name}_{self.cdump}' in task_config: walltime = task_config[f'wtime_{task_name}_{self.cdump}']