Simplify resource-related variables, remove CDUMP where unneeded (NOA…

…A-EMC#2727) This overhauls resource-related variables to use a common set of variables for each job. In the process, this also removed the use of CDUMP in most cases. Resolves NOAA-EMC#1299 NOAA-EMC#2693
AntonMFernando-NOAA · Jul 30, 2024 · 1cf8b44 · 1cf8b44
1 parent 61875f2
commit 1cf8b44
Show file tree

Hide file tree

Showing 138 changed files with 2,100 additions and 2,669 deletions.
diff --git a/ci/scripts/tests/test_rocotostat.py b/ci/scripts/tests/test_rocotostat.py
@@ -25,17 +25,16 @@
     database_destination = os.path.join(testdata_full_path, 'database.db')
     wget.download(database_url, database_destination)
 
-try:
-    rocotostat = which('rocotostat')
-except CommandNotFoundError:
+rocotostat_cmd = which('rocotostat')
+if not rocotostat_cmd:
     raise CommandNotFoundError("rocotostat not found in PATH")
 
-rocotostat.add_default_arg(['-w', os.path.join(testdata_path, 'workflow.xml'), '-d', os.path.join(testdata_path, 'database.db')])
+rocotostat_cmd.add_default_arg(['-w', os.path.join(testdata_path, 'workflow.xml'), '-d', os.path.join(testdata_path, 'database.db')])
 
 
 def test_rocoto_statcount():
 
-    result = rocoto_statcount(rocotostat)
+    result = rocoto_statcount(rocotostat_cmd)
 
     assert result['SUCCEEDED'] == 20
     assert result['FAIL'] == 0
@@ -47,15 +46,15 @@ def test_rocoto_statcount():
 
 def test_rocoto_summary():
 
-    result = rocotostat_summary(rocotostat)
+    result = rocotostat_summary(rocotostat_cmd)
 
     assert result['CYCLES_TOTAL'] == 1
     assert result['CYCLES_DONE'] == 1
 
 
 def test_rocoto_done():
 
-    result = rocotostat_summary(rocotostat)
+    result = rocotostat_summary(rocotostat_cmd)
 
     assert is_done(result)
 
@@ -79,10 +78,10 @@ def test_rocoto_stalled():
         database_destination = os.path.join(testdata_full_path, 'stalled.db')
         wget.download(database_url, database_destination)
 
-    rocotostat = which('rocotostat')
-    rocotostat.add_default_arg(['-w', xml, '-d', db])
+    rocotostat_cmd = which('rocotostat')
+    rocotostat_cmd.add_default_arg(['-w', xml, '-d', db])
 
-    result = rocoto_statcount(rocotostat)
+    result = rocoto_statcount(rocotostat_cmd)
 
     assert result['SUCCEEDED'] == 11
     assert is_stalled(result)

diff --git a/docs/doxygen/mainpage.h b/docs/doxygen/mainpage.h
@@ -24,7 +24,7 @@ To setup an experiment, a python script <b>\c setup_expt.py</b> (located in <b>\
     usage: setup_expt.py [-h] --pslot PSLOT
                      [--configdir CONFIGDIR] [--idate IDATE] [--icsdir ICSDIR]
                      [--resdetatmos RESDET] [--resensatmos RESENS] [--comroot COMROOT]
-                     [--expdir EXPDIR] [--nens NENS] [--cdump CDUMP]
+                     [--expdir EXPDIR] [--nens NENS] [--run RUN]
 
     Setup files and directories to start a GFS parallel. Create EXPDIR, copy
     config files Create ROTDIR experiment directory structure, link initial
@@ -52,7 +52,7 @@ To setup an experiment, a python script <b>\c setup_expt.py</b> (located in <b>\
                         (default: None)
         --nens       number of ensemble members
                         (default: 80)
-        --cdump      CDUMP to start the experiment
+        --run        RUN to start the experiment
                         (default: gdas)
 
 The above script creates directories <b>\c EXPDIR</b> and <b>\c ROTDIR</b>. It will make links for initial conditions from a location provided via the <b>\c --icsdir</b> argument for a chosen resolution for the control <b>\c --resdetatmos</b> and the ensemble <b>\c --resensatmos</b>. Experiment name is controlled by the input argument <b>\c --pslot</b>. The script will ask user input in case any of the directories already exist. It will copy experiment configuration files into the <b>\c EXPDIR</b> from <b>\c CONFIGDIR</b>.

diff --git a/docs/source/init.rst b/docs/source/init.rst
@@ -384,7 +384,7 @@ The warm starts and other output from production are at C768 deterministic and C
 What files should you pull for starting a new experiment with warm starts from production?
 ------------------------------------------------------------------------------------------
 
-That depends on what mode you want to run -- forecast-only or cycled. Whichever mode, navigate to the top of your ``ROTDIR`` and pull the entirety of the tarball(s) listed below for your mode. The files within the tarball are already in the ``$CDUMP.$PDY/$CYC/$ATMOS`` folder format expected by the system.
+That depends on what mode you want to run -- forecast-only or cycled. Whichever mode, navigate to the top of your ``ROTDIR`` and pull the entirety of the tarball(s) listed below for your mode. The files within the tarball are already in the ``$RUN.$PDY/$CYC/$ATMOS`` folder format expected by the system.
 
 For forecast-only there are two tarballs to pull
 

diff --git a/docs/source/setup.rst b/docs/source/setup.rst
@@ -145,7 +145,7 @@ The following command examples include variables for reference but users should
 
    cd workflow
    ./setup_expt.py gfs cycled --idate $IDATE --edate $EDATE [--app $APP] [--start $START] [--gfs_cyc $GFS_CYC]
-     [--resdetatmos $RESDETATMOS] [--resdetocean $RESDETOCEAN] [--resensatmos $RESENSATMOS] [--nens $NENS] [--cdump $CDUMP]
+     [--resdetatmos $RESDETATMOS] [--resdetocean $RESDETOCEAN] [--resensatmos $RESENSATMOS] [--nens $NENS] [--run $RUN]
      [--pslot $PSLOT] [--configdir $CONFIGDIR] [--comroot $COMROOT] [--expdir $EXPDIR] [--icsdir $ICSDIR]
 
 where:
@@ -170,7 +170,7 @@ where:
    * ``$RESDETOCEAN`` is the resolution of the ocean component of the deterministic forecast [default: 0.; determined based on atmosphere resolution]
    * ``$RESENSATMOS`` is the resolution of the atmosphere component of the ensemble forecast [default: 192]
    * ``$NENS`` is the number of ensemble members [default: 20]
-   * ``$CDUMP`` is the starting phase [default: gdas]
+   * ``$RUN`` is the starting phase [default: gdas]
    * ``$PSLOT`` is the name of your experiment [default: test]
    * ``$CONFIGDIR`` is the path to the config folder under the copy of the system you're using [default: $TOP_OF_CLONE/parm/config/]
    * ``$COMROOT`` is the path to your experiment output directory. Your ``ROTDIR`` (rotating com directory) will be created using ``COMROOT`` and ``PSLOT``. [default: $HOME]

diff --git a/ecf/scripts/enkfgdas/analysis/create/jenkfgdas_diag.ecf b/ecf/scripts/enkfgdas/analysis/create/jenkfgdas_diag.ecf
@@ -16,7 +16,6 @@ set -x
 
 export NET=%NET:gfs%
 export RUN=%RUN%
-export CDUMP=%RUN%
 
 ############################################################
 # Load modules

diff --git a/ecf/scripts/enkfgdas/analysis/create/jenkfgdas_select_obs.ecf b/ecf/scripts/enkfgdas/analysis/create/jenkfgdas_select_obs.ecf
@@ -16,7 +16,6 @@ set -x
 
 export NET=%NET:gfs%
 export RUN=%RUN%
-export CDUMP=%RUN%
 
 ############################################################
 # Load modules

diff --git a/ecf/scripts/enkfgdas/analysis/create/jenkfgdas_update.ecf b/ecf/scripts/enkfgdas/analysis/create/jenkfgdas_update.ecf
@@ -16,7 +16,6 @@ set -x
 
 export NET=%NET:gfs%
 export RUN=%RUN%
-export CDUMP=%RUN%
 
 ############################################################
 # Load modules

diff --git a/ecf/scripts/enkfgdas/analysis/recenter/ecen/jenkfgdas_ecen.ecf b/ecf/scripts/enkfgdas/analysis/recenter/ecen/jenkfgdas_ecen.ecf
@@ -16,7 +16,6 @@ set -x
 
 export NET=%NET:gfs%
 export RUN=%RUN%
-export CDUMP=%RUN%
 
 ############################################################
 # Load modules

diff --git a/ecf/scripts/enkfgdas/analysis/recenter/jenkfgdas_sfc.ecf b/ecf/scripts/enkfgdas/analysis/recenter/jenkfgdas_sfc.ecf
@@ -16,7 +16,6 @@ set -x
 
 export NET=%NET:gfs%
 export RUN=%RUN%
-export CDUMP=%RUN%
 
 ############################################################
 # Load modules

diff --git a/ecf/scripts/enkfgdas/forecast/jenkfgdas_fcst.ecf b/ecf/scripts/enkfgdas/forecast/jenkfgdas_fcst.ecf
@@ -16,7 +16,6 @@ set -x
 
 export NET=%NET:gfs%
 export RUN=%RUN%
-export CDUMP=%RUN%
 
 ############################################################
 # Load modules

diff --git a/ecf/scripts/enkfgdas/post/jenkfgdas_post_master.ecf b/ecf/scripts/enkfgdas/post/jenkfgdas_post_master.ecf
@@ -16,7 +16,6 @@ set -x
 
 export NET=%NET:gfs%
 export RUN=%RUN%
-export CDUMP=%RUN%
 
 ############################################################
 # Load modules

diff --git a/ecf/scripts/gdas/atmos/analysis/jgdas_atmos_analysis.ecf b/ecf/scripts/gdas/atmos/analysis/jgdas_atmos_analysis.ecf
@@ -16,7 +16,6 @@ set -x
 
 export NET=%NET:gfs%
 export RUN=%RUN%
-export CDUMP=%RUN%
 
 ############################################################
 # Load modules

diff --git a/ecf/scripts/gdas/atmos/analysis/jgdas_atmos_analysis_calc.ecf b/ecf/scripts/gdas/atmos/analysis/jgdas_atmos_analysis_calc.ecf
@@ -16,7 +16,6 @@ set -x
 
 export NET=%NET:gfs%
 export RUN=%RUN%
-export CDUMP=%RUN%
 
 ############################################################
 # Load modules

diff --git a/ecf/scripts/gdas/atmos/analysis/jgdas_atmos_analysis_diag.ecf b/ecf/scripts/gdas/atmos/analysis/jgdas_atmos_analysis_diag.ecf
@@ -16,7 +16,6 @@ set -x
 
 export NET=%NET:gfs%
 export RUN=%RUN%
-export CDUMP=%RUN%
 
 ############################################################
 # Load modules

diff --git a/ecf/scripts/gdas/atmos/gempak/jgdas_atmos_gempak.ecf b/ecf/scripts/gdas/atmos/gempak/jgdas_atmos_gempak.ecf
@@ -16,7 +16,6 @@ set -x
 
 export NET=%NET:gfs%
 export RUN=%RUN%
-export CDUMP=%RUN%
 
 ############################################################
 # Load modules

diff --git a/ecf/scripts/gdas/atmos/gempak/jgdas_atmos_gempak_meta_ncdc.ecf b/ecf/scripts/gdas/atmos/gempak/jgdas_atmos_gempak_meta_ncdc.ecf
@@ -17,7 +17,6 @@ set -x
 export model=%model:gdas%
 export NET=%NET:gfs%
 export RUN=%RUN%
-export CDUMP=%RUN%
 
 ############################################################
 # Load modules

diff --git a/ecf/scripts/gdas/atmos/obsproc/dump/jgdas_atmos_tropcy_qc_reloc.ecf b/ecf/scripts/gdas/atmos/obsproc/dump/jgdas_atmos_tropcy_qc_reloc.ecf
@@ -16,7 +16,6 @@ set -x
 
 export NET=%NET:gfs%
 export RUN=%RUN%
-export CDUMP=%RUN%
 
 ############################################################
 # Load modules

diff --git a/ecf/scripts/gdas/atmos/obsproc/prep/jgdas_atmos_emcsfc_sfc_prep.ecf b/ecf/scripts/gdas/atmos/obsproc/prep/jgdas_atmos_emcsfc_sfc_prep.ecf
@@ -16,7 +16,6 @@ set -x
 
 export NET=%NET:gfs%
 export RUN=%RUN%
-export CDUMP=%RUN%
 
 ############################################################
 # Load modules

diff --git a/ecf/scripts/gdas/atmos/post/jgdas_atmos_post_manager.ecf b/ecf/scripts/gdas/atmos/post/jgdas_atmos_post_manager.ecf
@@ -16,7 +16,6 @@ set -x
 
 export NET=%NET:gfs%
 export RUN=%RUN%
-export CDUMP=%RUN%
 
 ############################################################
 # Load modules

diff --git a/ecf/scripts/gdas/atmos/post/jgdas_atmos_post_master.ecf b/ecf/scripts/gdas/atmos/post/jgdas_atmos_post_master.ecf
@@ -16,7 +16,6 @@ set -x
 
 export NET=%NET:gfs%
 export RUN=%RUN%
-export CDUMP=%RUN%
 
 ############################################################
 # Load modules

diff --git a/ecf/scripts/gdas/atmos/post_processing/jgdas_atmos_chgres_forenkf.ecf b/ecf/scripts/gdas/atmos/post_processing/jgdas_atmos_chgres_forenkf.ecf
@@ -16,7 +16,6 @@ set -x
 
 export NET=%NET:gfs%
 export RUN=%RUN%
-export CDUMP=%RUN%
 
 ############################################################
 # Load modules

diff --git a/ecf/scripts/gdas/atmos/verf/jgdas_atmos_verfozn.ecf b/ecf/scripts/gdas/atmos/verf/jgdas_atmos_verfozn.ecf
@@ -16,7 +16,6 @@ set -x
 
 export NET=%NET:gfs%
 export RUN=%RUN%
-export CDUMP=%RUN%
 
 ############################################################
 # Load modules

diff --git a/ecf/scripts/gdas/atmos/verf/jgdas_atmos_verfrad.ecf b/ecf/scripts/gdas/atmos/verf/jgdas_atmos_verfrad.ecf
@@ -16,7 +16,6 @@ set -x
 
 export NET=%NET:gfs%
 export RUN=%RUN%
-export CDUMP=%RUN%
 
 ############################################################
 # Load modules

diff --git a/ecf/scripts/gdas/atmos/verf/jgdas_atmos_vminmon.ecf b/ecf/scripts/gdas/atmos/verf/jgdas_atmos_vminmon.ecf
@@ -16,7 +16,6 @@ set -x
 
 export NET=%NET:gfs%
 export RUN=%RUN%
-export CDUMP=%RUN%
 
 ############################################################
 # Load modules

diff --git a/ecf/scripts/gdas/jgdas_forecast.ecf b/ecf/scripts/gdas/jgdas_forecast.ecf
@@ -16,7 +16,6 @@ set -x
 
 export NET=%NET:gfs%
 export RUN=%RUN%
-export CDUMP=%RUN%
 
 ############################################################
 # Load modules

diff --git a/ecf/scripts/gdas/wave/init/jgdas_wave_init.ecf b/ecf/scripts/gdas/wave/init/jgdas_wave_init.ecf
@@ -16,7 +16,6 @@ set -x
 
 export NET=%NET:gfs%
 export RUN=%RUN%
-export CDUMP=%RUN%
 
 ############################################################
 ## Load modules

diff --git a/ecf/scripts/gdas/wave/post/jgdas_wave_postpnt.ecf b/ecf/scripts/gdas/wave/post/jgdas_wave_postpnt.ecf
@@ -16,7 +16,6 @@ set -x
 
 export NET=%NET:gfs%
 export RUN=%RUN%
-export CDUMP=%RUN%
 
 ############################################################
 ## Load modules

diff --git a/ecf/scripts/gdas/wave/post/jgdas_wave_postsbs.ecf b/ecf/scripts/gdas/wave/post/jgdas_wave_postsbs.ecf
@@ -16,7 +16,6 @@ set -x
 
 export NET=%NET:gfs%
 export RUN=%RUN%
-export CDUMP=%RUN%
 
 ############################################################
 ## Load modules

diff --git a/ecf/scripts/gdas/wave/prep/jgdas_wave_prep.ecf b/ecf/scripts/gdas/wave/prep/jgdas_wave_prep.ecf
@@ -16,7 +16,6 @@ set -x
 
 export NET=%NET:gfs%
 export RUN=%RUN%
-export CDUMP=%RUN%
 
 ############################################################
 ## Load modules

diff --git a/ecf/scripts/gfs/atmos/analysis/jgfs_atmos_analysis.ecf b/ecf/scripts/gfs/atmos/analysis/jgfs_atmos_analysis.ecf
@@ -16,7 +16,6 @@ set -x
 
 export NET=%NET:gfs%
 export RUN=%RUN%
-export CDUMP=%RUN%
 
 ############################################################
 # Load modules

diff --git a/ecf/scripts/gfs/atmos/analysis/jgfs_atmos_analysis_calc.ecf b/ecf/scripts/gfs/atmos/analysis/jgfs_atmos_analysis_calc.ecf
@@ -16,7 +16,6 @@ set -x
 
 export NET=%NET:gfs%
 export RUN=%RUN%
-export CDUMP=%RUN%
 
 ############################################################
 # Load modules

diff --git a/ecf/scripts/gfs/atmos/gempak/jgfs_atmos_gempak.ecf b/ecf/scripts/gfs/atmos/gempak/jgfs_atmos_gempak.ecf
@@ -16,7 +16,6 @@ set -x
 
 export NET=%NET:gfs%
 export RUN=%RUN%
-export CDUMP=%RUN%
 
 ############################################################
 # Load modules

diff --git a/ecf/scripts/gfs/atmos/gempak/jgfs_atmos_gempak_meta.ecf b/ecf/scripts/gfs/atmos/gempak/jgfs_atmos_gempak_meta.ecf
@@ -16,7 +16,6 @@ set -x
 
 export NET=%NET:gfs%
 export RUN=%RUN%
-export CDUMP=%RUN%
 
 ############################################################
 # Load modules

diff --git a/ecf/scripts/gfs/atmos/gempak/jgfs_atmos_gempak_ncdc_upapgif.ecf b/ecf/scripts/gfs/atmos/gempak/jgfs_atmos_gempak_ncdc_upapgif.ecf
@@ -16,7 +16,6 @@ set -x
 
 export NET=%NET:gfs%
 export RUN=%RUN%
-export CDUMP=%RUN%
 
 ############################################################
 # Load modules

diff --git a/ecf/scripts/gfs/atmos/gempak/jgfs_atmos_npoess_pgrb2_0p5deg.ecf b/ecf/scripts/gfs/atmos/gempak/jgfs_atmos_npoess_pgrb2_0p5deg.ecf
@@ -16,7 +16,6 @@ set -x
 
 export NET=%NET:gfs%
 export RUN=%RUN%
-export CDUMP=%RUN%
 
 ############################################################
 # Load modules

diff --git a/ecf/scripts/gfs/atmos/gempak/jgfs_atmos_pgrb2_spec_gempak.ecf b/ecf/scripts/gfs/atmos/gempak/jgfs_atmos_pgrb2_spec_gempak.ecf
@@ -16,7 +16,6 @@ set -x
 
 export NET=%NET:gfs%
 export RUN=%RUN%
-export CDUMP=%RUN%
 
 ############################################################
 # Load modules

diff --git a/ecf/scripts/gfs/atmos/obsproc/dump/jgfs_atmos_tropcy_qc_reloc.ecf b/ecf/scripts/gfs/atmos/obsproc/dump/jgfs_atmos_tropcy_qc_reloc.ecf
@@ -16,7 +16,6 @@ set -x
 
 export NET=%NET:gfs%
 export RUN=%RUN%
-export CDUMP=%RUN%
 
 ############################################################
 # Load modules

diff --git a/ecf/scripts/gfs/atmos/obsproc/prep/jgfs_atmos_emcsfc_sfc_prep.ecf b/ecf/scripts/gfs/atmos/obsproc/prep/jgfs_atmos_emcsfc_sfc_prep.ecf
@@ -16,7 +16,6 @@ set -x
 
 export NET=%NET:gfs%
 export RUN=%RUN%
-export CDUMP=%RUN%
 
 ############################################################
 # Load modules

diff --git a/ecf/scripts/gfs/atmos/post/jgfs_atmos_post_manager.ecf b/ecf/scripts/gfs/atmos/post/jgfs_atmos_post_manager.ecf
@@ -16,7 +16,6 @@ set -x
 
 export NET=%NET:gfs%
 export RUN=%RUN%
-export CDUMP=%RUN%
 
 ############################################################
 # Load modules

diff --git a/ecf/scripts/gfs/atmos/post/jgfs_atmos_post_master.ecf b/ecf/scripts/gfs/atmos/post/jgfs_atmos_post_master.ecf
@@ -16,7 +16,6 @@ set -x
 
 export NET=%NET:gfs%
 export RUN=%RUN%
-export CDUMP=%RUN%
 
 export FHRGRP=%FHRGRP%
 export FHRLST=%FHRLST%

diff --git a/ecf/scripts/gfs/atmos/post_processing/awips_20km_1p0/jgfs_atmos_awips_master.ecf b/ecf/scripts/gfs/atmos/post_processing/awips_20km_1p0/jgfs_atmos_awips_master.ecf
@@ -16,7 +16,6 @@ set -x
 
 export NET=%NET:gfs%
 export RUN=%RUN%
-export CDUMP=%RUN%
 
 ############################################################
 # Load modules