Simplify downstream atmos product generation scripts (#1822)

This PR came about attempting to understand how the atmos downstream products are
generated from the master grib2 files before they are updated to python-based scripts
and improved control.

This PR:
- removes explicit if blocks over the two downsets that largely repeats the same
operations and condenses into a loop
- introduces a generic `run_mpmd.sh` that is tested on Hera and WCOSS2 and can be
used elsewhere in the global-workflow.  Its input is a script that needs the instructions
to be executed, one-per-line.

No change in information content is expected from this update to the products.

This PR is necessary if the global-workflow team is to mange the post processing and
product generation scripts.

env/HERA.env

@@ -16,7 +16,7 @@ step=$1
 
 export npe_node_max=40
 export launcher="srun -l --export=ALL"
-export mpmd_opt="--multi-prog --output=${step}.%J.%t.out"
+export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out"
 
 # Configure MPI environment
 #export I_MPI_ADJUST_ALLREDUCE=5
@@ -215,6 +215,7 @@ elif [[ "${step}" = "post" ]]; then
     [[ ${NTHREADS_NP} -gt ${nth_max} ]] && export NTHREADS_NP=${nth_max}
     export APRUN_NP="${launcher} -n ${npe_post}"
 
+    export USE_CFP="YES"  # Use MPMD for downstream product generation on Hera
     export NTHREADS_DWN=${nth_dwn:-1}
     [[ ${NTHREADS_DWN} -gt ${nth_max} ]] && export NTHREADS_DWN=${nth_max}
     export APRUN_DWN="${launcher} -n ${npe_dwn}"
@@ -302,3 +303,4 @@ elif [[ "${step}" = "fit2obs" ]]; then
     export MPIRUN="${launcher} -n ${npe_fit2obs}"
 
 fi
+

env/JET.env

@@ -22,7 +22,7 @@ elif [[ "${PARTITION_BATCH}" = "kjet" ]]; then
   export npe_node_max=40
 fi
 export launcher="srun -l --epilog=/apps/local/bin/report-mem --export=ALL"
-export mpmd_opt="--multi-prog --output=${step}.%J.%t.out"
+export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out"
 
 # Configure MPI environment
 export OMP_STACKSIZE=2048000
@@ -201,6 +201,7 @@ elif [[ "${step}" = "post" ]]; then
     [[ ${NTHREADS_NP} -gt ${nth_max} ]] && export NTHREADS_NP=${nth_max}
     export APRUN_NP="${launcher} -n ${npe_post}"
 
+    export USE_CFP="YES"  # Use MPMD for downstream product generation on Jet
     export NTHREADS_DWN=${nth_dwn:-1}
     [[ ${NTHREADS_DWN} -gt ${nth_max} ]] && export NTHREADS_DWN=${nth_max}
     export APRUN_DWN="${launcher} -n ${npe_dwn}"

env/ORION.env

@@ -16,7 +16,7 @@ step=$1
 
 export npe_node_max=40
 export launcher="srun -l --export=ALL"
-export mpmd_opt="--multi-prog --output=${step}.%J.%t.out"
+export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out"
 
 # Configure MPI environment
 export MPI_BUFS_PER_PROC=2048
@@ -214,6 +214,7 @@ elif [[ "${step}" = "post" ]]; then
     [[ ${NTHREADS_NP} -gt ${nth_max} ]] && export NTHREADS_NP=${nth_max}
     export APRUN_NP="${launcher} -n ${npe_post}"
 
+    export USE_CFP="YES"  # Use MPMD for downstream product generation on Orion
     export NTHREADS_DWN=${nth_dwn:-1}
     [[ ${NTHREADS_DWN} -gt ${nth_max} ]] && export NTHREADS_DWN=${nth_max}
     export APRUN_DWN="${launcher} -n ${npe_dwn}"

env/S4.env

@@ -21,7 +21,7 @@ elif [[ ${PARTITION_BATCH} = "ivy" ]]; then
    export npe_node_max=20
 fi
 export launcher="srun -l --export=ALL"
-export mpmd_opt="--multi-prog --output=${step}.%J.%t.out"
+export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out"
 
 # Configure MPI environment
 export OMP_STACKSIZE=2048000
@@ -187,6 +187,7 @@ elif [[ "${step}" = "post" ]]; then
     [[ ${NTHREADS_NP} -gt ${nth_max} ]] && export NTHREADS_NP=${nth_max}
     export APRUN_NP="${launcher} -n ${npe_post}"
 
+    export USE_CFP="YES"  # Use MPMD for downstream product generation on S4
     export NTHREADS_DWN=${nth_dwn:-1}
     [[ ${NTHREADS_DWN} -gt ${nth_max} ]] && export NTHREADS_DWN=${nth_max}
     export APRUN_DWN="${launcher} -n ${npe_dwn}"

env/WCOSS2.env

@@ -198,6 +198,7 @@ elif [[ "${step}" = "post" ]]; then
     [[ ${NTHREADS_NP} -gt ${nth_max} ]] && export NTHREADS_NP=${nth_max}
     export APRUN_NP="${launcher} -n ${npe_np:-${npe_post}} -ppn ${npe_node_post} --cpu-bind depth --depth ${NTHREADS_NP}"
 
+    export USE_CFP="YES"  # Use MPMD for downstream product generation on WCOSS2
     export NTHREADS_DWN=${nth_dwn:-1}
     [[ ${NTHREADS_DWN} -gt ${nth_max} ]] && export NTHREADS_DWN=${nth_max}
     export APRUN_DWN="${launcher} -np ${npe_dwn} ${mpmd_opt}"

jobs/JGLOBAL_ATMOS_POST

@@ -51,7 +51,7 @@ fi
 for grid in '0p25' '0p50' '1p00'; do
   prod_dir="COM_ATMOS_GRIB_${grid}"
   GRID=${grid} YMD=${PDY} HH=${cyc} generate_com -rx "${prod_dir}:COM_ATMOS_GRIB_GRID_TMPL"
-  if [[ ! -d "${prod_dir}" ]]; then mkdir -m 775 -p "${!prod_dir}"; fi
+  if [[ ! -d "${!prod_dir}" ]]; then mkdir -m 775 -p "${!prod_dir}"; fi
 done
 
 if [ "${RUN}" = gfs ];then