Ref NOAA-EMC#45

Save work in progress.
EdwardSafford-NOAA · Sep 16, 2024 · 36549bb · 36549bb
1 parent b7a5424
commit 36549bb
Show file tree

Hide file tree

Showing 5 changed files with 251 additions and 324 deletions.
diff --git a/parm/templates/oznHoriz.yaml b/parm/templates/oznHoriz.yaml
@@ -7,12 +7,17 @@ datasets:
 
     type: MonDataSpace
     control_file:
-      - {{DATA}}/ozn_data/horiz/{{SENSOR}}_{{SAT}}.{{COMPONENT}}.ctl
+      - {{DATA}}/{{SENSOR}}_{{SAT}}.{{COMPONENT}}.ctl
+#     - {{DATA}}/ozn_data/horiz/{{SENSOR}}_{{SAT}}.{{COMPONENT}}.ctl
     filenames:
-      - {{DATA}}/ozn_data/horiz/{{SENSOR}}_{{SAT}}.{{COMPONENT}}.{{ PDATE | to_YMDH }}.ieee_d
-      - {{DATA}}/ozn_data/horiz/{{SENSOR}}_{{SAT}}.{{COMPONENT}}.{{ PDATEm6 | to_YMDH }}.ieee_d
-      - {{DATA}}/ozn_data/horiz/{{SENSOR}}_{{SAT}}.{{COMPONENT}}.{{ PDATEm12 | to_YMDH }}.ieee_d
-      - {{DATA}}/ozn_data/horiz/{{SENSOR}}_{{SAT}}.{{COMPONENT}}.{{ PDATEm18 | to_YMDH }}.ieee_d
+      - {{DATA}}/{{SENSOR}}_{{SAT}}.{{COMPONENT}}.{{ PDATE | to_YMDH }}.ieee_d
+      - {{DATA}}/{{SENSOR}}_{{SAT}}.{{COMPONENT}}.{{ PDATEm6 | to_YMDH }}.ieee_d
+      - {{DATA}}/{{SENSOR}}_{{SAT}}.{{COMPONENT}}.{{ PDATEm12 | to_YMDH }}.ieee_d
+      - {{DATA}}/{{SENSOR}}_{{SAT}}.{{COMPONENT}}.{{ PDATEm18 | to_YMDH }}.ieee_d
+#     - {{DATA}}/ozn_data/horiz/{{SENSOR}}_{{SAT}}.{{COMPONENT}}.{{ PDATE | to_YMDH }}.ieee_d
+#     - {{DATA}}/ozn_data/horiz/{{SENSOR}}_{{SAT}}.{{COMPONENT}}.{{ PDATEm6 | to_YMDH }}.ieee_d
+#     - {{DATA}}/ozn_data/horiz/{{SENSOR}}_{{SAT}}.{{COMPONENT}}.{{ PDATEm12 | to_YMDH }}.ieee_d
+#     - {{DATA}}/ozn_data/horiz/{{SENSOR}}_{{SAT}}.{{COMPONENT}}.{{ PDATEm18 | to_YMDH }}.ieee_d
     levels: {{LEVELS}}
     groups:
       - name: GsiIeee

diff --git a/scripts/exobsmon_plot.sh b/scripts/exobsmon_plot.sh
@@ -19,6 +19,12 @@ if [[ ! -e ${chan_yaml} ]]; then
    exit 2
 fi
 
+#-----------------------------------------------------
+# get data location and setup input data as necessary
+#-----------------------------------------------------
+#echo PDATE: $PDATE
+#${APRUN_PY} ${USHobsmon}/setUpData.py -i ${yaml_file} -p ${PDATE}
+
 #---------------------------------------------------------------
 # split $yaml_file into sat/instr[/plot], minimization, and obs
 # in order to reduce the plot jobs to a more managable size 
@@ -59,13 +65,14 @@ if compgen -G "${DATA}/OM_PLOT*.yaml" > /dev/null; then
       case ${MACHINE_ID} in
          hera|orion|hercules)
             # submit plot job
-            plotjob_id=$(${SUB} --account ${ACCOUNT} -n ${ctr}  -o ${logfile} -D . -J ${jobname} --time=1:00:00 \
+#           plotjob_id=$(${SUB} --account ${ACCOUNT} -n ${ctr}  -o ${logfile} -D . -J ${jobname} --time=1:00:00 \
+            plotjob_id=$(${SUB} --account ${ACCOUNT} -n ${ctr}  -o ${logfile} -D . -J ${jobname} --time=0:05:00 \
                    --mem=80000M --wrap "srun -l --multi-prog ${cmdfile}")
 
             # submit cleanup job to run after plot job
-            plotjob_id=`echo ${plotjob_id} | gawk '{ print $4 }'`
-            ${SUB} --account ${ACCOUNT} -n 1 -o ${logfile_clnup} -D . -J "OM_cleanup" --time=0:10:00 \
-                   -p ${SERVICE_PARTITION} --dependency=afterok:${plotjob_id} ${USHobsmon}/om_cleanup.sh
+#           plotjob_id=`echo ${plotjob_id} | gawk '{ print $4 }'`
+#           ${SUB} --account ${ACCOUNT} -n 1 -o ${logfile_clnup} -D . -J "OM_cleanup" --time=0:10:00 \
+#                  -p ${SERVICE_PARTITION} --dependency=afterok:${plotjob_id} ${USHobsmon}/om_cleanup.sh
 
          ;;
 
@@ -88,3 +95,4 @@ if compgen -G "${DATA}/OM_PLOT*.yaml" > /dev/null; then
       esac
    fi
 fi
+
diff --git a/ush/plotObsMon.py b/ush/plotObsMon.py
@@ -8,7 +8,8 @@
 import os
 from re import sub
 import yaml
-from setupdata import setupdata
+from setupdata import OM_data
+
 from wxflow import parse_j2yaml, save_as_yaml
 from wxflow import add_to_datetime, to_timedelta, to_datetime
 from eva.eva_driver import eva
@@ -139,8 +140,8 @@ def loadConfig(satname, instrument, obstype, plot, cycle_tm, cycle_interval,
     args = parser.parse_args()
     cycle_tm = to_datetime(args.pdate)
 
-    data = os.environ.get('DATA', '.')
-    os.chdir(data)
+    workdir = os.environ.get('DATA', '.')
+    os.chdir(workdir)
 
     try:
         mon_sources = args.input
@@ -188,11 +189,23 @@ def loadConfig(satname, instrument, obstype, plot, cycle_tm, cycle_interval,
                     parm_location = os.path.join(parm, 'templates')
                     plot_template = os.path.join(parm_location, plot_template)
 
+                    # move to unique directory based on plot_yaml file
+                    plot_dir = os.path.join(workdir, plot_yaml.split('.')[0])
+                    os.makedirs(plot_dir)
+                    os.chdir(plot_dir)
+
+                    config['DATA'] = plot_dir 
+#                   logger.info(f'plot_dir: {plot_dir}')
+
                     genYaml(plot_template, plot_yaml, config)
-#                   logger.info(f'config: {config}')
-                    setupdata(config, logger)
+
+                    # need to send data_location into setupdata maybe instead of config?  files we need should be specified in plot_yaml file
+                    plotData = OM_data(data_location, config, plot_yaml, logger)
+
+#                    setupdata(data_location, config, plot_yaml, logger)
+
                     eva(plot_yaml)
-                    os.remove(plot_yaml)
+#                   os.remove(plot_yaml)
 
     if 'minimization' in mon_dict.keys():
         satname = None
@@ -214,6 +227,7 @@ def loadConfig(satname, instrument, obstype, plot, cycle_tm, cycle_interval,
                 plot_template = os.path.join(parm_location, plot_template)
 
                 genYaml(plot_template, plot_yaml, config)
+                setupdata(config, plot_yaml, logger)
                 eva(plot_yaml)
                 os.remove(plot_yaml)
 

diff --git a/ush/setUpData.py.mk1 b/ush/setUpData.py.mk1