From 461137bc9cb1ba7100a2022576df52129c28251c Mon Sep 17 00:00:00 2001
From: Ed Safford <62339196+EdwardSafford-NOAA@users.noreply.github.com>
Date: Fri, 26 Apr 2024 08:34:06 -0400
Subject: [PATCH] Expand parallel job submissions (#28)

Expand the parallel job submissions by submitting minimization and
conventional obs plots as separate jobs using `ush/splitPlotYaml.py`.
All GFS plot jobs now to completion on hera in ~45 min using this
parallel strategy.

The `conTime.yaml` and `conVert.yaml` files have been generalized to use
DATATYPES, and use of 'all' for datatypes is now supported using the
parm/observation_datatypes.yaml master list.

The change to `ush/diagnosticAverager.py` is a pycode fix and is
unrelated to the parallel job changes.

Note that this has not been tested yet on wcoss2. I figured to do that
as a separate issue, and likely next.

Closes #25
---
 parm/gfs/conTime.yaml           |  10 +-
 parm/gfs/conVert.yaml           |   6 +-
 parm/gfs/gfs_plots.yaml         | 551 +++++++++++++++++++++++---------
 parm/observation_datatypes.yaml |  34 ++
 scripts/exobsmon_plot.sh        |  92 ++++--
 ush/diagnosticAverager.py       |   3 +-
 ush/plotObsMon.py               |  25 +-
 ush/splitPlotYaml.py            |  15 +
 8 files changed, 552 insertions(+), 184 deletions(-)
 create mode 100644 parm/observation_datatypes.yaml

diff --git a/parm/gfs/conTime.yaml b/parm/gfs/conTime.yaml
index e378397..a8bfbdd 100644
--- a/parm/gfs/conTime.yaml
+++ b/parm/gfs/conTime.yaml
@@ -146,6 +146,8 @@ datasets:
 
     regions: 1
 
+    datatypes: {{DATATYPES}}
+
     groups:
       - name: ges
 
@@ -278,8 +280,8 @@ datasets:
 
     regions: 1
 
-    datatypes: &datatypes ['q120_0', 'q130_0']
-
+    datatypes: {{DATATYPES}}
+    
     groups:
       - name: anl
 
@@ -452,7 +454,7 @@ graphics:
     # 4 plot ConMon time series
     # -------------------------
     - batch figure:
-        datatypes: *datatypes
+        datatypes: {{DATATYPES}}
         variables:  ['count1_allev']
 
       figure:
@@ -576,7 +578,7 @@ graphics:
             label: 'final outloop'
 
     - batch figure:
-        datatypes: *datatypes
+        datatypes: {{DATATYPES}}
         variables:  ['bias1_allev']
 
       figure:
diff --git a/parm/gfs/conVert.yaml b/parm/gfs/conVert.yaml
index 3b88bf9..8b20fe2 100644
--- a/parm/gfs/conVert.yaml
+++ b/parm/gfs/conVert.yaml
@@ -24,6 +24,8 @@ datasets:
 
     regions: 1
 
+    datatypes: {{DATATYPES}}
+
     groups:
       - name: ges
 
@@ -37,7 +39,7 @@ datasets:
 
     regions: 1
 
-    datatypes: &datatypes ['q120_0', 'q130_0']
+    datatypes: {{DATATYPES}}
 
     groups:
       - name: anl
@@ -59,7 +61,7 @@ graphics:
     # 4 plot ConMon time series
     # -------------------------
     - batch figure:
-        datatypes: *datatypes
+        datatypes: {{DATATYPES}}
         variables:  ['count1', 'count2', 'count_vqc1', 'count3']
 
       figure:
diff --git a/parm/gfs/gfs_plots.yaml b/parm/gfs/gfs_plots.yaml
index 0e739ae..aa39643 100644
--- a/parm/gfs/gfs_plots.yaml
+++ b/parm/gfs/gfs_plots.yaml
@@ -1,112 +1,106 @@
 # DA Monitoring source dictionary
 
+model: gfs
 cycle_interval: 6
-#data: '/scratch1/NCEPDEV/da/Edward.Safford/noscrub/git/eva/src/eva/tests/data'
 data: '/scratch1/NCEPDEV/da/Edward.Safford/noscrub/test_data/gfs'
-model: gfs
 
 satellites:
-#  - name: aura
-#    instruments:
-#      - name: omi
-#        plot_list:
-#          - plot: ozn horiz
-#            times: 4
-#            levels: '1'
-#            component: ges
-#
-#          - plot: ozn time
-#            times: 121 
-#            levels: '1'
-#            component: ges
-#
-#  - name: npp
-#    instruments:
-#      - name: ompsnp
-#        plot_list:
-#          - plot: ozn horiz
-#            times: 4
-#            levels: '1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22'
-#            component: ges
-#
-#          - plot: ozn time
-#            times: 121 
-#            levels: '1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22'
-#            component: ges
-#
-#          - plot: ozn summary
-#            times: 121
-#            levels: '1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22'
-#            component: ges
-#
-#      - name: ompslp
-#        plot_list:
-#          - plot: ozn horiz
-#            times: 4
-#            levels: '1'
-#            component: ges
-#
-#          - plot: ozn time
-#            times: 121 
-#            levels: '1'
-#            component: ges
-#
-#      - name: ompstc8
-#        plot_list:
-#          - plot: ozn horiz
-#            times: 4
-#            levels: '1'
-#            component: ges
-#
-#          - plot: ozn time
-#            times: 121 
-#            levels: '1'
-#            component: ges
-#
-#  - name: n20
-#    instruments:
-#      - name: ompsnp
-#        plot_list:
-#          - plot: ozn horiz
-#            times: 4
-#            levels: '1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22'
-#            component: ges
-#
-#          - plot: ozn time
-#            times: 121 
-#            levels: '1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22'
-#            component: ges
-#
-#          - plot: ozn summary
-#            times: 121
-#            levels: '1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22'
-#            component: ges
-#
-#      - name: ompstc8
-#        plot_list:
-#          - plot: ozn horiz
-#            times: 4
-#            levels: '1'
-#            component: ges
-#
-#          - plot: ozn time
-#            times: 121 
-#            levels: '1'
-#            component: ges
-#
-#  - name: n20
-#    instruments:
-#      - name: ompstc8
-#        plot_list:
-#          - plot: ozn horiz
-#            times: 4
-#            levels: '1'
-#            component: ges
-#
-#          - plot: ozn time
-#            times: 121 
-#            levels: '1'
-#            component: ges
+  - name: aura
+    instruments:
+      - name: omi
+        plot_list:
+          - plot: ozn horiz
+            times: 4
+            levels: '1'
+            component: ges
+
+          - plot: ozn time
+            times: 121 
+            levels: '1'
+            component: ges
+
+  - name: npp
+    instruments:
+      - name: ompsnp
+        plot_list:
+          - plot: ozn horiz
+            times: 4
+            levels: '1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22'
+            component: ges
+
+          - plot: ozn summary
+            times: 121
+            levels: '1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22'
+            component: ges
+
+      - name: ompslp
+        plot_list:
+          - plot: ozn horiz
+            times: 4
+            levels: '1'
+            component: ges
+
+          - plot: ozn time
+            times: 121 
+            levels: '1'
+            component: ges
+
+      - name: ompstc8
+        plot_list:
+          - plot: ozn horiz
+            times: 4
+            levels: '1'
+            component: ges
+
+          - plot: ozn time
+            times: 121 
+            levels: '1'
+            component: ges
+
+  - name: n20
+    instruments:
+      - name: ompsnp
+        plot_list:
+          - plot: ozn horiz
+            times: 4
+            levels: '1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22'
+            component: ges
+
+          - plot: ozn time
+            times: 121 
+            levels: '1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22'
+            component: ges
+
+          - plot: ozn summary
+            times: 121
+            levels: '1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22'
+            component: ges
+
+      - name: ompstc8
+        plot_list:
+          - plot: ozn horiz
+            times: 4
+            levels: '1'
+            component: ges
+
+          - plot: ozn time
+            times: 121 
+            levels: '1'
+            component: ges
+
+  - name: n20
+    instruments:
+      - name: ompstc8
+        plot_list:
+          - plot: ozn horiz
+            times: 4
+            levels: '1'
+            component: ges
+
+          - plot: ozn time
+            times: 121 
+            levels: '1'
+            component: ges
 
 
 # Radiance
@@ -119,6 +113,14 @@ satellites:
             times: 121
             channels: 'all'
             component: ges
+          - plot: rad bcoef
+            times: 121
+            channels: 'all'
+            component: ges
+          - plot: rad angle
+            times: 121
+            channels: 'all'
+            component: ges
 
   - name: g18
     instruments:
@@ -128,6 +130,14 @@ satellites:
             times: 121
             channels: 'all'
             component: ges
+          - plot: rad bcoef
+            times: 121
+            channels: 'all'
+            component: ges
+          - plot: rad angle
+            times: 121
+            channels: 'all'
+            component: ges
 
   - name: himawari9
     instruments:
@@ -137,6 +147,14 @@ satellites:
             times: 121
             channels: 'all'
             component: ges
+          - plot: rad bcoef
+            times: 121
+            channels: 'all'
+            component: ges
+          - plot: rad angle
+            times: 121
+            channels: 'all'
+            component: ges
 
   - name: metop-b
     instruments:
@@ -146,6 +164,14 @@ satellites:
             times: 121
             channels: 'all'
             component: ges
+          - plot: rad bcoef
+            times: 121
+            channels: 'all'
+            component: ges
+          - plot: rad angle
+            times: 121
+            channels: 'all'
+            component: ges
 
       - name: iasi
         plot_list:
@@ -153,35 +179,45 @@ satellites:
             times: 121
             channels: 'all'
             component: ges
-
-      - name: mhs
-        plot_list:
-          - plot: rad time
+          - plot: rad bcoef
             times: 121
             channels: 'all'
             component: ges
+#     file corruption here?
 #          - plot: rad angle
 #            times: 121
-#            channels: '1,2,3,4,5'
-#            component: ges
-#          - plot: rad bcoef
-#            times: 121
-#            channels: '1,2,3,4,5'
+#            channels: 'all'
 #            component: ges
 
-      - name: avhrr
+      - name: mhs
         plot_list:
           - plot: rad time
             times: 121
             channels: 'all'
             component: ges
+          - plot: rad bcoef
+            times: 121
+            channels: '1,2,3,4,5'
+            component: ges
+          - plot: rad angle
+            times: 121
+            channels: 'all'
+            component: ges
 
-      - name: mhs
+      - name: avhrr
         plot_list:
           - plot: rad time
             times: 121
             channels: 'all'
             component: ges
+          - plot: rad bcoef
+            times: 121
+            channels: 'all'
+            component: ges
+          - plot: rad angle
+            times: 121
+            channels: 'all'
+            component: ges
 
   - name: metop-c
     instruments:
@@ -191,6 +227,14 @@ satellites:
             times: 121
             channels: 'all'
             component: ges
+          - plot: rad bcoef
+            times: 121
+            channels: 'all'
+            component: ges
+          - plot: rad angle
+            times: 121
+            channels: 'all'
+            component: ges
 
       - name: iasi
         plot_list:
@@ -198,6 +242,14 @@ satellites:
             times: 121
             channels: 'all'
             component: ges
+          - plot: rad bcoef
+            times: 121
+            channels: 'all'
+            component: ges
+          - plot: rad angle
+            times: 121
+            channels: 'all'
+            component: ges
 
       - name: avhrr
         plot_list:
@@ -205,6 +257,14 @@ satellites:
             times: 121
             channels: 'all'
             component: ges
+          - plot: rad bcoef
+            times: 121
+            channels: 'all'
+            component: ges
+          - plot: rad angle
+            times: 121
+            channels: 'all'
+            component: ges
 
       - name: mhs
         plot_list:
@@ -212,6 +272,14 @@ satellites:
             times: 121
             channels: 'all'
             component: ges
+          - plot: rad bcoef
+            times: 121
+            channels: 'all'
+            component: ges
+          - plot: rad angle
+            times: 121
+            channels: 'all'
+            component: ges
 
   - name: n15
     instruments:
@@ -221,6 +289,14 @@ satellites:
             times: 121
             channels: 'all'
             component: ges
+          - plot: rad bcoef
+            times: 121
+            channels: 'all'
+            component: ges
+          - plot: rad angle
+            times: 121
+            channels: 'all'
+            component: ges
 
   - name: n18
     instruments:
@@ -230,19 +306,11 @@ satellites:
             times: 121
             channels: 'all'
             component: ges
-
-      - name: avhrr
-        plot_list:
-          - plot: rad time
+          - plot: rad bcoef
             times: 121
             channels: 'all'
             component: ges
-
-  - name: n19
-    instruments:
-      - name: amsua
-        plot_list:
-          - plot: rad time
+          - plot: rad angle
             times: 121
             channels: 'all'
             component: ges
@@ -253,14 +321,61 @@ satellites:
             times: 121
             channels: 'all'
             component: ges
-
-      - name: mhs
-        plot_list:
-          - plot: rad time
+          - plot: rad bcoef
+            times: 121
+            channels: 'all'
+            component: ges
+          - plot: rad angle
             times: 121
             channels: 'all'
             component: ges
 
+#  file corruption? 
+#  - name: n19
+#    instruments:
+#      - name: amsua
+#        plot_list:
+#          - plot: rad time
+#            times: 121
+#            channels: 'all'
+#            component: ges
+#          - plot: rad bcoef
+#            times: 121
+#            channels: 'all'
+#            component: ges
+#          - plot: rad angle
+#            times: 121
+#            channels: 'all'
+#            component: ges
+#
+#      - name: avhrr
+#        plot_list:
+#          - plot: rad time
+#            times: 121
+#            channels: 'all'
+#            component: ges
+#          - plot: rad bcoef
+#            times: 121
+#            channels: 'all'
+#            component: ges
+#          - plot: rad angle
+#            times: 121
+#            channels: 'all'
+#            component: ges
+
+#      - name: mhs
+#        plot_list:
+#          - plot: rad time
+#            times: 121
+#            channels: 'all'
+#            component: ges
+#          - plot: rad bcoef
+#            times: 121
+#            channels: 'all'
+#            component: ges
+#          - plot: rad angle
+#            times: 121
+
   - name: n20
     instruments:
       - name: atms
@@ -269,6 +384,14 @@ satellites:
             times: 121
             channels: 'all'
             component: ges
+          - plot: rad bcoef
+            times: 121
+            channels: 'all'
+            component: ges
+          - plot: rad angle
+            times: 121
+            channels: 'all'
+            component: ges
 
       - name: cris-fsr
         plot_list:
@@ -276,7 +399,14 @@ satellites:
             times: 121
             channels: 'all'
             component: ges
-
+          - plot: rad bcoef
+            times: 121
+            channels: 'all'
+            component: ges
+          - plot: rad angle
+            times: 121
+            channels: 'all'
+            component: ges
 
   - name: n21
     instruments:
@@ -286,6 +416,14 @@ satellites:
             times: 121
             channels: 'all'
             component: ges
+          - plot: rad bcoef
+            times: 121
+            channels: 'all'
+            component: ges
+          - plot: rad angle
+            times: 121
+            channels: 'all'
+            component: ges
 
       - name: cris-fsr
         plot_list:
@@ -293,6 +431,14 @@ satellites:
             times: 121
             channels: 'all'
             component: ges
+          - plot: rad bcoef
+            times: 121
+            channels: 'all'
+            component: ges
+          - plot: rad angle
+            times: 121
+            channels: 'all'
+            component: ges
 
   - name: npp
     instruments:
@@ -302,6 +448,14 @@ satellites:
             times: 121
             channels: 'all'
             component: ges
+          - plot: rad bcoef
+            times: 121
+            channels: 'all'
+            component: ges
+          - plot: rad angle
+            times: 121
+            channels: 'all'
+            component: ges
 
       - name: viirs-m
         plot_list:
@@ -309,6 +463,14 @@ satellites:
             times: 121
             channels: 'all'
             component: ges
+          - plot: rad bcoef
+            times: 121
+            channels: 'all'
+            component: ges
+          - plot: rad angle
+            times: 121
+            channels: 'all'
+            component: ges
 
   - name: j1
     instruments:
@@ -318,12 +480,15 @@ satellites:
             times: 121
             channels: 'all'
             component: ges
+          - plot: rad bcoef
+            times: 121
+            channels: 'all'
+            component: ges
+          - plot: rad angle
+            times: 121
+            channels: 'all'
+            component: ges
 
-#          - plot: rad bcoef
-#            times: 121
-#            channels: '12,13,14,15,16'
-#            component: ges
-#
   - name: f17
     instruments:
       - name: ssmis
@@ -332,6 +497,14 @@ satellites:
             times: 121
             channels: 'all'
             component: ges
+          - plot: rad bcoef
+            times: 121
+            channels: 'all'
+            component: ges
+          - plot: rad angle
+            times: 121
+            channels: 'all'
+            component: ges
 
   - name: f18
     instruments:
@@ -341,27 +514,93 @@ satellites:
             times: 121
             channels: 'all'
             component: ges
+          - plot: rad bcoef
+            times: 121
+            channels: 'all'
+            component: ges
+          - plot: rad angle
+            times: 121
+            channels: 'all'
+            component: ges
 
 
-#minimization:
-#  - model: gfs
-#    plot_list:
-#      - plot: min summary
-#        run: gdas
-#        times: 28
-#
-#      - plot: min gnorm four cycle
-#        run: gdas
-#        times: 28
-#
-#      - plot: min gnorm one cycle
-#        run: gdas
-#        times: 28
-#
-#observations:
-#  - obstype: q
-#    plot_list:
-#      - plot: con time
-#        times: 121
-#      - plot: con vert
-#        times: 1
+  - name: f18
+    instruments:
+      - name: ssmis
+        plot_list:
+          - plot: rad time
+            times: 121
+            channels: 'all'
+            component: ges
+
+
+minimization:
+  - model: gfs
+    plot_list:
+      - plot: min summary
+        run: gdas
+        times: 28
+
+      - plot: min gnorm four cycle
+        run: gdas
+        times: 28
+
+      - plot: min gnorm one cycle
+        run: gdas
+        times: 28
+
+
+observations:
+  - obstype: ps
+    plot_list:
+      - plot: con time
+        times: 121
+        datatypes: 'all'
+  - obstype: gps
+    plot_list:
+      - plot: con time
+        times: 121
+        datatypes: 'all'
+      - plot: con vert
+        times: 1
+        datatypes: 'all'
+  - obstype: q
+    plot_list:
+      - plot: con time
+        times: 121
+        datatypes: 'all'
+      - plot: con vert
+        times: 1
+        datatypes: 'all'
+  - obstype: t
+    plot_list:
+      - plot: con time
+        times: 121
+        datatypes: 'all'
+      - plot: con vert
+        times: 1
+        datatypes: 'all'
+  - obstype: u
+    plot_list:
+      - plot: con time
+        times: 121
+        datatypes: 'all'
+      - plot: con vert
+        times: 1
+        datatypes: 'all'
+  - obstype: uv
+    plot_list:
+      - plot: con time
+        times: 121
+        datatypes: 'all'
+      - plot: con vert
+        times: 1
+        datatypes: 'all'
+  - obstype: v
+    plot_list:
+      - plot: con time
+        times: 121
+        datatypes: 'all'
+      - plot: con vert
+        times: 1
+        datatypes: 'all'
diff --git a/parm/observation_datatypes.yaml b/parm/observation_datatypes.yaml
new file mode 100644
index 0000000..695e0bc
--- /dev/null
+++ b/parm/observation_datatypes.yaml
@@ -0,0 +1,34 @@
+gps: [  'gps3_0',   'gps4_0',   'gps5_0',  'gps41_0',  'gps42_0',  'gps43_0',  'gps44_0', 
+      'gps265_0', 'gps266_0', 'gps267_0', 'gps268_0', 'gps269_0', 'gps421_0', 'gps440_0', 
+      'gps722_0', 'gps723_0', 'gps724_0', 'gps725_0', 'gps726_0', 'gps727_0', 'gps728_0',
+      'gps729_0', 'gps740_0', 'gps741_0', 'gps742_0', 'gps743_0', 'gps744_0', 'gps745_0',
+      'gps750_0', 'gps751_0', 'gps752_0', 'gps753_0', 'gps754_0', 'gps755_0', 'gps820_0',
+      'gps821_0', 'gps825_0', 'gps786_0']
+
+ps: ['ps120_0', 'ps132_0', 'ps180_0', 'ps180_1', 'ps181_0', 'ps182_0', 'ps183_0',
+     'ps187_0']
+
+q: ['q120_0', 'q130_0', 'q131_0', 'q132_0', 'q133_0', 'q134_0', 'q135_0', 
+    'q136_0', 'q180_0', 'q180_1', 'q181_0', 'q182_0', 'q183_0', 'q187_0']
+
+
+t: ['t120_0', 't126_0', 't130_0', 't131_0', 't132_0', 't133_0', 't134_0', 't135_0', 
+    't136_0', 't180_0', 't180_1', 't181_0', 't182_0', 't183_0', 't187_0']
+
+u: ['uv210_0', 'uv220_0', 'uv221_0', 'uv223_0', 'uv224_0', 'uv228_0', 'uv229_0', 'uv230_0',
+    'uv231_0', 'uv232_0', 'uv233_0', 'uv234_0', 'uv235_0', 'uv236_0', 'uv240_0', 'uv240_257',
+    'uv240_259', 'uv240_270', 'uv240_271', 'uv241_0', 'uv242_0', 'uv242_171', 'uv242_172',
+    'uv242_173', 'uv242_174', 'uv243_0', 'uv243_54', 'uv243_55', 'uv243_56', 'uv243_57',
+    'uv243_70', 'uv244_0', 'uv244_3', 'uv244_4', 'uv244_206', 'uv244_207', 'uv244_209' ]
+
+uv: ['uv210_0', 'uv220_0', 'uv221_0', 'uv223_0', 'uv224_0', 'uv228_0', 'uv229_0', 'uv230_0',
+     'uv231_0', 'uv232_0', 'uv233_0', 'uv234_0', 'uv235_0', 'uv236_0', 'uv240_0', 'uv240_257',
+     'uv240_259', 'uv240_270', 'uv240_271', 'uv241_0', 'uv242_0', 'uv242_171', 'uv242_172',
+     'uv242_173', 'uv242_174', 'uv243_0', 'uv243_54', 'uv243_55', 'uv243_56', 'uv243_57',
+     'uv243_70', 'uv244_0', 'uv244_3', 'uv244_4', 'uv244_206', 'uv244_207', 'uv244_209' ]
+
+v: ['uv210_0', 'uv220_0', 'uv221_0', 'uv223_0', 'uv224_0', 'uv228_0', 'uv229_0', 'uv230_0',
+    'uv231_0', 'uv232_0', 'uv233_0', 'uv234_0', 'uv235_0', 'uv236_0', 'uv240_0', 'uv240_257',
+    'uv240_259', 'uv240_270', 'uv240_271', 'uv241_0', 'uv242_0', 'uv242_171', 'uv242_172',
+    'uv242_173', 'uv242_174', 'uv243_0', 'uv243_54', 'uv243_55', 'uv243_56', 'uv243_57',
+    'uv243_70', 'uv244_0', 'uv244_3', 'uv244_4', 'uv244_206', 'uv244_207', 'uv244_209' ]
diff --git a/scripts/exobsmon_plot.sh b/scripts/exobsmon_plot.sh
index a229fe5..078974a 100755
--- a/scripts/exobsmon_plot.sh
+++ b/scripts/exobsmon_plot.sh
@@ -4,10 +4,6 @@
 # exobsmon_plot.sh
 #------------------
 
-jobname="OM_test"
-logfile="${OM_LOGS}/${MODEL}/OM_log_test"
-if [[ -e ${logfile} ]]; then rm ${logfile}; fi
-
 #-------------------------------------------------------------
 # locate $model_plots.yaml and instrument_channels.yaml files
 #-------------------------------------------------------------
@@ -24,31 +20,89 @@ if [[ ! -e ${chan_yaml} ]]; then
    exit 2
 fi
 
-#----------------------------------------
-# split $plot_yaml into sat/instr[/plot]
+#-----------------------------------------------------------
+# split $plot_yaml into sat/instr[/plot], minimization, obs
 #
 ${APRUN_PY} ${USHobsmon}/splitPlotYaml.py -i ${plot_yaml} -c ${chan_yaml}
 
-#exit 0
 
-cmdfile="OM_test_jobscript"
->$cmdfile
+#--------------------------------------------------------------
+# Submit OM_sat_plots job if split yields any sat_*.yaml files
+#
+if compgen -G "${DATA}/sat_*.yaml" > /dev/null; then
+
+   jobname="OM_sat_plots"
+   logfile="${OM_LOGS}/${MODEL}/OM_sat_plot.log"
+   if [[ -e ${logfile} ]]; then rm ${logfile}; fi
+
+   cmdfile="OM_sat_jobscript"
+   >$cmdfile
+   ctr=0
+
+   for yaml in ${DATA}/sat_*.yaml; do
+      echo "${ctr} $yaml"
+      echo "${ctr} ${APRUN_PY} ${USHobsmon}/plotObsMon.py -i ${yaml}  -p ${PDATE}" >> $cmdfile
+      ((ctr+=1))
+   done 
+   chmod 755 $cmdfile
+
+   echo "ctr: $ctr"
+   echo "submitting job ${jobname}"
+
+   if [[ ${ctr} > 0 ]]; then
+      $SUB --account ${ACCOUNT} -n ${ctr}  -o ${logfile} -D . -J ${jobname} --time=1:00:00 \
+           --mem=80000M --wrap "srun -l --multi-prog ${cmdfile}"
+   fi
+
+fi
+
+#------------------------------------------------------------------
+# Submit OM_min_plots job if split yields a minimization.yaml file
+#
+if compgen -G "${DATA}/minimization.yaml" > /dev/null; then
+
+   jobname="OM_min_plots"
+   logfile="${OM_LOGS}/${MODEL}/OM_min_plot.log"
+   if [[ -e ${logfile} ]]; then rm ${logfile}; fi
 
-ctr=0
-for yaml in ./sat_*.yaml; do
-   echo "${ctr} $yaml"
-   echo "${ctr} ${APRUN_PY} ${USHobsmon}/plotObsMon.py -i ${yaml}  -p ${PDATE}" >> $cmdfile
-   ((ctr+=1))
-done 
-chmod 755 $cmdfile
+   cmdfile="OM_min_jobscript"
+   echo "0 ${APRUN_PY} ${USHobsmon}/plotObsMon.py -i ${DATA}/minimization.yaml  -p ${PDATE}" > $cmdfile
+   chmod 755 $cmdfile
 
-echo "submitting job $jobname"
+   echo "submitting job ${jobname}"
 
-if [[ ${ctr} > 0 ]]; then
-   $SUB --account ${ACCOUNT} -n ${ctr}  -o ${logfile} -D . -J ${jobname} --time=1:00:00 \
+   $SUB --account ${ACCOUNT} -n 1  -o ${logfile} -D . -J ${jobname} --time=0:05:00 \
         --mem=80000M --wrap "srun -l --multi-prog ${cmdfile}"
 fi
 
+#------------------------------------------------------------------
+# Submit OM_con_plots job if split yields any obs_*.yaml files
+#
+if compgen -G "${DATA}/obs*.yaml" > /dev/null; then
+   echo "have OBS plots"
+   jobname="OM_obs_plots"
+   logfile="${OM_LOGS}/${MODEL}/OM_obs_plot.log"
+   if [[ -e ${logfile} ]]; then rm ${logfile}; fi
+
+   cmdfile="OM_obs_jobscript"
+   >$cmdfile
+   ctr=0
+
+   for yaml in ${DATA}/obs*.yaml; do
+      echo "${ctr} $yaml"
+      echo "${ctr} ${APRUN_PY} ${USHobsmon}/plotObsMon.py -i ${yaml}  -p ${PDATE}" >> $cmdfile
+      ((ctr+=1))
+   done 
+   chmod 755 $cmdfile
+
+   echo "ctr: $ctr"
+   echo "submitting job ${jobname}"
+
+   if [[ ${ctr} > 0 ]]; then
+      $SUB --account ${ACCOUNT} -n ${ctr}  -o ${logfile} -D . -J ${jobname} --time=1:00:00 \
+           --mem=80000M --wrap "srun -l --multi-prog ${cmdfile}"
+   fi
+fi
 
 #-----------------------------
 # Copy output to COMOUTplots
diff --git a/ush/diagnosticAverager.py b/ush/diagnosticAverager.py
index a04e3bf..e7450fe 100644
--- a/ush/diagnosticAverager.py
+++ b/ush/diagnosticAverager.py
@@ -180,7 +180,8 @@ def calculate_penalty(filename, inputdict, logger):
 
         for key in omf_dict.keys():
 
-            return_dict[outgroups[i]]['data'] = omf_dict[key]['data'] / efferr_dict[f'EffectiveError{loop}'][variable]
+            return_dict[outgroups[i]]['data'] = omf_dict[key]['data'] /
+            efferr_dict[f'EffectiveError{loop}'][variable]
             return_dict[outgroups[i]]['qc var'] = f'EffectiveQC{loop}'
 
     return return_dict
diff --git a/ush/plotObsMon.py b/ush/plotObsMon.py
index 047609a..6d11a41 100755
--- a/ush/plotObsMon.py
+++ b/ush/plotObsMon.py
@@ -49,7 +49,7 @@ def camelCase(s):
 
 
 def loadConfig(satname, instrument, obstype, plot, cycle_tm, cycle_interval,
-               data_location, model=None, chan_dict=None):
+               data_location, model=None, chan_dict=None, datatype_dict=None):
     """
     Load configuration dictionary.
 
@@ -62,6 +62,8 @@ def loadConfig(satname, instrument, obstype, plot, cycle_tm, cycle_interval,
         cycle_interval (int): number of hours between cycles
         data_location (str): path to directory containing data files
         model (str): model|experiment name
+        chan_dict (dict): dictionary with full channel definitions
+        datatype_dict (dict): dictionary with full datatype definitions
     Return:
         config(dict): Dictionary containing configuration information
     """
@@ -71,6 +73,7 @@ def loadConfig(satname, instrument, obstype, plot, cycle_tm, cycle_interval,
         'OBSTYPE': obstype,
         'LEVELS': plot.get('levels'),
         'CHANNELS': plot.get('channels'),
+        'DATATYPES': plot.get('datatypes'),
         'MODEL': model,
         'RUN': plot.get('run'),
         'COMPONENT': plot.get('component'),
@@ -107,6 +110,10 @@ def loadConfig(satname, instrument, obstype, plot, cycle_tm, cycle_interval,
                 ctr += interval
             config['XTICKS'] = xticks[:-1]
 
+    if config['DATATYPES'] is not None:
+        if config['DATATYPES'] == 'all':
+            config['DATATYPES'] = obs_dict[config['OBSTYPE']]
+
     return config
 
 # --------------------------------------------------------------------------------------------
@@ -208,12 +215,26 @@ def loadConfig(satname, instrument, obstype, plot, cycle_tm, cycle_interval,
         satname = None
         instrument = None
         obstype = None
+        chan_dict = None
+
+        # Load the obs_dict, which will contain all the datatypes for a given
+        # observation.  This is used if the input yaml file uses 'all' for datatypes.
+        try:
+            parm_location = os.environ.get('PARMobsmon', '../parm')
+            obs_types = os.path.join(parm_location, 'observation_datatypes.yaml')
+            with open(obs_types, 'r') as obs_types_opened:
+                obs_dict = yaml.safe_load(obs_types_opened)
+
+        except Exception as e:
+            logger.info('Warning: unable to load observation datatype information ' +
+                        f'errors when attempting to load: {obs_types}, error: {e}')
+
         for obs in mon_dict.get('observations'):
             obstype = obs.get('obstype')
 
             for plot in obs.get('plot_list'):
                 config = loadConfig(satname, instrument, obstype, plot, cycle_tm, cycle_interval,
-                                    data_location)
+                                    data_location, model, chan_dict, obs_dict)
 
                 plot_template = f"{config['PLOT_TEMPLATE']}.yaml"
                 plot_yaml = f"{config['OBSTYPE']}_{plot_template}"
diff --git a/ush/splitPlotYaml.py b/ush/splitPlotYaml.py
index 7aa2c0d..8d4ccb9 100644
--- a/ush/splitPlotYaml.py
+++ b/ush/splitPlotYaml.py
@@ -110,3 +110,18 @@ def removeKey(d, keys):
                     file = open(fname, "w")
                     yaml.dump(pd, file)
                     file.close()
+
+    if 'minimization' in mon_dict.keys():
+        md = removeKey(mon_dict, ['satellites', 'observations'])
+        fname = f'minimization.yaml'
+        file = open(fname, "w")
+        yaml.dump(md, file)
+        file.close()
+
+    if 'observations' in mon_dict.keys():
+        logger.info(f'HAVE OBS')
+        od = removeKey(mon_dict, ['satellites', 'minimization'])
+        fname = f'observations.yaml'
+        file = open(fname, "w")
+        yaml.dump(od, file)
+        file.close()