Merge branch 'develop' into feature/faster-better-recentering

NOAA-EMC · Sep 2, 2023 · 840027a · 840027a
2 parents 1f5ffbb + 10bcec5
commit 840027a
Show file tree

Hide file tree

Showing 15 changed files with 112 additions and 26 deletions.
diff --git a/parm/soca/obs/config/adt_all.yaml b/parm/soca/obs/config/adt_all.yaml
@@ -0,0 +1,50 @@
+obs space:
+  name: adt_all
+  obsdatain:
+    engine:
+      type: H5File
+      obsfile: !ENV ${DATA}/obs/${OPREFIX}adt_all.${PDY}${cyc}.nc4
+  obsdataout:
+    engine:
+      type: H5File
+      obsfile: !ENV ${DATA}/diags/adt_all.${PDY}${cyc}.nc4
+  simulated variables: [absoluteDynamicTopography]
+  io pool:
+    max pool size: 1
+obs operator:
+  name: ADT
+obs error:
+  covariance model: diagonal
+obs filters:
+- filter: Domain Check
+  where:
+  - variable: {name: GeoVaLs/sea_area_fraction}
+    minvalue: 0.9
+- filter: Domain Check
+  where:
+  - variable: { name: GeoVaLs/sea_surface_temperature}
+    minvalue: 5.0
+- filter: Background Check
+  absolute threshold: 1.0
+- filter: Domain Check
+  where:
+  - variable: {name: GeoVaLs/sea_floor_depth_below_sea_surface}
+    minvalue: 500
+- filter: Perform Action
+  action:
+    name: assign error
+    error function:
+      name: ObsFunction/LinearCombination
+      options:
+        variables: [GeoVaLs/mesoscale_representation_error,
+                    ObsError/absoluteDynamicTopography]
+        coefs: [0.1,
+                0.5]
+- filter: Domain Check
+  where:
+  - variable: { name: GeoVaLs/sea_ice_area_fraction}
+    maxvalue: 0.00001
+- filter: Domain Check
+  where:
+  - variable: {name: GeoVaLs/distance_from_coast}
+    minvalue: 100e3
diff --git a/parm/soca/obs/config/adt_j2.yaml b/parm/soca/obs/config/adt_j2.yaml
@@ -0,0 +1,15 @@
+obs space:
+  name: adt_j2
+  obsdatain:
+    engine:
+      type: H5File
+      obsfile: !ENV ${DATA}/obs/${OPREFIX}adt_j2.${PDY}${cyc}.nc4
+  obsdataout:
+    engine:
+      type: H5File
+      obsfile: !ENV ${DATA}/diags/adt_j3.${PDY}${cyc}.nc4
+  simulated variables: [absoluteDynamicTopography]
+obs operator:
+  name: ADT
+obs error:
+  covariance model: diagonal
diff --git a/parm/soca/obs/config/sst_metopa_l3u_so025.yaml b/parm/soca/obs/config/sst_metopa_l3u_so025.yaml
@@ -29,15 +29,15 @@ obs filters:
 - filter: Domain Check
   where:
   - variable: {name: ObsError/seaSurfaceTemperature}
-    minvalue: 0.001
+    minvalue: 1.0e-8
 - filter: Domain Check
   where:
   - variable: { name: GeoVaLs/sea_ice_area_fraction}
-    maxvalue: 0.00001
+    maxvalue: 1.0e-5
 - filter: Domain Check
   where:
   - variable: {name: GeoVaLs/sea_surface_temperature}
-    minvalue: 5.0
+    minvalue: -1.0
 - filter: Domain Check
   where:
   - variable: {name: GeoVaLs/distance_from_coast}

diff --git a/parm/soca/obs/config/sst_metopb_l3u_so025.yaml b/parm/soca/obs/config/sst_metopb_l3u_so025.yaml
@@ -29,15 +29,15 @@ obs filters:
 - filter: Domain Check
   where:
   - variable: {name: ObsError/seaSurfaceTemperature}
-    minvalue: 0.001
+    minvalue: 1.0e-8
 - filter: Domain Check
   where:
   - variable: { name: GeoVaLs/sea_ice_area_fraction}
-    maxvalue: 0.00001
+    maxvalue: 1.0e-5
 - filter: Domain Check
   where:
   - variable: {name: GeoVaLs/sea_surface_temperature}
-    minvalue: 5.0
+    minvalue: -1.0
 - filter: Domain Check
   where:
   - variable: {name: GeoVaLs/distance_from_coast}

diff --git a/parm/soca/obs/config/sst_metopc_l3u_so025.yaml b/parm/soca/obs/config/sst_metopc_l3u_so025.yaml
@@ -29,15 +29,15 @@ obs filters:
 - filter: Domain Check
   where:
   - variable: {name: ObsError/seaSurfaceTemperature}
-    minvalue: 0.001
+    minvalue: 1.0e-8
 - filter: Domain Check
   where:
   - variable: { name: GeoVaLs/sea_ice_area_fraction}
-    maxvalue: 0.00001
+    maxvalue: 1.0e-5
 - filter: Domain Check
   where:
   - variable: {name: GeoVaLs/sea_surface_temperature}
-    minvalue: 5.0
+    minvalue: -1.0
 - filter: Domain Check
   where:
   - variable: {name: GeoVaLs/distance_from_coast}

diff --git a/parm/soca/obs/config/sst_noaa18_l3u_so025.yaml b/parm/soca/obs/config/sst_noaa18_l3u_so025.yaml
@@ -29,15 +29,15 @@ obs filters:
 - filter: Domain Check
   where:
   - variable: {name: ObsError/seaSurfaceTemperature}
-    minvalue: 0.001
+    minvalue: 1.0e-8
 - filter: Domain Check
   where:
   - variable: { name: GeoVaLs/sea_ice_area_fraction}
-    maxvalue: 0.00001
+    maxvalue: 1.0e-5
 - filter: Domain Check
   where:
   - variable: {name: GeoVaLs/sea_surface_temperature}
-    minvalue: 5.0
+    minvalue: -1.0
 - filter: Domain Check
   where:
   - variable: {name: GeoVaLs/distance_from_coast}

diff --git a/parm/soca/obs/config/sst_noaa19_l3u_so025.yaml b/parm/soca/obs/config/sst_noaa19_l3u_so025.yaml
@@ -13,6 +13,7 @@ obs space:
     max pool size: 1
 obs operator:
   name: Identity
+  observation alias file: obsop_name_map.yaml
 obs error:
   covariance model: diagonal
 obs filters:
@@ -28,15 +29,15 @@ obs filters:
 - filter: Domain Check
   where:
   - variable: {name: ObsError/seaSurfaceTemperature}
-    minvalue: 0.001
+    minvalue: 1.0e-8
 - filter: Domain Check
   where:
   - variable: { name: GeoVaLs/sea_ice_area_fraction}
-    maxvalue: 0.00001
+    maxvalue: 1.0e-5
 - filter: Domain Check
   where:
   - variable: {name: GeoVaLs/sea_surface_temperature}
-    minvalue: 5.0
+    minvalue: -1.0
 - filter: Domain Check
   where:
   - variable: {name: GeoVaLs/distance_from_coast}

diff --git a/parm/soca/obs/config/sst_viirs_n20_l3u_so025.yaml b/parm/soca/obs/config/sst_viirs_n20_l3u_so025.yaml
@@ -29,15 +29,15 @@ obs filters:
 - filter: Domain Check
   where:
   - variable: {name: ObsError/seaSurfaceTemperature}
-    minvalue: 0.001
+    minvalue: 1.0e-8
 - filter: Domain Check
   where:
   - variable: { name: GeoVaLs/sea_ice_area_fraction}
-    maxvalue: 0.00001
+    maxvalue: 1.0e-5
 - filter: Domain Check
   where:
   - variable: {name: GeoVaLs/sea_surface_temperature}
-    minvalue: 5.0
+    minvalue: -1.0
 - filter: Domain Check
   where:
   - variable: {name: GeoVaLs/distance_from_coast}

diff --git a/parm/soca/obs/config/sst_viirs_npp_l3u_so025.yaml b/parm/soca/obs/config/sst_viirs_npp_l3u_so025.yaml
@@ -29,15 +29,15 @@ obs filters:
 - filter: Domain Check
   where:
   - variable: {name: ObsError/seaSurfaceTemperature}
-    minvalue: 0.001
+    minvalue: 1.0e-8
 - filter: Domain Check
   where:
   - variable: { name: GeoVaLs/sea_ice_area_fraction}
-    maxvalue: 0.00001
+    maxvalue: 1.0e-5
 - filter: Domain Check
   where:
   - variable: {name: GeoVaLs/sea_surface_temperature}
-    minvalue: 5.0
+    minvalue: -1.0
 - filter: Domain Check
   where:
   - variable: {name: GeoVaLs/distance_from_coast}

diff --git a/parm/soca/obs/obs_list.yaml b/parm/soca/obs/obs_list.yaml
@@ -1,5 +1,7 @@
 observers:
+- !INC ${OBS_YAML_DIR}/adt_all.yaml
 - !INC ${OBS_YAML_DIR}/adt_j3.yaml
+- !INC ${OBS_YAML_DIR}/adt_j2.yaml
 #- !INC ${OBS_YAML_DIR}/salt_profile_fnmoc.yaml
 - !INC ${OBS_YAML_DIR}/sss_smap.yaml
 - !INC ${OBS_YAML_DIR}/sst_noaa19_l3u.yaml

diff --git a/scripts/exgdas_global_marine_analysis_prep.py b/scripts/exgdas_global_marine_analysis_prep.py
@@ -268,6 +268,11 @@ def find_clim_ens(input_date):
 # stage observations from R2D2 COMIN_OBS to COM_OBS
 ufsda.stage.obs(stage_cfg)
 
+# concatenate altimeters into one obs space
+# TODO (SAMG)temporary, move this into the obs procecing eventually
+adt_obs = f"{os.getenv('COM_OBS')}/{RUN}.t{cyc}z.adt"
+ufsda.soca_utils.concatenate_ioda(adt_obs, wildcard="*.nc4", output_suffix=f"_all.{PDY}{cyc}.nc4", clean=True)
+
 # get the list of observations
 obs_files = []
 for ob in stage_cfg['observations']['observers']:

diff --git a/test/soca/create_obsdb.py b/test/soca/create_obsdb.py
@@ -36,6 +36,7 @@ def cdl2nc(cdl_filename, nc4_filename):
     # Change the obs file name format
     obsdir = os.getenv('SOCA_TEST_OBS')
     cdl2nc(os.path.join(obsdir, 'adt.nc.cdl'), 'adt_j3_20180415.nc4')
+    cdl2nc(os.path.join(obsdir, 'adt.nc.cdl'), 'adt_j2_20180415.nc4')
     cdl2nc(os.path.join(obsdir, 'sst.nc.cdl'), 'sst_noaa19_l3u_20180415.nc4')
     cdl2nc(os.path.join(obsdir, 'sss.nc.cdl'), 'sss_smap_20180415.nc4')
     cdl2nc(os.path.join(obsdir, 'prof.nc.cdl'), 'temp_profile_fnmoc_20180415.nc4')
@@ -53,6 +54,7 @@ def cdl2nc(cdl_filename, nc4_filename):
                          'provider': 'gdasapp',
                          'experiment': 'soca',
                          'obs_types': ['adt_j3',
+                                       'adt_j2',
                                        'sst_noaa19_l3u',
                                        'sss_smap',
                                        'temp_profile_fnmoc',

diff --git a/test/soca/gw/run_jjobs.yaml.test b/test/soca/gw/run_jjobs.yaml.test
@@ -56,4 +56,4 @@ job options:
   nodes: 1
   ntasks: 16
   partition: @PARTITION@
-  time: 00:05:00
+  time: 00:10:00
diff --git a/ush/soca/soca_vrfy.py b/ush/soca/soca_vrfy.py
@@ -152,6 +152,10 @@ def plotMeridionalSlice(config):
     """
     pcolormesh of a Meridional slice of an ocean field
     """
+    variable = config['variable']
+    exp = config['exp']
+    PDY = config['PDY']
+    cyc = config['cyc']
     lon = float(config['lon'])
     grid = xr.open_dataset(config['grid file'])
     data = xr.open_dataset(config['fields file'])
@@ -168,6 +172,8 @@ def plotMeridionalSlice(config):
                    cmap=config['colormap'])
     plt.colorbar(label=config['variable']+' Lon '+str(lon), shrink=0.5, orientation='horizontal')
     ax.set_ylim(-config['max depth'], 0)
+    title = f"{exp} {PDY} {cyc} {variable} lon {int(lon)}"
+    ax.set_title(title)
     dirname = os.path.join(config['comout'], config['variable'])
     os.makedirs(dirname, exist_ok=True)
     figname = os.path.join(dirname, config['variable'] +

diff --git a/ush/ufsda/soca_utils.py b/ush/ufsda/soca_utils.py
@@ -53,8 +53,8 @@ def obs_specs(iodafname):
     raise Exception(f"No known obs type in {iodafname}.")
 
 
-def concatenate_ioda(iodafname):
-    flist = glob.glob(iodafname+'.*')
+def concatenate_ioda(iodafname, wildcard=".*", output_suffix="", clean=False):
+    flist = glob.glob(f"{iodafname}{wildcard}")
     flist.sort()
     nfiles = len(flist)
     if nfiles == 0:
@@ -63,7 +63,7 @@ def concatenate_ioda(iodafname):
 
     if len(flist) == 1:
         logging.info(f"Only file is {flist[0]}, rename to {iodafname}. No need to concatenate.")
-        shutil.move(flist[0], iodafname)
+        shutil.move(flist[0], iodafname+output_suffix)
         return
 
     logging.info(f"Concatenating {nfiles} files from globbing {iodafname}.*")
@@ -107,7 +107,12 @@ def concatenate_ioda(iodafname):
     globalattrs = {}
 
     # Write
-    writer = iconv.IodaWriter(iodafname, locationkeylist, dimdict)
+    writer = iconv.IodaWriter(iodafname+output_suffix, locationkeylist, dimdict)
     writer.BuildIoda(outdata, vardims, varattrs, globalattrs)
 
+    # Remove files
+    if clean:
+        for file in flist:
+            os.remove(file)
+
     return