Merge branch 'NOAA-EMC:develop' into develop

rrfs-test/ush/fv3jedi_gsi_increment_singleob.py

@@ -89,7 +89,6 @@
 jncdiag = Dataset(jdiag, mode='r')
 oberr_input = jncdiag.groups["EffectiveError0"].variables[f"{variable}"][:][0]
 oberr_final = jncdiag.groups["EffectiveError2"].variables[f"{variable}"][:][0]
-#pdb.set_trace()
 ob = jncdiag.groups["ObsValue"].variables[f"{variable}"][:][0]
 omf= jncdiag.groups["ombg"].variables[f"{variable}"][:][0]
 fmo= -1*omf
@@ -100,26 +99,35 @@
 job = np.around(ob.astype(np.float64), decimals)
 jomf = np.around(omf.astype(np.float64), decimals)
 jhofx = np.around(hofx.astype(np.float64), decimals)
+
 print(f"JEDI:")
 print(f"  oberr_input: {oberr_input}")
 print(f"  oberr_final: {oberr_final}")
 subtitle1_hofx = f"  ob:   {job}\n  omf:  {jomf}\n  hofx: {jhofx}\n  oberr_final: {oberr_final}\n"
 print(subtitle1_hofx)
 
 # FROM GSI diag
-prefix = ""
-if variable == "windWestard":
-    prefix == "u_"
-if variable == "windNorthward":
-    prefix == "v_"
+#prefix = ""
+#if variable == "windEastward":
+#    prefix = "u_"
+#if variable == "windNorthward":
+#    prefix = "v_"
+
 gncdiag = Dataset(gdiag, mode='r')
-oberr_input = 1.0/(gncdiag.variables["Errinv_Input"][:][0])
-oberr_final = 1.0/(gncdiag.variables["Errinv_Final"][:][0])
-ob = gncdiag.variables[f"{prefix}Observation"][:][0]
-omf= gncdiag.variables[f"{prefix}Obs_Minus_Forecast_unadjusted"][:][0]
+oberr_input = jncdiag.groups["GsiInputObsError"].variables[f"{variable}"][:][0]
+oberr_final = jncdiag.groups["GsiFinalObsError"].variables[f"{variable}"][:][0]
+ob = jncdiag.groups["ObsValue"].variables[f"{variable}"][:][0]
+hofx = jncdiag.groups["GsiHofXBc"].variables[f"{variable}"][:][0]
+#hofx = jncdiag.groups["GsiHofX"].variables[f"{variable}"][:][0]
+omf = ob - hofx
+
+#oberr_input = 1.0/(gncdiag.variables["Errinv_Input"][:][0])
+#oberr_final = 1.0/(gncdiag.variables["Errinv_Final"][:][0])
+#ob = gncdiag.variables[f"{prefix}Observation"][:][0]
+#omf= gncdiag.variables[f"{prefix}Obs_Minus_Forecast_unadjusted"][:][0]
 #omf= gncdiag.variables["Obs_Minus_Forecast_adjusted"][:][0]
-fmo= -1*omf
-hofx= fmo+ob
+#fmo= -1*omf
+#hofx= fmo+ob
 oberr_input = np.around(oberr_input.astype(np.float64), decimals)
 oberr_final = np.around(oberr_final.astype(np.float64), decimals)
 gob = np.around(ob.astype(np.float64), decimals)

rrfs-test/ush/fv3jedi_gsi_validation.py

@@ -46,17 +46,17 @@
 print(f"min/max of jedi errf: {np.min(jerrf)} {np.max(jerrf)}")
 print(f"min/max of gsi errf: {np.min(gerrf)} {np.max(gerrf)}")
 
-xmin = np.min(jerrf)
-xmax = np.max(jerrf)
-# Round to nearest 10 that contains all errf values.
-errf_xmin = np.floor(xmin/10)*10
-errf_xmax = np.ceil(xmax/10)*10
+# Function to calculate the floor and ceiling based on the order of magnitude
+def order_of_magnitude_bounds(value):
+    if value == 0:
+        return 0, 0
+    order_of_magnitude = 10 ** np.floor(np.log10(np.abs(value)))
+    floor_value = np.floor(value / order_of_magnitude) * order_of_magnitude
+    ceiling_value = np.ceil(value / order_of_magnitude) * order_of_magnitude
+    return floor_value, ceiling_value
 
-xmin = np.min(jhofx)
-xmax = np.max(jhofx)
-# Round to nearest 10 that contains all hofx values.
-hofx_xmin = np.floor(xmin/10)*10
-hofx_xmax = np.ceil(xmax/10)*10
+errf_xmin, errf_xmax = order_of_magnitude_bounds(jerrf)
+hofx_xmin, hofx_xmax = order_of_magnitude_bounds(jhofx)
 
 # CREATE PLOT ##############################
 fig = plt.figure(figsize = (6, 3))

rrfs-test/validated_yamls/gen_yaml.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+#use this script to combine the basic config yaml with an obtype config yaml.
+basic_config="fv3jedi_hyb3denvar.yaml"
+obtype_config="msonet_airTemperature_obtype188.yaml"
+
+# Don't edit below this line.
+
+# copy the basic configuration yaml into the new yaml
+cp -p templates/basic_config/$basic_config ./$obtype_config
+
+# use the stread editor to replace the instance of @OBSERVATION@ placeholder
+# in the basic configuration file with the contents of the obtype config yaml.
+sed -i '/@OBSERVATIONS@/{
+        r templates/obtype_config/'"${obtype_config}"'
+        d
+}' ./$obtype_config

rrfs-test/validated_yamls/templates/basic_config/fv3jedi_hyb3denvar.yaml

@@ -0,0 +1,153 @@
+cost function:
+  cost type: 3D-Var
+  time window:
+      begin: 2022-05-26T18:00:00Z
+      length: PT2H
+  analysis variables: &3dvars [eastward_wind,northward_wind,air_temperature,air_pressure_thickness,specific_humidity,cloud_liquid_ice,cloud_liquid_water,ozone_mass_mixing_ratio]
+
+  geometry:
+    fms initialization:
+      namelist filename: DataFix/fmsmpp.nml
+      field table filename: DataFix/field_table
+    namelist filename: DataFix/input_lam_C775.nml
+    field metadata override: Data/fieldmetadata/tlei-gfs-restart.yaml
+    akbk: DataFix/fix/akbk61.nc
+    # input.nml
+    npz: 60
+    layout: [8,10]
+    io_layout: [1,1]
+    ntiles: 1
+
+  background:
+    datetime: 2022-05-26T19:00:00Z
+    filetype: fms restart
+    datapath: Data/bkg
+    filename_core: fv3_dynvars.nc
+    filename_trcr: fv3_tracer.nc
+    filename_sfcd: fv3_sfcdata.nc
+    filename_sfcw: fv3_srf_wnd.nc
+    filename_cplr: coupler.res
+    state variables: [eastward_wind,northward_wind,u_component_of_native_D_grid_wind,v_component_of_native_D_grid_wind,air_temperature,air_pressure_thickness,specific_humidity,cloud_liquid_ice,cloud_liquid_water,ozone_mass_mixing_ratio,phis,f10m]
+  background error:
+    covariance model: hybrid
+    components:
+    - covariance:
+        covariance model: SABER
+        saber central block:
+          saber block name: BUMP_NICAS
+          active variables: *3dvars
+          read:
+            general:
+              color log: true
+              #sampling method: potential
+            io:
+              data directory: DataFix/bump_new
+              files prefix: fv3jedi_bumpparameters_nicas_lam_atm
+              alias:
+              - in code: common
+                in file: fixed_250km_0.3
+            drivers:
+               multivariate strategy: duplicated
+               read local nicas: true
+        saber outer blocks:
+        - saber block name: StdDev
+          read:
+            model file:
+              datetime: 2022-05-26T19:00:00Z
+              set datetime on read: true
+              filetype: fms restart
+              psinfile: true
+              datapath: Data_static/staticb_rrfs_step2/
+              filename_core: 20220526.190000.stddev.fv_core.res.nc
+              filename_trcr: 20220526.190000.stddev.fv_tracer.res.nc
+              filename_cplr: 20220526.190000.stddev.coupler.res
+              date: 2022-05-26T19:00:00Z
+      weight:
+        value: 0.1
+
+    - covariance:
+        covariance model: ensemble
+        members from template:
+          template:
+            datetime: 2022-05-26T19:00:00Z
+            filetype: fms restart
+            state variables: *3dvars
+            datapath: Data/inputs/memMEM/
+            filename_core: 20220526.190000.fv_core.res.tile1.nc
+            filename_trcr: 20220526.190000.fv_tracer.res.tile1.nc
+            filename_sfcd: 20220526.190000.sfc_data.nc
+            filename_sfcw: 20220526.190000.fv_srf_wnd.res.nc
+            filename_cplr: 20220526.190000.coupler.res
+          pattern: MEM
+          nmembers: 20
+          zero padding: 3
+        localization:
+          localization method: SABER
+          saber central block:
+            saber block name: BUMP_NICAS
+            active variables: *3dvars
+            read:
+              io:
+                files prefix: DataFix/bump_new/fv3jedi_bumpparameters_nicas_lam_atm
+                alias:
+                - in code: common
+                  in file: fixed_250km_0.3
+
+              drivers:
+                 multivariate strategy: duplicated
+                 read local nicas: true
+      weight:
+        value: 0.9
+
+  observations:
+     observers:
+@OBSERVATIONS@
+
+variational:
+  minimizer:
+    algorithm: DRPCG    # Derber-Rosati Conjugate Gradients
+  iterations:
+  - ninner: 50
+    gradient norm reduction: 1e-60
+    geometry:
+      fms initialization:
+        namelist filename: DataFix/fmsmpp.nml
+        field table filename: DataFix/field_table
+      namelist filename: DataFix/input_lam_C775.nml
+      field metadata override: Data/fieldmetadata/tlei-gfs-restart.yaml
+      akbk: DataFix/fix/akbk61.nc
+      # input.nml
+      npz: 60
+      layout: [8,10]
+      io_layout: [1,1]
+      ntiles: 1
+      fieldsets:
+      - fieldset: DataFix/fix/dynamics_lam_cmaq.yaml
+    diagnostics:
+      departures: ombg
+  - ninner: 50
+    gradient norm reduction: 1e-60
+    geometry:
+      fms initialization:
+        namelist filename: DataFix/fmsmpp.nml
+        field table filename: DataFix/field_table
+      namelist filename: DataFix/input_lam_C775.nml
+      field metadata override: Data/fieldmetadata/tlei-gfs-restart.yaml
+      akbk: DataFix/fix/akbk61.nc
+      # input.nml
+      npz: 60
+      layout: [8,10]
+      io_layout: [1,1]
+      ntiles: 1
+      fieldsets:
+      - fieldset: DataFix/fix/dynamics_lam_cmaq.yaml
+final:
+  diagnostics:
+    departures: oman
+
+output:
+  filetype: fms restart
+  datapath: ./
+  prefix: hybens3dvar-fv3_lam-C775
+  frequency: PT1H
+

rrfs-test/validated_yamls/templates/obtype_config/msonet_airTemperature_obtype188.yaml

@@ -0,0 +1,126 @@
+     - obs space:
+         name: mesonet
+         obsdatain:
+           engine:
+             type: H5File
+             obsfile: "obsout/sfc_tsen_obs_2022052619.nc4"
+         obsdataout:
+           engine:
+             type: H5File
+             obsfile: MSONET_hofxs_airTemperature_2022052619.nc4
+             allow overwrite: true
+         io pool:
+           max pool size: 1
+         observed variables: [airTemperature]
+         simulated variables: [airTemperature]
+
+       obs operator:
+         name: Composite
+         components:
+         - name: VertInterp
+           vertical coordinate: air_pressure
+           observation vertical coordinate: pressure
+           observation vertical coordinate group: MetaData
+           interpolation method: log-linear
+           gsi geovals:
+             filename: "obsout/sfc_tsen_geoval_2022052619.nc4"
+             levels_are_top_down: False
+           variables:
+           - name: airTemperature
+
+       obs error:
+         covariance model: diagonal
+
+       obs filters:
+         # ------------------
+         # airTemperature
+         # ------------------
+         # Initial error assignment
+         - filter: Perform Action
+           filter variables:
+           - name: airTemperature
+           where:
+           - variable: ObsType/airTemperature
+             is_in: 188
+           action:
+             name: assign error
+             error parameter: 4.0  # K
+           defer to post: true
+
+         # Ajusted error after initial assignment (qcmod.f90)
+         - filter: Perform Action
+           filter variables:
+           - name: airTemperature
+           where:
+           - variable: ObsType/airTemperature
+             is_in: 188
+           action:
+             name: inflate error
+             inflation variable:
+               name: ObsFunction/ObsErrorFactorConventional
+               options:
+                 test QCflag: PreQC
+                 test QCthreshold: 3
+                 inflate variables: [airTemperature]
+                 pressure: MetaData/pressure
+                 distance threshold: -1.
+           defer to post: true
+
+         # error inflation based on pressure check (setupt.f90)
+         - filter: Perform Action
+           filter variables:
+           - name: airTemperature
+           where:
+           - variable: ObsType/airTemperature
+             is_in: 188
+           action:
+             name: inflate error
+             #inflation factor: 1.09757
+             inflation variable:
+               name: ObsFunction/ObsErrorFactorPressureCheck
+               options:
+                 variable: airTemperature
+                 inflation factor: 8.0
+                 geovar_sfc_geomz: surface_geometric_height
+           defer to post: true
+
+         # Reject airTemperature with ObsType=195
+         - filter: Perform Action
+           filter variables:
+           - name: airTemperature
+           where:
+           - variable: ObsType/airTemperature
+             is_in: 195
+           action:
+             name: reject
+
+         - filter: Bounds Check
+           filter variables:
+           - name: airTemperature
+           minvalue: 200
+           maxvalue: 400
+
+         - filter: Background Check
+           filter variables:
+           - name: airTemperature
+           threshold: 4.0
+           where:
+           - variable: ObsType/airTemperature
+             is_in: 188
+           action:
+             name: reject
+
+        # Print filter data
+         - filter: Print Filter Data
+           message: Printing filter data
+           summary: true
+           variables:
+           - variable: MetaData/latitude
+           - variable: MetaData/longitude
+           - variable: MetaData/pressure
+           - variable: ObsType/airTemperature
+           - variable: ObsValue/airTemperature
+           - variable: QCflagsData/airTemperature
+
+         - filter: GOMsaver
+           filename: ./Data/geovals/msonet_geovals_rrfs_2022052619.nc4