Added yaml file that include bias correction for atms_npp. (#66)

this pull request include a new yaml file includes the bias correction
part on top of rrfs_mpasjedi_2022052619_atms_npp_qc.yaml. This new yaml
file named as 'rrfs_mpasjedi_2022052619_atms_npp_qc_**bc**.yaml'
Also, three fixed files have been added into the test data Data/obs

rrfs-test/scripts/link_mpasjedi_expr.sh

@@ -53,4 +53,5 @@ ln -snf ${RDASApp}/fix/expr_data/${exprname}/bkg/restart.2024-05-27_00.00.00.nc
 ln -snf ${RDASApp}/fix/expr_data/${exprname}/bkg/restart.2024-05-27_00.00.00.nc static.nc
 ln -snf ${RDASApp}/fix/expr_data/${exprname}/obs/* obs/
 ln -snf ${RDASApp}/fix/expr_data/${exprname}/ens/* ens/
+ln -snf ${RDASApp}/fix/crtm/2.4.0 crtm
 

rrfs-test/testinput/rrfs_mpasjedi_2022052619_atms_npp_qc.yaml → rrfs-test/validated_yamls/templates/obtype_config/atms_npp_qc_bc.yaml

@@ -1,189 +1,13 @@
-
-# application-agnostic anchors that specify observation errors
-# applicable to Variational, HofX3D
-
-# reusable latitude bands for all observation types
-_conventional obs localizations: &heightAndHorizObsLoc
-  _blank: null
-
-_nonconventional obs localizations: &horizObsLoc
-  _blank: null
-
-_obs space: &ObsSpace
-  obs perturbations seed: 1
-  io pool:
-    max pool size: 10
-  distribution:
-    name: RoundRobin
-
-_obs error diagonal: &ObsErrorDiagonal
-  covariance model: diagonal
-  # Note: the same 'obs perturbations seed' must be used for all members for the 'zero-mean perturbations' option to work
-  zero-mean perturbations: true
-  member: 1
-  number of members: 1
-
-_get values: &GetValues
-  nnearest: 3
-
-_multi iteration filter: &multiIterationFilter
-  apply at iterations: 0,1,2,3,4,5
-# ObsAnchors and ObsErrorAnchors are automatically prepended above this line
-_iteration: &iterationConfig
-  geometry:
-    nml_file: ./namelist.atmosphere_15km
-    streams_file: ./streams.atmosphere_15km
-    deallocate non-da fields: true
-    interpolation type: unstructured
-  gradient norm reduction: 1e-3
-_member: &memberConfig
-  date: &analysisDate '2022-05-26T19:00:00Z'
-  state variables: &incvars [spechum,surface_pressure,temperature,uReconstructMeridional,uReconstructZonal]
-  stream name: ensemble
-
-output:
-  filename: ./an.$Y-$M-$D_$h.$m.$s.nc
-  stream name: analysis
-variational:
-  minimizer:
-    algorithm: DRPCG
-  iterations:
-  - <<: *iterationConfig
-    diagnostics:
-      departures: ombg
-    ninner: 50
-  - <<: *iterationConfig
-    ninner: 50
-final:
-  diagnostics:
-    departures: oman
-cost function:
-  cost type: 3D-Var
-  time window:
-     begin: '2022-05-26T18:00:00Z'
-     length: PT2H
-  jb evaluation: false
-  geometry:
-    nml_file: ./namelist.atmosphere_15km
-    streams_file: ./streams.atmosphere_15km
-    deallocate non-da fields: true
-    interpolation type: unstructured
-  analysis variables: *incvars 
-  background:
-    state variables: [spechum,surface_pressure,temperature,uReconstructMeridional,uReconstructZonal,theta,rho,u,qv,pressure,landmask,xice,snowc,skintemp,ivgtyp,isltyp,snowh,vegfra,u10,v10,lai,smois,tslb,pressure_p,qc,qi,qg,qr,qs,cldfrac]
-    filename: Data/bkg/bg.2022-05-26_19.00.00.nc
-    date: *analysisDate 
-  background error:
-    covariance model: ensemble
-    localization:
-      localization method: SABER
-      saber central block:
-        saber block name: BUMP_NICAS
-        active variables: *incvars
-        read:
-          io:
-            data directory: DataFix/bump_new
-            files prefix: bumploc_1000km6km 
-          drivers:
-            multivariate strategy: duplicated
-            read local nicas: true
-          model:
-            level for 2d variables: last
-    members from template:
-      template:
-        <<: *memberConfig
-        filename: ./Data/inputs/%iMember%/restart.nc
-      pattern: %iMember%
-      start: 1
-      zero padding: 1 
-      nmembers: 5 
-  observations:
-     observers:
-     - obs space:
-         name: Aircraft
-         obsdatain:
-           engine:
-             type: H5File
-             obsfile: Data/obs/rass_tsen_obs_2022052619.nc4
-         obsdataout:
-           engine:
-             type: H5File
-             obsfile: Data/hofx/rass_tsen_obs_2022052619.nc4
-         simulated variables: [airTemperature]
-         observed variables: [airTemperature]
-       obs operator:
-         name: VertInterp
-       obs error:
-         covariance model: diagonal
-       obs filters:
-       - filter: Bounds Check
-         filter variables:
-         - name: airTemperature@GsiUseFlag 
-         minvalue: 0.5 
-         maxvalue: 1.5 
-         action:
-            name: accept 
-
-     - obs space:
-         name: sonde  
-         obsdatain:
-           engine:
-             type: H5File
-             obsfile: Data/obs/sondes_tsen_obs_2022052619.nc4
-         obsdataout:
-           engine:
-             type: H5File
-             obsfile: Data/hofx/rass_tsen_obs_2022052619.nc4
-         simulated variables: [airTemperature]
-         observed variables: [airTemperature]
-       obs operator:
-         name: VertInterp
-       obs error:
-         covariance model: diagonal
-       obs filters:
-       - filter: Bounds Check
-         filter variables:
-         - name: airTemperature@GsiUseFlag 
-         minvalue: 0.5 
-         maxvalue: 1.5 
-         action:
-            name: accept 
-
-
-     - obs space:
-         name: sonde
-         obsdatain:
-           engine:
-             type: H5File
-             obsfile: Data/obs/sondes_uv_obs_2022052619.nc4
-         simulated variables: [windEastward, windNorthward]
-       obs operator:
-         name: VertInterp
-         vertical coordinate: air_pressure
-         observation vertical coordinate: pressure
-         interpolation method: log-linear
-       obs error:
-         covariance model: diagonal ufo
-       obs filters:
-       - filter: PreQC
-         maxvalue: 3
-       - filter: Background Check
-         filter variables:
-         - name: windEastward
-         - name: windNorthward
-         threshold: 6.0
-       monitoring only: true
-
      - obs space:
          name: atms_npp
          obsdatain:
            engine:
              type: H5File
-             obsfile: Data/obs/atms_npp_obs_2022052619.nc
+             obsfile: data/obs/atms_npp_obs_2024052700.nc
          obsdataout:
            engine:
              type: H5File
-             obsfile: Data/hofx/rass_atms_obs_2022052619.nc4
+             obsfile: ./rass_atms_obs_2024052700.nc4
          simulated variables: [brightnessTemperature]
          observed variables: [brightnessTemperature]
          channels: &atms_npp_channels 1-22
@@ -194,11 +18,42 @@ cost function:
            obs options:
              Sensor_ID: atms_npp
              EndianType: little_endian
-             CoefficientPath: Data/crtm_2.4.0/
+             CoefficientPath: data/crtm/
              IRVISlandCoeff: IGBP
            linear obs operator:
              Absorbers: [H2O, O3]
 
+       obs bias:
+         input file:  data/obs/atms_npp.satbias.nc4
+         output file: ./out_atms_npp.satbias.nc
+         variational bc:
+           predictors:
+           - name: constant
+           - name: lapseRate
+             order: 2
+             tlapse: &atms_npp_tlapse  data/obs/atms_npp.tlapse.txt
+           - name: lapseRate
+             tlapse: *atms_npp_tlapse
+           - name: emissivityJacobian
+           - name: sensorScanAngle
+             order: 4
+           - name: sensorScanAngle
+             order: 3
+           - name: sensorScanAngle
+             order: 2
+           - name: sensorScanAngle
+         covariance:
+           minimal required obs number: 20
+           variance range: [1.0e-6, 10.0]
+           step size: 1.0e-4
+           largest analysis variance: 10000.0
+           prior:
+             input file: data/obs/atms_npp.satbias_cov.nc4
+             inflation:
+               ratio: 1.1
+               ratio for small dataset: 2.0
+           output file: ./out_atms_npp.satbias_cov.nc
+
        obs pre filters:
        # Step 0-A: Create Diagnostic Flags
        - filter: Create Diagnostic Flags
@@ -391,8 +246,8 @@ cost function:
              save: true
              cenlat: 38.5 
              cenlon: -97.5
-             #radius: 1700000
-             radius: 1700
+             #radius: 1700 #km
+             radius: 2500 #km
          minvalue: 1.0
 
        # Step 0-C: Assign Initial All-Sky Observation Error
@@ -421,7 +276,7 @@ cost function:
              flag: ScanEdgeRemoval
            - name: reject
 
-       # Step 2: Data Thinning
+       # Step 2: data Thinning
        - filter: Gaussian Thinning
          horizontal_mesh: 145
          use_reduced_horizontal_grid: true
@@ -723,8 +578,4 @@ cost function:
            - name: reject
 
 
-test:
-  reference filename: Data/testinput/rrfs-mpasjedi-atms_npp.ref
-  test output filename: ./rrfs-mpasjedi-atms_npp.out
-  float relative tolerance: 1.0e-3
-  float absolute tolerance: 1.0e-6
+