diff --git a/parm/atm/obs/config/conv_ps.yaml b/parm/atm/obs/config/conv_ps.yaml new file mode 100644 index 000000000..8fa4ab185 --- /dev/null +++ b/parm/atm/obs/config/conv_ps.yaml @@ -0,0 +1,295 @@ +obs space: + name: surface_ps + obsdatain: + engine: + type: H5File + obsfile: $(DATA)/obs/$(OPREFIX)sondes.{{ current_cycle | to_YMDH }}.nc4 + obsdataout: + engine: + type: H5File + obsfile: $(DATA)/diags/diag_sondes_{{ current_cycle | to_YMDH }}.nc4 + io pool: + max pool size: 1 + simulated variables: [stationPressure] + +obs operator: + name: SfcPCorrected + variables: + - name: stationPressure + da_psfc_scheme: GSI + station_altitude: height + geovar_sfc_geomz: surface_altitude + geovar_geomz: geopotential_height + +obs prior filters: +# Initial Error Assignments for SFC Observations +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: [181] + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + round_to_the_nearest_integer: true + xvar: + name: ObsValue/stationPressure + xvals: [80000, 75000, 70000, 65000, 60000, 55000 ] + errors: [110, 120, 120, 120, 120, 1.0e+11] + +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: [187] + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + round_to_the_nearest_integer: true + xvar: + name: ObsValue/stationPressure + xvals: [85000, 80000, 75000, 70000, 65000, 60000, 55000 ] + errors: [ 120, 140, 140, 140, 140, 140, 1.0e+11] + +# Initial Error Assignments for SFCSHIP Observations +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: [180] + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + round_to_the_nearest_integer: true + xvar: + name: ObsValue/stationPressure + xvals: [60000, 55000 ] + errors: [ 130, 1.0e+11] + +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: [183] + action: + name: assign error + error parameter: 1.0e+11 + +# Initial Error Assignments for Radiosonde +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: [120] + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + round_to_the_nearest_integer: true + xvar: + name: ObsValue/stationPressure + xvals: [80000, 75000, 70000, 65000, 60000, 55000 ] + errors: [ 110, 120, 120, 120, 120, 1.0e+11] + +obs post filters: +# Observation range sanity check +- filter: Bounds Check + filter variables: + - name: stationPressure + minvalue: 37499.0 + maxvalue: 106999.0 + action: + name: reject + +# Reject all ObsType 183 +- filter: RejectList + where: + - variable: + name: ObsType/stationPressure + is_in: 183 + +# Reject surface pressure below 500 hPa +- filter: Bounds Check + filter variables: + - name: stationPressure + minvalue: 50000.00 + action: + name: reject + +- filter: RejectList + where: + - variable: + name: PreQC/stationPressure + is_in: 4-15 + +# Inflate obs error based on obs type +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: PreQC/stationPressure + is_in: 3, 7 + action: + name: inflate error + inflation factor: 1.2 + +# Calculate obs error inflation factors for duplicated observations at the same location +- filter: Variable Assignment + assignments: + - name: ObsErrorFactorDuplicateCheck/stationPressure + type: float + function: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: false + variable: stationPressure + +# Reduce effective observation error based on obs type and subtype +# In this case: reduce effective obs error for buoy +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: + name: ObsType/stationPressure + is_in: 180 + - variable: + name: ObsSubType/stationPressure + is_in: 0 + action: + name: inflate error + inflation factor: 0.7 + +# Reduce original observation error based on obs type and subtype +# In this case: reduce original obs error for buoy +- filter: Variable Assignment + where: + - variable: + name: ObsType/stationPressure + is_in: 180 + - variable: + name: ObsSubType/stationPressure + is_in: 0 + assignments: + - name: ObsError/stationPressure + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsError/stationPressure + coefs: [0.7] + +# Calculate obs error inflation factors for large discrepancies between model and observations +- filter: Variable Assignment + assignments: + - name: ObsErrorFactorSfcPressure/stationPressure + type: float + function: + name: ObsFunction/ObsErrorFactorSfcPressure + options: + geovar_sfc_geomz: surface_altitude + geovar_geomz: geopotential_height + station_altitude: height + +# Inflate surface pressure observation based on discrepancies between +# model and observations due to terrian +- filter: Perform Action + filter variables: + - name: stationPressure + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSfcPressure/stationPressure + +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/Innovation + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsValue/stationPressure + - name: HofX/stationPressure + coefs: [1, -1] + +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorBoundSfcPressure1 + type: float + function: + name: ObsFunction/ObsErrorBoundConventional + options: + obsvar: stationPressure + obserr_bound_min: 100 + obserr_bound_max: 300 + obserr_bound_factor: 5.0 + +- filter: Background Check + filter variables: + - name: stationPressure + where: + - variable: PreQC/stationPressure + is_not_in: 3 + function absolute threshold: + - name: DerivedMetaData/ObsErrorBoundSfcPressure1 + action: + name: reject + +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorBoundSfcPressure2 + type: float + function: + name: ObsFunction/ObsErrorBoundConventional + options: + obsvar: stationPressure + obserr_bound_min: 100 + obserr_bound_max: 300 + obserr_bound_factor: 3.5 + +- filter: Background Check + filter variables: + - name: stationPressure + where: + - variable: PreQC/stationPressure + is_in: 3 + function absolute threshold: + - name: DerivedMetaData/ObsErrorBoundSfcPressure2 + action: + name: reject + +# Inflate obs error based on duplicate check +- filter: Perform Action + filter variables: + - name: stationPressure + action: + name: inflate error + inflation variable: + name: ObsErrorFactorDuplicateCheck/stationPressure + +# Reject data based on PreUseFlag (usage in GSI) +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: PreUseFlag/stationPressure + is_not_in: 0, 1 + action: + name: reject + +# End of Filters + diff --git a/parm/atm/obs/testing/conv_ps.yaml b/parm/atm/obs/testing/conv_ps.yaml index f43bc9f41..20a449a8a 100644 --- a/parm/atm/obs/testing/conv_ps.yaml +++ b/parm/atm/obs/testing/conv_ps.yaml @@ -8,8 +8,8 @@ obs space: engine: type: H5File obsfile: !ENV conv_ps_diag_${CDATE}.nc4 - overwrite: true simulated variables: [stationPressure] + geovals: filename: !ENV conv_ps_geoval_${CDATE}.nc4 @@ -22,191 +22,276 @@ obs operator: geovar_sfc_geomz: surface_altitude geovar_geomz: geopotential_height +obs prior filters: +# Initial Error Assignments for SFC Observations +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: [181] + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + round_to_the_nearest_integer: true + xvar: + name: ObsValue/stationPressure + xvals: [80000, 75000, 70000, 65000, 60000, 55000 ] + errors: [110, 120, 120, 120, 120, 1.0e+11] + +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: [187] + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + round_to_the_nearest_integer: true + xvar: + name: ObsValue/stationPressure + xvals: [85000, 80000, 75000, 70000, 65000, 60000, 55000 ] + errors: [ 120, 140, 140, 140, 140, 140, 1.0e+11] + +# Initial Error Assignments for SFCSHIP Observations +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: [180] + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + round_to_the_nearest_integer: true + xvar: + name: ObsValue/stationPressure + xvals: [60000, 55000 ] + errors: [ 130, 1.0e+11] + +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: [183] + action: + name: assign error + error parameter: 1.0e+11 + +# Initial Error Assignments for Radiosonde +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: [120] + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + round_to_the_nearest_integer: true + xvar: + name: ObsValue/stationPressure + xvals: [80000, 75000, 70000, 65000, 60000, 55000 ] + errors: [ 110, 120, 120, 120, 120, 1.0e+11] + obs post filters: - # Observation range sanity check - - filter: Bounds Check - filter variables: - - name: stationPressure - minvalue: 37499.0 - maxvalue: 106999.0 - action: - name: reject - - # Reject all ObsType 183 - - filter: RejectList - where: - - variable: - name: ObsType/stationPressure - is_in: 183 - - # Reject surface pressure below 500 hPa - - filter: Bounds Check - filter variables: - - name: stationPressure - minvalue: 50000.00 - action: - name: reject - - - filter: RejectList - where: - - variable: - name: PreQC/stationPressure - is_in: 4-15 - - # Inflate obs error based on obs type - - filter: Perform Action - filter variables: - - name: stationPressure - where: - - variable: PreQC/stationPressure - is_in: 3, 7 - action: - name: inflate error - inflation factor: 1.2 - - # Calculate obs error inflation factors for duplicated observations at the same location - - filter: Variable Assignment - assignments: - - name: ObsErrorFactorDuplicateCheck/stationPressure - type: float - function: - name: ObsFunction/ObsErrorFactorDuplicateCheck - options: - use_air_pressure: false - variable: stationPressure - - # Reduce effective observation error based on obs type and subtype - # In this case: reduce effective obs error for buoy - - filter: Perform Action - filter variables: - - name: stationPressure - where: - - variable: - name: ObsType/stationPressure - is_in: 180 - - variable: - name: ObsSubType/stationPressure - is_in: 0 - action: - name: inflate error - inflation factor: 0.7 - - # Reduce original observation error based on obs type and subtype - # In this case: reduce original obs error for buoy - - filter: Variable Assignment - where: - - variable: - name: ObsType/stationPressure - is_in: 180 - - variable: - name: ObsSubType/stationPressure - is_in: 0 - assignments: - - name: ObsError/stationPressure - type: float - function: - name: ObsFunction/Arithmetic - options: - variables: - - name: ObsError/stationPressure - coefs: [0.7] - - # Calculate obs error inflation factors for large discrepancies between model and observations - - filter: Variable Assignment - assignments: - - name: ObsErrorFactorSfcPressure/stationPressure - type: float - function: - name: ObsFunction/ObsErrorFactorSfcPressure - options: - geovar_sfc_geomz: surface_altitude - - # Inflate surface pressure observation based on discrepancies between - # model and observations due to terrian - - filter: Perform Action - filter variables: - - name: stationPressure - action: - name: inflate error - inflation variable: - name: ObsErrorFactorSfcPressure/stationPressure - - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/Innovation - type: float - function: - name: ObsFunction/Arithmetic - options: - variables: - - name: ObsValue/stationPressure - - name: HofX/stationPressure - coefs: [1, -1] - - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/ObsErrorBoundSfcPressure1 - type: float - function: - name: ObsFunction/ObsErrorBoundConventional - options: - obsvar: stationPressure - obserr_bound_min: 100 - obserr_bound_max: 300 - obserr_bound_factor: 5.0 - - - filter: Background Check - filter variables: - - name: stationPressure - where: - - variable: PreQC/stationPressure - is_not_in: 3 - function absolute threshold: - - name: DerivedMetaData/ObsErrorBoundSfcPressure1 - action: - name: reject - - - filter: Variable Assignment - assignments: - - name: DerivedMetaData/ObsErrorBoundSfcPressure2 - type: float - function: - name: ObsFunction/ObsErrorBoundConventional - options: - obsvar: stationPressure - obserr_bound_min: 100 - obserr_bound_max: 300 - obserr_bound_factor: 3.5 - - - filter: Background Check - filter variables: - - name: stationPressure - where: - - variable: PreQC/stationPressure - is_in: 3 - function absolute threshold: - - name: DerivedMetaData/ObsErrorBoundSfcPressure2 - action: - name: reject - - # Inflate obs error based on duplicate check - - filter: Perform Action - filter variables: - - name: stationPressure - action: - name: inflate error - inflation variable: - name: ObsErrorFactorDuplicateCheck/stationPressure - - # Reject data based on PreUseFlag (usage in GSI) - - filter: Perform Action - filter variables: - - name: stationPressure - where: - - variable: PreUseFlag/stationPressure - is_not_in: 0, 1 - action: - name: reject +# Observation range sanity check +- filter: Bounds Check + filter variables: + - name: stationPressure + minvalue: 37499.0 + maxvalue: 106999.0 + action: + name: reject + +# Reject all ObsType 183 +- filter: RejectList + where: + - variable: + name: ObsType/stationPressure + is_in: 183 + +# Reject surface pressure below 500 hPa +- filter: Bounds Check + filter variables: + - name: stationPressure + minvalue: 50000.00 + action: + name: reject + +- filter: RejectList + where: + - variable: + name: PreQC/stationPressure + is_in: 4-15 + +# Inflate obs error based on obs type +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: PreQC/stationPressure + is_in: 3, 7 + action: + name: inflate error + inflation factor: 1.2 + +# Calculate obs error inflation factors for duplicated observations at the same location +- filter: Variable Assignment + assignments: + - name: ObsErrorFactorDuplicateCheck/stationPressure + type: float + function: + name: ObsFunction/ObsErrorFactorDuplicateCheck + options: + use_air_pressure: false + variable: stationPressure + +# Reduce effective observation error based on obs type and subtype +# In this case: reduce effective obs error for buoy +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: + name: ObsType/stationPressure + is_in: 180 + - variable: + name: ObsSubType/stationPressure + is_in: 0 + action: + name: inflate error + inflation factor: 0.7 + +# Reduce original observation error based on obs type and subtype +# In this case: reduce original obs error for buoy +- filter: Variable Assignment + where: + - variable: + name: ObsType/stationPressure + is_in: 180 + - variable: + name: ObsSubType/stationPressure + is_in: 0 + assignments: + - name: ObsError/stationPressure + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsError/stationPressure + coefs: [0.7] + +# Calculate obs error inflation factors for large discrepancies between model and observations +- filter: Variable Assignment + assignments: + - name: ObsErrorFactorSfcPressure/stationPressure + type: float + function: + name: ObsFunction/ObsErrorFactorSfcPressure + options: + geovar_sfc_geomz: surface_altitude + geovar_geomz: geopotential_height + station_altitude: height + +# Inflate surface pressure observation based on discrepancies between +# model and observations due to terrian +- filter: Perform Action + filter variables: + - name: stationPressure + action: + name: inflate error + inflation variable: + name: ObsErrorFactorSfcPressure/stationPressure + +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/Innovation + type: float + function: + name: ObsFunction/Arithmetic + options: + variables: + - name: ObsValue/stationPressure + - name: HofX/stationPressure + coefs: [1, -1] + +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorBoundSfcPressure1 + type: float + function: + name: ObsFunction/ObsErrorBoundConventional + options: + obsvar: stationPressure + obserr_bound_min: 100 + obserr_bound_max: 300 + obserr_bound_factor: 5.0 + +- filter: Background Check + filter variables: + - name: stationPressure + where: + - variable: PreQC/stationPressure + is_not_in: 3 + function absolute threshold: + - name: DerivedMetaData/ObsErrorBoundSfcPressure1 + action: + name: reject + +- filter: Variable Assignment + assignments: + - name: DerivedMetaData/ObsErrorBoundSfcPressure2 + type: float + function: + name: ObsFunction/ObsErrorBoundConventional + options: + obsvar: stationPressure + obserr_bound_min: 100 + obserr_bound_max: 300 + obserr_bound_factor: 3.5 + +- filter: Background Check + filter variables: + - name: stationPressure + where: + - variable: PreQC/stationPressure + is_in: 3 + function absolute threshold: + - name: DerivedMetaData/ObsErrorBoundSfcPressure2 + action: + name: reject + +# Inflate obs error based on duplicate check +- filter: Perform Action + filter variables: + - name: stationPressure + action: + name: inflate error + inflation variable: + name: ObsErrorFactorDuplicateCheck/stationPressure + +# Reject data based on PreUseFlag (usage in GSI) +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: PreUseFlag/stationPressure + is_not_in: 0, 1 + action: + name: reject +# End of Filters passedBenchmark: 85378 diff --git a/parm/atm/obs/testing/conv_ps_noqc.yaml b/parm/atm/obs/testing/conv_ps_noqc.yaml index 44a6ef4c8..8d469729f 100644 --- a/parm/atm/obs/testing/conv_ps_noqc.yaml +++ b/parm/atm/obs/testing/conv_ps_noqc.yaml @@ -8,8 +8,8 @@ obs space: engine: type: H5File obsfile: !ENV conv_ps_diag_${CDATE}.nc4 - overwrite: true simulated variables: [stationPressure] + geovals: filename: !ENV conv_ps_geoval_${CDATE}.nc4 @@ -22,10 +22,89 @@ obs operator: geovar_sfc_geomz: surface_altitude geovar_geomz: geopotential_height -vector ref: GsiHofXBc -tolerance: 1.e-4 -#linear obs operator test: -# coef TL: 0.1 -# tolerance TL: 1.0e-13 -# tolerance AD: 1.0e-11 +obs prior filters: +# Initial Error Assignments for SFC Observations +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: [181] + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + round_to_the_nearest_integer: true + xvar: + name: ObsValue/stationPressure + xvals: [80000, 75000, 70000, 65000, 60000, 55000 ] + errors: [110, 120, 120, 120, 120, 1.0e+11] + +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: [187] + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + round_to_the_nearest_integer: true + xvar: + name: ObsValue/stationPressure + xvals: [85000, 80000, 75000, 70000, 65000, 60000, 55000 ] + errors: [ 120, 140, 140, 140, 140, 140, 1.0e+11] + +# Initial Error Assignments for SFCSHIP Observations +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: [180] + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + round_to_the_nearest_integer: true + xvar: + name: ObsValue/stationPressure + xvals: [60000, 55000 ] + errors: [ 130, 1.0e+11] + +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: [183] + action: + name: assign error + error parameter: 1.0e+11 + + # Initial Error Assignments for Radiosonde +- filter: Perform Action + filter variables: + - name: stationPressure + where: + - variable: ObsType/stationPressure + is_in: [120] + action: + name: assign error + error function: + name: ObsFunction/ObsErrorModelStepwiseLinear + options: + round_to_the_nearest_integer: true + xvar: + name: ObsValue/stationPressure + xvals: [80000, 75000, 70000, 65000, 60000, 55000 ] + errors: [ 110, 120, 120, 120, 120, 1.0e+11] + +passedBenchmark: 92824 # total: 92842; missing: 18 +#vector ref: GsiHofXBc +#tolerance: 1.e-4 diff --git a/ush/ufoeval/run_ufo_hofx_test.sh b/ush/ufoeval/run_ufo_hofx_test.sh index ccc043bd4..230966284 100755 --- a/ush/ufoeval/run_ufo_hofx_test.sh +++ b/ush/ufoeval/run_ufo_hofx_test.sh @@ -33,6 +33,7 @@ run_filtering=YES run_eva=YES eva_stats_only=NO keep_output=NO +window_shift=False while getopts "c:hsxq" opt; do case $opt in @@ -99,7 +100,6 @@ if [ $run_filtering == NO ]; then else yamlpath=$GDASApp/parm/atm/obs/testing/${obtype}.yaml fi - exename=test_ObsFilters.x #-------------- Do not modify below this line ---------------- @@ -178,11 +178,13 @@ export OPREFIX=gdas.t${cyc}z export APREFIX=gdas.t${cyc}z export GPREFIX=gdas.t${gcyc}z +if [ $obtype == "conv_ps" ]; then window_shift=True; fi cat > $workdir/temp.yaml << EOF window begin: '{{ WINDOW_BEGIN | to_isotime }}' window end: '{{ WINDOW_END | to_isotime }}' observations: - !INC $yamlpath +window shift: ${window_shift} EOF $GDASApp/ush/genYAML --input $workdir/temp.yaml --output $workdir/${obtype}_${cycle}.yaml diff --git a/ush/ufoeval/test_yamls.sh b/ush/ufoeval/test_yamls.sh index 7e98d80e2..d46056bd8 100755 --- a/ush/ufoeval/test_yamls.sh +++ b/ush/ufoeval/test_yamls.sh @@ -13,10 +13,23 @@ if [ $? -ne 0 ]; then exit 1 fi -for file in `ls ../../parm/atm/obs/testing/*yaml`; do +# Process tests wiht QC +for file in `find ../../parm/atm/obs/testing/*.yaml -type f -not -name "*noqc*"`; do basefile=${file##*/} - inst="${basefile%.*}" - ./run_ufo_hofx_test.sh -x $inst > $WORKDIR/$inst.log 2> $WORKDIR/$inst.err + obtype="${basefile%.*}" + ./run_ufo_hofx_test.sh -x $obtype > $WORKDIR/$obtype.log 2> $WORKDIR/$obtype.err + if [ $? == 0 ]; then + echo $basefile Passes \(yay!\) + else + echo $basefile Fails \(boo!\) + fi +done + +# Process tests without QC (HofX + Observation error assignment) +for file in `ls ../../parm/atm/obs/testing/*_noqc.yaml`; do + basefile=${file##*/} + obtype="${basefile%_noqc.*}" + ./run_ufo_hofx_test.sh -x -q $obtype > $WORKDIR/${obtype}_noqc.log 2> $WORKDIR/${obtype}_noqc.err if [ $? == 0 ]; then echo $basefile Passes \(yay!\) else