From a5e1c2fd67604ca145c96efcf1fa5ec9363b7ee0 Mon Sep 17 00:00:00 2001
From: Samuel Degelia <samuel.degelia@noaa.gov>
Date: Wed, 23 Oct 2024 16:12:38 -0400
Subject: [PATCH] Add bufr2ioda ctest (#207)

A new ctest is added called `rrfs_bufr2ioda_msonet`. It runs `bufr2ioda.x` for a given prepbufr file. Then,`iodaconv_comp_rrfs.sh` is run to compare the output between the `ioda_msonet.nc` generated during the ctest and a reference version.
---
 rrfs-test/CMakeLists.txt                      |  36 +-
 .../testinput/rrfs_bufr2ioda_msonet.yaml      | 369 ++++++++++++++++++
 rrfs-test/ush/iodaconv_comp_rrfs.sh           |  48 +++
 ush/run_rrfs_tests.sh                         |   2 +-
 4 files changed, 451 insertions(+), 4 deletions(-)
 create mode 100644 rrfs-test/testinput/rrfs_bufr2ioda_msonet.yaml
 create mode 100755 rrfs-test/ush/iodaconv_comp_rrfs.sh

diff --git a/rrfs-test/CMakeLists.txt b/rrfs-test/CMakeLists.txt
index 420727f..5b013cd 100644
--- a/rrfs-test/CMakeLists.txt
+++ b/rrfs-test/CMakeLists.txt
@@ -62,10 +62,9 @@ if(FV3_DYCORE)
           set(MPI_ARGS ${RESTORE_MPI_ARGS})
       endif()
 
+endif()
 
-   endif()
-
-   if(MPAS_DYCORE)
+if(MPAS_DYCORE)
 
       message(STATUS "MACHINE_ID is " ${MACHINE_ID})
       if("${MACHINE_ID}" STREQUAL "orion")
@@ -117,4 +116,35 @@ if(FV3_DYCORE)
           set(MPI_ARGS ${RESTORE_MPI_ARGS})
       endif()
 
+      ########################
+      ### Bufr2ioda ctests ###
+      ########################
+
+      # YAML-based bufr2ioda
+      set(case "rrfs_bufr2ioda_msonet")
+      set(casedir "${CMAKE_CURRENT_BINARY_DIR}/rundir-${case}")
+      set(src_casedir "${rrfs-test_data_local}/rrfs-data_mpasjedi_2024052700")
+      if (NOT EXISTS "${casedir}")
+         file(MAKE_DIRECTORY ${casedir})
+      endif()
+      file(CREATE_LINK ${src_casedir}/data ${casedir}/data SYMBOLIC)
+      configure_file(${src_casedir}/data/obs/bufr/rap.t00z.prepbufr.tm00 ${casedir}/prepbufr COPYONLY)
+      file(COPY ${src_yaml}/${case}.yaml DESTINATION ${casedir})
+      set(MPI_ARGS "--time=00:05:00 --mem=64G")
+      set(IODA_CONV_COMP_TOL "5.0e-4")
+      ecbuild_add_test( TARGET  ${case}
+	                MPI     1
+                        TYPE    SCRIPT
+                        COMMAND bash
+          		WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/rundir-${case}
+           		ARGS    ${CMAKE_SOURCE_DIR}/rrfs-test/ush/iodaconv_comp_rrfs.sh
+                                netcdf
+                                "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x ${case}.yaml"
+                                data/obs/ioda_mesonet.nc
+                                ioda_mesonet.nc 
+			        ${IODA_CONV_COMP_TOL}
+                        DEPENDS bufr2ioda.x )
+
+
 endif()
+
diff --git a/rrfs-test/testinput/rrfs_bufr2ioda_msonet.yaml b/rrfs-test/testinput/rrfs_bufr2ioda_msonet.yaml
new file mode 100644
index 0000000..2d67555
--- /dev/null
+++ b/rrfs-test/testinput/rrfs_bufr2ioda_msonet.yaml
@@ -0,0 +1,369 @@
+# (C) Copyright 2020 NOAA/NWS/NCEP/EMC
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+
+observations:
+  - obs space:
+      name: bufr
+
+      obsdatain: "prepbufr"
+
+      exports:
+        #group_by_variable: prepbufrDataLvlCat
+        subsets:
+          - MSONET    # MESONET SURFACE REPORTS
+        variables:
+          timestamp:
+            timeoffset:
+              timeOffset: "*/DHR"
+              transforms:
+                - scale: 3600
+              referenceTime: 2024-05-27T00:00:00Z
+          timeOffset:
+            query: "*/DHR"
+          longitude:
+            query: "*/XOB"
+          latitude:
+            query: "*/YOB"
+          stationIdentification:
+            query: "*/SID"
+          stationElevation:
+            query: "*/ELV"
+          prepbufrReportType:
+            query: "*/TYP"
+          dumpReportType:
+            query: "*/T29"
+
+          mesonetProvider:
+            query: "*/PRVSTG"
+          prepbufrDataLvlCat:
+            query: "*/CAT"
+
+          pressure:
+            query: "*/P___INFO/P__EVENT{1}/POB"
+            transforms:
+              - scale: 100
+          pressureQualityMarker:
+            query: "*/P___INFO/P__EVENT{1}/PQM"
+          pressureError:
+            query: "*/P___INFO/P__BACKG/POE"
+            transforms:
+              - scale: 100
+          pressureTunedError:
+            query: "*/P___INFO/P__POSTP/POETU"
+            transforms:
+              - scale: 100
+
+          heightOfObservation:
+            query: "*/Z___INFO/Z__EVENT{1}/ZOB"
+          heightOfObservationQualityMark:
+            query: "*/Z___INFO/Z__EVENT{1}/ZQM"
+
+          specificHumidity:
+            query: "*/Q___INFO/Q__EVENT{1}/QOB"
+            type: float
+            transforms:
+              - scale: 0.000001
+          specificHumidityQualityMarker:
+            query: "*/Q___INFO/Q__EVENT{1}/QQM"
+          relativeHumidityError:
+            query: "*/Q___INFO/Q__BACKG/QOE"
+            transforms:
+              - scale: 0.1
+          relativeHumidityTunedError:
+            query: "*/Q___INFO/Q__POSTP/QOETU"
+            transforms:
+              - scale: 0.1
+
+          dewpointTemperature:
+            query: "*/Q___INFO/TDO"
+            transforms:
+              - offset: 273.15
+
+          airTemperature:
+            query: "*/T___INFO/T__EVENT{1}/TOB"
+            transforms:
+              - offset: 273.15
+          airTemperatureQualityMarker:
+            query: "*/T___INFO/T__EVENT{1}/TQM"
+          airTemperatureError:
+            query: "*/T___INFO/T__BACKG/TOE"
+          airTemperatureTunedError:
+            query: "*/T___INFO/T__POSTP/TOETU"
+
+          windEastward:
+            query: "*/W___INFO/W__EVENT{1}/UOB"
+          windNorthward:
+            query: "*/W___INFO/W__EVENT{1}/VOB"
+          windQualityMarker:
+            query: "*/W___INFO/W__EVENT{1}/WQM"
+          windError:
+            query: "*/W___INFO/W__BACKG/WOE"
+          windTunedError:
+            query: "*/W___INFO/W__POSTP/WOETU"
+            transform:
+              - scale: 0.514444
+
+          pressureReducedToMeanSeaLevel:
+            query: "*/PMSL_SEQ/PMO"
+            transforms:
+              - scale: 100
+          pressureReducedToMeanSeaLevelQualityMarker:
+            query: "*/PMSL_SEQ/PMQ"
+
+          horizontalVisibility:
+            query: "*/VISB2SEQ/HOVI"
+            type: float
+          maximumWindGustSpeed:
+            query: "*/GUST2SEQ/MXGS"
+          
+    ioda:
+      backend: netcdf
+      obsdataout: "./ioda_mesonet.nc"
+
+
+      dimensions:
+        - name: pevent_Dim
+          path: "*/P___INFO/P__EVENT"
+        - name: qevent_Dim
+          path: "*/Q___INFO/Q__EVENT"
+        - name: tevent_Dim
+          path: "*/T___INFO/T__EVENT"
+        - name: wevent_Dim
+          path: "*/W___INFO/W__EVENT"
+        - name: zevent_Dim
+          path: "*/Z___INFO/Z__EVENT"
+
+
+      variables:
+
+        # MetaData
+        - name: "MetaData/dateTime"
+          coordinates: "longitude latitude"
+          source: variables/timestamp
+          longName: "Observation Time"
+          units: "seconds since 1970-01-01T00:00:00Z"
+
+        - name: "MetaData/timeOffset"
+          coordinates: "longitude latitude"
+          source: variables/timeOffset
+          longName: "Observation Time Offset from Reference Time"
+          units: "Seconds"
+
+        - name: "MetaData/stationIdentification"
+          coordinates: "longitude latitude"
+          source: variables/stationIdentification
+          longName: "Station ID"
+          units: ""
+
+        - name: "MetaData/longitude"
+          coordinates: "longitude latitude"
+          source: variables/longitude
+          longName: "Longitude"
+          units: "degrees_east"
+          range: [0, 360]
+
+        - name: "MetaData/latitude"
+          coordinates: "longitude latitude"
+          source: variables/latitude
+          longName: "Latitude"
+          units: "degrees_north"
+          range: [-90, 90]
+
+        - name: "MetaData/stationElevation"
+          coordinates: "longitude latitude"
+          source: variables/stationElevation
+          longName: "Height of Station"
+          units: "Meter"
+
+        - name: "MetaData/prepbufrReportType"
+          coordinates: "longitude latitude"
+          source: variables/prepbufrReportType
+          longName: "Prepbufr Report Type"
+          units: ""
+
+        - name: "MetaData/dumpReportType"
+          coordinates: "longitude latitude"
+          source: variables/dumpReportType
+          longName: "Data Dump Report Type"
+          units: ""
+
+        - name: "MetaData/mesonetProvider"
+          coordinates: "longitude latitude"
+          source: variables/mesonetProvider
+          longName: "Mesonet Provider ID String"
+          units: ""
+
+        - name: "MetaData/prepbufrDataLvlCat"
+          coordinates: "longitude latitude"
+          source: variables/prepbufrDataLvlCat
+          longName: "Prepbufr Data Level Category"
+          units: ""
+
+        - name: "MetaData/pressure"
+          coordinates: "longitude latitude"
+          source: variables/pressure
+          longName: "Pressure"
+          units: "Pa"
+
+        - name: "MetaData/height"
+          coordinates: "longitude latitude"
+          source: variables/heightOfObservation
+          longName: "Height"
+          units: "Meter"
+
+        # ObsType
+        - name: "ObsType/specificHumidity"
+          coordinates: "longitude latitude"
+          source: variables/prepbufrReportType
+          longName: "specificHumidity Report Type"
+          units: ""
+
+        - name: "ObsType/airTemperature"
+          coordinates: "longitude latitude"
+          source: variables/prepbufrReportType
+          longName: "airTemperature report Type"
+          units: ""
+
+        - name: "ObsType/windEastward"
+          coordinates: "longitude latitude"
+          source: variables/prepbufrReportType
+          longName: "windEastward Report Type"
+          units: ""
+
+        - name: "ObsType/windNorthward"
+          coordinates: "longitude latitude"
+          source: variables/prepbufrReportType
+          longName: "windNorthward Report Type"
+          units: ""
+
+        # ObsValue
+        - name: "ObsValue/specificHumidity"
+          coordinates: "longitude latitude"
+          source: variables/specificHumidity
+          longName: "Specific Humidity"
+          units: "Kilogram Kilogram-1"
+
+        - name: "ObsValue/airTemperature"
+          coordinates: "longitude latitude"
+          source: variables/airTemperature
+          longName: "Temperature"
+          units: "Kelvin"
+
+        - name: "ObsValue/dewpointTemperature"
+          coordinates: "longitude latitude"
+          source: variables/dewpointTemperature
+          longName: "Dewpoint Temperature"
+          units: "Kelvin"
+
+        - name: "ObsValue/windEastward"
+          coordinates: "longitude latitude"
+          source: variables/windEastward
+          longName: "Eastward Wind"
+          units: "Meter Second-1"
+
+        - name: "ObsValue/windNorthward"
+          coordinates: "longitude latitude"
+          source: variables/windNorthward
+          longName: "Northward Wind"
+          units: "Meter Second-1"
+
+        - name: "ObsValue/pressureReducedToMeanSeaLevel"
+          coordinates: "longitude latitude"
+          source: variables/pressureReducedToMeanSeaLevel
+          longName: "Mean Sea-Level Pressure"
+          units: "Pa"
+
+        - name: "ObsValue/horizontalVisibility"
+          coordinates: "longitude latitude"
+          source: variables/horizontalVisibility
+          longName: "Horizontal Visibility"
+          units: "Meter"
+
+        - name: "ObsValue/maximumWindGustSpeed"
+          coordinates: "longitude latitude"
+          source: variables/maximumWindGustSpeed
+          longName: "Maximum Wind Gust Speed"
+          units: "Meter Second-1"
+        
+        # Quality Marker
+        - name: "QualityMarker/pressure"
+          coordinates: "longitude latitude"
+          source: variables/pressureQualityMarker
+          longName: "Pressure Quality Marker"
+          
+        - name: "QualityMarker/height"
+          coordinates: "longitude latitude"
+          source: variables/heightOfObservationQualityMark
+          longName: "Height Quality Marker"
+          
+        - name: "QualityMarker/specificHumidity"
+          coordinates: "longitude latitude"
+          source: variables/specificHumidityQualityMarker
+          longName: "Specific Humidity Quality Marker"
+          
+        - name: "QualityMarker/airTemperature"
+          coordinates: "longitude latitude"
+          source: variables/airTemperatureQualityMarker
+          longName: "Temperature Quality Marker"
+          
+        - name: "QualityMarker/wind"
+          coordinates: "longitude latitude"
+          source: variables/windQualityMarker
+          longName: "U, V-Component of Wind Quality Marker"
+
+        - name: "QualityMarker/pressureReducedToMeanSeaLevel"
+          coordinates: "longitude latitude"
+          source: variables/pressureReducedToMeanSeaLevelQualityMarker
+          longName: "Mean Sea Level Pressure Quality Marker"
+          
+        # ObsError
+        - name: "ObsError/pressure"
+          coordinates: "longitude latitude"
+          source: variables/pressureError
+          longName: "Pressure Observation Error"
+          units: "Pa"
+          
+        - name: "ObsError/relativeHumidity"
+          coordinates: "longitude latitude"
+          source: variables/relativeHumidityError
+          longName: "Relative Humidity Error"
+          units: "1"
+          
+        - name: "ObsError/airTemperature"
+          coordinates: "longitude latitude"
+          source: variables/airTemperatureError
+          longName: "Temperature Error"
+          units: "Kelvin"
+          
+        - name: "ObsError/wind"
+          coordinates: "longitude latitude"
+          source: variables/windError
+          longName: "East and Northward wind error"
+          units: "Meter Second-1"        
+          
+        # Tuned ObsError
+        - name: "TunedObsError/pressure"
+          coordinates: "longitude latitude"
+          source: variables/pressureTunedError
+          longName: "Analysis-Tuned Pressure Observation Error"
+          units: "Pa"
+          
+        - name: "TunedObsError/relativeHumidity"
+          coordinates: "longitude latitude"
+          source: variables/relativeHumidityTunedError
+          longName: "Analysis-Tuned Relative Humidity Error"
+          units: "1"
+          
+        - name: "TunedObsError/airTemperature"
+          coordinates: "longitude latitude"
+          source: variables/airTemperatureTunedError
+          longName: "Analysis-Tuned Temperature Error"
+          units: "Kelvin"
+          
+        - name: "TunedObsError/wind"
+          coordinates: "longitude latitude"
+          source: variables/windTunedError
+          longName: "Analysis-Tuned East and Northward wind error"
+          units: "Meter Second-1"
diff --git a/rrfs-test/ush/iodaconv_comp_rrfs.sh b/rrfs-test/ush/iodaconv_comp_rrfs.sh
new file mode 100755
index 0000000..31b9390
--- /dev/null
+++ b/rrfs-test/ush/iodaconv_comp_rrfs.sh
@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# use nccmp or odc to compare the output of a ioda-converter
+#
+# argument 1: what type of file to compare; netcdf or odb
+# argument 2: the command to run the ioda converter
+# argument 3: the reference file name 
+# argument 4: the test file name to compare against
+# argument 5: the tolerance for differences between the files
+# argument 6: verbose option
+
+set -eu
+
+file_type=$1
+cmd=$2
+file_name_ref=$3
+file_name_test=$4
+tol=${5:-"0.0"}
+verbose=${6:-${VERBOSE:-"N"}}
+
+[[ $verbose == [YyTt] || \
+   $verbose == [Yy][Ee][Ss] || \
+   $verbose == [Tt][Rr][Uu][Ee] ]] && set -x
+
+rc="-1"
+case $file_type in
+  netcdf)
+    $cmd && \
+    nccmp $file_name_ref $file_name_test -d -m -g -f -S -T ${tol}
+    rc=${?}
+    ;;
+   odb)
+    $cmd && \
+    odc compare $file_name_ref $file_name_test
+    rc=${?}
+    ;;
+  ascii)
+    $cmd && \
+    diff $file_name_ref $file_name_test
+    rc=${?}
+    ;;
+   *)
+    echo "ERROR: iodaconv_comp_rrfs.sh: Unrecognized file type: ${file_type}"
+    rc="-2"
+    ;;
+esac
+
+exit $rc
diff --git a/ush/run_rrfs_tests.sh b/ush/run_rrfs_tests.sh
index 173036f..b73422b 100755
--- a/ush/run_rrfs_tests.sh
+++ b/ush/run_rrfs_tests.sh
@@ -14,5 +14,5 @@ source ${ushdir}/load_rdas.sh
 set -x
 cd ${ushdir}/../build/rrfs-test
 pwd
-ctest -j6 # or ctest -VV for verbose outputs
+ctest -j7 # or ctest -VV for verbose outputs
 exit $?