diff --git a/rrfs-test/CMakeLists.txt b/rrfs-test/CMakeLists.txt index 420727f..5b013cd 100644 --- a/rrfs-test/CMakeLists.txt +++ b/rrfs-test/CMakeLists.txt @@ -62,10 +62,9 @@ if(FV3_DYCORE) set(MPI_ARGS ${RESTORE_MPI_ARGS}) endif() +endif() - endif() - - if(MPAS_DYCORE) +if(MPAS_DYCORE) message(STATUS "MACHINE_ID is " ${MACHINE_ID}) if("${MACHINE_ID}" STREQUAL "orion") @@ -117,4 +116,35 @@ if(FV3_DYCORE) set(MPI_ARGS ${RESTORE_MPI_ARGS}) endif() + ######################## + ### Bufr2ioda ctests ### + ######################## + + # YAML-based bufr2ioda + set(case "rrfs_bufr2ioda_msonet") + set(casedir "${CMAKE_CURRENT_BINARY_DIR}/rundir-${case}") + set(src_casedir "${rrfs-test_data_local}/rrfs-data_mpasjedi_2024052700") + if (NOT EXISTS "${casedir}") + file(MAKE_DIRECTORY ${casedir}) + endif() + file(CREATE_LINK ${src_casedir}/data ${casedir}/data SYMBOLIC) + configure_file(${src_casedir}/data/obs/bufr/rap.t00z.prepbufr.tm00 ${casedir}/prepbufr COPYONLY) + file(COPY ${src_yaml}/${case}.yaml DESTINATION ${casedir}) + set(MPI_ARGS "--time=00:05:00 --mem=64G") + set(IODA_CONV_COMP_TOL "5.0e-4") + ecbuild_add_test( TARGET ${case} + MPI 1 + TYPE SCRIPT + COMMAND bash + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/rundir-${case} + ARGS ${CMAKE_SOURCE_DIR}/rrfs-test/ush/iodaconv_comp_rrfs.sh + netcdf + "${CMAKE_BINARY_DIR}/bin/bufr2ioda.x ${case}.yaml" + data/obs/ioda_mesonet.nc + ioda_mesonet.nc + ${IODA_CONV_COMP_TOL} + DEPENDS bufr2ioda.x ) + + endif() + diff --git a/rrfs-test/testinput/rrfs_bufr2ioda_msonet.yaml b/rrfs-test/testinput/rrfs_bufr2ioda_msonet.yaml new file mode 100644 index 0000000..2d67555 --- /dev/null +++ b/rrfs-test/testinput/rrfs_bufr2ioda_msonet.yaml @@ -0,0 +1,369 @@ +# (C) Copyright 2020 NOAA/NWS/NCEP/EMC +# +# This software is licensed under the terms of the Apache Licence Version 2.0 +# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + +observations: + - obs space: + name: bufr + + obsdatain: "prepbufr" + + exports: + #group_by_variable: prepbufrDataLvlCat + subsets: + - MSONET # MESONET SURFACE REPORTS + variables: + timestamp: + timeoffset: + timeOffset: "*/DHR" + transforms: + - scale: 3600 + referenceTime: 2024-05-27T00:00:00Z + timeOffset: + query: "*/DHR" + longitude: + query: "*/XOB" + latitude: + query: "*/YOB" + stationIdentification: + query: "*/SID" + stationElevation: + query: "*/ELV" + prepbufrReportType: + query: "*/TYP" + dumpReportType: + query: "*/T29" + + mesonetProvider: + query: "*/PRVSTG" + prepbufrDataLvlCat: + query: "*/CAT" + + pressure: + query: "*/P___INFO/P__EVENT{1}/POB" + transforms: + - scale: 100 + pressureQualityMarker: + query: "*/P___INFO/P__EVENT{1}/PQM" + pressureError: + query: "*/P___INFO/P__BACKG/POE" + transforms: + - scale: 100 + pressureTunedError: + query: "*/P___INFO/P__POSTP/POETU" + transforms: + - scale: 100 + + heightOfObservation: + query: "*/Z___INFO/Z__EVENT{1}/ZOB" + heightOfObservationQualityMark: + query: "*/Z___INFO/Z__EVENT{1}/ZQM" + + specificHumidity: + query: "*/Q___INFO/Q__EVENT{1}/QOB" + type: float + transforms: + - scale: 0.000001 + specificHumidityQualityMarker: + query: "*/Q___INFO/Q__EVENT{1}/QQM" + relativeHumidityError: + query: "*/Q___INFO/Q__BACKG/QOE" + transforms: + - scale: 0.1 + relativeHumidityTunedError: + query: "*/Q___INFO/Q__POSTP/QOETU" + transforms: + - scale: 0.1 + + dewpointTemperature: + query: "*/Q___INFO/TDO" + transforms: + - offset: 273.15 + + airTemperature: + query: "*/T___INFO/T__EVENT{1}/TOB" + transforms: + - offset: 273.15 + airTemperatureQualityMarker: + query: "*/T___INFO/T__EVENT{1}/TQM" + airTemperatureError: + query: "*/T___INFO/T__BACKG/TOE" + airTemperatureTunedError: + query: "*/T___INFO/T__POSTP/TOETU" + + windEastward: + query: "*/W___INFO/W__EVENT{1}/UOB" + windNorthward: + query: "*/W___INFO/W__EVENT{1}/VOB" + windQualityMarker: + query: "*/W___INFO/W__EVENT{1}/WQM" + windError: + query: "*/W___INFO/W__BACKG/WOE" + windTunedError: + query: "*/W___INFO/W__POSTP/WOETU" + transform: + - scale: 0.514444 + + pressureReducedToMeanSeaLevel: + query: "*/PMSL_SEQ/PMO" + transforms: + - scale: 100 + pressureReducedToMeanSeaLevelQualityMarker: + query: "*/PMSL_SEQ/PMQ" + + horizontalVisibility: + query: "*/VISB2SEQ/HOVI" + type: float + maximumWindGustSpeed: + query: "*/GUST2SEQ/MXGS" + + ioda: + backend: netcdf + obsdataout: "./ioda_mesonet.nc" + + + dimensions: + - name: pevent_Dim + path: "*/P___INFO/P__EVENT" + - name: qevent_Dim + path: "*/Q___INFO/Q__EVENT" + - name: tevent_Dim + path: "*/T___INFO/T__EVENT" + - name: wevent_Dim + path: "*/W___INFO/W__EVENT" + - name: zevent_Dim + path: "*/Z___INFO/Z__EVENT" + + + variables: + + # MetaData + - name: "MetaData/dateTime" + coordinates: "longitude latitude" + source: variables/timestamp + longName: "Observation Time" + units: "seconds since 1970-01-01T00:00:00Z" + + - name: "MetaData/timeOffset" + coordinates: "longitude latitude" + source: variables/timeOffset + longName: "Observation Time Offset from Reference Time" + units: "Seconds" + + - name: "MetaData/stationIdentification" + coordinates: "longitude latitude" + source: variables/stationIdentification + longName: "Station ID" + units: "" + + - name: "MetaData/longitude" + coordinates: "longitude latitude" + source: variables/longitude + longName: "Longitude" + units: "degrees_east" + range: [0, 360] + + - name: "MetaData/latitude" + coordinates: "longitude latitude" + source: variables/latitude + longName: "Latitude" + units: "degrees_north" + range: [-90, 90] + + - name: "MetaData/stationElevation" + coordinates: "longitude latitude" + source: variables/stationElevation + longName: "Height of Station" + units: "Meter" + + - name: "MetaData/prepbufrReportType" + coordinates: "longitude latitude" + source: variables/prepbufrReportType + longName: "Prepbufr Report Type" + units: "" + + - name: "MetaData/dumpReportType" + coordinates: "longitude latitude" + source: variables/dumpReportType + longName: "Data Dump Report Type" + units: "" + + - name: "MetaData/mesonetProvider" + coordinates: "longitude latitude" + source: variables/mesonetProvider + longName: "Mesonet Provider ID String" + units: "" + + - name: "MetaData/prepbufrDataLvlCat" + coordinates: "longitude latitude" + source: variables/prepbufrDataLvlCat + longName: "Prepbufr Data Level Category" + units: "" + + - name: "MetaData/pressure" + coordinates: "longitude latitude" + source: variables/pressure + longName: "Pressure" + units: "Pa" + + - name: "MetaData/height" + coordinates: "longitude latitude" + source: variables/heightOfObservation + longName: "Height" + units: "Meter" + + # ObsType + - name: "ObsType/specificHumidity" + coordinates: "longitude latitude" + source: variables/prepbufrReportType + longName: "specificHumidity Report Type" + units: "" + + - name: "ObsType/airTemperature" + coordinates: "longitude latitude" + source: variables/prepbufrReportType + longName: "airTemperature report Type" + units: "" + + - name: "ObsType/windEastward" + coordinates: "longitude latitude" + source: variables/prepbufrReportType + longName: "windEastward Report Type" + units: "" + + - name: "ObsType/windNorthward" + coordinates: "longitude latitude" + source: variables/prepbufrReportType + longName: "windNorthward Report Type" + units: "" + + # ObsValue + - name: "ObsValue/specificHumidity" + coordinates: "longitude latitude" + source: variables/specificHumidity + longName: "Specific Humidity" + units: "Kilogram Kilogram-1" + + - name: "ObsValue/airTemperature" + coordinates: "longitude latitude" + source: variables/airTemperature + longName: "Temperature" + units: "Kelvin" + + - name: "ObsValue/dewpointTemperature" + coordinates: "longitude latitude" + source: variables/dewpointTemperature + longName: "Dewpoint Temperature" + units: "Kelvin" + + - name: "ObsValue/windEastward" + coordinates: "longitude latitude" + source: variables/windEastward + longName: "Eastward Wind" + units: "Meter Second-1" + + - name: "ObsValue/windNorthward" + coordinates: "longitude latitude" + source: variables/windNorthward + longName: "Northward Wind" + units: "Meter Second-1" + + - name: "ObsValue/pressureReducedToMeanSeaLevel" + coordinates: "longitude latitude" + source: variables/pressureReducedToMeanSeaLevel + longName: "Mean Sea-Level Pressure" + units: "Pa" + + - name: "ObsValue/horizontalVisibility" + coordinates: "longitude latitude" + source: variables/horizontalVisibility + longName: "Horizontal Visibility" + units: "Meter" + + - name: "ObsValue/maximumWindGustSpeed" + coordinates: "longitude latitude" + source: variables/maximumWindGustSpeed + longName: "Maximum Wind Gust Speed" + units: "Meter Second-1" + + # Quality Marker + - name: "QualityMarker/pressure" + coordinates: "longitude latitude" + source: variables/pressureQualityMarker + longName: "Pressure Quality Marker" + + - name: "QualityMarker/height" + coordinates: "longitude latitude" + source: variables/heightOfObservationQualityMark + longName: "Height Quality Marker" + + - name: "QualityMarker/specificHumidity" + coordinates: "longitude latitude" + source: variables/specificHumidityQualityMarker + longName: "Specific Humidity Quality Marker" + + - name: "QualityMarker/airTemperature" + coordinates: "longitude latitude" + source: variables/airTemperatureQualityMarker + longName: "Temperature Quality Marker" + + - name: "QualityMarker/wind" + coordinates: "longitude latitude" + source: variables/windQualityMarker + longName: "U, V-Component of Wind Quality Marker" + + - name: "QualityMarker/pressureReducedToMeanSeaLevel" + coordinates: "longitude latitude" + source: variables/pressureReducedToMeanSeaLevelQualityMarker + longName: "Mean Sea Level Pressure Quality Marker" + + # ObsError + - name: "ObsError/pressure" + coordinates: "longitude latitude" + source: variables/pressureError + longName: "Pressure Observation Error" + units: "Pa" + + - name: "ObsError/relativeHumidity" + coordinates: "longitude latitude" + source: variables/relativeHumidityError + longName: "Relative Humidity Error" + units: "1" + + - name: "ObsError/airTemperature" + coordinates: "longitude latitude" + source: variables/airTemperatureError + longName: "Temperature Error" + units: "Kelvin" + + - name: "ObsError/wind" + coordinates: "longitude latitude" + source: variables/windError + longName: "East and Northward wind error" + units: "Meter Second-1" + + # Tuned ObsError + - name: "TunedObsError/pressure" + coordinates: "longitude latitude" + source: variables/pressureTunedError + longName: "Analysis-Tuned Pressure Observation Error" + units: "Pa" + + - name: "TunedObsError/relativeHumidity" + coordinates: "longitude latitude" + source: variables/relativeHumidityTunedError + longName: "Analysis-Tuned Relative Humidity Error" + units: "1" + + - name: "TunedObsError/airTemperature" + coordinates: "longitude latitude" + source: variables/airTemperatureTunedError + longName: "Analysis-Tuned Temperature Error" + units: "Kelvin" + + - name: "TunedObsError/wind" + coordinates: "longitude latitude" + source: variables/windTunedError + longName: "Analysis-Tuned East and Northward wind error" + units: "Meter Second-1" diff --git a/rrfs-test/ush/iodaconv_comp_rrfs.sh b/rrfs-test/ush/iodaconv_comp_rrfs.sh new file mode 100755 index 0000000..31b9390 --- /dev/null +++ b/rrfs-test/ush/iodaconv_comp_rrfs.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +# use nccmp or odc to compare the output of a ioda-converter +# +# argument 1: what type of file to compare; netcdf or odb +# argument 2: the command to run the ioda converter +# argument 3: the reference file name +# argument 4: the test file name to compare against +# argument 5: the tolerance for differences between the files +# argument 6: verbose option + +set -eu + +file_type=$1 +cmd=$2 +file_name_ref=$3 +file_name_test=$4 +tol=${5:-"0.0"} +verbose=${6:-${VERBOSE:-"N"}} + +[[ $verbose == [YyTt] || \ + $verbose == [Yy][Ee][Ss] || \ + $verbose == [Tt][Rr][Uu][Ee] ]] && set -x + +rc="-1" +case $file_type in + netcdf) + $cmd && \ + nccmp $file_name_ref $file_name_test -d -m -g -f -S -T ${tol} + rc=${?} + ;; + odb) + $cmd && \ + odc compare $file_name_ref $file_name_test + rc=${?} + ;; + ascii) + $cmd && \ + diff $file_name_ref $file_name_test + rc=${?} + ;; + *) + echo "ERROR: iodaconv_comp_rrfs.sh: Unrecognized file type: ${file_type}" + rc="-2" + ;; +esac + +exit $rc diff --git a/ush/run_rrfs_tests.sh b/ush/run_rrfs_tests.sh index 173036f..b73422b 100755 --- a/ush/run_rrfs_tests.sh +++ b/ush/run_rrfs_tests.sh @@ -14,5 +14,5 @@ source ${ushdir}/load_rdas.sh set -x cd ${ushdir}/../build/rrfs-test pwd -ctest -j6 # or ctest -VV for verbose outputs +ctest -j7 # or ctest -VV for verbose outputs exit $?