diff --git a/.github/workflows/norms.yaml b/.github/workflows/norms.yaml index 7d180f88f..2e721e7af 100644 --- a/.github/workflows/norms.yaml +++ b/.github/workflows/norms.yaml @@ -25,4 +25,4 @@ jobs: - name: Run C++ linter on utils run: | cd $GITHUB_WORKSPACE/GDASApp/utils/test/ - ./cpplint.py --quiet --recursive $GITHUB_WORKSPACE/GDASApp/utils + ./cpplint.py --quiet --recursive --exclude=$GITHUB_WORKSPACE/GDASApp/utils/obsproc/rtofs/*.cc $GITHUB_WORKSPACE/GDASApp/utils diff --git a/CMakeLists.txt b/CMakeLists.txt index 60c85bbf7..ef34501af 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,6 +8,8 @@ cmake_minimum_required( VERSION 3.20 FATAL_ERROR ) find_package(ecbuild 3.5 REQUIRED HINTS ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../ecbuild) project(GDASApp VERSION 1.0.0 LANGUAGES C CXX Fortran ) +# include_directories(${CMAKE_SOURCE_DIR}) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}) include(GNUInstallDirs) enable_testing() diff --git a/sorc/fv3 b/sorc/fv3 index 61450b4e3..47b766295 160000 --- a/sorc/fv3 +++ b/sorc/fv3 @@ -1 +1 @@ -Subproject commit 61450b4e3e80bb96b26c5f3808ce60b5e5cb4207 +Subproject commit 47b766295682df2e53b59d0dd07b5c511b84dd67 diff --git a/sorc/gsibec b/sorc/gsibec index c8ac58d9b..e6e1944e1 160000 --- a/sorc/gsibec +++ b/sorc/gsibec @@ -1 +1 @@ -Subproject commit c8ac58d9b43eb8f890e565d12c88f1b0579c9ccd +Subproject commit e6e1944e1cb8f339b4eb79acffb5470bb7723b42 diff --git a/sorc/gsw b/sorc/gsw index 1a02ebaf6..697cbeb76 160000 --- a/sorc/gsw +++ b/sorc/gsw @@ -1 +1 @@ -Subproject commit 1a02ebaf6f7a4e9f2c2d2dd973fb050e697bcc74 +Subproject commit 697cbeb7605d70ed3857664c5f54a5c05346e31f diff --git a/sorc/land-imsproc b/sorc/land-imsproc index 2ac7c0d90..a3b233446 160000 --- a/sorc/land-imsproc +++ b/sorc/land-imsproc @@ -1 +1 @@ -Subproject commit 2ac7c0d90d3ebd449ad11bce155a71d381b4534a +Subproject commit a3b233446002af7643956bc31a7e68b89bf1a9d9 diff --git a/sorc/land-jediincr b/sorc/land-jediincr index 79576b79d..7c1f6a3f8 160000 --- a/sorc/land-jediincr +++ b/sorc/land-jediincr @@ -1 +1 @@ -Subproject commit 79576b79df6a3f1ab8e7dde4425821f2915006b3 +Subproject commit 7c1f6a3f8f949e376786eef7dba55a9e10e9778d diff --git a/utils/obsproc/CMakeLists.txt b/utils/obsproc/CMakeLists.txt index 9cce213e1..c092a8758 100644 --- a/utils/obsproc/CMakeLists.txt +++ b/utils/obsproc/CMakeLists.txt @@ -1,2 +1,2 @@ -# add_subdirectory(rtofs) +add_subdirectory(rtofs) add_subdirectory(applications) diff --git a/utils/obsproc/NetCDFToIodaConverter.h b/utils/obsproc/NetCDFToIodaConverter.h index 89b55b227..214fa25bf 100644 --- a/utils/obsproc/NetCDFToIodaConverter.h +++ b/utils/obsproc/NetCDFToIodaConverter.h @@ -49,6 +49,9 @@ namespace gdasapp { oops::Log::trace() << "NetCDFToIodaConverter::NetCDFToIodaConverter created." << std::endl; } + + + // Method to write out a IODA file (writter called in destructor) void writeToIoda() { // Extract ioda variables from the provider's files @@ -60,6 +63,8 @@ namespace gdasapp { // Read the provider's netcdf file gdasapp::obsproc::iodavars::IodaVars iodaVars = providerToIodaVars(inputFilenames_[myrank]); + + for (int i = myrank + comm_.size(); i < inputFilenames_.size(); i += comm_.size()) { iodaVars.append(providerToIodaVars(inputFilenames_[i])); oops::Log::info() << " appending: " << inputFilenames_[i] << std::endl; @@ -200,6 +205,9 @@ namespace gdasapp { } } + + + private: // Virtual method that reads the provider's netcdf file and store the relevant // info in a IodaVars struct diff --git a/utils/obsproc/RTOFSInSitu.h b/utils/obsproc/RTOFSInSitu.h index 32b9b967f..aab33579a 100644 --- a/utils/obsproc/RTOFSInSitu.h +++ b/utils/obsproc/RTOFSInSitu.h @@ -1,151 +1,20 @@ #pragma once -#include -// #define _BSD_SOURCE -#include -#include // used in file_exists - -#include -#include -using std::ofstream; -#include // std::get_time -using std::setprecision; -#include -using std::cerr; -using std::endl; -using std::cout; -#include // NOLINT (using C API) #include -using std::string; -#include #include #include "NetCDFToIodaConverter.h" -#include "oops/util/dateFunctions.h" -#include "oops/util/DateTime.h" - - - - - - - -namespace rtofs -{ - -bool file_exists(const std::string& name); - -void skip8bytes(FILE * f); -int read_int(FILE * f); -void read_float_array(FILE * f, float * a, int n); -void read_int_array(FILE * f, int * a, int n); - -void print_float_array(std::string filename, float * a, int n); -void print_int_array(std::string filename, int * a, int n); -void print_2d_float_array(std::string filename, float ** a, int n, int * dim2); -float * alloc_read_float_array(FILE * f, int n); - - - -class RTOFSOb -{ - public: - RTOFSOb(int n, int mx_lvl, int vrsn); - - typedef char char12[12]; - typedef char char7[7]; - - void read(FILE * f); - int NumberOfObservations() { return n; } - void print(std::string DirectoryName); - - - float * btm; // bottom depth - float * lat; // latitude - float * lon; // longitude - int * ls; // ?? - int * lt; // array of dimensions, the number of levels - int * sal_typ; // ?? salinity type ?? - float * sqc; // salinity qc - int * tmp_typ; // ?? temperature type ?? - float * tqc; // temperature qc - - // observations per level: - float ** lvl; // depth - float ** sal; // salinity (PSU) - float ** sal_err; // salinity error (std deviation, PSU) - float ** sprb; // ?? salinity ... ?? - float ** tmp; // in situ temperature (C) - float ** tmp_err; // in situ temperature error (C) - float ** tprb; // ?? temperature ... ?? - float ** clm_sal; // climatology of salinity - float ** cssd; // climatological std deviation for salinity - float ** clm_tmp; // climatology of temperature - float ** ctsd; // climatological std deviation for temperature - int ** flg; // ?? qc flags ?? - - char12 * dtg; // date (Julian, seconds) - // std::time_t * dtg; // date (Julian, seconds) - - private: - int n; - int mx_lvl; - int vrsn; - - void allocate(); - void allocate2d(); -}; // class RTOFSOb - - - -class RTOFSDataFile -{ - public: - explicit RTOFSDataFile(std::string filename); - int NumberOfObservations() { return nobs; } - RTOFSOb & observations() { return * ob; } - - - private: - std::string filename; - int nobs; - FILE * f; - RTOFSOb * ob; - - void read_file(); -}; // class RTOFSDataFile - -} // namespace rtofs - - +#include "rtofs/RTOFSDataFile.h" +#include "rtofs/RTOFSOb.h" +#include "rtofs/util.h" namespace gdasapp { -int64_t - SecondsSinceReferenceTime(rtofs::RTOFSOb::char12 time) +namespace rtofs { - // parse the date - std::string s(time); - int year = std::stoi(s.substr(0, 4)); - int month = std::stoi(s.substr(4, 2)); - int day = std::stoi(s.substr(6, 2)); - int hour = std::stoi(s.substr(8, 2)); - int minute = std::stoi(s.substr(10, 2)); - int second = 0; - - uint64_t julianDate = - util::datefunctions::dateToJulian(year, month, day); - - // 2440588 = Julian date for January 1, 1970. - int daysSinceEpoch = julianDate - 2440588; - int secondsOffset = util::datefunctions::hmsToSeconds(hour, minute, second); - return static_cast(daysSinceEpoch * 86400.0f) + secondsOffset; -} // SecondsSinceReferenceTime - - class RTOFSInSitu: public NetCDFToIodaConverter @@ -155,448 +24,48 @@ class RTOFSInSitu: const eckit::Configuration & fullConfig, const eckit::mpi::Comm & comm): NetCDFToIodaConverter(fullConfig, comm) - { - variable_ = "waterTemperature"; - } - - // Read binary file and populate iodaVars - gdasapp::obsproc::iodavars::IodaVars - providerToIodaVars(const std::string fileName) final - { - oops::Log::info() - << "Processing files provided by RTOFS" - << std::endl; - - // Set the int metadata names - std::vector intMetadataNames = {"temperature"}; - - // Set the float metadata name - std::vector floatMetadataNames = {}; + {} - oops::Log::info() - << "Processing file " - << fileName - << std::endl; - // read the file - rtofs::RTOFSDataFile RTOFSFile(fileName); - int n = RTOFSFile.NumberOfObservations(); - rtofs::RTOFSOb & ob = RTOFSFile.observations(); + protected: + void ProcessFile(std::string filename); - int NumberOfTemperatureValues = 0; - for (int i = 0; i < n; i ++) - NumberOfTemperatureValues += ob.lt[i]; + rtofs::RTOFSOb * pob; - gdasapp::obsproc::iodavars::IodaVars iodaVars( - NumberOfTemperatureValues, - floatMetadataNames, - intMetadataNames); - iodaVars.referenceDate_ = "seconds since 1970-01-01T00:00:00Z"; - - int k = 0; - for (int i = 0; i < n; i ++) - { - int64_t date = SecondsSinceReferenceTime(ob.dtg[i]); - - for (int j = 0; j < ob.lt[i]; j ++) - { - iodaVars.longitude_(k) = ob.lon[i]; - iodaVars.latitude_(k) = ob.lat[i]; - iodaVars.obsVal_(k) = ob.tmp[i][j]; - iodaVars.obsError_(k) = ob.tmp_err[i][j]; - iodaVars.datetime_(k) = date; - // iodaVars.preQc_(k) = oneDimFlagsVal[i]; - // iodaVars.intMetadata_.row(k) << -999; - - k++; - } - } - - return iodaVars; - }; // providerToIodaVars + private: + // Read binary file and populate iodaVars + virtual gdasapp::obsproc::iodavars::IodaVars + providerToIodaVars(const std::string filename) = 0; }; // class RTOFSInSitu -} // namespace gdasapp - - - -namespace rtofs -{ - -// the variables marked "skipped" are arrays which are present -// in the binary file, but were skipped by the fortran code -// that was reading the files. - -RTOFSOb:: - RTOFSOb(int n, int mx_lvl, int vrsn): - n(n), - mx_lvl(mx_lvl), - vrsn(vrsn) -{ - allocate(); -} - - - -void -RTOFSOb:: - allocate() -{ - dtg = new char12[n]; - - lat = new float[n]; - lon = new float[n]; - btm = new float[n]; - ls = new int[n]; - lt = new int[n]; - sal_typ = new int[n]; - sqc = new float[n]; - tmp_typ = new int[n]; - tqc = new float[n]; - - lvl = new float * [n]; - sal = new float * [n]; - sal_err = new float * [n]; - sprb = new float * [n]; - tmp = new float * [n]; - tmp_err = new float * [n]; - clm_sal = new float * [n]; // skipped? - tprb = new float * [n]; - cssd = new float * [n]; - clm_tmp = new float * [n]; // skipped? - ctsd = new float * [n]; - flg = new int * [n]; -} - - -void -RTOFSOb:: - allocate2d() -{ - for (int i = 0; i < n; i ++) - { - int k = lt[i]; - - lvl[i] = new float[k]; - sal[i] = new float[k]; - sal_err[i] = new float[k]; - sprb[i] = new float[k]; - tmp[i] = new float[k]; - tmp_err[i] = new float[k]; - tprb[i] = new float[k]; - clm_sal[i] = new float[k]; - cssd[i] = new float[k]; - clm_tmp[i] = new float[k]; - ctsd[i] = new float[k]; - flg[i] = new int[k]; - } -} - - - -void -RTOFSOb:: - read(FILE * f) -{ - read_float_array(f, btm, n); - read_float_array(f, lat, n); - read_float_array(f, lon, n); - read_int_array(f, ls, n); - read_int_array(f, lt, n); - read_int_array(f, sal_typ, n); - read_float_array(f, sqc, n); - read_int_array(f, tmp_typ, n); - read_float_array(f, tqc, n); - - allocate2d(); - - for (int i = 0; i < n; i ++) - { - int k = lt[i]; - - read_float_array(f, lvl[i], k); - read_float_array(f, sal[i], k); - read_float_array(f, sal_err[i], k); - read_float_array(f, sprb[i], k); - read_float_array(f, tmp[i], k); - read_float_array(f, tmp_err[i], k); - read_float_array(f, tprb[i], k); - read_float_array(f, clm_sal[i], k); - read_float_array(f, cssd[i], k); - read_float_array(f, clm_tmp[i], k); - read_float_array(f, ctsd[i], k); - read_int_array(f, flg[i], k); - } -} - - -void -RTOFSOb:: - print(std::string DirectoryName) -{ - if (!file_exists(DirectoryName)) - { - cerr << "Directory " << DirectoryName << "doesn't exist" << endl; - exit(1); - } - print_float_array(DirectoryName + "/latitude", lat, n); - print_float_array(DirectoryName + "/longitude", lon, n); - print_float_array(DirectoryName + "/btm", btm, n); - print_float_array(DirectoryName + "/tqc", tqc, n); - print_float_array(DirectoryName + "/sqc", sqc, n); - print_int_array(DirectoryName + "/lt", lt, n); - print_int_array(DirectoryName + "/ls", ls, n); - print_int_array(DirectoryName + "/sal_typ", sal_typ, n); - print_int_array(DirectoryName + "/tmp_typ", sal_typ, n); - - print_2d_float_array(DirectoryName + "/tmp", tmp, n, lt); - print_2d_float_array(DirectoryName + "/sal", sal, n, lt); - - // print lvl2d array - { - ofstream o; - o.open(DirectoryName + "/lvl2d"); - for (int i = 0; i < n; i ++) - for (int j = 0; j < lt[i]; j ++) - o - << j - << endl; - o.close(); - } -} // RTOFSOb::print - - - -RTOFSDataFile:: -RTOFSDataFile(std::string filename): - filename(filename) -{ - if (!file_exists(filename)) - { - cerr << "File not found" << endl; - exit(1); - } - - - const char * fn = filename.c_str(); - f = fopen(fn, "rb"); - if (!f) - { - cerr << "Error opening file " << fn << endl; - exit(1); - } - - read_file(); -} // RTOFSDataFile::RTOFSDataFile - - +// read the binary file and possibly do additional filtering +// to prepare an rtofs::RTOFSOb to be converted to iodavars object void -RTOFSDataFile:: - read_file() +RTOFSInSitu:: + ProcessFile(std::string filename) { - fseek(f, 4, SEEK_CUR); - - int n_read = read_int(f); - int mx_lvl = read_int(f); - int vrsn = read_int(f); - - ob = new RTOFSOb(n_read, mx_lvl, vrsn); - nobs = n_read; - - ob->read(f); - - skip8bytes(f); - - fread(ob->dtg, sizeof(RTOFSOb::char12), n_read, f); - - skip8bytes(f); - - RTOFSOb::char12 * ob_rct = new RTOFSOb::char12[n_read]; - fread(ob_rct, sizeof(RTOFSOb::char12), n_read, f); - - skip8bytes(f); + oops::Log::info() + << "Processing RTOFS file " + << filename + << std::endl; - RTOFSOb::char7 * ob_sgn = new RTOFSOb::char7[n_read]; - fread(ob_sgn, sizeof(RTOFSOb::char7), n_read, f); + // read the file + rtofs::RTOFSDataFile RTOFSFile(filename); + pob = & RTOFSFile.observations(); + rtofs::RTOFSOb & ob = * pob; - if (vrsn == 2) - { - float ** glb_sal = new float * [n_read]; - for (int i = 0; i < n_read; i ++) - { - int k = ob->lt[i]; - glb_sal[i] = alloc_read_float_array(f, k); - } - } -} // read_file + int n = ob.NumberOfObservations(); + // to be used with selection by instrument: + std::vector tmp_instrument = + rtofs::find_unique_values(ob.tmp_typ, n); - -void skip8bytes(FILE * f) -{ - fseek(f, 8, SEEK_CUR); -} - - -void - read_float_array(FILE * f, float * a, int n) -{ - skip8bytes(f); - - fread(a, sizeof(float), n, f); - - uint32_t * data = reinterpret_cast(a); - - for (int i = 0; i < n; i ++) - data[i] = be32toh(data[i]); -} // read_float_array - - -void - read_int_array(FILE * f, int * a, int n) -{ - skip8bytes(f); - - fread(a, sizeof(int), n, f); - - uint32_t * data = reinterpret_cast(a); - - for (int i = 0; i < n; i ++) - data[i] = be32toh(data[i]); -} // read_int_array - - - -void - print_level(std::string filename, float ** a, int n, int level) -{ - ofstream o(filename); - o - << std::fixed - << std::setprecision(9); - - for (int i = 0; i < n; i ++) - o - << a[i] [level] - << endl; - - o.close(); -} - - -void - print_int_array(std::string filename, int * a, int n) -{ - ofstream o(filename); - for (int i = 0; i < n; i ++) - o - << a[i] - << endl; - o.close(); -} - - -void - print_float_array(std::string filename, float * a, int n) -{ - ofstream o(filename); - o - << std::fixed - << std::setprecision(9); - for (int i = 0; i < n; i ++) - o - << a[i] - << endl; -} - -void - print_2d_float_array(std::string filename, float ** a, int n, int * dim2) -{ - ofstream o(filename); - o - << std::fixed - << std::setprecision(9); - for (int i = 0; i < n; i ++) - for (int j = 0; j < dim2[i]; j ++) - o - << a[i] [j] - << endl; -} - - -bool - file_exists(const std::string& name) -{ - struct stat buffer; - return (stat (name.c_str(), &buffer) == 0); -} - - -int - read_int(FILE * f) -{ - int dummy; - fread(&dummy, sizeof(int), 1, f); - dummy = static_cast(be32toh(dummy)); - return dummy; -} // read_int - - - -void - read_real_array(FILE * f, float * a, int n) -{ - fread(a, sizeof(float), n, f); - - uint32_t * data = reinterpret_cast(a); - - for (int i = 0; i < n; i ++) - data[i] = be32toh(data[i]); -} // read_real_array - - -float * - alloc_read_float_array(FILE * f, int n) -{ - float * a = new float[n]; - skip8bytes(f); - read_real_array(f, a, n); - return a; -} - - -void - print_int_array(int * a, int n) -{ - for (int i = 0; i < n; i ++) - cout - << a[i] - << endl; -} - - -int * - alloc_read_int_array(FILE * f, int n) -{ - int * a = new int[n]; - skip8bytes(f); - read_int_array(f, a, n); - return a; -} - - -void - print_real_array(float * a, int n) -{ - cout - << std::fixed - << std::setprecision(9); - for (int i = 0; i < n; i ++) - cout - << a[i] - << endl; -} + std::vector sal_instrument = + rtofs::find_unique_values(ob.sal_typ, n); +} // RTOFSInSitu::ProcessFile } // namespace rtofs + +} // namespace gdasapp diff --git a/utils/obsproc/RTOFSSalinity.h b/utils/obsproc/RTOFSSalinity.h new file mode 100644 index 000000000..d13073c31 --- /dev/null +++ b/utils/obsproc/RTOFSSalinity.h @@ -0,0 +1,83 @@ +#pragma once + + +#include +#include + +#include "rtofs/RTOFSOb.h" +#include "RTOFSInSitu.h" + + +namespace gdasapp +{ + + +class RTOFSSalinity: + public rtofs::RTOFSInSitu +{ + public: + explicit RTOFSSalinity( + const eckit::Configuration & fullConfig, + const eckit::mpi::Comm & comm): + RTOFSInSitu(fullConfig, comm) + { + variable_ = "waterSalinity"; + } + + // Read binary file and populate iodaVars + gdasapp::obsproc::iodavars::IodaVars + providerToIodaVars(const std::string filename) final; +}; // class RTOFSSalinity + + +// Read binary file and populate iodaVars +gdasapp::obsproc::iodavars::IodaVars +RTOFSSalinity:: + providerToIodaVars(const std::string filename) +{ + ProcessFile(filename); + + // Set the int metadata names + std::vector intMetadataNames = {"sal_typ"}; + + // Set the float metadata name + std::vector floatMetadataNames = {"level"}; + + rtofs::RTOFSOb & ob = * pob; + + gdasapp::obsproc::iodavars::IodaVars iodaVars( + ob.TotalNumberOfValues(), + floatMetadataNames, + intMetadataNames); + + iodaVars.referenceDate_ = "seconds since 1970-01-01T00:00:00Z"; + iodaVars.niMetadata_ = 1; + iodaVars.nfMetadata_ = 1; + + int n = ob.NumberOfObservations(); + + int k = 0; + for (int i = 0; i < n; i ++) + { + int64_t date = rtofs::RTOFSOb::SecondsSinceReferenceTime(ob.dtg[i]); + + for (int j = 0; j < ob.lt[i]; j ++) + { + iodaVars.longitude_(k) = ob.lon[i]; + iodaVars.latitude_(k) = ob.lat[i]; + iodaVars.obsVal_(k) = ob.sal[i][j]; + iodaVars.obsError_(k) = ob.sal_err[i][j]; + iodaVars.datetime_(k) = date; + iodaVars.preQc_(k) = ob.flg[i][j]; + iodaVars.intMetadata_.row(k) << ob.sal_typ[i]; + iodaVars.floatMetadata_.row(k) << ob.lvl[i][j]; + + k++; + } + } + + return iodaVars; +} // RTOFSInSitu::providerToIodaVars + + +} // namespace gdasapp diff --git a/utils/obsproc/RTOFSTemperature.h b/utils/obsproc/RTOFSTemperature.h new file mode 100644 index 000000000..c66b39dc2 --- /dev/null +++ b/utils/obsproc/RTOFSTemperature.h @@ -0,0 +1,81 @@ +#pragma once + +#include +#include + +#include "rtofs/RTOFSOb.h" +#include "RTOFSInSitu.h" + + +namespace gdasapp +{ + + +class RTOFSTemperature: + public rtofs::RTOFSInSitu +{ + public: + explicit RTOFSTemperature( + const eckit::Configuration & fullConfig, + const eckit::mpi::Comm & comm): + RTOFSInSitu(fullConfig, comm) + { + variable_ = "waterTemperature"; + } + + // Read binary file and populate iodaVars + gdasapp::obsproc::iodavars::IodaVars + providerToIodaVars(const std::string filename) final; +}; // class RTOFSTemperature + + + +gdasapp::obsproc::iodavars::IodaVars +RTOFSTemperature:: + providerToIodaVars(const std::string filename) +{ + ProcessFile(filename); + + // Set the int metadata names + std::vector intMetadataNames = {"tmp_typ"}; + + // Set the float metadata name + std::vector floatMetadataNames = {"level"}; + rtofs::RTOFSOb & ob = * pob; + + gdasapp::obsproc::iodavars::IodaVars iodaVars( + ob.TotalNumberOfValues(), + floatMetadataNames, + intMetadataNames); + + iodaVars.referenceDate_ = "seconds since 1970-01-01T00:00:00Z"; + iodaVars.niMetadata_ = 1; + iodaVars.nfMetadata_ = 1; + + int n = ob.NumberOfObservations(); + + int k = 0; + for (int i = 0; i < n; i ++) + { + int64_t date = rtofs::RTOFSOb::SecondsSinceReferenceTime(ob.dtg[i]); + + for (int j = 0; j < ob.lt[i]; j ++) + { + iodaVars.longitude_(k) = ob.lon[i]; + iodaVars.latitude_(k) = ob.lat[i]; + iodaVars.obsVal_(k) = ob.tmp[i][j]; + iodaVars.obsError_(k) = ob.tmp_err[i][j]; + iodaVars.datetime_(k) = date; + iodaVars.preQc_(k) = ob.flg[i][j]; + iodaVars.intMetadata_.row(k) << ob.tmp_typ[i]; + iodaVars.floatMetadata_.row(k) << ob.lvl[i][j]; + + k++; + } + } + + return iodaVars; +}; // class RTOFSTemperature + + +} // namespace gdasapp diff --git a/utils/obsproc/applications/CMakeLists.txt b/utils/obsproc/applications/CMakeLists.txt index f03631855..99b517aa2 100644 --- a/utils/obsproc/applications/CMakeLists.txt +++ b/utils/obsproc/applications/CMakeLists.txt @@ -10,6 +10,5 @@ target_compile_features( gdas_obsprovider2ioda.x PUBLIC cxx_std_17) target_link_libraries( gdas_obsprovider2ioda.x PUBLIC oops ioda NetCDF::NetCDF_CXX) -# to be used when rtofs is a static library: -# link_directories(${CMAKE_SOURCE_DIR}/rtofs) -# target_link_libraries( gdas_obsprovider2ioda.x PRIVATE rtofs) +link_directories(${CMAKE_SOURCE_DIR}/rtofs) +target_link_libraries(gdas_obsprovider2ioda.x PRIVATE rtofs) diff --git a/utils/obsproc/applications/gdas_obsprovider2ioda.h b/utils/obsproc/applications/gdas_obsprovider2ioda.h index b3bcd5c89..ac08b19c6 100644 --- a/utils/obsproc/applications/gdas_obsprovider2ioda.h +++ b/utils/obsproc/applications/gdas_obsprovider2ioda.h @@ -9,7 +9,8 @@ #include "../Ghrsst2Ioda.h" #include "../IcecAmsr2Ioda.h" #include "../Rads2Ioda.h" -#include "../RTOFSInSitu.h" +#include "../RTOFSSalinity.h" +#include "../RTOFSTemperature.h" #include "../Smap2Ioda.h" #include "../Smos2Ioda.h" #include "../Viirsaod2Ioda.h" @@ -19,8 +20,10 @@ namespace gdasapp { public: explicit ObsProvider2IodaApp(const eckit::mpi::Comm & comm = oops::mpi::world()) : Application(comm) {} + static const std::string classname() {return "gdasapp::ObsProvider2IodaApp";} + int execute(const eckit::Configuration & fullConfig, bool /*validate*/) const { // Get the file provider string identifier from the config std::string provider; @@ -32,8 +35,11 @@ namespace gdasapp { } else if (provider == "GHRSST") { Ghrsst2Ioda conv2ioda(fullConfig, this->getComm()); conv2ioda.writeToIoda(); - } else if (provider == "RTOFS") { - RTOFSInSitu conv2ioda(fullConfig, this->getComm()); + } else if (provider == "RTOFStmp") { + RTOFSTemperature conv2ioda(fullConfig, this->getComm()); + conv2ioda.writeToIoda(); + } else if (provider == "RTOFSsal") { + RTOFSSalinity conv2ioda(fullConfig, this->getComm()); conv2ioda.writeToIoda(); } else if (provider == "SMAP") { Smap2Ioda conv2ioda(fullConfig, this->getComm()); diff --git a/utils/obsproc/rtofs/CMakeLists.txt b/utils/obsproc/rtofs/CMakeLists.txt new file mode 100644 index 000000000..d524b22a3 --- /dev/null +++ b/utils/obsproc/rtofs/CMakeLists.txt @@ -0,0 +1,26 @@ +set( + rtofs_src_files + RTOFSDataFile.h + RTOFSDataFile.cc + RTOFSOb.h + RTOFSOb.cc + util.cc + util.h +) + +list( + APPEND gdasapp_provider2ioda_src_files + ${rtofs_src_files} +) + + +add_library(rtofs STATIC + ${rtofs_src_files} +) + +target_compile_features( + rtofs + PUBLIC cxx_std_17 +) + +target_link_libraries(rtofs PUBLIC oops ioda NetCDF::NetCDF_CXX) diff --git a/utils/obsproc/rtofs/README b/utils/obsproc/rtofs/README new file mode 100644 index 000000000..32cfbb5a4 --- /dev/null +++ b/utils/obsproc/rtofs/README @@ -0,0 +1,14 @@ +Author: E. Givelberg +Tue Jun 4 13:33:48 CDT 2024 + +The code in this rtofs directory reads a binary Fortran +file and generates a data object that can be ingested into ioda. + +The fortran binary file consists of records, including additional +information that needs to be skipped. +There are typically 8 bytes to be skipped between 2 data arrays. +The read data is converted from the big endian to the host +format, which is little endian in linux. + +There are currently several data objects that are read in RTOFSDataFile +and are discarded. diff --git a/utils/obsproc/rtofs/RTOFSDataFile.cc b/utils/obsproc/rtofs/RTOFSDataFile.cc new file mode 100644 index 000000000..cd7c97d01 --- /dev/null +++ b/utils/obsproc/rtofs/RTOFSDataFile.cc @@ -0,0 +1,95 @@ +#include +using std::cerr; +using std::endl; + +#include +using std::ofstream; + +#include "RTOFSDataFile.h" +#include "util.h" + + +namespace gdasapp +{ +namespace rtofs +{ + + +RTOFSDataFile:: +RTOFSDataFile(std::string filename) +{ + if (!file_exists(filename)) + { + cerr << "File not found" << endl; + exit(1); + } + + + const char * fn = filename.c_str(); + f = fopen(fn, "rb"); + if (!f) + { + cerr << "Error opening file " << fn << endl; + exit(1); + } + + read_file(); +} // RTOFSDataFile::RTOFSDataFile + + +void +RTOFSDataFile:: + read_file() +{ + fseek(f, 4, SEEK_CUR); + + int n_read = read_int(f); + int mx_lvl = read_int(f); + int vrsn = read_int(f); + + ob = new RTOFSOb(n_read, mx_lvl, vrsn); + + ob->read(f); + + skip8bytes(f); + + fread(ob->dtg, sizeof(RTOFSOb::char12), n_read, f); + + skip8bytes(f); + + RTOFSOb::char12 * ob_rct = new RTOFSOb::char12[n_read]; + fread(ob_rct, sizeof(RTOFSOb::char12), n_read, f); + + skip8bytes(f); + + RTOFSOb::char7 * ob_sgn = new RTOFSOb::char7[n_read]; + fread(ob_sgn, sizeof(RTOFSOb::char7), n_read, f); + +#ifdef RTOFS_OUTPUT_SGN + // we don't know what sgn is. + // by default, it is not output + { + ofstream o("sgn"); + for (int i = 0; i < n_read; i ++) + { + for (int k = 0; k < 7; k ++) + o << ob_sgn[i][k]; + o << std::endl; + } + } +#endif // RTOFS_OUTPUT_SGN + + // we never had an input file with vrsn == 2 + if (vrsn == 2) + { + float ** glb_sal = new float * [n_read]; + for (int i = 0; i < n_read; i ++) + { + int k = ob->lt[i]; + glb_sal[i] = alloc_read_float_array(f, k); + } + } +} // RTOFSDataFile::read_file + +} // namespace rtofs +} // namespace gdasapp diff --git a/utils/obsproc/rtofs/RTOFSDataFile.h b/utils/obsproc/rtofs/RTOFSDataFile.h new file mode 100644 index 000000000..39d344ac3 --- /dev/null +++ b/utils/obsproc/rtofs/RTOFSDataFile.h @@ -0,0 +1,29 @@ +#pragma once + +#include +#include + +#include "RTOFSOb.h" + + +namespace gdasapp +{ +namespace rtofs +{ + +class RTOFSDataFile +{ + public: + explicit RTOFSDataFile(std::string filename); + RTOFSOb & observations() { return * ob; } + + private: + FILE * f; + RTOFSOb * ob; + + void read_file(); +}; // class RTOFSDataFile + +} // namespace rtofs + +} // namespace gdasapp diff --git a/utils/obsproc/rtofs/RTOFSOb.cc b/utils/obsproc/rtofs/RTOFSOb.cc new file mode 100644 index 000000000..43bdabea8 --- /dev/null +++ b/utils/obsproc/rtofs/RTOFSOb.cc @@ -0,0 +1,218 @@ +// NOLINT +#include +using std::cerr; +using std::endl; +using std::cout; + +#include +using std::ofstream; + +#include "RTOFSOb.h" +#include "util.h" + + +#include "oops/util/dateFunctions.h" + + +namespace gdasapp +{ +namespace rtofs +{ + + + +int64_t +RTOFSOb:: + SecondsSinceReferenceTime(rtofs::RTOFSOb::char12 time) +{ + // parse the date + std::string s(time); + int year = std::stoi(s.substr(0, 4)); + int month = std::stoi(s.substr(4, 2)); + int day = std::stoi(s.substr(6, 2)); + int hour = std::stoi(s.substr(8, 2)); + int minute = std::stoi(s.substr(10, 2)); + int second = 0; + + uint64_t julianDate = + util::datefunctions::dateToJulian(year, month, day); + + // 2440588 = Julian date for January 1, 1970. + int daysSinceEpoch = julianDate - 2440588; + int secondsOffset = util::datefunctions::hmsToSeconds(hour, minute, second); + return static_cast(daysSinceEpoch * 86400.0f) + secondsOffset; +} // SecondsSinceReferenceTime + + + +// the variables marked "skipped" are arrays which are present +// in the binary file, but were skipped by the fortran code +// that was reading the files. + +RTOFSOb:: + RTOFSOb(int n, int mx_lvl, int vrsn): + n(n), + mx_lvl(mx_lvl), + vrsn(vrsn) +{ + allocate(); +} + + +int +RTOFSOb:: + TotalNumberOfValues() +{ + int NumberOfValues = 0; + for (int i = 0; i < n; i ++) + NumberOfValues += lt[i]; + return NumberOfValues; +} + + + +void +RTOFSOb:: + allocate() +{ + dtg = new char12[n]; + + lat = new float[n]; + lon = new float[n]; + btm = new float[n]; + ls = new int[n]; + lt = new int[n]; + sal_typ = new int[n]; + sqc = new float[n]; + tmp_typ = new int[n]; + tqc = new float[n]; + + lvl = new float * [n]; + sal = new float * [n]; + sal_err = new float * [n]; + sprb = new float * [n]; + tmp = new float * [n]; + tmp_err = new float * [n]; + clm_sal = new float * [n]; // skipped + tprb = new float * [n]; + cssd = new float * [n]; + clm_tmp = new float * [n]; // skipped + ctsd = new float * [n]; + flg = new int * [n]; +} + + +void +RTOFSOb:: + allocate2d() +{ + for (int i = 0; i < n; i ++) + { + int k = lt[i]; + + lvl[i] = new float[k]; + sal[i] = new float[k]; + sal_err[i] = new float[k]; + sprb[i] = new float[k]; + tmp[i] = new float[k]; + tmp_err[i] = new float[k]; + tprb[i] = new float[k]; + clm_sal[i] = new float[k]; + cssd[i] = new float[k]; + clm_tmp[i] = new float[k]; + ctsd[i] = new float[k]; + flg[i] = new int[k]; + } +} + + + +void +RTOFSOb:: + read(FILE * f) +{ + read_float_array(f, btm, n); + read_float_array(f, lat, n); + read_float_array(f, lon, n); + read_int_array(f, ls, n); + read_int_array(f, lt, n); + read_int_array(f, sal_typ, n); + read_float_array(f, sqc, n); + read_int_array(f, tmp_typ, n); + read_float_array(f, tqc, n); + + allocate2d(); + + for (int i = 0; i < n; i ++) + { + int k = lt[i]; + + read_float_array(f, lvl[i], k); + read_float_array(f, sal[i], k); + read_float_array(f, sal_err[i], k); + read_float_array(f, sprb[i], k); + read_float_array(f, tmp[i], k); + read_float_array(f, tmp_err[i], k); + read_float_array(f, tprb[i], k); + read_float_array(f, clm_sal[i], k); + read_float_array(f, cssd[i], k); + read_float_array(f, clm_tmp[i], k); + read_float_array(f, ctsd[i], k); + read_int_array(f, flg[i], k); + } +} + + +// helpful function to dump ascii files that can be visualized +// with python scripts +void +RTOFSOb:: + print(std::string DirectoryName) +{ + if (!file_exists(DirectoryName)) + { + cerr << "Directory " << DirectoryName << "doesn't exist" << endl; + exit(1); + } + print_float_array(DirectoryName + "/latitude", lat, n); + print_float_array(DirectoryName + "/longitude", lon, n); + print_float_array(DirectoryName + "/btm", btm, n); + print_float_array(DirectoryName + "/tqc", tqc, n); + print_float_array(DirectoryName + "/sqc", sqc, n); + print_int_array(DirectoryName + "/lt", lt, n); + print_int_array(DirectoryName + "/ls", ls, n); + print_int_array(DirectoryName + "/sal_typ", sal_typ, n); + print_int_array(DirectoryName + "/tmp_typ", tmp_typ, n); + + print_2d_array(DirectoryName + "/tmp", tmp, n, lt); + print_2d_array(DirectoryName + "/sal", sal, n, lt); + print_2d_array(DirectoryName + "/lvl", lvl, n, lt); + print_2d_array(DirectoryName + "/cssd", cssd, n, lt); + print_2d_array(DirectoryName + "/ctsd", ctsd, n, lt); + print_2d_array(DirectoryName + "/tprb", tprb, n, lt); + print_2d_array(DirectoryName + "/sprb", sprb, n, lt); + print_2d_array(DirectoryName + "/clm_tmp", clm_tmp, n, lt); + + print_level(DirectoryName + "/tmp0", tmp, n, 0, lt); + print_level(DirectoryName + "/clm_tmp0", clm_tmp, n, 0, lt); + print_level(DirectoryName + "/tmp10", tmp, n, 10, lt); + print_level(DirectoryName + "/clm_tmp10", clm_tmp, n, 10, lt); + print_level(DirectoryName + "/tprb0", tprb, n, 0, lt); + + print_2d_array(DirectoryName + "/flg", flg, n, lt); + + // print lvl2d array + { + ofstream o; + o.open(DirectoryName + "/lvl2d"); + for (int i = 0; i < n; i ++) + for (int j = 0; j < lt[i]; j ++) + o + << j + << endl; + o.close(); + } +} // RTOFSOb::print + +} // namespace rtofs +} // namespace gdasapp diff --git a/utils/obsproc/rtofs/RTOFSOb.h b/utils/obsproc/rtofs/RTOFSOb.h new file mode 100644 index 000000000..fc7cc6c3b --- /dev/null +++ b/utils/obsproc/rtofs/RTOFSOb.h @@ -0,0 +1,66 @@ +#pragma once + +#include +#include + + +namespace gdasapp +{ + +namespace rtofs +{ + +class RTOFSOb +{ + public: + RTOFSOb(int n, int mx_lvl, int vrsn); + + typedef char char12[12]; + typedef char char7[7]; + + static int64_t SecondsSinceReferenceTime(char12 time); + + void read(FILE * f); + int NumberOfObservations() { return n; } + int TotalNumberOfValues(); + void print(std::string DirectoryName); + + + float * btm; // bottom depth + float * lat; // latitude + float * lon; // longitude + int * ls; // ?? + int * lt; // array of dimensions, the number of levels + int * sal_typ; // ?? salinity type ?? + float * sqc; // salinity qc + int * tmp_typ; // ?? temperature type ?? + float * tqc; // temperature qc + + // observations per level: + float ** lvl; // depth + float ** sal; // salinity (PSU) + float ** sal_err; // salinity error (std deviation, PSU) + float ** sprb; // ?? salinity ... ?? + float ** tmp; // in situ temperature (C) + float ** tmp_err; // in situ temperature error (C) + float ** tprb; // ?? temperature ... ?? + float ** clm_sal; // climatology of salinity + float ** cssd; // climatological std deviation for salinity + float ** clm_tmp; // climatology of temperature + float ** ctsd; // climatological std deviation for temperature + int ** flg; // ?? qc flags ?? + + char12 * dtg; // date (Julian, seconds) + + private: + int n; + int mx_lvl; + int vrsn; + + void allocate(); + void allocate2d(); +}; // class RTOFSOb + +} // namespace rtofs + +} // namespace gdasapp diff --git a/utils/obsproc/rtofs/util.cc b/utils/obsproc/rtofs/util.cc new file mode 100644 index 000000000..ae96fc655 --- /dev/null +++ b/utils/obsproc/rtofs/util.cc @@ -0,0 +1,216 @@ +#include "util.h" + +#include // used in file_exists + +#include +using std::ofstream; + +#include +using std::setprecision; + +#include +using std::cerr; +using std::endl; +using std::cout; + + + +namespace gdasapp +{ + +namespace rtofs +{ + + +void skip8bytes(FILE * f) +{ + fseek(f, 8, SEEK_CUR); +} + + +void + read_float_array(FILE * f, float * a, int n) +{ + skip8bytes(f); + + fread(a, sizeof(float), n, f); + + uint32_t * data = reinterpret_cast(a); + + for (int i = 0; i < n; i ++) + data[i] = be32toh(data[i]); +} // read_float_array + + +void + read_int_array(FILE * f, int * a, int n) +{ + skip8bytes(f); + + fread(a, sizeof(int), n, f); + + uint32_t * data = reinterpret_cast(a); + + for (int i = 0; i < n; i ++) + data[i] = be32toh(data[i]); +} // read_int_array + + + +void + print_level(std::string filename, float ** a, int n, int level, int * dim2) +{ + ofstream o(filename); + o + << std::fixed + << std::setprecision(9); + + for (int i = 0; i < n; i ++) + if (level < dim2[i]) + o + << a[i] [level] + << endl; + + o.close(); +} + + +void + print_int_array(std::string filename, int * a, int n) +{ + ofstream o(filename); + for (int i = 0; i < n; i ++) + o + << a[i] + << endl; + o.close(); +} + + +void + print_float_array(std::string filename, float * a, int n) +{ + ofstream o(filename); + o + << std::fixed + << std::setprecision(9); + for (int i = 0; i < n; i ++) + o + << a[i] + << endl; +} + +void + print_2d_float_array(std::string filename, float ** a, int n, int * dim2) +{ + ofstream o(filename); + o + << std::fixed + << std::setprecision(9); + for (int i = 0; i < n; i ++) + for (int j = 0; j < dim2[i]; j ++) + o + << a[i] [j] + << endl; +} + + + +bool + file_exists(const std::string& name) +{ + struct stat buffer; + return (stat (name.c_str(), &buffer) == 0); +} + + +int + read_int(FILE * f) +{ + int dummy; + fread(&dummy, sizeof(int), 1, f); + dummy = static_cast(be32toh(dummy)); + return dummy; +} // read_int + + + +void + read_real_array(FILE * f, float * a, int n) +{ + fread(a, sizeof(float), n, f); + + uint32_t * data = reinterpret_cast(a); + + for (int i = 0; i < n; i ++) + data[i] = be32toh(data[i]); +} // read_real_array + + +float * + alloc_read_float_array(FILE * f, int n) +{ + float * a = new float[n]; + skip8bytes(f); + read_real_array(f, a, n); + return a; +} + + +void + print_int_array(int * a, int n) +{ + for (int i = 0; i < n; i ++) + cout + << a[i] + << endl; +} + + +int * + alloc_read_int_array(FILE * f, int n) +{ + int * a = new int[n]; + skip8bytes(f); + read_int_array(f, a, n); + return a; +} + + +void + print_real_array(float * a, int n) +{ + cout + << std::fixed + << std::setprecision(9); + for (int i = 0; i < n; i ++) + cout + << a[i] + << endl; +} + + +std::vector + find_unique_values(int * instrument_list, int n) +{ + std::vector instrument; + + // Function to insert a value while keeping + // the vector sorted and without duplicates + auto insertUniqueSorted = [&](int value) + { + auto it = std::lower_bound(instrument.begin(), instrument.end(), value); + if (it == instrument.end() || *it != value) + instrument.insert(it, value); + }; + + for (int i = 0; i < n; i ++) + insertUniqueSorted(instrument_list[i]); + + return instrument; +} + + +} // namespace rtofs + +} // namespace gdasapp diff --git a/utils/obsproc/rtofs/util.h b/utils/obsproc/rtofs/util.h new file mode 100644 index 000000000..5dee3d6ea --- /dev/null +++ b/utils/obsproc/rtofs/util.h @@ -0,0 +1,61 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + + +namespace gdasapp +{ + + +namespace rtofs +{ + +bool file_exists(const std::string& name); + +void skip8bytes(FILE * f); +int read_int(FILE * f); +void read_float_array(FILE * f, float * a, int n); +void read_int_array(FILE * f, int * a, int n); + +void print_float_array(std::string filename, float * a, int n); +void print_int_array(std::string filename, int * a, int n); + + +// void print_2d_float_array(std::string filename, float ** a, int n, int * dim2); + +// template +// void print_2d_array(std::string filename, NUMBER ** a, int n, int * dim2); + +void print_level(std::string filename, float ** a, int n, int level, int * dim2); +float * alloc_read_float_array(FILE * f, int n); + + +std::vector find_unique_values(int * instrument_list, int n); + + + + +template +void + print_2d_array(std::string filename, NUMBER ** a, int n, int * dim2) +{ + std::ofstream o(filename); + o + << std::fixed + << std::setprecision(9); + for (int i = 0; i < n; i ++) + for (int j = 0; j < dim2[i]; j ++) + o + << a[i] [j] + << std::endl; +} + + +} // namespace rtofs + +} // namespace gdasapp diff --git a/utils/test/CMakeLists.txt b/utils/test/CMakeLists.txt index 2d912c2d2..ce9eae1ba 100644 --- a/utils/test/CMakeLists.txt +++ b/utils/test/CMakeLists.txt @@ -3,7 +3,8 @@ list( APPEND utils_test_input testinput/gdas_meanioda.yaml testinput/gdas_rads2ioda.yaml testinput/gdas_ghrsst2ioda.yaml - testinput/gdas_rtofsinsitu.yaml + testinput/gdas_rtofstmp.yaml + testinput/gdas_rtofssal.yaml testinput/gdas_smap2ioda.yaml testinput/gdas_smos2ioda.yaml testinput/gdas_icecamsr2ioda.yaml @@ -12,7 +13,8 @@ list( APPEND utils_test_input set( gdas_utils_test_ref testref/ghrsst2ioda.test - # testref/rtofsinsitu.test + testref/rtofstmp.test + testref/rtofssal.test testref/rads2ioda.test testref/smap2ioda.test testref/smos2ioda.test @@ -32,10 +34,13 @@ CREATE_SYMLINK( ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DIR} ${utils_ execute_process( COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_SOURCE_DIR}/cpplint.py ${CMAKE_BINARY_DIR}/bin/${PROJECT_NAME}_cpplint.py) # add linter for the utils +# TODO: the linter complains about local .h files included in the .cc +# in the rtofs directory +# the exclude argument needs to be removed ecbuild_add_test( TARGET test_gdasapp_util_coding_norms TYPE SCRIPT COMMAND ${CMAKE_BINARY_DIR}/bin/${PROJECT_NAME}_cpplint.py - ARGS --quiet --recursive ${CMAKE_CURRENT_SOURCE_DIR}/../ + ARGS --quiet --recursive --exclude=${CMAKE_CURRENT_SOURCE_DIR}/../obsproc/rtofs/*.cc ${CMAKE_CURRENT_SOURCE_DIR}/../ WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/bin ) # Test example IODA utility that computes the mean of a variable @@ -65,14 +70,53 @@ ecbuild_add_test( TARGET test_gdasapp_util_ghrsst2ioda LIBS gdas-utils WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc) -#TODO: -# add binary files for the test -# Test the RTOFS in Situ converter -# ecbuild_add_test( TARGET test_gdasapp_util_rtofsinsitu - # COMMAND ${CMAKE_BINARY_DIR}/bin/gdas_obsprovider2ioda.x - # ARGS "../testinput/gdas_rtofsinsitu.yaml" - # LIBS gdas-utils - # WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc) +# copy rtofs binary input files to the testing area +# and generate the tests + +set(RTOFS_INPUT_FILE1 "rtofsinsitu_2024032600.profile") +set(RTOFS_INPUT_FILE2 "rtofsinsitu_2024032700.profile") + +# rtofs directory on orion: +set(RTOFS_FILES_PATH + "/work/noaa/da/marineda/gfs-marine/data/obs/ci/obs" +) +if (NOT EXISTS ${RTOFS_FILES_PATH}) + # rtofs directory on hera: + set(RTOFS_FILES_PATH + "/scratch1/NCEPDEV/da/common/ci/obs" + ) +endif() +if (NOT EXISTS ${RTOFS_FILES_PATH}) + message("Error: staging directory for RTOFS test files not found") +else() + set(RTOFS_FILE1 "${RTOFS_FILES_PATH}/${RTOFS_INPUT_FILE1}") + set(RTOFS_FILE2 "${RTOFS_FILES_PATH}/${RTOFS_INPUT_FILE2}") + set(DESTINATION_DIR "${CMAKE_CURRENT_BINARY_DIR}/obsproc") + + if (EXISTS ${RTOFS_FILE1} AND EXISTS ${RTOFS_FILE2}) + message("Found RTOFS files; generating tests") + + file(COPY ${RTOFS_FILE1} DESTINATION ${DESTINATION_DIR}) + file(COPY ${RTOFS_FILE2} DESTINATION ${DESTINATION_DIR}) + + # Test the RTOFStmp to IODA converter + ecbuild_add_test( TARGET test_gdasapp_util_rtofstmp + COMMAND ${CMAKE_BINARY_DIR}/bin/gdas_obsprovider2ioda.x + ARGS "../testinput/gdas_rtofstmp.yaml" + LIBS gdas-utils + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc) + + # Test the RTOFSsal to IODA converter + ecbuild_add_test( TARGET test_gdasapp_util_rtofssal + COMMAND ${CMAKE_BINARY_DIR}/bin/gdas_obsprovider2ioda.x + ARGS "../testinput/gdas_rtofssal.yaml" + LIBS gdas-utils + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/obsproc) + else() + message("Error: RTOFS input files not found; no test generated.") + endif() +endif() + # Test the SMAP to IODA converter ecbuild_add_test( TARGET test_gdasapp_util_smap2ioda diff --git a/utils/test/testinput/gdas_rtofsinsitu.yaml b/utils/test/testinput/gdas_rtofsinsitu.yaml deleted file mode 100644 index 4c8684d9d..000000000 --- a/utils/test/testinput/gdas_rtofsinsitu.yaml +++ /dev/null @@ -1,13 +0,0 @@ -provider: RTOFS -window begin: 2021-03-24T15:00:00Z -window end: 2021-03-24T21:00:00Z -output file: rtofsinsitu_2024032600.profile.nc4 -input files: -- rtofsinsitu_2024032600.profile -# - 2024032600.profile -# - 2024032600.sfc - -# test: - # reference filename: testref/rtofsinsitu.test - # test output filename: testoutput/rtofsinsitu.test - # float relative tolerance: 1e-6 diff --git a/utils/test/testinput/gdas_rtofssal.yaml b/utils/test/testinput/gdas_rtofssal.yaml new file mode 100644 index 000000000..8bf0165eb --- /dev/null +++ b/utils/test/testinput/gdas_rtofssal.yaml @@ -0,0 +1,12 @@ +provider: RTOFSsal +window begin: 2021-03-24T15:00:00Z +window end: 2021-03-24T21:00:00Z +output file: rtofssal_2024032600.profile.nc4 +input files: +- rtofsinsitu_2024032600.profile +- rtofsinsitu_2024032700.profile + +test: + reference filename: testref/rtofssal.test + test output filename: testoutput/rtofssal.test + float relative tolerance: 1e-6 diff --git a/utils/test/testinput/gdas_rtofstmp.yaml b/utils/test/testinput/gdas_rtofstmp.yaml new file mode 100644 index 000000000..2b92319bb --- /dev/null +++ b/utils/test/testinput/gdas_rtofstmp.yaml @@ -0,0 +1,12 @@ +provider: RTOFStmp +window begin: 2021-03-24T15:00:00Z +window end: 2021-03-24T21:00:00Z +output file: rtofstmp_2024032600.profile.nc4 +input files: +- rtofsinsitu_2024032600.profile +- rtofsinsitu_2024032700.profile + +test: + reference filename: testref/rtofstmp.test + test output filename: testoutput/rtofstmp.test + float relative tolerance: 1e-6 diff --git a/utils/test/testref/rtofssal.test b/utils/test/testref/rtofssal.test new file mode 100644 index 000000000..a63dacdb9 --- /dev/null +++ b/utils/test/testref/rtofssal.test @@ -0,0 +1,26 @@ +Reading: [rtofsinsitu_2024032600.profile,rtofsinsitu_2024032700.profile] +seconds since 1970-01-01T00:00:00Z +obsVal: + Min: -999 + Max: 43.2123 + Sum: 2.51883e+07 +obsError: + Min: 0.0101875 + Max: 1.74092 + Sum: 32320.7 +preQc: + Min: 0 + Max: 1042 + Sum: 3100218 +longitude: + Min: -179.959 + Max: 179.732 + Sum: -2.48938e+07 +latitude: + Min: -70.1871 + Max: 82.324 + Sum: 1.25108e+06 +datetime: + Min: 1711411200 + Max: 1711583940 + Sum: 1247390658115740 diff --git a/utils/test/testref/rtofstmp.test b/utils/test/testref/rtofstmp.test new file mode 100644 index 000000000..4c65720fb --- /dev/null +++ b/utils/test/testref/rtofstmp.test @@ -0,0 +1,26 @@ +Reading: [rtofsinsitu_2024032600.profile,rtofsinsitu_2024032700.profile] +seconds since 1970-01-01T00:00:00Z +obsVal: + Min: -1.98999 + Max: 34.2 + Sum: 5.97847e+06 +obsError: + Min: 0.0146336 + Max: 1.35821 + Sum: 86632.5 +preQc: + Min: 0 + Max: 1042 + Sum: 3100218 +longitude: + Min: -179.959 + Max: 179.732 + Sum: -2.48938e+07 +latitude: + Min: -70.1871 + Max: 82.324 + Sum: 1.25108e+06 +datetime: + Min: 1711411200 + Max: 1711583940 + Sum: 1247390658115740