diff --git a/src/nemo-feedback/NemoFeedback.cc b/src/nemo-feedback/NemoFeedback.cc index 96d7eea..f441ff5 100644 --- a/src/nemo-feedback/NemoFeedback.cc +++ b/src/nemo-feedback/NemoFeedback.cc @@ -6,8 +6,8 @@ #include "nemo-feedback/NemoFeedback.h" -#include #include +#include #include #include @@ -46,35 +46,6 @@ namespace nemo_feedback { constexpr std::string_view defaultDepthGroup{"MetaData"}; constexpr std::string_view defaultDepthVariable{"depthBelowWaterSurface"}; -/// \brief create whole report variables from a diagnostic flag -void wholeReportQCFromDiagnosticFlags(const - NemoFeedbackDataCreator& creator, const std::string& group, const - std::string& name, feedback_io::Data& wholeReportQCData) { - if (wholeReportQCData.n_obs() == 0) { - wholeReportQCData = feedback_io::Data(creator.indexer(), - std::vector(creator.indexer()->n_source_data(), 0)); - } - feedback_io::Data QCData = - creator.create(group, name, - ufo::DiagnosticFlag(0), 4, 1); - for (size_t iProfile = 0; - iProfile < QCData.n_obs(); ++iProfile) { - size_t nBadObs = 0; - for (size_t iLevel = 0; - iLevel < QCData.length(iProfile); - ++iLevel) { - if (QCData(iProfile, iLevel) == 4) { - ++nBadObs; - } - } - if (wholeReportQCData[iProfile] == 0 || - wholeReportQCData[iProfile] == 4) { - wholeReportQCData[iProfile] = - (nBadObs == QCData.length(iProfile) ? 4 : 1); - } - } -} - NemoFeedback::NemoFeedback( ioda::ObsSpace & obsdb, @@ -222,10 +193,11 @@ template void NemoFeedback::write_all_data(feedback_io::Writer& writer, const NemoFeedbackDataCreator& creator) const { - feedback_io::Data wholeReportQCData(creator.indexer(), - std::vector(creator.indexer()->n_source_data(), 0)); - feedback_io::Data wholeReportPositionQCData; - feedback_io::Data wholeReportTimeQCData; + feedback_io::Data wholeReportQCData(creator.indexer(), + std::vector(creator.indexer()->n_source_data(), + feedback_io::QC::Level::None)); + feedback_io::Data wholeReportPositionQCData; + feedback_io::Data wholeReportTimeQCData; std::vector do_not_assimilate; for (const NemoFeedbackVariableParameters& nemoVariableParams : parameters_.variables.value()) { @@ -270,70 +242,78 @@ void NemoFeedback::write_all_data(feedback_io::Writer& writer, // Quality control rank variables if (obsdb_.has("DiagnosticFlags/FinalReject", ufo_name)) { - - feedback_io::Data variableFinalQCData = + feedback_io::Data variableFinalQCData = creator.create("DiagnosticFlags/FinalReject", ufo_name, - ufo::DiagnosticFlag(0), 4, 1); + ufo::DiagnosticFlag(0), feedback_io::QC::Level::Bad, + feedback_io::QC::Level::Good); - // Add do not assimilate flag if required. + // Add do not assimilate flag if required to the final QC information. if (obsdb_.has("DiagnosticFlags/DoNotAssimilate", ufo_name)) { - feedback_io::Data variableDoNotAssimilateData( + feedback_io::Data variableDoNotAssimilateData( creator.create("DiagnosticFlags/DoNotAssimilate", ufo_name, - ufo::DiagnosticFlag(0), 128, 0)); + ufo::DiagnosticFlag(0), feedback_io::QC::Level::DoNotAssimilate, + feedback_io::QC::Level::None)); for (size_t iProfile = 0; iProfile < variableDoNotAssimilateData.n_obs(); ++iProfile) { for (size_t iLevel = 0; iLevel < variableDoNotAssimilateData.length(iProfile); ++iLevel) { - variableFinalQCData(iProfile, iLevel) += - variableDoNotAssimilateData(iProfile, iLevel); + if (variableDoNotAssimilateData(iProfile, iLevel) == + feedback_io::QC::Level::DoNotAssimilate) { + variableFinalQCData(iProfile, iLevel) = + static_cast( + static_cast(variableFinalQCData(iProfile, iLevel)) + + static_cast(feedback_io::QC::Level::DoNotAssimilate)); + } } } } - // Per-profile quality rank - feedback_io::Data wholeVariableQCData(creator.indexer(), - std::vector(creator.indexer()->n_source_data(), 0)); - for (size_t iProfile = 0; - iProfile < variableFinalQCData.n_obs(); ++iProfile) { - size_t nBadObs = 0; - for (size_t iLevel = 0; - iLevel < variableFinalQCData.length(iProfile); - ++iLevel) { - if (variableFinalQCData(iProfile, iLevel) == 4) { - ++nBadObs; - } - } - wholeVariableQCData[iProfile] = - (nBadObs == variableFinalQCData.length(iProfile) ? 4 : 1); - } + // set per-profile quality rank for the variable + feedback_io::Data wholeVariableQCData( + creator.indexer(), std::vector( + creator.indexer()->n_source_data(), feedback_io::QC::Level::None)); + feedback_io::wholeReportFromPerProfile(variableFinalQCData, + wholeVariableQCData); + // update set per-profile quality rank for the whole report + feedback_io::wholeReportFromPerProfile(variableFinalQCData, + wholeReportQCData); writer.write_variable_surf_qc(nemo_name + "_QC", wholeVariableQCData); writer.write_variable_level_qc(nemo_name + "_LEVEL_QC", variableFinalQCData); - // Whole Observation report QC + // Whole Observation Position report QC const std::string positionQCGroup("DiagnosticFlags/PositionReject"); if (obsdb_.has(positionQCGroup, ufo_name)) { - wholeReportQCFromDiagnosticFlags(creator, positionQCGroup, ufo_name, + feedback_io::Data QCData = creator.create( + positionQCGroup, ufo_name, ufo::DiagnosticFlag(0), + feedback_io::QC::Level::Bad, feedback_io::QC::Level::Good); + if (wholeReportPositionQCData.n_obs() == 0) { + wholeReportPositionQCData = feedback_io::Data( + creator.indexer(), std::vector( + creator.indexer()->n_source_data(), + feedback_io::QC::Level::None)); + } + feedback_io::wholeReportFromPerProfile(QCData, wholeReportPositionQCData); } + // Whole Observation time report QC const std::string timeQCGroup("DiagnosticFlags/TimeReject"); if (obsdb_.has(timeQCGroup, ufo_name)) { - wholeReportQCFromDiagnosticFlags(creator, timeQCGroup, ufo_name, - wholeReportTimeQCData); - } - - { - for (size_t iProfile = 0; iProfile < variableData.n_obs(); ++iProfile) { - if (wholeReportQCData[iProfile] == 0 || - wholeReportQCData[iProfile] == 4) { - wholeReportQCData[iProfile] = - (4 == wholeVariableQCData[iProfile] ? 4 : 1); - } + feedback_io::Data QCData = + creator.create(timeQCGroup, ufo_name, ufo::DiagnosticFlag(0), + feedback_io::QC::Level::Bad, feedback_io::QC::Level::Good); + if (wholeReportTimeQCData.n_obs() == 0) { + wholeReportTimeQCData = feedback_io::Data( + creator.indexer(), std::vector( + creator.indexer()->n_source_data(), + feedback_io::QC::Level::None)); } + feedback_io::wholeReportFromPerProfile(QCData, + wholeReportTimeQCData); } } @@ -356,9 +336,9 @@ void NemoFeedback::write_all_data(feedback_io::Writer& writer, const std::string depthVariable = parameters_.depthVariable.value() .value_or(static_cast(defaultDepthVariable)); if (obsdb_.has(depthQCGroup, depthVariable)) { - feedback_io::Data depthQCData( - creator.create(depthQCGroup, depthVariable, - ufo::DiagnosticFlag(0), 4, 1)); + feedback_io::Data depthQCData( + creator.create(depthQCGroup, depthVariable, ufo::DiagnosticFlag(0), + feedback_io::QC::Level::Bad, feedback_io::QC::Level::Good)); writer.write_variable_level_qc("DEPTH_QC", depthQCData); } if (wholeReportPositionQCData.n_obs() != 0) { @@ -424,7 +404,6 @@ std::tuple, feedback_io::Data> stationIdentificationAvailable = true; } - const int32_t buoyIDMissingValueInt = util::missingValue(); constexpr size_t buoyIDWidth = 8; const std::string missingStringFeedback = feedback_io::typeToFill::value(); diff --git a/src/nemo-feedback/NemoFeedbackDataCreator.cc b/src/nemo-feedback/NemoFeedbackDataCreator.cc index f55e071..241ced6 100644 --- a/src/nemo-feedback/NemoFeedbackDataCreator.cc +++ b/src/nemo-feedback/NemoFeedbackDataCreator.cc @@ -144,14 +144,15 @@ feedback_io::Data NemoFeedbackDataCreator::create_from_obsdb( return feedback_io::Data(indexer_, std::move(data)); } -feedback_io::Data NemoFeedbackDataCreator::create_from_obsdb(const - std::string& obsGroup, const std::string& ufoName, const - ufo::DiagnosticFlag TypeInstance, int32_t whenTrue, - int32_t whenFalse) const { +feedback_io::Data + NemoFeedbackDataCreator::create_from_obsdb( + const std::string& obsGroup, const std::string& ufoName, const + ufo::DiagnosticFlag TypeInstance, feedback_io::QC::Level whenTrue, + feedback_io::QC::Level whenFalse) const { oops::Log::trace() << NemoFeedbackDataCreator::className() << ":create_from_obsdb ufo::DiagnosticFlag " << obsGroup << "/" << ufoName << std::endl; - std::vector data; + std::vector data; std::vector flagData(obsdb_.nlocs(), 0); obsdb_.get_db(obsGroup, ufoName, flagData); for (ufo::DiagnosticFlag flag : flagData) { @@ -161,7 +162,7 @@ feedback_io::Data NemoFeedbackDataCreator::create_from_obsdb(const throw eckit::BadValue(NemoFeedbackDataCreator::className() + ":create_from_obsdb ufo::DiagnosticFlag no data.", Here()); - return feedback_io::Data(indexer_, std::move(data)); + return feedback_io::Data(indexer_, std::move(data)); } template @@ -304,9 +305,10 @@ NemoFeedbackDataCreator::create_altimeter_IDs() const { feedback_io::Data(indexer_, std::move(station_types))); } -feedback_io::Data NemoFeedbackDataCreator::create(const std::string& - obsGroup, const std::string& ufoName, const ufo::DiagnosticFlag - typeInstance, int32_t whenTrue, int32_t whenFalse) const { +feedback_io::Data NemoFeedbackDataCreator::create( + const std::string& obsGroup, const std::string& ufoName, const + ufo::DiagnosticFlag typeInstance, feedback_io::QC::Level whenTrue, + feedback_io::QC::Level whenFalse) const { if (obsGroup == "hofx" || obsGroup == "HofX") { std::ostringstream err_stream; err_stream << NemoFeedbackDataCreator::className() diff --git a/src/nemo-feedback/NemoFeedbackDataCreator.h b/src/nemo-feedback/NemoFeedbackDataCreator.h index 6a6b4c1..b5d4261 100644 --- a/src/nemo-feedback/NemoFeedbackDataCreator.h +++ b/src/nemo-feedback/NemoFeedbackDataCreator.h @@ -63,11 +63,11 @@ class NemoFeedbackDataCreator { create_altimeter_IDs() const; /// \brief create DiagnosticFlags feedbackData - feedback_io::Data create(const std::string& obsGroup, + feedback_io::Data create(const std::string& obsGroup, const std::string& ufoName, const ufo::DiagnosticFlag typeInstance, - const int32_t whenTrue, - const int32_t whenFalse) const; + const feedback_io::QC::Level whenTrue, + const feedback_io::QC::Level whenFalse) const; /// \brief create feedbackData template @@ -91,11 +91,12 @@ std::shared_ptr indexer() const {return indexer_;} size_t width, bool leftJustify = false) const; /// \brief create DiagnosticFlags feedbackData from obsdb - feedback_io::Data create_from_obsdb(const std::string& obsGroup, + feedback_io::Data create_from_obsdb( + const std::string& obsGroup, const std::string& ufoName, const ufo::DiagnosticFlag TypeInstance, - int32_t whenTrue, - int32_t whenFalse) const; + feedback_io::QC::Level whenTrue, + feedback_io::QC::Level whenFalse) const; /// \brief create feedbackData from obsdb template diff --git a/src/nemo-feedback/feedback_io/Data.cc b/src/nemo-feedback/feedback_io/Data.cc index a232c7c..0b4f395 100644 --- a/src/nemo-feedback/feedback_io/Data.cc +++ b/src/nemo-feedback/feedback_io/Data.cc @@ -88,7 +88,71 @@ std::vector Data::raw_profile(size_t iProfile) const { template class Data; template class Data; template class Data; +template class Data; template class Data; template class Data; + +/// \brief update the whole report QC information for a profile based on the +/// current whole variable QC information +feedback_io::QC::Level wholeReportUpdate(feedback_io::QC::Level current, const + size_t length, const size_t nGoodObs, const size_t nBadObs, const size_t + nBadDoNotAssimilate, const size_t nDoNotAssimilate) { + // If good obs were previously found, the report is good + if (current == QC::Level::Good) { + return QC::Level::Good; + } + // Upgrade any reports if they contain any good obs (goodDoNotAssimilate -> + // good) + if (nGoodObs > 0) { + return QC::Level::Good; + } + // Check currently bad, do-not-assimilate or not-yet-checked reports and + // downgrade where necessary + if (nBadObs == length) { + return QC::Level::Bad; + } else if (nBadDoNotAssimilate == length) { + // No good observations, all bad-do-not-assimilate + return QC::Level::BadDoNotAssimilate; + } else if (nDoNotAssimilate == length) { + // No good observations, some mix of good and bad do-not-assimilate types + // -> GoodDoNotAssimilate + return QC::Level::GoodDoNotAssimilate; + } else { + // No good observations, some mix of bad and do-not-assimilate -> Bad + return QC::Level::Bad; + } +} + +void wholeReportFromPerProfile(const Data& QCData, + Data& wholeReportQCData) { + for (size_t iProfile = 0; + iProfile < QCData.n_obs(); ++iProfile) { + size_t nGoodObs = 0; + size_t nBadObs = 0; + size_t nBadDoNotAssimilate = 0; + size_t nDoNotAssimilate = 0; + for (size_t iLevel = 0; + iLevel < QCData.length(iProfile); + ++iLevel) { + if (QCData(iProfile, iLevel) == QC::Level::Bad) { + ++nBadObs; + } else if (QCData(iProfile, iLevel) == + QC::Level::BadDoNotAssimilate) { + ++nBadDoNotAssimilate; + } + if (QCData(iProfile, iLevel) == QC::Level::BadDoNotAssimilate || + QCData(iProfile, iLevel) == QC::Level::GoodDoNotAssimilate || + QCData(iProfile, iLevel) == QC::Level::DoNotAssimilate) { + ++nDoNotAssimilate; + } + if (QCData(iProfile, iLevel) == QC::Level::Good) { + ++nGoodObs; + } + } + wholeReportQCData[iProfile] = wholeReportUpdate( + wholeReportQCData[iProfile], QCData.length(iProfile), nGoodObs, + nBadObs, nBadDoNotAssimilate, nDoNotAssimilate); + } +} } // namespace feedback_io } // namespace nemo_feedback diff --git a/src/nemo-feedback/feedback_io/Data.h b/src/nemo-feedback/feedback_io/Data.h index d88bdaa..3306ff9 100644 --- a/src/nemo-feedback/feedback_io/Data.h +++ b/src/nemo-feedback/feedback_io/Data.h @@ -10,10 +10,12 @@ #include -#include #include +#include #include +#include + namespace nemo_feedback { namespace feedback_io { @@ -119,5 +121,9 @@ class Data { std::shared_ptr indexer_; std::shared_ptr> data_; }; + +/// \brief create whole report data from per-profile QC data +void wholeReportFromPerProfile(const Data& QCData, + Data& wholeReportQCData); } // namespace feedback_io } // namespace nemo_feedback diff --git a/src/nemo-feedback/feedback_io/Utils.cc b/src/nemo-feedback/feedback_io/Utils.cc index 0a9a50c..95f0f57 100644 --- a/src/nemo-feedback/feedback_io/Utils.cc +++ b/src/nemo-feedback/feedback_io/Utils.cc @@ -20,6 +20,12 @@ #include "oops/util/Logger.h" #include "oops/util/Duration.h" +#define DOUBLE_FILLVALUE 99999.0 +#define FLOAT_FILLVALUE 99999.0 +#define INT32_FILLVALUE 0 +#define STRING_FILLVALUE "MISSING " + + namespace nemo_feedback { namespace feedback_io { @@ -41,6 +47,9 @@ template <> const float typeToFill::value() { template <> const int32_t typeToFill::value() { return INT32_FILLVALUE; } +template <> const QC::Level typeToFill::value() { + return QC::Level::None; +} template <> const std::string typeToFill::value() { return STRING_FILLVALUE; } diff --git a/src/nemo-feedback/feedback_io/Utils.h b/src/nemo-feedback/feedback_io/Utils.h index fd0ae1f..c95c1e2 100644 --- a/src/nemo-feedback/feedback_io/Utils.h +++ b/src/nemo-feedback/feedback_io/Utils.h @@ -16,11 +16,6 @@ #include "oops/util/DateTime.h" -#define DOUBLE_FILLVALUE 99999.0 -#define FLOAT_FILLVALUE 99999.0 -#define INT32_FILLVALUE 0 -#define STRING_FILLVALUE "MISSING " - namespace nemo_feedback { namespace feedback_io { @@ -61,5 +56,18 @@ struct NameData { void validate() const; }; + +namespace QC { + +enum class Level : int32_t { + None = 0, + Good = 1, + Bad = 4, + DoNotAssimilate = 128, + GoodDoNotAssimilate = 129, + BadDoNotAssimilate = 132 +}; + +} // namespace QC } // namespace feedback_io } // namespace nemo_feedback diff --git a/src/nemo-feedback/feedback_io/Writer.cc b/src/nemo-feedback/feedback_io/Writer.cc index b48991e..67a2d80 100644 --- a/src/nemo-feedback/feedback_io/Writer.cc +++ b/src/nemo-feedback/feedback_io/Writer.cc @@ -219,6 +219,9 @@ void Writer::write_metadata_variables( /// JULD_QC_FLAGS, POSITION_QC, POSITION_QC_FLAGS template void Writer::define_whole_report_variables() { + // Note: This ought to be true/deleted, but it is set to false for OPS + // compatibilty reasons until OS46 release and to make our intentions clear + constexpr bool kSetQCFillValues = false; std::string flag_conventions = "JEDI UFO QC flag conventions"; { const std::vector dims{*nobs_dim, ncFile->getDim(STRINGWMO)}; @@ -237,6 +240,11 @@ void Writer::define_whole_report_variables() { { const std::vector dims{*nobs_dim, *nlevels_dim}; netCDF::NcVar tmp_var = ncFile->addVar("DEPTH_QC", netCDF::ncInt, dims); + if (kSetQCFillValues) { + const auto fill = + static_cast(typeToFill::value()); + tmp_var.setFill(true, fill); + } tmp_var.putAtt("long_name", "Quality on depth"); tmp_var.putAtt("Conventions", QC_CONVENTIONS); } @@ -253,6 +261,11 @@ void Writer::define_whole_report_variables() { { netCDF::NcVar tmp_var = ncFile->addVar("OBSERVATION_QC", netCDF::ncInt, *nobs_dim); + if (kSetQCFillValues) { + const auto fill = + static_cast(typeToFill::value()); + tmp_var.setFill(true, fill); + } tmp_var.putAtt("long_name", "Quality on observation"); tmp_var.putAtt("Conventions", QC_CONVENTIONS); } @@ -268,6 +281,11 @@ void Writer::define_whole_report_variables() { { netCDF::NcVar tmp_var = ncFile->addVar("POSITION_QC", netCDF::ncInt, *nobs_dim); + if (kSetQCFillValues) { + const auto fill = + static_cast(typeToFill::value()); + tmp_var.setFill(true, fill); + } tmp_var.putAtt("long_name", "Quality on position"); tmp_var.putAtt("Conventions", QC_CONVENTIONS); } @@ -282,6 +300,11 @@ void Writer::define_whole_report_variables() { { netCDF::NcVar tmp_var = ncFile->addVar("JULD_QC", netCDF::ncInt, *nobs_dim); + if (kSetQCFillValues) { + const auto fill = + static_cast(typeToFill::value()); + tmp_var.setFill(true, fill); + } tmp_var.putAtt("long_name", "Quality on date and time"); tmp_var.putAtt("Conventions", QC_CONVENTIONS); } @@ -324,7 +347,7 @@ void Writer::define_variable( typeToNcType(), dims); obs_var.putAtt("long_name", longName); obs_var.putAtt("units", units); - obs_var.setFill(true, feedback_io::typeToFill::value()); + obs_var.setFill(true, typeToFill::value()); for (const auto &name : name_data_.additional_names) { netCDF::NcVar add_var; @@ -332,14 +355,14 @@ void Writer::define_variable( typeToNcType(), dims); add_var.putAtt("long_name", longName + " " + name); add_var.putAtt("units", units); - add_var.setFill(true, feedback_io::typeToFill::value()); + add_var.setFill(true, typeToFill::value()); } { const std::vector qcf_dims{*nobs_dim, *nqcf_dim}; netCDF::NcVar qc_flags_var = ncFile->addVar(variable_name + "_QC_FLAGS", netCDF::ncInt, qcf_dims); - qc_flags_var.setFill(true, feedback_io::typeToFill::value()); + qc_flags_var.setFill(true, typeToFill::value()); qc_flags_var.putAtt("long_name", std::string("quality flags on ") + longName); qc_flags_var.putAtt("Conventions", flag_conventions); @@ -350,7 +373,7 @@ void Writer::define_variable( *nqcf_dim}; netCDF::NcVar level_qc_flags_var = ncFile->addVar(variable_name + "_LEVEL_QC_FLAGS", netCDF::ncInt, lvl_qcf_dims); - level_qc_flags_var.setFill(true, feedback_io::typeToFill::value()); + level_qc_flags_var.setFill(true, typeToFill::value()); level_qc_flags_var.putAtt("long_name", std::string("quality flags for each level on ") + longName); level_qc_flags_var.putAtt("Conventions", flag_conventions); @@ -363,7 +386,9 @@ void Writer::define_variable( netCDF::NcVar level_qc_var = ncFile->addVar(variable_name + "_LEVEL_QC", netCDF::ncInt, dims); - level_qc_var.setFill(true, feedback_io::typeToFill::value()); + const auto fill = + static_cast(typeToFill::value()); + level_qc_var.setFill(true, fill); level_qc_var.putAtt("long_name", std::string("quality for each level on ") + longName); level_qc_var.putAtt("Conventions", QC_CONVENTIONS); @@ -421,6 +446,7 @@ void Writer::write_coord_variables() { const std::vector dims{*nobs_dim, *nlevels_dim}; netCDF::NcVar depth_var = ncFile->addVar("DEPTH", netCDF::ncDouble, dims); + depth_var.setFill(true, typeToFill::value()); depth_var.putAtt("units", "metre"); depth_var.putAtt("long_name", "Depth"); @@ -526,6 +552,19 @@ void Writer::write_variable_surf_qc( surf_var.putVar(buffer.data()); } +template +void Writer::write_variable_surf_qc( + const std::string & variable_name, + const Data& data) { + oops::Log::trace() << Writer::className() + << "::write_variable_surf_qc: writing " + << variable_name << std::endl; + auto surf_var = ncFile->getVar(variable_name); + if (data.n_locations() == 0) return; + std::vector buffer = data.raw_surface(); + surf_var.putVar(buffer.data()); +} + template void Writer::write_variable_surf_qc( const std::string& variable_name, @@ -600,7 +639,44 @@ void Writer::write_variable_level_qc( throw eckit::BadValue(err_stream.str(), Here()); } for (size_t iProfile = 0; iProfile < data.n_obs(); ++iProfile) { - auto profileBuffer = data.raw_profile(iProfile); + std::vector profileBuffer = data.raw_profile(iProfile); + var.putVar({iProfile, 0}, + {1, profileBuffer.size()}, + profileBuffer.data()); + } +} + +template +void Writer::write_variable_level_qc( + const std::string & variable_name, + const Data& data) { + oops::Log::trace() << Writer::className() + << "::write_variable_level_qc: writing " + << variable_name << std::endl; + auto var = ncFile->getVar(variable_name); + if (var.isNull()) { + std::ostringstream err_stream; + err_stream << Writer::className() << "::write_variable_level_qc" + << " ncVar '" << variable_name << "' is not present in " + <<"NetCDF file"; + throw eckit::BadValue(err_stream.str(), Here()); + } + + if (data.n_locations() == 0) return; + + if (metaData_.nLocations != data.n_locations()) { + std::ostringstream err_stream; + err_stream << Writer::className() << "::write_variable_level_qc " + << "index range out of bounds '" << variable_name << "' " + << metaData_.nLocations << " != " << data.n_locations(); + throw eckit::BadValue(err_stream.str(), Here()); + } + for (size_t iProfile = 0; iProfile < data.n_obs(); ++iProfile) { + std::vector buffer = data.raw_profile(iProfile); + std::vector profileBuffer; + std::transform(buffer.begin(), buffer.end(), + std::back_inserter(profileBuffer), + [](QC::Level value) {return static_cast(value);}); var.putVar({iProfile, 0}, {1, profileBuffer.size()}, profileBuffer.data()); diff --git a/src/nemo-feedback/feedback_io/Writer.h b/src/nemo-feedback/feedback_io/Writer.h index db8545e..40a91e3 100644 --- a/src/nemo-feedback/feedback_io/Writer.h +++ b/src/nemo-feedback/feedback_io/Writer.h @@ -104,6 +104,10 @@ class Writer { const std::string & variable_name, const Data& data); + void write_variable_surf_qc( + const std::string & variable_name, + const Data& data); + /// \brief Write surface QC data variable with specified flag void write_variable_surf_qc( const std::string & variable_name, @@ -115,6 +119,10 @@ class Writer { const std::string & variable_name, const Data& data); + void write_variable_level_qc( + const std::string & variable_name, + const Data& data); + /// \brief Write level QC data variable with specified flag void write_variable_level_qc( const std::string & variable_name, diff --git a/src/tests/Data/hofx_prof_2var_obs.nc b/src/tests/Data/hofx_prof_2var_obs.nc index 3962158..35f7979 100644 --- a/src/tests/Data/hofx_prof_2var_obs.nc +++ b/src/tests/Data/hofx_prof_2var_obs.nc @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:8fa42ed770be485ed1ee27d1d8e75cff73483de9afe59d56d0b2320246bef78a -size 233383 +oid sha256:fbb9a9417c3bcfa5499b58dacb14bb7e2634244748099082dfd1e7cb7f1ecb11 +size 246824 diff --git a/src/tests/Data/hofx_sic_obs.nc b/src/tests/Data/hofx_sic_obs.nc index 2914435..28c065c 100644 --- a/src/tests/Data/hofx_sic_obs.nc +++ b/src/tests/Data/hofx_sic_obs.nc @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:03d12931b2d7d672d3f8b4fde70740fd4531e92d0f802eded075b04254a9f7b6 -size 147567 +oid sha256:1f65b618aebbbdc8d1399b24e3b5fe547629fd1a3e3137c5a9b8265dfd73393b +size 154548 diff --git a/src/tests/Data/hofx_two_vars_obs.nc b/src/tests/Data/hofx_two_vars_obs.nc index 7792872..3357dd1 100644 --- a/src/tests/Data/hofx_two_vars_obs.nc +++ b/src/tests/Data/hofx_two_vars_obs.nc @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:f2505bc219488a3e4a33d75d64ae271e528ece40fc3cacc4857c19fc603d2857 -size 75082 +oid sha256:28066d9230508ffaeb8803dbc57db4a049e8569e0ba61fe1d741075aeb81407e +size 199925 diff --git a/src/tests/testoutput/test_hofx3d_nc_prof_2vars_writer_out_ref.cdl b/src/tests/testoutput/test_hofx3d_nc_prof_2vars_writer_out_ref.cdl index 574eaee..b606fef 100644 --- a/src/tests/testoutput/test_hofx3d_nc_prof_2vars_writer_out_ref.cdl +++ b/src/tests/testoutput/test_hofx3d_nc_prof_2vars_writer_out_ref.cdl @@ -27,6 +27,7 @@ variables: LONGITUDE:units = "degrees_east" ; LONGITUDE:long_name = "longitude" ; double DEPTH(N_OBS, N_LEVELS) ; + DEPTH:_FillValue = 99999. ; DEPTH:units = "metre" ; DEPTH:long_name = "Depth" ; double JULD(N_OBS) ; @@ -181,7 +182,7 @@ data: _, _, _, _ ; - OBSERVATION_QC = 1, 1 ; + OBSERVATION_QC = 1, 129 ; OBSERVATION_QC_FLAGS = _, _, @@ -227,11 +228,11 @@ data: 1, _, 1, _ ; - POTM_QC = 1, 1 ; + POTM_QC = 1, 129 ; POTM_LEVEL_QC = 1, 1, 1, 4, 1, 1, - 1, 1, 1, 4, 1, 1 ; + 129, 129, 129, 132, 129, 129 ; POTM_IOBSI = _, _ ; @@ -269,11 +270,11 @@ data: 1, _, 1, _ ; - PSAL_QC = 1, 1 ; + PSAL_QC = 1, 129 ; PSAL_LEVEL_QC = 1, 1, 1, 4, 1, 1, - 1, 1, 1, 4, 1, 1 ; + 129, 129, 129, 132, 129, 129 ; PSAL_IOBSI = _, _ ; diff --git a/src/tests/testoutput/test_hofx_profiles_writer_out_ref.cdl b/src/tests/testoutput/test_hofx_profiles_writer_out_ref.cdl index f74405f..f3e2343 100644 --- a/src/tests/testoutput/test_hofx_profiles_writer_out_ref.cdl +++ b/src/tests/testoutput/test_hofx_profiles_writer_out_ref.cdl @@ -25,6 +25,7 @@ variables: LONGITUDE:units = "degrees_east" ; LONGITUDE:long_name = "longitude" ; double DEPTH(N_OBS, N_LEVELS) ; + DEPTH:_FillValue = 99999. ; DEPTH:units = "metre" ; DEPTH:long_name = "Depth" ; double JULD(N_OBS) ; diff --git a/src/tests/testoutput/test_hofx_sic_writer_out_ref.cdl b/src/tests/testoutput/test_hofx_sic_writer_out_ref.cdl index ef3bafc..9c723fd 100644 --- a/src/tests/testoutput/test_hofx_sic_writer_out_ref.cdl +++ b/src/tests/testoutput/test_hofx_sic_writer_out_ref.cdl @@ -27,6 +27,7 @@ variables: LONGITUDE:units = "degrees_east" ; LONGITUDE:long_name = "longitude" ; double DEPTH(N_OBS, N_LEVELS) ; + DEPTH:_FillValue = 99999. ; DEPTH:units = "metre" ; DEPTH:long_name = "Depth" ; double JULD(N_OBS) ; @@ -186,7 +187,7 @@ data: _, _, _, _ ; - OBSERVATION_QC = 4, 4, 4, 1, 4, 1, 4, 1, 1, 1, 4 ; + OBSERVATION_QC = 4, 4, 4, 1, 4, 1, 132, 129, 1, 1, 132 ; OBSERVATION_QC_FLAGS = _, _, @@ -285,7 +286,7 @@ data: _, _, _, _ ; - ICECONC_QC = 4, 4, 4, 1, 4, 1, 4, 1, 1, 1, 4 ; + ICECONC_QC = 4, 4, 4, 1, 4, 1, 132, 129, 1, 1, 132 ; ICECONC_LEVEL_QC = 4, @@ -294,11 +295,11 @@ data: 1, 4, 1, - 4, - 1, + 132, + 129, 1, 1, - 4 ; + 132 ; ICECONC_IOBSI = _, _, _, _, _, _, _, _, _, _, _ ; diff --git a/src/tests/testoutput/test_hofx_two_vars_writer_out_ref.cdl b/src/tests/testoutput/test_hofx_two_vars_writer_out_ref.cdl index 9472de5..6546f46 100644 --- a/src/tests/testoutput/test_hofx_two_vars_writer_out_ref.cdl +++ b/src/tests/testoutput/test_hofx_two_vars_writer_out_ref.cdl @@ -25,6 +25,7 @@ variables: LONGITUDE:units = "degrees_east" ; LONGITUDE:long_name = "longitude" ; double DEPTH(N_OBS, N_LEVELS) ; + DEPTH:_FillValue = 99999. ; DEPTH:units = "metre" ; DEPTH:long_name = "Depth" ; double JULD(N_OBS) ; @@ -210,7 +211,7 @@ data: _, _, _, _ ; - OBSERVATION_QC = 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ; + OBSERVATION_QC = 1, 1, 1, 1, 1, 1, 132, 1, 1, 1, 129 ; OBSERVATION_QC_FLAGS = _, _, @@ -309,7 +310,7 @@ data: _, _, _, _ ; - ICECONC_QC = 1, 4, 1, 1, 4, 1, 4, 1, 1, 1, 4 ; + ICECONC_QC = 1, 4, 1, 1, 4, 1, 132, 129, 1, 1, 132 ; ICECONC_LEVEL_QC = 1, @@ -318,11 +319,11 @@ data: 1, 4, 1, - 4, - 1, + 132, + 129, 1, 1, - 4 ; + 132 ; ICECONC_IOBSI = _, _, _, _, _, _, _, _, _, _, _ ; @@ -395,7 +396,7 @@ data: _, _, _, _ ; - SST_QC = 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 ; + SST_QC = 1, 1, 1, 1, 1, 129, 132, 1, 1, 1, 129 ; SST_LEVEL_QC = 1, @@ -403,12 +404,12 @@ data: 1, 1, 1, + 129, + 132, 1, 1, 1, - 1, - 1, - 1 ; + 129 ; SST_IOBSI = _, _, _, _, _, _, _, _, _, _, _ ;