From 7483c36ce1a402af33b6363804a1424f68a443af Mon Sep 17 00:00:00 2001 From: Howard Soh Date: Mon, 23 Dec 2024 11:40:56 +0000 Subject: [PATCH] #3007 Reduced the complexiity of read_time. Added read_time_as_number --- src/libcode/vx_ioda/ioda.cc | 203 +++++++++++++++++++----------------- 1 file changed, 110 insertions(+), 93 deletions(-) diff --git a/src/libcode/vx_ioda/ioda.cc b/src/libcode/vx_ioda/ioda.cc index da7f20b7fa..2523dfe742 100644 --- a/src/libcode/vx_ioda/ioda.cc +++ b/src/libcode/vx_ioda/ioda.cc @@ -38,32 +38,12 @@ using namespace netCDF; static bool has_postfix(const std::string &, std::string const &); -//////////////////////////////////////////////////////////////////////// - -//////////////////////////////////////////////////////////////////////// - - -iodaReader::iodaReader() { - clear(); - conf_info = new IODADataConfInfo(); -} - - -//////////////////////////////////////////////////////////////////////// - - -iodaReader::~iodaReader() { - clear(); - delete conf_info; -} - - //////////////////////////////////////////////////////////////////////// bool iodaReader::check_missing_thresh(double value) const { bool check = false; - for(int idx=0; idxmissing_thresh.n(); idx++) { - if(conf_info->missing_thresh[idx].check(value)) { + for(int idx=0; idxmetadata_map[meta_key]; + StringArray alt_names = conf_info.metadata_map[meta_key]; + mlog << Debug(9) << method_name + << "looking for " << meta_key << " from " << alt_names.n()<< " names\n"; for (int idx =0; idx " << "trouble getting " << GET_NC_NAME(var) << "\n\n"; exit(1); @@ -151,27 +140,25 @@ bool iodaReader::get_meta_data_chars(NcVar &var, char *metadata_buf) const { bool iodaReader::get_meta_data_double(const char *metadata_key, double *metadata_buf) { bool status = false; - static const char *method_name = "iodaReader::get_meta_data_double() -> "; ConcatString metadata_name = find_meta_name(metadata_key, metadata_vars); if(metadata_name.nonempty()) { NcVar meta_var = get_var(f_in, metadata_name.c_str(), metadata_group_name); if(IS_VALID_NC(meta_var)) { + static const char *method_name = "iodaReader::get_meta_data_double() -> "; status = get_nc_data(&meta_var, metadata_buf, nlocs); if (!status) mlog << Debug(3) << method_name << "trouble getting " << metadata_name << "\n"; } } - else mlog << Debug(4) << method_name - << "Metadata for " << metadata_key << " does not exist!\n"; if(status) { for(int idx=0; idx &iodaReader::get_point_pairs( +vector *iodaReader::get_point_pairs( const char *var_name_f, const char *var_name_o, const char *group_name_f, const char *group_name_o, const int channel) { ConcatString log_var_name_f = var_name_f; ConcatString log_var_name_o = var_name_o; + vector obs_val_f(nlocs, bad_data_double); + vector obs_val_o(nlocs, bad_data_double); static const char *method_name = "iodaReader::get_point_pairs -> "; - vector obs_val_f(nlocs, bad_data_float); - vector obs_val_o(nlocs, bad_data_float); - if (!read_obs_data(obs_val_f.data(), var_name_f, group_name_f, channel)) { clear(); exit(-1); @@ -250,23 +236,29 @@ vector &iodaReader::get_point_pairs( point_pairs.clear(); point_pairs.resize(nlocs); for (int i=0; imetadata_map[metadata_key]; + StringArray alt_names = conf_info.metadata_map[metadata_key]; if (alt_names.n() > 0) { for (int idx=0; idx hdr_ut_arr(nlocs); - bool no_leap_year = get_att_no_leap_year(&in_hdr_vld_var); - status = get_nc_data(&in_hdr_vld_var, hdr_ut_arr.data()); - if (get_var_units(&in_hdr_vld_var, units) && units.nonempty()) { - parse_cf_time_string(units.c_str(), base_ut, sec_per_unit); - } - else { - mlog << Error << "\n" << method_name << "Fail to get time units from " - << GET_NC_NAME(in_hdr_vld_var) << "\n\n"; - return status; - } - for (int i=0; i< nlocs; i++) { - vld_arr[i] = add_to_unixtime(base_ut, sec_per_unit, - hdr_ut_arr[i], no_leap_year); - } - } - else if (is_time_string) { + if (NC_STRING == GET_NC_TYPE_ID(in_hdr_vld_var)) { char valid_time[ndatetime+1]; vector hdr_vld_block(nlocs*(ndatetime+1), 0); vector hdr_vld_block2(nlocs, nullptr); @@ -408,7 +363,17 @@ bool iodaReader::read_time() { hdr_vld_block.clear(); hdr_vld_block2.clear(); } - else { + else if (NC_CHAR == GET_NC_TYPE_ID(in_hdr_vld_var)) { + if(dim_names.has("ndatetime")) ndatetime = get_dim_value(f_in, "ndatetime", error_out); + else { + NcDim datetime_dim = get_nc_dim(&in_hdr_vld_var, 1); + ndatetime = IS_VALID_NC(datetime_dim) ? get_dim_size(&datetime_dim) : nstring; + mlog << Debug(3) << "\n" << method_name + << "ndatetime dimension does not exist!\n"; + } + mlog << Debug(5) << method_name << "dimensions: nvars=" << nvars << ", nlocs=" << nlocs + << ", nrecs=" << nrecs << ", nstring=" << nstring << ", ndatetime=" << ndatetime << "\n"; + char valid_time[ndatetime+1]; vector hdr_vld_block(nlocs*(ndatetime+1), 0); status = get_nc_data(&in_hdr_vld_var, hdr_vld_block.data()); @@ -420,6 +385,8 @@ bool iodaReader::read_time() { vld_arr[i] = yyyymmddThhmmss_to_unix(valid_time); } } + else status = read_time_as_number(&in_hdr_vld_var); + if(!status) { mlog << Error << "\n" << method_name << "trouble getting datetime\n\n"; @@ -430,6 +397,51 @@ bool iodaReader::read_time() { //////////////////////////////////////////////////////////////////////// +bool iodaReader::read_time_as_number(NcVar *hdr_vld_var) { + bool status = false; + static const char *method_name = "iodaReader::read_time_as_number() -> "; + static const char *method_name_s = "iodaReader::read_time_as_number() "; + + unixtime base_ut; + int sec_per_unit; + ConcatString units; + bool no_leap_year = get_att_no_leap_year(hdr_vld_var); + if (!get_var_units(hdr_vld_var, units)) { + mlog << Error << "\n" << method_name << "Missing time units at " + << GET_NC_NAME_P(hdr_vld_var) << "\n\n"; + return false; + } + else if (units.nonempty()) parse_cf_time_string(units.c_str(), base_ut, sec_per_unit); + else { + mlog << Error << "\n" << method_name << "Empty time units at " + << GET_NC_NAME_P(hdr_vld_var) << "\n\n"; + return false; + } + if (NC_INT64 == GET_NC_TYPE_ID_P(hdr_vld_var)) { + vector hdr_ut_arr(nlocs); + status = get_nc_data(hdr_vld_var, hdr_ut_arr.data()); + if (status) { + for (int i=0; i hdr_ut_arr(nlocs); + status = get_nc_data(hdr_vld_var, hdr_ut_arr.data()); + if (status) { + for (int i=0; i< nlocs; i++) { + vld_arr[i] = add_to_unixtime(base_ut, sec_per_unit, + hdr_ut_arr[i], no_leap_year); + } + } + } + return status; +} + +//////////////////////////////////////////////////////////////////////// + void iodaReader::read_ioda(netCDF::NcFile *_f_in) { static const char *method_name = "iodaReader::read_ioda() -> "; bool error_out = true; @@ -479,6 +491,7 @@ void iodaReader::read_ioda(netCDF::NcFile *_f_in) { //////////////////////////////////////////////////////////////////////// void iodaReader::read_metadata_names() { + static const char *method_name = "iodaReader::read_metadata_names -> "; datetime_name = find_meta_name(conf_key_datetime, metadata_vars); lon_name = find_meta_name("longitude", metadata_vars); lat_name = find_meta_name("latitude", metadata_vars); @@ -563,12 +576,13 @@ bool iodaReader::read_obs_data(double *data_buf, const char *var_name, bool iodaReader::read_string_data(const char *var_name, vector &hdr_data, int str_length) { bool status = false; char hdr_val[512]; + static const char *method_name = "iodaReader::read_string_data -> "; static const char *method_name_s = "iodaReader::read_string_data() "; hdr_data.clear(); NcVar hdr_var = get_var(f_in, var_name, metadata_group_name); if (IS_INVALID_NC(hdr_var)) return status; - + hdr_data.resize(nlocs, ""); if (NC_STRING == GET_NC_TYPE_ID(hdr_var)) { vector hdr_data2(nlocs, nullptr); for (int i=0; i &hdr_data for (int i=0; i &hdr_data for (int i=0; i &hdr_data void iodaReader::set_data_config(const char *default_file_name, const char *user_file_name) { // Read the config files - conf_info->read_data_config(default_file_name, user_file_name); + conf_info.read_data_config(default_file_name, user_file_name); // Process the configuration - conf_info->process_data_config(); + conf_info.process_data_config(); } //////////////////////////////////////////////////////////////////////// @@ -618,6 +636,5 @@ static bool has_postfix(const std::string &str_buf, std::string const &postfix) } } - ////////////////////////////////////////////////////////////////////////