diff --git a/src/tools/other/ioda2nc/ioda2nc.cc b/src/tools/other/ioda2nc/ioda2nc.cc index 6e2d57c25b..905cd32efe 100644 --- a/src/tools/other/ioda2nc/ioda2nc.cc +++ b/src/tools/other/ioda2nc/ioda2nc.cc @@ -37,6 +37,7 @@ #include "main.h" #include "apply_mask.h" +#include "ioda.h" #include "ioda2nc_conf_info.h" #include "vx_log.h" #include "vx_nc_util.h" @@ -60,20 +61,11 @@ using namespace netCDF; // Constants // -static const char *DEF_CONFIG_NAME = "MET_BASE/config/IODA2NCConfig_default"; +static constexpr char DEF_CONFIG_NAME[] = "MET_BASE/config/IODA2NCConfig_default"; -static const char *program_name = "ioda2nc"; +static constexpr char program_name[] = "ioda2nc"; -static const int REJECT_DEBUG_LEVEL = 9; -static const int string_data_len = 512; - -static const char *metadata_group_name = "MetaData"; -static const char *qc_group_name = "QCFlags"; -static const char *qc_postfix = "PreQC"; -static const char *obs_group_name = "ObsValue"; -static const char *derived_obs_group_name = "DerivedObsValue"; - -enum class e_ioda_format { v1, v2 }; +static constexpr int REJECT_DEBUG_LEVEL = 9; //////////////////////////////////////////////////////////////////////// @@ -101,7 +93,8 @@ static MaskPoly mask_poly; static StringArray mask_sid; // Beginning and ending retention times -static unixtime valid_beg_ut, valid_end_ut; +static unixtime valid_beg_ut; +static unixtime valid_end_ut; // Number of IODA messages to process from the command line static int nmsg = -1; @@ -133,12 +126,13 @@ static vector observations; // Output NetCDF file, dimensions, and variables // static NcFile *f_out = (NcFile *) nullptr; +static iodaFile ioda_file; //////////////////////////////////////////////////////////////////////// static void initialize(); static void process_command_line(int, char **); -static void open_netcdf(); +static void open_netcdf_output(); static void process_ioda_file(int); static void write_netcdf_hdr_data(); static void clean_up(); @@ -146,7 +140,7 @@ static void clean_up(); static void addObservation(const float *obs_arr, const ConcatString &hdr_typ, const ConcatString &hdr_sid, const time_t hdr_vld, const float hdr_lat, const float hdr_lon, const float hdr_elv, - const float quality_mark, const int buf_size); + const float quality_mark); static bool keep_message_type(const char *); static bool keep_station_id(const char *); @@ -167,17 +161,15 @@ static void set_valid_end_time(const StringArray &); static void set_verbosity(const StringArray &); static bool check_core_data(const bool, const bool, - StringArray &, StringArray &, e_ioda_format); -static bool check_missing_thresh(float value); + const StringArray &, const StringArray &, e_ioda_format); static ConcatString find_meta_name(string meta_key, StringArray available_names); static bool get_meta_data_float(NcFile *, StringArray &, const char *, float *, const int); static bool get_meta_data_strings(NcVar &, char *); static bool get_meta_data_strings(NcVar &, char **); -static bool get_obs_data_float(NcFile *, const ConcatString, NcVar *, +static bool get_obs_data_float(NcFile *, const ConcatString &, NcVar *, float *, int *, const int, const e_ioda_format); static bool has_postfix(const std::string &, std::string const &); -static bool is_in_metadata_map(string metadata_key, StringArray &available_list); //////////////////////////////////////////////////////////////////////// @@ -191,11 +183,10 @@ int met_main(int argc, char *argv[]) { process_command_line(argc, argv); // Open the NetCDF file - open_netcdf(); + open_netcdf_output(); // Process each IODA file for(int i=0; i= valid_beg_ut if(valid_beg_ut != (unixtime) 0 && valid_end_ut != (unixtime) 0 && valid_beg_ut > valid_end_ut) { - mlog << Error << "\nprocess_command_line() -> " + mlog << Error << "\n" << method_name << "the ending time (" << unix_to_yyyymmdd_hhmmss(valid_end_ut) << ") must be greater than the beginning time (" << unix_to_yyyymmdd_hhmmss(valid_beg_ut) << ").\n\n"; @@ -324,12 +314,13 @@ void process_command_line(int argc, char **argv) { if(do_summary) save_summary_only = !conf_info.getSummaryInfo().raw_data; else save_summary_only = false; + ioda_file.set_data_config(&conf_info); return; } //////////////////////////////////////////////////////////////////////// -void open_netcdf() { +static void open_netcdf_output() { // Create the output netCDF file for writing mlog << Debug(1) << "Creating NetCDF File:\t\t" << ncfile << "\n"; @@ -337,7 +328,7 @@ void open_netcdf() { // Check for a valid file if(IS_INVALID_NC_P(f_out)) { - mlog << Error << "\nopen_netcdf() -> " + mlog << Error << "\nopen_netcdf_output() -> " << "trouble opening output file: " << ncfile << "\n\n"; delete f_out; @@ -360,36 +351,52 @@ void open_netcdf() { //////////////////////////////////////////////////////////////////////// -void process_ioda_file(int i_pb) { - int npbmsg, npbmsg_total; - int idx, i_msg, i_read, n_file_obs, n_hdr_obs; - int rej_typ, rej_sid, rej_vld, rej_grid, rej_poly; - int rej_elv, rej_nobs; - double x, y; +static void process_ioda_file(int i_pb) { + int npbmsg; + int npbmsg_total; + int idx; + int i_msg; + int n_file_obs; + int n_hdr_obs; + int rej_typ; + int rej_sid; + int rej_vld; + int rej_grid; + int rej_poly; + int rej_elv; + int rej_nobs; + double x; + double y; - bool status; - bool is_time_offset = false; - bool is_time_string = false; unixtime file_ut; unixtime adjusted_file_ut; - unixtime msg_ut, beg_ut, end_ut; - unixtime min_msg_ut, max_msg_ut; + unixtime msg_ut; + unixtime beg_ut; + unixtime end_ut; + unixtime min_msg_ut; + unixtime max_msg_ut; - ConcatString file_name, blk_prefix, blk_file, log_message; + ConcatString log_message; ConcatString prefix; - ConcatString start_time_str, end_time_str; - char min_time_str[max_str_len], max_time_str[max_str_len]; + ConcatString start_time_str; + ConcatString end_time_str; + char max_time_str[max_str_len]; + char min_time_str[max_str_len]; char hdr_typ[max_str_len]; ConcatString hdr_sid; char modified_hdr_typ[max_str_len]; - double hdr_lat, hdr_lon, hdr_elv; + double hdr_lat; + double hdr_lon; + double hdr_elv; unixtime hdr_vld_ut; float obs_arr[OBS_ARRAY_LEN]; const int debug_level_for_performance = 3; - int start_t, end_t, method_start, method_end; - start_t = end_t = method_start = method_end = clock(); + clock_t start_t; + clock_t end_t; + clock_t method_start; + clock_t method_end; IntArray diff_file_times; int diff_file_time_count; @@ -403,6 +410,8 @@ void process_ioda_file(int i_pb) { conf_info.area_mask.ny() > 0); bool apply_poly_mask = (conf_info.poly_mask.n_points() > 0); + start_t = end_t = method_start = method_end = clock(); + // List the IODA file being processed mlog << Debug(1) << "Processing IODA File:\t" << ioda_files[i_pb]<< "\n"; @@ -415,153 +424,41 @@ void process_ioda_file(int i_pb) { << "\" for reading.\n\n"; delete f_in; f_in = (NcFile *) nullptr; + clean_up(); exit(1); } - // Initialize - hdr_typ[0] = 0; - file_ut = beg_ut = end_ut = hdr_vld_ut = (unixtime) 0; - filtered_times.clear(); - min_msg_ut = max_msg_ut = (unixtime) 0; - min_time_str[0] = 0; - max_time_str[0] = 0; - modified_hdr_typ[0] = 0; - - // Set the file name for the IODA file - file_name << ioda_files[i_pb]; - - int nrecs = 0; - int nlocs = 0; - int nstring = 0; - int nvars = 0; - StringArray dim_names; - StringArray metadata_vars; - StringArray obs_value_vars; - bool error_out = true; - e_ioda_format ioda_format = e_ioda_format::v2; - - get_dim_names(f_in, &dim_names); - ConcatString nlocs_name = find_meta_name("nlocs", dim_names); - if(0 < nlocs_name.length()) nlocs = get_dim_value(f_in, nlocs_name.c_str(), error_out); // number of locations - - nvars = bad_data_int ; - nstring = string_data_len; - if (! has_nc_group(f_in, obs_group_name)) ioda_format = e_ioda_format::v1; - - if ( ioda_format == e_ioda_format::v1 ) { - StringArray var_names; - get_var_names(f_in, &var_names); - for(idx=0; idx= 8) { - for(idx=0; idx= 6) { - for(idx=0; idx hdr_lat_arr (nlocs); - vector hdr_lon_arr (nlocs); - vector hdr_elv_arr (nlocs); - vector obs_pres_arr (nlocs); - vector obs_hght_arr (nlocs); - vector hdr_time_arr (nlocs); - char *hdr_vld_block = new char[nlocs*ndatetime]; - char *hdr_msg_types = nullptr; - char *hdr_station_ids = nullptr; - char **hdr_vld_block2 = nullptr; - char **hdr_msg_types2 = nullptr; - char **hdr_station_ids2 = nullptr; vector v_qc_data; vector v_obs_data; - if (is_time_string) { - hdr_vld_block2 = (char**) calloc(nlocs, sizeof(char*)); - for (int i=0; i message_type_map = conf_info.getMessageTypeMap(); // Initialize @@ -720,41 +541,21 @@ void process_ioda_file(int i_pb) { for(idx=0; idx 0) { - if(bin_count > 0 && (i_read+1)%bin_count == 0) { - cout << nint((double) (i_read+1)/npbmsg*100.0) << "% " << flush; - showed_progress = true; - if(mlog.verbosity_level() >= debug_level_for_performance) { - end_t = clock(); - cout << (end_t-start_t)/double(CLOCKS_PER_SEC) - << " seconds\n"; - start_t = clock(); - } + for(int i_read=0; i_read 0 && bin_count > 0 + && (i_read+1)%bin_count == 0) { + cout << nint((double) (i_read+1)/npbmsg*100.0) << "% " << flush; + showed_progress = true; + if(mlog.verbosity_level() >= debug_level_for_performance) { + end_t = clock(); + cout << (end_t-start_t)/double(CLOCKS_PER_SEC) + << " seconds\n"; + start_t = clock(); } } - if (is_time_offset) { - msg_ut = add_to_unixtime(base_ut, sec_per_unit, - hdr_time_arr[i_read], no_leap_year); - } - else if (is_time_string) { - char valid_time[nstring+1]; - - m_strncpy(valid_time, (const char *)hdr_vld_block2[i_read], - nstring, method_name_s, "valid_time", true); - valid_time[nstring] = 0; - msg_ut = yyyymmddThhmmss_to_unix(valid_time); - } - else { - char valid_time[ndatetime+1]; - m_strncpy(valid_time, (const char *)(hdr_vld_block + (i_read * ndatetime)), - ndatetime, method_name_s, "valid_time", true); - valid_time[ndatetime] = 0; - msg_ut = yyyymmddThhmmss_to_unix(valid_time); - } + msg_ut = ioda_file.vld_arr[i_read]; // Check to make sure that the message time hasn't changed // from one IODA message to the next @@ -800,13 +601,8 @@ void process_ioda_file(int i_pb) { } if(has_msg_type) { - if (nullptr != hdr_msg_types2) { - m_strncpy(hdr_typ, hdr_msg_types2[i_read], nstring, method_name_s, "hdr_typ2"); - } - else { - m_strncpy(hdr_typ, hdr_msg_types+(i_read*nstring), nstring, method_name_s, "hdr_typ"); - - } + m_strncpy(hdr_typ, ioda_file.msg_types.data()+(i_read*nstring), + nstring, method_name_s, "hdr_typ"); m_rstrip(hdr_typ, nstring); // If the message type is not listed in the configuration @@ -832,12 +628,8 @@ void process_ioda_file(int i_pb) { if(has_station_id) { char tmp_sid[nstring+1]; - if (nullptr != hdr_station_ids2) { - m_strncpy(tmp_sid, hdr_station_ids2[i_read], nstring, method_name_s, "tmp_sid2"); - } - else { - m_strncpy(tmp_sid, hdr_station_ids+(i_read*nstring), nstring, method_name_s, "tmp_sid"); - } + m_strncpy(tmp_sid, ioda_file.station_ids.data()+(i_read*nstring), + nstring, method_name_s, "tmp_sid"); m_rstrip(tmp_sid, nstring, false); m_replace_char(tmp_sid, ' ', '_'); hdr_sid = tmp_sid; @@ -856,13 +648,13 @@ void process_ioda_file(int i_pb) { // LON LAT DHR ELV TYP T29 ITP // Longitude - hdr_lon = hdr_lon_arr[i_read]; + hdr_lon = ioda_file.lon_arr[i_read]; // Latitude - hdr_lat = hdr_lat_arr[i_read]; + hdr_lat = ioda_file.lat_arr[i_read]; // Elevation - hdr_elv = hdr_elv_arr[i_read]; + hdr_elv = ioda_file.elv_arr[i_read]; // Compute the valid time and check if it is within the // specified valid range @@ -912,7 +704,7 @@ void process_ioda_file(int i_pb) { // Check if the message elevation is within the specified range. // Missing data values for elevation are retained. - if(!check_missing_thresh(hdr_elv) && + if(!ioda_file.check_missing_thresh(hdr_elv) && (hdr_elv < conf_info.beg_elev || hdr_elv > conf_info.end_elev) ) { rej_elv++; continue; @@ -931,12 +723,11 @@ void process_ioda_file(int i_pb) { continue; } obs_arr[1] = var_idx; - obs_arr[2] = obs_pres_arr[i_read]; - obs_arr[3] = obs_hght_arr[i_read]; + obs_arr[2] = ioda_file.obs_pres_arr[i_read]; + obs_arr[3] = ioda_file.obs_hght_arr[i_read]; obs_arr[4] = v_obs_data[idx][i_read]; addObservation(obs_arr, (string)hdr_typ, (string)hdr_sid, hdr_vld_ut, - hdr_lat, hdr_lon, hdr_elv, (float)v_qc_data[idx][i_read], - OBS_BUFFER_SIZE); + hdr_lat, hdr_lon, hdr_elv, (float)v_qc_data[idx][i_read]); // Increment the current and total observations counts n_file_obs++; @@ -1014,22 +805,7 @@ void process_ioda_file(int i_pb) { } } - delete [] hdr_vld_block; - - if (hdr_msg_types) delete [] hdr_msg_types; - if (hdr_station_ids) delete [] hdr_station_ids; - if (nullptr != hdr_msg_types2) { - for (int i=0; i " + static const string method_name = "write_netcdf_hdr_data()"; + mlog << Error << "\n" << method_name << " -> " << "No IODA records retained. Nothing to write.\n\n"; // Delete the NetCDF file remove_temp_file(ncfile); exit(1); } + nc_point_obs.set_nc_out_data(observations, summary_obs, conf_info.getSummaryInfo()); + nc_point_obs.init_netcdf(obs_cnt, hdr_cnt, program_name); + // Make sure all obs data is processed before handling header StringArray nc_var_name_arr; StringArray nc_var_unit_arr; @@ -1105,30 +883,30 @@ void write_netcdf_hdr_data() { //////////////////////////////////////////////////////////////////////// -void addObservation(const float *obs_arr, const ConcatString &hdr_typ, +static void addObservation(const float *obs_arr, const ConcatString &hdr_typ, const ConcatString &hdr_sid, const time_t hdr_vld, const float hdr_lat, const float hdr_lon, const float hdr_elv, - const float quality_mark, const int buf_size) + const float quality_mark) { // Write the quality flag to the netCDF file ConcatString obs_qty; - if(check_missing_thresh(quality_mark)) + if(ioda_file.check_missing_thresh(quality_mark)) obs_qty.add("NA"); else obs_qty.format("%d", nint(quality_mark)); - int var_index = obs_arr[1]; + auto var_index = (int)obs_arr[1]; map name_map = conf_info.getObsVarMap(); string var_name = obs_var_names[var_index]; string out_name = name_map[var_name]; - Observation obs = Observation(hdr_typ.text(), - hdr_sid.text(), - hdr_vld, - hdr_lat, hdr_lon, hdr_elv, - obs_qty.text(), - var_index, - obs_arr[2], obs_arr[3], obs_arr[4], - (0addObservationObj(obs); return; @@ -1136,7 +914,7 @@ void addObservation(const float *obs_arr, const ConcatString &hdr_typ, //////////////////////////////////////////////////////////////////////// -void clean_up() { +static void clean_up() { nc_point_obs.close(); @@ -1150,7 +928,7 @@ void clean_up() { //////////////////////////////////////////////////////////////////////// -bool keep_message_type(const char *mt_str) { +static bool keep_message_type(const char *mt_str) { bool keep = conf_info.message_type.n_elements() == 0 || conf_info.message_type.has(mt_str, false); @@ -1163,7 +941,7 @@ bool keep_message_type(const char *mt_str) { //////////////////////////////////////////////////////////////////////// -bool keep_station_id(const char *sid_str) { +static bool keep_station_id(const char *sid_str) { bool keep = (conf_info.station_id.n_elements() == 0 || conf_info.station_id.has(sid_str, false)); @@ -1177,8 +955,8 @@ bool keep_station_id(const char *sid_str) { //////////////////////////////////////////////////////////////////////// -bool keep_valid_time(const unixtime ut, - const unixtime min_ut, const unixtime max_ut) { +static bool keep_valid_time(const unixtime ut, + const unixtime min_ut, const unixtime max_ut) { bool keep = true; // If min_ut and max_ut both set, check the range @@ -1204,34 +982,32 @@ bool keep_valid_time(const unixtime ut, //////////////////////////////////////////////////////////////////////// -bool check_core_data(const bool has_msg_type, const bool has_station_id, - StringArray &dim_names, StringArray &metadata_vars, - e_ioda_format ioda_format) { +static bool check_core_data(const bool has_msg_type, const bool has_station_id, + const StringArray &dim_names, const StringArray &metadata_vars, + e_ioda_format ioda_format) { bool is_netcdf_ready = true; static const char *method_name = "check_core_data() -> "; StringArray &t_core_dims = (ioda_format == e_ioda_format::v2) ? core_dims : core_dims_v1; for(int idx=0; idx " << "core dimension \"" << t_core_dims[idx] << "\" is missing.\n\n"; is_netcdf_ready = false; } } - if (ioda_format == e_ioda_format::v1) { - if(has_msg_type || has_station_id) { - if (!is_in_metadata_map("nstring", dim_names)) { - mlog << Error << "\n" << method_name << "-> " - << "core dimension \"nstring\" is missing.\n\n"; - is_netcdf_ready = false; - } + if ((ioda_format == e_ioda_format::v1) && (has_msg_type || has_station_id)) { + if (!ioda_file.is_in_metadata_map("nstring", dim_names)) { + mlog << Error << "\n" << method_name << "-> " + << "core dimension \"nstring\" is missing.\n\n"; + is_netcdf_ready = false; } } for(int idx=0; idx " << "core variable \"" << core_meta_vars[idx] << "\" is missing.\n\n"; is_netcdf_ready = false; @@ -1242,68 +1018,7 @@ bool check_core_data(const bool has_msg_type, const bool has_station_id, //////////////////////////////////////////////////////////////////////// -bool check_missing_thresh(float value) { - bool check = false; - for(int idx=0; idx 0) { - NcVar meta_var = get_var(f_in, metadata_name.c_str(), metadata_group_name); - if(IS_VALID_NC(meta_var)) { - status = get_nc_data(&meta_var, metadata_buf, nlocs); - if(!status) mlog << Debug(3) << method_name - << "trouble getting " << metadata_name << "\n"; - } - } - else mlog << Debug(4) << method_name - << "Metadata for " << metadata_key << " does not exist!\n"; - if(status) { - for(int idx=0; idx 0) { + if(var_name.nonempty()) { ConcatString qc_name = var_name; ConcatString qc_group = qc_postfix; if (ioda_format == e_ioda_format::v2) { @@ -1370,7 +1085,7 @@ bool get_obs_data_float(NcFile *f_in, const ConcatString var_name, } if(status) { for(int idx=0; idx= postfix_len) { - return (0 == str_buf.compare(buf_len - postfix_len, postfix_len, postfix)); - } else { - return false; - } -} - -//////////////////////////////////////////////////////////////////////// - -bool is_in_metadata_map(std::string metadata_key, StringArray &available_list) { - bool found = available_list.has(metadata_key); - - if (!found) { - StringArray alt_names = conf_info.metadata_map[metadata_key]; - if (alt_names.n() > 0) { - for (int idx=0; idx