diff --git a/oneflux_steps/all.sln b/oneflux_steps/all.sln new file mode 100644 index 00000000..d1d6445b --- /dev/null +++ b/oneflux_steps/all.sln @@ -0,0 +1,53 @@ + +Microsoft Visual Studio Solution File, Format Version 10.00 +# Visual C++ Express 2008 +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "nee_proc", "nee_proc\nee_proc.vcproj", "{34171ED1-CCD6-412E-8C74-F487928AAB00}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "qc_auto", "qc_auto\qc_auto.vcproj", "{A122D8D9-88F9-4060-B217-4069A9E57CB5}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "ustar_mp", "ustar_mp\ustar_mp.vcproj", "{17ED534B-8266-4746-8A32-A256DD0B8639}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "energy_proc", "energy_proc\energy_proc.vcproj", "{7F5AE56B-6D82-45FD-8817-E2AA1FB9DF38}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "meteo_proc", "meteo_proc\meteo_proc.vcproj", "{F9FBC974-D3DE-4913-8A1D-2D8C8B20C0E9}" +EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "ure", "ure\ure.vcproj", "{DD57FEC5-6AB0-4F98-BE4B-972A6B36C088}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Win32 = Debug|Win32 + Release|Win32 = Release|Win32 + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {34171ED1-CCD6-412E-8C74-F487928AAB00}.Debug|Win32.ActiveCfg = Debug|Win32 + {34171ED1-CCD6-412E-8C74-F487928AAB00}.Debug|Win32.Build.0 = Debug|Win32 + {34171ED1-CCD6-412E-8C74-F487928AAB00}.Release|Win32.ActiveCfg = Release|Win32 + {34171ED1-CCD6-412E-8C74-F487928AAB00}.Release|Win32.Build.0 = Release|Win32 + {A122D8D9-88F9-4060-B217-4069A9E57CB5}.Debug|Win32.ActiveCfg = Debug|Win32 + {A122D8D9-88F9-4060-B217-4069A9E57CB5}.Debug|Win32.Build.0 = Debug|Win32 + {A122D8D9-88F9-4060-B217-4069A9E57CB5}.Release|Win32.ActiveCfg = Release|Win32 + {A122D8D9-88F9-4060-B217-4069A9E57CB5}.Release|Win32.Build.0 = Release|Win32 + {17ED534B-8266-4746-8A32-A256DD0B8639}.Debug|Win32.ActiveCfg = Debug|Win32 + {17ED534B-8266-4746-8A32-A256DD0B8639}.Debug|Win32.Build.0 = Debug|Win32 + {17ED534B-8266-4746-8A32-A256DD0B8639}.Release|Win32.ActiveCfg = Release|Win32 + {17ED534B-8266-4746-8A32-A256DD0B8639}.Release|Win32.Build.0 = Release|Win32 + {7F5AE56B-6D82-45FD-8817-E2AA1FB9DF38}.Debug|Win32.ActiveCfg = Debug|Win32 + {7F5AE56B-6D82-45FD-8817-E2AA1FB9DF38}.Debug|Win32.Build.0 = Debug|Win32 + {7F5AE56B-6D82-45FD-8817-E2AA1FB9DF38}.Release|Win32.ActiveCfg = Release|Win32 + {7F5AE56B-6D82-45FD-8817-E2AA1FB9DF38}.Release|Win32.Build.0 = Release|Win32 + {F9FBC974-D3DE-4913-8A1D-2D8C8B20C0E9}.Debug|Win32.ActiveCfg = Debug|Win32 + {F9FBC974-D3DE-4913-8A1D-2D8C8B20C0E9}.Debug|Win32.Build.0 = Debug|Win32 + {F9FBC974-D3DE-4913-8A1D-2D8C8B20C0E9}.Release|Win32.ActiveCfg = Release|Win32 + {F9FBC974-D3DE-4913-8A1D-2D8C8B20C0E9}.Release|Win32.Build.0 = Release|Win32 + {DD57FEC5-6AB0-4F98-BE4B-972A6B36C088}.Debug|Win32.ActiveCfg = Debug|Win32 + {DD57FEC5-6AB0-4F98-BE4B-972A6B36C088}.Debug|Win32.Build.0 = Debug|Win32 + {DD57FEC5-6AB0-4F98-BE4B-972A6B36C088}.Release|Win32.ActiveCfg = Release|Win32 + {DD57FEC5-6AB0-4F98-BE4B-972A6B36C088}.Release|Win32.Build.0 = Release|Win32 + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + VisualSVNWorkingCopyRoot = . + EndGlobalSection +EndGlobal diff --git a/oneflux_steps/common/common.c b/oneflux_steps/common/common.c index edd3d84b..c1f251bb 100644 --- a/oneflux_steps/common/common.c +++ b/oneflux_steps/common/common.c @@ -32,7 +32,7 @@ #include #include #endif /* _DEBUG */ -#pragma comment(lib, "kernel32.lib") +#pragma comment(lib, "kernel32") static WIN32_FIND_DATA wfd; static HANDLE handle; #elif defined (linux) || defined (__linux) || defined (__linux__) || defined (__APPLE__) @@ -78,7 +78,7 @@ int isinf_f (float x) { return !isnan (x) && isnan (x - x); } int isinf_d (double x) { return !isnan (x) && isnan (x - x); } int isinf_ld (long double x) { return !isnan (x) && isnan (x - x); } -/* stolen to http://www.codeproject.com/Articles/10606/Folder-Utility-Create-Path-Remove-Folder-Remove-Al */ +/* */ int create_dir(char *Path) { #if defined (_WIN32) char DirName[256]; @@ -186,7 +186,7 @@ static int get_files_from_path(const char *const path, FILES **files, int *const strcpy((*files)[*count-1].list[(*files)[*count-1].count-1].path, path); strcpy((*files)[*count-1].list[(*files)[*count-1].count-1].fullpath, path); - if ( !mystrcat((*files)[*count-1].list[(*files)[*count-1].count-1].fullpath, wfd.cFileName, PATH_SIZE) ) { + if ( !string_concat((*files)[*count-1].list[(*files)[*count-1].count-1].fullpath, wfd.cFileName, PATH_SIZE) ) { printf(err_filename_too_big, wfd.cFileName); free_files(*files, *count); return 0; @@ -244,7 +244,7 @@ static int get_files_from_path(const char *const path, FILES **files, int *const strcpy((*files)[*count-1].list[(*files)[*count-1].count-1].path, path); strcpy((*files)[*count-1].list[(*files)[*count-1].count-1].fullpath, path); - if ( !mystrcat((*files)[*count-1].list[(*files)[*count-1].count-1].fullpath, dit->d_name, PATH_SIZE) ) { + if ( !string_concat((*files)[*count-1].list[(*files)[*count-1].count-1].fullpath, dit->d_name, PATH_SIZE) ) { printf(err_filename_too_big, dit->d_name); free_files(*files, *count); return 0; @@ -273,6 +273,7 @@ void free_files(FILES *files, const int count) { } /* + TODO CHECK: on ubuntu fopen erroneously open a path (maybe a bug on NTFS partition driver ?) */ FILES *get_files(const char *const program_path, char *string, int *const count, int *const error) { @@ -495,7 +496,7 @@ FILES *get_files(const char *const program_path, char *string, int *const count, if ( program_path ) { strcpy(files[*count-1].list->path, program_path); strcpy(files[*count-1].list->fullpath, program_path); - if ( !mystrcat(files[*count-1].list->fullpath, token_by_comma, PATH_SIZE) ) { + if ( !string_concat(files[*count-1].list->fullpath, token_by_comma, PATH_SIZE) ) { printf(err_filename_too_big, token_by_comma); free_files(files, *count); return 0; @@ -526,7 +527,7 @@ FILES *get_files(const char *const program_path, char *string, int *const count, /* get token length */ i = strlen(token_by_plus); /* token is a path ? */ - if ( token_by_plus[token_length-1] == FOLDER_DELIMITER ) { + if ( token_by_plus[i-1] == FOLDER_DELIMITER ) { /* add length of filter */ token_length += strlen(filter); @@ -642,7 +643,7 @@ FILES *get_files(const char *const program_path, char *string, int *const count, if ( program_path ) { strcpy(files[*count-1].list[files[*count-1].count-1].path, program_path); strcpy(files[*count-1].list[files[*count-1].count-1].fullpath, program_path); - if ( !mystrcat(files[*count-1].list[files[*count-1].count-1].fullpath, token_by_plus, PATH_SIZE) ) { + if ( !string_concat(files[*count-1].list[files[*count-1].count-1].fullpath, token_by_plus, PATH_SIZE) ) { printf(err_filename_too_big, token_by_plus); free_files(files, *count); return 0; @@ -1029,7 +1030,7 @@ int add_char_to_string(char *const string, char c, const int size) { } /* */ -int mystrcat(char *const string, const char *const string_to_add, const int size) { +int string_concat(char *const string, const char *const string_to_add, const int size) { int i; int y; @@ -1737,6 +1738,70 @@ void info_free(INFO *info) { * */ +/* */ +int get_rows_count_by_timeres(const int timeres, const int year) { + int rows_count; + + assert((timeres > SPOT_TIMERES) && (timeres <= HOURLY_TIMERES)); + + /* assume HALFHOUR_TIMERES */ + rows_count = IS_LEAP_YEAR(year) ? LEAP_YEAR_ROWS : YEAR_ROWS; + + switch ( timeres ) { + case QUATERHOURLY_TIMERES: + rows_count *= 2; + break; + + case HOURLY_TIMERES: + rows_count /= 2; + break; + } + + return rows_count; +} + +/* */ +int get_rows_per_day_by_timeres(const int timeres) { + int rows_per_day; + + assert((timeres > SPOT_TIMERES) && (timeres <= HOURLY_TIMERES)); + + rows_per_day = 48; /* assume HALFHOUR_TIMERES */ + + switch ( timeres ) { + case QUATERHOURLY_TIMERES: + rows_per_day *= 2; + break; + + case HOURLY_TIMERES: + rows_per_day /= 2; + break; + } + + return rows_per_day; +} + +/* */ +int get_rows_per_hour_by_timeres(const int timeres) { + int rows_per_hour; + + assert((timeres > SPOT_TIMERES) && (timeres <= HOURLY_TIMERES)); + + rows_per_hour = 2; /* assume HALFHOUR_TIMERES */ + + switch ( timeres ) { + case QUATERHOURLY_TIMERES: + rows_per_hour *= 2; + break; + + case HOURLY_TIMERES: + rows_per_hour /= 2; + break; + } + + return rows_per_hour; +} + /* */ TIMESTAMP *get_timestamp(const char *const string) { int i; @@ -1749,6 +1814,12 @@ TIMESTAMP *get_timestamp(const char *const string) { const char *field[] = { "year", "month", "day", "hour", "minute", "second" }; const int field_size[] = { 4, 2, 2, 2, 2, 2 }; + for ( i = 0; string[i]; i++ ); + if ( (i < 0) || (i > 14) || (i & 1) ) { + puts("bad length for timestamp"); + return NULL; + } + t = malloc(sizeof*t); if ( ! t ) { puts(err_out_of_memory); @@ -1761,12 +1832,6 @@ TIMESTAMP *get_timestamp(const char *const string) { t->mm = 0; t->ss = 0; - for ( i = 0; string[i]; i++ ); - if ( (i < 0) || (i > 14) || (i & 1) ) { - printf("bad length for %s\n", TIMESTAMP_STRING); - return NULL; - } - p = (char *)string; index = 0; while ( 1 ) { @@ -1820,41 +1885,29 @@ TIMESTAMP *get_timestamp(const char *const string) { } /* */ -int get_row_by_timestamp(const TIMESTAMP *const t, const int hourly_dataset) { +int get_row_by_timestamp(const TIMESTAMP *const t, const int timeres) { int i; int y; int rows_per_day; int rows_per_hour; const int days_in_month[] = { 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 }; + assert((timeres > SPOT_TIMERES) && (timeres <= HOURLY_TIMERES)); + /* */ if ( ! t ) { return -1; } - /* */ - rows_per_day = 48; - if ( hourly_dataset ) { - rows_per_day = 24; - } - rows_per_hour = 2; - if ( hourly_dataset ) { - rows_per_hour = 1; - } + rows_per_day = get_rows_per_day_by_timeres(timeres); + rows_per_hour = get_rows_per_hour_by_timeres(timeres); /* check for last row */ if ( (1 == t->DD) && (1 == t->MM) && (0 == t->hh) && (0 == t->mm) ) { - if ( IS_LEAP_YEAR(t->YYYY-1) ) { - i = LEAP_YEAR_ROWS; - } else { - i = YEAR_ROWS; - } - if ( hourly_dataset ) { - i /= 2; - } + i = get_rows_count_by_timeres(timeres, t->YYYY-1); } else { i = 0; for (y = 0; y < t->MM - 1; y++ ) { @@ -1901,13 +1954,84 @@ int get_year_from_timestamp_string(const char *const string) { } /* */ -TIMESTAMP *timestamp_get_by_row(int row, int yy, const int hourly_dataset, const int start) { +int check_timestamp(const TIMESTAMP* const p) +{ + int hour; + + const int days_per_month[] = { 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 }; + + assert(p); + + if ( p->YYYY <= 0 ) return 0; + if ( (p->MM <= 0) || (p->MM > 12) ) return 0; + if ( p->DD <= 0 ) return 0; + + if ( 2 == p->MM ) // check for february leap + { + if ( p->DD > days_per_month[p->MM-1] + + IS_LEAP_YEAR(p->YYYY) ) + { + return 0; + } + } + else if ( p->DD > days_per_month[p->MM-1] ) + { + return 0; + } + + // fix 24 hh + hour = p->hh; + if ( (hour < 0) || (hour > 23) ) return 0; + if ( (p->mm < 0) || (p->hh > 59) ) return 0; + if ( (p->ss < 0) || (p->ss > 59) ) return 0; + + return 1; +} + +/* static - used by timestamp_difference_in_seconds */ +static int timestamp_to_epoch(const TIMESTAMP* const p) +{ +#define YEAR_0 (1900) + int iYear; + int iDoy; + + const int _iDoy[] = { 0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334 }; + + iYear = p->YYYY - YEAR_0; + iDoy = _iDoy[p->MM-1] + (p->DD-1); + if ( IS_LEAP_YEAR(p->YYYY) && (p->MM > 2) ) ++iDoy; + + return p->ss + p->mm*60 + p->hh*3600 + iDoy*86400 + + (iYear-70)*31536000 + ((iYear-69)/4)*86400 - + ((iYear-1)/100)*86400 + ((iYear+299)/400)*86400; +#undef YEAR_0 +} + +/* */ +int timestamp_difference_in_seconds(const TIMESTAMP* const p1, const TIMESTAMP* const p2) +{ + int s1; + int s2; + + assert(p1); + assert(p2); + + s1 = timestamp_to_epoch(p1); + s2 = timestamp_to_epoch(p2); + + return s1-s2; +} + +/* */ +TIMESTAMP *timestamp_get_by_row(int row, int yy, const int timeres, const int start) { int i; int is_leap; int rows_per_day; int days_per_month[12] = { 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 }; static TIMESTAMP t = { 0 }; + assert((timeres > SPOT_TIMERES) && (timeres <= HOURLY_TIMERES)); + /* reset */ t.YYYY = yy; t.ss = 0; @@ -1924,7 +2048,7 @@ TIMESTAMP *timestamp_get_by_row(int row, int yy, const int hourly_dataset, const } /* compute rows per day */ - rows_per_day = hourly_dataset ? 24: 48; + rows_per_day = get_rows_per_day_by_timeres(timeres); /* get day and month */ t.DD = row / rows_per_day; @@ -1945,10 +2069,12 @@ TIMESTAMP *timestamp_get_by_row(int row, int yy, const int hourly_dataset, const ++t.MM; /* get hour */ - if ( hourly_dataset ) { - t.hh = row % rows_per_day; - t.mm = 0; - } else { + if ( QUATERHOURLY_TIMERES == timeres ) { + // TODO + // CHECK THIS! + t.hh = (row % rows_per_day) / 4; + t.mm = (row % 15) * 15; + } else if ( HALFHOURLY_TIMERES == timeres ) { t.hh = (row % rows_per_day) / 2; /* even row ? */ if ( row & 1 ) { @@ -1956,6 +2082,9 @@ TIMESTAMP *timestamp_get_by_row(int row, int yy, const int hourly_dataset, const } else { t.mm = 0; } + } else if ( HOURLY_TIMERES == timeres ) { + t.hh = row % rows_per_day; + t.mm = 0; } /* ok */ @@ -1963,35 +2092,35 @@ TIMESTAMP *timestamp_get_by_row(int row, int yy, const int hourly_dataset, const } /* */ -char *timestamp_get_by_row_s(int row, int yy, const int hourly_dataset, const int start) { +char *timestamp_get_by_row_s(int row, int yy, const int timeres, const int start) { TIMESTAMP *t; static char buffer[12+1] = { 0 }; - t = timestamp_get_by_row(row, yy, hourly_dataset, start); + assert((timeres > SPOT_TIMERES) && (timeres <= HOURLY_TIMERES)); + + t = timestamp_get_by_row(row, yy, timeres, start); sprintf(buffer, "%04d%02d%02d%02d%02d", t->YYYY, t->MM, t->DD, t->hh, t->mm); return buffer; } /* */ -TIMESTAMP *timestamp_ww_get_by_row(int row, int year, const int hourly_dataset, int start) { +TIMESTAMP *timestamp_ww_get_by_row(int row, int year, const int timeres, int start) { int i; int last; /* */ assert((row >= 0) &&(row < 52)); + assert((timeres > SPOT_TIMERES) && (timeres <= HOURLY_TIMERES)); /* */ last = (52-1 == row); - i = 7 * (hourly_dataset ? 24 : 48); + i = 7 * get_rows_per_day_by_timeres(timeres); row *= i; /* */ if ( ! start ) { if ( last ) { - row = IS_LEAP_YEAR(year) ? 17568 : 17520; - if ( hourly_dataset ) { - row /= 2; - } + row = get_rows_count_by_timeres(timeres, year); } else { row += i; } @@ -2000,20 +2129,22 @@ TIMESTAMP *timestamp_ww_get_by_row(int row, int year, const int hourly_dataset, } /* */ - return timestamp_get_by_row(row, year, hourly_dataset, start); + return timestamp_get_by_row(row, year, timeres, start); } /* */ -char *timestamp_ww_get_by_row_s(int row, int yy, const int hourly_dataset, const int start) { +char *timestamp_ww_get_by_row_s(int row, int yy, const int timeres, const int start) { TIMESTAMP *t; static char buffer[8+1] = { 0 }; - t = timestamp_ww_get_by_row(row, yy, hourly_dataset, start); + assert((timeres > SPOT_TIMERES) && (timeres <= HOURLY_TIMERES)); + + t = timestamp_ww_get_by_row(row, yy, timeres, start); sprintf(buffer, "%04d%02d%02d", t->YYYY, t->MM, t->DD); return buffer; } -/* private function for gapfilling */ +/* static function for gapfilling. Calculates the mean */ static PREC gf_get_similiar_mean(const GF_ROW *const gf_rows, const int rows_count) { int i; PREC mean; @@ -2037,7 +2168,72 @@ static PREC gf_get_similiar_mean(const GF_ROW *const gf_rows, const int rows_cou return mean; } -/* gapfilling */ +/* static function for gapfilling. Calculates the symmetric mean */ +static void gf_get_similiar_mean_sym_mean(GF_ROW *const gf_rows, const int rows_count, const int current_row) { + int i; + int n; + int n_above; + int n_below; + PREC mean; + PREC mean_above; + PREC mean_below; + + /* check parameter */ + assert(gf_rows); + + /* reset */ + mean = 0.0; + mean_above = 0.0; + mean_below = 0.0; + n = 0; + n_above = 0; + n_below = 0; + + /* get means */ + for ( i = 0; i < rows_count; i++ ) { + if ( gf_rows[i].value1_w_sym_mean >= gf_rows[i].value1_sym_mean ) { + mean_above += gf_rows[i].similiar; + ++n_above; + } + + if ( gf_rows[i].value1_w_sym_mean <= gf_rows[i].value1_sym_mean ) { + mean_below += gf_rows[i].similiar; + ++n_below; + } + } + + mean_above /= n_above; + mean_below /= n_below; + + if ( n_above ) { + mean += mean_above; + ++n; + } + + if ( n_below ) { + mean += mean_below; + ++n; + } + + if ( ! n ) { + mean = INVALID_VALUE; + } else { + mean /= n; + } + + /* check for NAN */ + if ( mean != mean ) { + mean = INVALID_VALUE; + } + + gf_rows[current_row].filled_sym_mean = mean; + gf_rows[current_row].mean_above = mean_above; + gf_rows[current_row].n_above = n_above; + gf_rows[current_row].mean_below = mean_below; + gf_rows[current_row].n_below = n_below; +} + +/* gapfilling function to calculate the standard deviation */ PREC gf_get_similiar_standard_deviation(const GF_ROW *const gf_rows, const int rows_count) { int i; PREC mean; @@ -2073,7 +2269,7 @@ PREC gf_get_similiar_standard_deviation(const GF_ROW *const gf_rows, const int r return sum2; } -/* gapfilling */ +/* gapfilling function to calculate the median */ PREC gf_get_similiar_median(const GF_ROW *const gf_rows, const int rows_count, int *const error) { int i; PREC *p_median; @@ -2123,7 +2319,7 @@ PREC gf_get_similiar_median(const GF_ROW *const gf_rows, const int rows_count, i return result; } -/* private function for gapfilling */ +/* static function for gapfilling. This is the core gapfillign function called only internally */ static int gapfill( PREC *values, const int struct_size, GF_ROW *const gf_rows, @@ -2134,15 +2330,21 @@ static int gapfill( PREC *values, const int end, const int step, const int method, - const int hourly_dataset, + const int timeres, const PREC value1_tolerance_min, const PREC value1_tolerance_max, - const PREC value2_tolerance, - const PREC value3_tolerance, + const PREC value2_tolerance_min, + const PREC value2_tolerance_max, + const PREC value3_tolerance_min, + const PREC value3_tolerance_max, const int tofill_column, const int value1_column, const int value2_column, - const int value3_column) { + const int value3_column, + const int sym_mean, + const int debug, + const char* debug_file_name, + int debug_start_year) { int i; int y; int j; @@ -2153,11 +2355,14 @@ static int gapfill( PREC *values, int window_current; int samples_count; PREC value1_tolerance; + PREC value2_tolerance; + PREC value3_tolerance; PREC *window_current_values; PREC *row_current_values; /* check parameter */ assert(values && gf_rows && (method >=0 && method < GF_METHODS)); + assert((timeres > SPOT_TIMERES) && (timeres <= HOURLY_TIMERES)); /* reset */ window = 0; @@ -2165,19 +2370,41 @@ static int gapfill( PREC *values, window_end = 0; window_current = 0; samples_count = 0; - value1_tolerance = 0.0; + value1_tolerance = value1_tolerance_min; + value2_tolerance = value2_tolerance_min; + value3_tolerance = value3_tolerance_min; - /* j is and index checker for hourly method */ - if ( hourly_dataset ) { - j = 3; - } else { - j = 5; + /* j is an index checker for timeres */ + switch ( timeres ) { + case QUATERHOURLY_TIMERES: + j = 9; + break; + + case HALFHOURLY_TIMERES: + j = 5; + break; + + case HOURLY_TIMERES: + j = 3; + break; } /* */ i = start; if ( GF_TOFILL_METHOD == method ) { - z = hourly_dataset ? 24 : 48; + switch ( timeres ) { + case QUATERHOURLY_TIMERES: + z = 96; + break; + + case HALFHOURLY_TIMERES: + z = 48; + break; + + case HOURLY_TIMERES: + z = 24; + break; + } } else { z = 1; } @@ -2186,18 +2413,35 @@ static int gapfill( PREC *values, samples_count = 0; /* compute window */ - window = 48 * i; - if ( hourly_dataset ) { - window /= 2; + switch ( timeres ) { + case QUATERHOURLY_TIMERES: + window = 96 * i; + break; + + case HALFHOURLY_TIMERES: + window = 48 * i; + break; + + case HOURLY_TIMERES: + window = 24 * i; + break; } /* get window start index */ window_start = current_row - window; if ( GF_TOFILL_METHOD == method ) { - if ( hourly_dataset ) { - window_start -= 1; - } else { - window_start -= 2; + switch ( timeres ) { + case QUATERHOURLY_TIMERES: + window_start -= 4; + break; + + case HALFHOURLY_TIMERES: + window_start -= 2; + break; + + case HOURLY_TIMERES: + window_start -= 1; + break; } } @@ -2209,17 +2453,22 @@ static int gapfill( PREC *values, /* get window end index */ window_end = current_row + window; if (GF_TOFILL_METHOD == method ) { - if ( hourly_dataset ) { - window_end += 2; - } else { - window_end += 3; + switch ( timeres ) { + case QUATERHOURLY_TIMERES: + window_end += 5; + break; + + case HALFHOURLY_TIMERES: + window_end += 3; + break; + + case HOURLY_TIMERES: + window_end += 2; + break; } } - /* fix bounds for first two methods - cause in hour method (NEE_METHOD) a window start at -32 and window end at 69, - it will be fixed to window start at 0 and this is an error... - */ + /* fix bounds for first two methods */ if ( GF_TOFILL_METHOD != method ) { if ( window_start < 0 ) { window_start = 0; @@ -2229,10 +2478,9 @@ static int gapfill( PREC *values, window_end = end_window; } - /* modified on June 25, 2013 */ /* compute tolerance for value1 */ if ( IS_INVALID_VALUE(value1_tolerance_min) ) { - value1_tolerance = value1_tolerance_max; + value1_tolerance = GF_DRIVER_1_TOLERANCE_MIN; } else if ( IS_INVALID_VALUE(value1_tolerance_max) ) { value1_tolerance = value1_tolerance_min; } else { @@ -2243,14 +2491,47 @@ static int gapfill( PREC *values, value1_tolerance = value1_tolerance_max; } } + + /* compute tolerance for value2 */ + if ( IS_INVALID_VALUE(value2_tolerance_min) ) { + value2_tolerance = GF_DRIVER_2A_TOLERANCE_MIN; + } else if ( IS_INVALID_VALUE(value2_tolerance_max) ) { + value2_tolerance = value2_tolerance_min; + } else { + value2_tolerance = ((PREC *)(((char *)values)+current_row*struct_size))[value2_column]; + if ( value2_tolerance < value2_tolerance_min ) { + value2_tolerance = value2_tolerance_min; + } else if ( value2_tolerance > value2_tolerance_max ) { + value2_tolerance = value2_tolerance_max; + } + } + + /* compute tolerance for value3 */ + if ( IS_INVALID_VALUE(value3_tolerance_min) ) { + value3_tolerance = GF_DRIVER_2B_TOLERANCE_MIN; + } else if ( IS_INVALID_VALUE(value3_tolerance_max) ) { + value3_tolerance = value3_tolerance_min; + } else { + value3_tolerance = ((PREC *)(((char *)values)+current_row*struct_size))[value3_column]; + if ( value3_tolerance < value3_tolerance_min ) { + value3_tolerance = value3_tolerance_min; + } else if ( value3_tolerance > value3_tolerance_max ) { + value3_tolerance = value3_tolerance_max; + } + } } - /* loop through window */ + assert(! IS_INVALID_VALUE(value1_tolerance)); + assert(! IS_INVALID_VALUE(value2_tolerance)); + assert(! IS_INVALID_VALUE(value3_tolerance)); + + /* gapfilling loop, it is called passing window size and method to use */ for ( window_current = window_start; window_current < window_end; window_current += z ) { window_current_values = ((PREC *)(((char *)values)+window_current*struct_size)); row_current_values = ((PREC *)(((char *)values)+current_row*struct_size)); switch ( method ) { + /* gapfilling method 1 when all the three drivers are available */ case GF_ALL_METHOD: if ( IS_FLAG_SET(gf_rows[window_current].mask, GF_ALL_VALID) ) { if ( @@ -2258,26 +2539,47 @@ static int gapfill( PREC *values, (FABS(window_current_values[value1_column]-row_current_values[value1_column]) < value1_tolerance) && (FABS(window_current_values[value3_column]-row_current_values[value3_column]) < value3_tolerance) ) { - gf_rows[samples_count++].similiar = window_current_values[tofill_column]; + gf_rows[samples_count].similiar = window_current_values[tofill_column]; + gf_rows[samples_count].index = window_current; + + /* if specified by parameter sets the information to use the symmetric mean */ + if ( sym_mean ) { + gf_rows[samples_count].value1_sym_mean = row_current_values[value1_column]; + gf_rows[samples_count].value1_w_sym_mean = window_current_values[value1_column]; + } + + ++samples_count; } } break; + /* gapfilling method 2 when only the main driver is available */ case GF_VALUE1_METHOD: if ( IS_FLAG_SET(gf_rows[window_current].mask, (GF_TOFILL_VALID|GF_VALUE1_VALID)) ) { if ( FABS(window_current_values[value1_column]-row_current_values[value1_column]) < value1_tolerance ) { - gf_rows[samples_count++].similiar = window_current_values[tofill_column]; + gf_rows[samples_count].similiar = window_current_values[tofill_column]; + gf_rows[samples_count].index = window_current; + + /* if specified by parameter sets the information to use the symmetric mean */ + if ( sym_mean ) { + gf_rows[samples_count].value1_sym_mean = row_current_values[value1_column]; + gf_rows[samples_count].value1_w_sym_mean = window_current_values[value1_column]; + } + + ++samples_count; } } break; + /* gapfilling method 3 based only on time, when the main driver is not available */ case GF_TOFILL_METHOD: for ( y = 0; y < j; y++ ) { if ( ((window_current+y) < 0) || (window_current+y) >= end_window ) { continue; } if ( IS_FLAG_SET(gf_rows[window_current+y].mask, GF_TOFILL_VALID) ) { - gf_rows[samples_count++].similiar = ((PREC *)(((char *)values)+((window_current+y)*struct_size)))[tofill_column]; + gf_rows[samples_count].similiar = ((PREC *)(((char *)values)+((window_current+y)*struct_size)))[tofill_column]; + gf_rows[samples_count++].index = window_current+y; } } break; @@ -2301,10 +2603,43 @@ static int gapfill( PREC *values, if ( GF_TOFILL_METHOD == method ) { ++gf_rows[current_row].time_window; } + + /* compute symmetrical mean (if enabled) */ + if ( sym_mean && (method < GF_TOFILL_METHOD) ) { + gf_get_similiar_mean_sym_mean(gf_rows, samples_count, current_row); + } /* set samples */ gf_rows[current_row].samples_count = samples_count; + /* debug file. only for internal use */ + if ( debug ) { + char buf[FILENAME_SIZE]; + FILE* f; + + sprintf(buf, "%s_%s.txt", debug_file_name, timestamp_end_by_row_s(current_row, debug_start_year, timeres)); + f = fopen(buf, "w"); + if ( ! f ) { + printf("unable to create %s\n", buf); + } else { + fprintf(f, "filled = %g\n", gf_rows[current_row].filled); + fprintf(f, "stddev = %g\n", gf_rows[current_row].stddev); + fprintf(f, "method = %d\n", gf_rows[current_row].method); + + if ( sym_mean && (method < GF_TOFILL_METHOD) ) { + fprintf(f, "filled_sym_mean = %g\n", gf_rows[current_row].filled_sym_mean); + } + + fprintf(f, "time_window = %d\n", gf_rows[current_row].time_window); + fprintf(f, "window_start=%d, window_end=%d\n", window_start, window_end); + fprintf(f, "samples_count = %d\n", gf_rows[current_row].samples_count); + for ( y = 0; y < samples_count; ++y ) { + fprintf(f, "%s,%g\n", timestamp_end_by_row_s(gf_rows[y].index, debug_start_year, timeres), gf_rows[y].similiar); + } + fclose(f); + } + } + /* ok */ return 1; } @@ -2322,294 +2657,91 @@ static int gapfill( PREC *values, return 0; } -/* DEV version */ -static int dev_mds_gf( PREC *values, + +/* gap-filling function called by the different tools and codes. Uses the gapfill function above */ +GF_ROW *gf_mds( PREC *values, const int struct_size, - GF_ROW *const gf_rows, - const int start_window, - const int end_window, - const int current_row, - const int hourly_dataset, - const PREC value1_tolerance_min, - const PREC value1_tolerance_max, - const PREC value2_tolerance, - const PREC value3_tolerance, + const int rows_count, + const int columns_count, + const int timeres, + PREC value1_tolerance_min, + PREC value1_tolerance_max, + PREC value2_tolerance_min, + PREC value2_tolerance_max, + PREC value3_tolerance_min, + PREC value3_tolerance_max, const int tofill_column, const int value1_column, const int value2_column, - const int value3_column) { + const int value3_column, + const int value1_qc_column, + const int value2_qc_column, + const int value3_qc_column, + const int qc_thrs1, + const int qc_thrs2, + const int qc_thrs3, + const int values_min, + const int compute_hat, + int start_row, + int end_row, + int *no_gaps_filled_count, + int sym_mean, + int max_mdv_win, + int debug, + const char* debug_file_name, + int debug_start_year) { int i; - int y; - int j; - int z; - int window; - int window_start; - int window_end; - int window_current; - int samples_count; - int current_method; - int start; - int end; - int step; - int method; - PREC value1_tolerance; - PREC *window_current_values; - PREC *row_current_values; - - struct { - int start; - int end; - int step; - int method; - } mds[] = { - { 7, 14, 7, GF_ALL_METHOD } - , { 7, 7, 7, GF_VALUE1_METHOD } - , { 0, 2, 1, GF_TOFILL_METHOD } - , { 21, 77, 7, GF_ALL_METHOD } - , { 14, 77, 7, GF_VALUE1_METHOD } - , { 3, 0, 3, GF_TOFILL_METHOD } - }; + int c; + int valids_count; + GF_ROW *gf_rows; - /* check parameter */ - assert(values && gf_rows); + /* */ + assert(values && rows_count && no_gaps_filled_count); /* reset */ - window = 0; - window_start = 0; - window_end = 0; - window_current = 0; - samples_count = 0; - value1_tolerance = 0.0; - - /* fix last method */ - mds[5].end = end_window + 1; - - /* j is and index checker for hourly method */ - if ( hourly_dataset ) { - j = 3; - } else { - j = 5; + *no_gaps_filled_count = 0; + if ( start_row < 0 ) { + start_row = 0; + } + if ( -1 == end_row ) { + end_row = rows_count; + } else if ( end_row > rows_count ) { + end_row = rows_count; } - /* */ - current_method = 0; - while ( current_method < SIZEOF_ARRAY(mds) ) { - start = mds[current_method].start; - end = mds[current_method].end; - step = mds[current_method].step; - method = mds[current_method].method; - - i = start; - if ( GF_TOFILL_METHOD == method ) { - z = hourly_dataset ? 24 : 48; - } else { - z = 1; - } - while ( i <= end ) { - /* reset */ - samples_count = 0; - - /* compute window */ - window = 48 * i; - if ( hourly_dataset ) { - window /= 2; - } + /* allocate memory */ + gf_rows = malloc(rows_count*sizeof*gf_rows); + if ( !gf_rows ) { + puts(err_out_of_memory); + return NULL; + } - /* get window start index */ - window_start = current_row - window; - if ( GF_TOFILL_METHOD == method ) { - if ( hourly_dataset ) { - window_start -= 1; - } else { - window_start -= 2; - } - } + /* reset */ + for ( i = 0; i < rows_count; i++ ) { + gf_rows[i].mask = 0; + gf_rows[i].similiar = INVALID_VALUE; + gf_rows[i].stddev = INVALID_VALUE; + gf_rows[i].filled = INVALID_VALUE; + gf_rows[i].quality = INVALID_VALUE; + gf_rows[i].time_window = 0; + gf_rows[i].samples_count = 0; + gf_rows[i].method = 0; - if ( GF_TOFILL_METHOD != method ) { - /* fix for recreate markus code */ - ++window_start; - } - - /* get window end index */ - window_end = current_row + window; - if (GF_TOFILL_METHOD == method ) { - if ( hourly_dataset ) { - window_end += 2; - } else { - window_end += 3; - } - } - - /* fix bounds for first two methods - cause in hour method (NEE_METHOD) a window start at -32 and window end at 69, - it will be fixed to window start at 0 and this is an error... - */ - if ( GF_TOFILL_METHOD != method ) { - if ( window_start < 0 ) { - window_start = 0; - } - - if ( window_end > end_window ) { - window_end = end_window; - } - - /* modified on June 25, 2013 */ - /* compute tolerance for value1 */ - if ( IS_INVALID_VALUE(value1_tolerance_min) ) { - value1_tolerance = value1_tolerance_max; - } else if ( IS_INVALID_VALUE(value1_tolerance_max) ) { - value1_tolerance = value1_tolerance_min; - } else { - value1_tolerance = ((PREC *)(((char *)values)+current_row*struct_size))[value1_column]; - if ( value1_tolerance < value1_tolerance_min ) { - value1_tolerance = value1_tolerance_min; - } else if ( value1_tolerance > value1_tolerance_max ) { - value1_tolerance = value1_tolerance_max; - } - } - } - - /* loop through window */ - for ( window_current = window_start; window_current < window_end; window_current += z ) { - window_current_values = ((PREC *)(((char *)values)+window_current*struct_size)); - row_current_values = ((PREC *)(((char *)values)+current_row*struct_size)); - - switch ( method ) { - case GF_ALL_METHOD: - if ( IS_FLAG_SET(gf_rows[window_current].mask, GF_ALL_VALID) ) { - if ( - (FABS(window_current_values[value2_column]-row_current_values[value2_column]) < value2_tolerance) && - (FABS(window_current_values[value1_column]-row_current_values[value1_column]) < value1_tolerance) && - (FABS(window_current_values[value3_column]-row_current_values[value3_column]) < value3_tolerance) - ) { - gf_rows[samples_count++].similiar = window_current_values[tofill_column]; - } - } - break; - - case GF_VALUE1_METHOD: - if ( IS_FLAG_SET(gf_rows[window_current].mask, (GF_TOFILL_VALID|GF_VALUE1_VALID)) ) { - if ( FABS(window_current_values[value1_column]-row_current_values[value1_column]) < value1_tolerance ) { - gf_rows[samples_count++].similiar = window_current_values[tofill_column]; - } - } - break; - - case GF_TOFILL_METHOD: - for ( y = 0; y < j; y++ ) { - if ( ((window_current+y) < 0) || (window_current+y) >= end_window ) { - continue; - } - if ( IS_FLAG_SET(gf_rows[window_current+y].mask, GF_TOFILL_VALID) ) { - gf_rows[samples_count++].similiar = ((PREC *)(((char *)values)+((window_current+y)*struct_size)))[tofill_column]; - } - } - break; - } - } - - if ( samples_count > 1 ) { - /* set mean */ - gf_rows[current_row].filled = gf_get_similiar_mean(gf_rows, samples_count); - - /* set standard deviation */ - gf_rows[current_row].stddev = gf_get_similiar_standard_deviation(gf_rows, samples_count); - - /* set method */ - gf_rows[current_row].method = method + 1; - - /* set time-window */ - gf_rows[current_row].time_window = i * 2; - - /* fix hour method timewindow */ - if ( GF_TOFILL_METHOD == method ) { - ++gf_rows[current_row].time_window; - } - - /* set samples */ - gf_rows[current_row].samples_count = samples_count; - - /* ok */ - return 1; - } - - /* inc loop */ - i += step; - - /* break if window bigger than */ - if ( (window_start < start_window) && (window_end > end_window) ) { - break; - } - } - ++current_method; - } - - /* */ - return 0; -} - -/* */ -GF_ROW *dev_gf_mds_with_bounds( PREC *values, - const int struct_size, - const int rows_count, - const int columns_count, - const int hourly_dataset, - PREC value1_tolerance_min, - PREC value1_tolerance_max, - PREC value2_tolerance, - PREC value3_tolerance, - const int tofill_column, - const int value1_column, - const int value2_column, - const int value3_column, - const int value1_qc_column, - const int value2_qc_column, - const int value3_qc_column, - const int qc_thrs, - const int values_min, - const int compute_hat, - int start_row, - int end_row, - int *no_gaps_filled_count) { - int i; - int c; - int valids_count; - GF_ROW *gf_rows; - - /* */ - assert(values && rows_count && no_gaps_filled_count); - - /* reset */ - *no_gaps_filled_count = 0; - if ( start_row < 0 ) { - start_row = 0; - } - if ( -1 == end_row ) { - end_row = rows_count; - } else if ( end_row > rows_count ) { - end_row = rows_count; - } - - /* allocate memory */ - gf_rows = malloc(rows_count*sizeof*gf_rows); - if ( !gf_rows ) { - puts(err_out_of_memory); - return NULL; - } - - /* reset */ - for ( i = 0; i < rows_count; i++ ) { - gf_rows[i].mask = 0; - gf_rows[i].similiar = INVALID_VALUE; - gf_rows[i].stddev = INVALID_VALUE; - gf_rows[i].filled = INVALID_VALUE; - gf_rows[i].quality = INVALID_VALUE; - gf_rows[i].time_window = 0; - gf_rows[i].samples_count = 0; - gf_rows[i].method = 0; - } + if ( sym_mean ) { + gf_rows[i].value1_sym_mean = INVALID_VALUE; + gf_rows[i].value1_w_sym_mean = INVALID_VALUE; + gf_rows[i].similiar_sym_mean = INVALID_VALUE; + gf_rows[i].filled_sym_mean = INVALID_VALUE; + gf_rows[i].mean_above = INVALID_VALUE; + gf_rows[i].mean_below = INVALID_VALUE; + gf_rows[i].n_above = INVALID_VALUE; + gf_rows[i].n_below = INVALID_VALUE; + } + } /* update mask and count valids TO FILL */ + /* creates a bitmask to identify for each record is target value + and the different drivers are available or not */ valids_count = 0; for ( i = start_row; i < end_row; i++ ) { for ( c = 0; c < columns_count; c++ ) { @@ -2627,26 +2759,27 @@ GF_ROW *dev_gf_mds_with_bounds( PREC *values, } /* check for QC */ - if ( !IS_INVALID_VALUE(qc_thrs) && + /* sets as invalid in the bitmask drivers with QC above the threshold */ + if ( !IS_INVALID_VALUE(qc_thrs1) && (value1_qc_column != -1) && !IS_INVALID_VALUE(((PREC *)(((char *)values)+i*struct_size))[value1_qc_column]) ) { - if ( ((PREC *)(((char *)values)+i*struct_size))[value1_qc_column] > qc_thrs ) { + if ( ((PREC *)(((char *)values)+i*struct_size))[value1_qc_column] > qc_thrs1 ) { gf_rows[i].mask &= ~GF_VALUE1_VALID; } } - if ( !IS_INVALID_VALUE(qc_thrs) && + if ( !IS_INVALID_VALUE(qc_thrs2) && (value2_qc_column != -1) && !IS_INVALID_VALUE(((PREC *)(((char *)values)+i*struct_size))[value2_qc_column]) ) { - if ( ((PREC *)(((char *)values)+i*struct_size))[value2_qc_column] > qc_thrs ) { + if ( ((PREC *)(((char *)values)+i*struct_size))[value2_qc_column] > qc_thrs2 ) { gf_rows[i].mask &= ~GF_VALUE2_VALID; } } - if ( !IS_INVALID_VALUE(qc_thrs) && + if ( !IS_INVALID_VALUE(qc_thrs3) && (value3_qc_column != -1) && !IS_INVALID_VALUE(((PREC *)(((char *)values)+i*struct_size))[value3_qc_column]) ) { - if ( ((PREC *)(((char *)values)+i*struct_size))[value3_qc_column] > qc_thrs ) { + if ( ((PREC *)(((char *)values)+i*struct_size))[value3_qc_column] > qc_thrs3 ) { gf_rows[i].mask &= ~GF_VALUE3_VALID; } } @@ -2662,23 +2795,29 @@ GF_ROW *dev_gf_mds_with_bounds( PREC *values, return NULL; } - /* */ + /* sets the tolerance values for the drivers */ if ( IS_INVALID_VALUE(value1_tolerance_min) && IS_INVALID_VALUE(value1_tolerance_max) ) { - value1_tolerance_min = GF_SW_IN_TOLERANCE_MIN; - value1_tolerance_max = GF_SW_IN_TOLERANCE_MAX; - } + value1_tolerance_min = GF_DRIVER_1_TOLERANCE_MIN; + value1_tolerance_max = GF_DRIVER_1_TOLERANCE_MAX; + } else if ( IS_INVALID_VALUE(value1_tolerance_min) ) { + value1_tolerance_min = GF_DRIVER_1_TOLERANCE_MIN; + } - /* */ - if ( IS_INVALID_VALUE(value2_tolerance) ) { - value2_tolerance = GF_TA_TOLERANCE; + if ( IS_INVALID_VALUE(value2_tolerance_min) && IS_INVALID_VALUE(value2_tolerance_max) ) { + value2_tolerance_min = GF_DRIVER_2A_TOLERANCE_MIN; + value2_tolerance_max = GF_DRIVER_2A_TOLERANCE_MAX; + } else if ( IS_INVALID_VALUE(value2_tolerance_min) ) { + value2_tolerance_min = GF_DRIVER_2A_TOLERANCE_MIN; } - /* */ - if ( IS_INVALID_VALUE(value3_tolerance) ) { - value3_tolerance = GF_VPD_TOLERANCE; + if ( IS_INVALID_VALUE(value3_tolerance_min) && IS_INVALID_VALUE(value3_tolerance_max) ) { + value3_tolerance_min = GF_DRIVER_2B_TOLERANCE_MIN; + value3_tolerance_max = GF_DRIVER_2B_TOLERANCE_MAX; + } else if ( IS_INVALID_VALUE(value3_tolerance_min) ) { + value3_tolerance_min = GF_DRIVER_2B_TOLERANCE_MIN; } - /* loop for each row */ + /* gapfilling call loop for each row */ for ( i = start_row; i < end_row; i++ ) { /* copy value from TOFILL to FILLED */ gf_rows[i].filled = ((PREC *)(((char *)values)+i*struct_size))[tofill_column]; @@ -2688,424 +2827,26 @@ GF_ROW *dev_gf_mds_with_bounds( PREC *values, continue; } - /* fill - Added 20140422: if a gap is impossible to fill, e.g. if with MDV there are no data in the whole dataset acquired in a range +/- one hour, - the data point is not filled and the qc is set to -9999 - */ - if ( ! dev_mds_gf(values, struct_size, gf_rows, start_row, end_row, i, hourly_dataset, value1_tolerance_min, value1_tolerance_max, value2_tolerance, value3_tolerance, tofill_column, value1_column, value2_column, value3_column) ) { - ++*no_gaps_filled_count; - continue; - - } - - /* compute quality */ - gf_rows[i].quality = (gf_rows[i].method > 0) + - ((gf_rows[i].method == 1 && gf_rows[i].time_window > 14) || (gf_rows[i].method == 2 && gf_rows[i].time_window > 14) || (gf_rows[i].method == 3 && gf_rows[i].time_window > 1)) + - ((gf_rows[i].method == 1 && gf_rows[i].time_window > 56) || (gf_rows[i].method == 2 && gf_rows[i].time_window > 28) || (gf_rows[i].method == 3 && gf_rows[i].time_window > 5)); - } - - /* ok */ - return gf_rows; -} - -/* */ -GF_ROW *gf_mds_with_bounds( PREC *values, - const int struct_size, - const int rows_count, - const int columns_count, - const int hourly_dataset, - PREC value1_tolerance_min, - PREC value1_tolerance_max, - PREC value2_tolerance, - PREC value3_tolerance, - const int tofill_column, - const int value1_column, - const int value2_column, - const int value3_column, - const int value1_qc_column, - const int value2_qc_column, - const int value3_qc_column, - const int qc_thrs, - const int values_min, - const int compute_hat, - int start_row, - int end_row, - int *no_gaps_filled_count) { - int i; - int c; - int valids_count; - GF_ROW *gf_rows; - - /* */ - assert(values && rows_count && no_gaps_filled_count); - - /* reset */ - *no_gaps_filled_count = 0; - if ( start_row < 0 ) { - start_row = 0; - } - if ( -1 == end_row ) { - end_row = rows_count; - } else if ( end_row > rows_count ) { - end_row = rows_count; - } - - /* allocate memory */ - gf_rows = malloc(rows_count*sizeof*gf_rows); - if ( !gf_rows ) { - puts(err_out_of_memory); - return NULL; - } - - /* reset */ - for ( i = 0; i < rows_count; i++ ) { - gf_rows[i].mask = 0; - gf_rows[i].similiar = INVALID_VALUE; - gf_rows[i].stddev = INVALID_VALUE; - gf_rows[i].filled = INVALID_VALUE; - gf_rows[i].quality = INVALID_VALUE; - gf_rows[i].time_window = 0; - gf_rows[i].samples_count = 0; - gf_rows[i].method = 0; - } - - /* update mask and count valids TO FILL */ - valids_count = 0; - for ( i = start_row; i < end_row; i++ ) { - for ( c = 0; c < columns_count; c++ ) { - if ( !IS_INVALID_VALUE(((PREC *)(((char *)values)+i*struct_size))[c]) ) { - if ( tofill_column == c ) { - gf_rows[i].mask |= GF_TOFILL_VALID; - } else if ( value1_column == c ) { - gf_rows[i].mask |= GF_VALUE1_VALID; - } else if ( value2_column == c ) { - gf_rows[i].mask |= GF_VALUE2_VALID; - } else if ( value3_column == c ) { - gf_rows[i].mask |= GF_VALUE3_VALID; - } - } - } - - /* check for QC */ - if ( !IS_INVALID_VALUE(qc_thrs) && - (value1_qc_column != -1) && - !IS_INVALID_VALUE(((PREC *)(((char *)values)+i*struct_size))[value1_qc_column]) ) { - if ( ((PREC *)(((char *)values)+i*struct_size))[value1_qc_column] > qc_thrs ) { - gf_rows[i].mask &= ~GF_VALUE1_VALID; - } - } - - if ( !IS_INVALID_VALUE(qc_thrs) && - (value2_qc_column != -1) && - !IS_INVALID_VALUE(((PREC *)(((char *)values)+i*struct_size))[value2_qc_column]) ) { - if ( ((PREC *)(((char *)values)+i*struct_size))[value2_qc_column] > qc_thrs ) { - gf_rows[i].mask &= ~GF_VALUE2_VALID; - } - } - - if ( !IS_INVALID_VALUE(qc_thrs) && - (value3_qc_column != -1) && - !IS_INVALID_VALUE(((PREC *)(((char *)values)+i*struct_size))[value3_qc_column]) ) { - if ( ((PREC *)(((char *)values)+i*struct_size))[value3_qc_column] > qc_thrs ) { - gf_rows[i].mask &= ~GF_VALUE3_VALID; - } - } - - if ( IS_FLAG_SET(gf_rows[i].mask, GF_TOFILL_VALID) ) { - ++valids_count; - } - } - - if ( valids_count < values_min ) { - puts(err_gf_too_less_values); - free(gf_rows); - return NULL; - } - - /* */ - if ( IS_INVALID_VALUE(value1_tolerance_min) && IS_INVALID_VALUE(value1_tolerance_max) ) { - value1_tolerance_min = GF_SW_IN_TOLERANCE_MIN; - value1_tolerance_max = GF_SW_IN_TOLERANCE_MAX; - } - - /* */ - if ( IS_INVALID_VALUE(value2_tolerance) ) { - value2_tolerance = GF_TA_TOLERANCE; - } - - /* */ - if ( IS_INVALID_VALUE(value3_tolerance) ) { - value3_tolerance = GF_VPD_TOLERANCE; - } - - /* loop for each row */ - for ( i = start_row; i < end_row; i++ ) { - /* copy value from TOFILL to FILLED */ - gf_rows[i].filled = ((PREC *)(((char *)values)+i*struct_size))[tofill_column]; - - /* compute hat ? */ - if ( !IS_INVALID_VALUE(gf_rows[i].filled) && !compute_hat ) { - continue; - } - - /* fill - Added 20140422: if a gap is impossible to fill, e.g. if with MDV there are no data in the whole dataset acquired in a range +/- one hour, - the data point is not filled and the qc is set to -9999 - */ - if ( !gapfill(values, struct_size, gf_rows, start_row, end_row, i, 7, 14, 7, GF_ALL_METHOD, hourly_dataset, value1_tolerance_min, value1_tolerance_max, value2_tolerance, value3_tolerance, tofill_column, value1_column, value2_column, value3_column) ) - if ( !gapfill(values, struct_size, gf_rows, start_row, end_row, i, 7, 7, 7, GF_VALUE1_METHOD, hourly_dataset, value1_tolerance_min, value1_tolerance_max, value2_tolerance, value3_tolerance, tofill_column, value1_column, value2_column, value3_column) ) - if ( !gapfill(values, struct_size, gf_rows, start_row, end_row, i, 0, 2, 1, GF_TOFILL_METHOD, hourly_dataset, value1_tolerance_min, value1_tolerance_max, value2_tolerance, value3_tolerance, tofill_column, value1_column, value2_column, value3_column) ) - if ( !gapfill(values, struct_size, gf_rows, start_row, end_row, i, 21, 77, 7, GF_ALL_METHOD, hourly_dataset, value1_tolerance_min, value1_tolerance_max, value2_tolerance, value3_tolerance, tofill_column, value1_column, value2_column, value3_column) ) - if ( !gapfill(values, struct_size, gf_rows, start_row, end_row, i, 14, 77, 7, GF_VALUE1_METHOD, hourly_dataset, value1_tolerance_min, value1_tolerance_max, value2_tolerance, value3_tolerance, tofill_column, value1_column, value2_column, value3_column) ) - if ( !gapfill(values, struct_size, gf_rows, start_row, end_row, i, 3, end_row + 1, 3, GF_TOFILL_METHOD, hourly_dataset, value1_tolerance_min, value1_tolerance_max, value2_tolerance, value3_tolerance, tofill_column, value1_column, value2_column, value3_column) ) { + /* fill gaps. If a values is impossible to fill it remains -9999 with QC also -9999 */ + if ( !gapfill(values, struct_size, gf_rows, start_row, end_row, i, 7, 14, 7, GF_ALL_METHOD, timeres, value1_tolerance_min, value1_tolerance_max, value2_tolerance_min, value2_tolerance_max, value3_tolerance_min, value3_tolerance_max, tofill_column, value1_column, value2_column, value3_column, sym_mean, debug, debug_file_name, debug_start_year) ) { + if ( !gapfill(values, struct_size, gf_rows, start_row, end_row, i, 7, 7, 7, GF_VALUE1_METHOD, timeres, value1_tolerance_min, value1_tolerance_max, value2_tolerance_min, value2_tolerance_max, value3_tolerance_min, value3_tolerance_max, tofill_column, value1_column, value2_column, value3_column, sym_mean, debug, debug_file_name, debug_start_year) ) { + if ( !gapfill(values, struct_size, gf_rows, start_row, end_row, i, 0, 2, 1, GF_TOFILL_METHOD, timeres, value1_tolerance_min, value1_tolerance_max, value2_tolerance_min, value2_tolerance_max, value3_tolerance_min, value3_tolerance_max, tofill_column, value1_column, value2_column, value3_column, sym_mean, debug, debug_file_name, debug_start_year) ) { + if ( !gapfill(values, struct_size, gf_rows, start_row, end_row, i, 21, 77, 7, GF_ALL_METHOD, timeres, value1_tolerance_min, value1_tolerance_max, value2_tolerance_min, value2_tolerance_max, value3_tolerance_min, value3_tolerance_max, tofill_column, value1_column, value2_column, value3_column, sym_mean, debug, debug_file_name, debug_start_year) ) { + if ( !gapfill(values, struct_size, gf_rows, start_row, end_row, i, 14, 77, 7, GF_VALUE1_METHOD, timeres, value1_tolerance_min, value1_tolerance_max, value2_tolerance_min, value2_tolerance_max, value3_tolerance_min, value3_tolerance_max, tofill_column, value1_column, value2_column, value3_column, sym_mean, debug, debug_file_name, debug_start_year) ) { + int max_win = end_row + 1; + if ( (max_mdv_win > 0) && ((max_mdv_win * 2) + 1 < end_row + 1) ) { + max_win = (max_mdv_win * 2) + 1; + } + if ( !gapfill(values, struct_size, gf_rows, start_row, end_row, i, 3, max_win, 3, GF_TOFILL_METHOD, timeres, value1_tolerance_min, value1_tolerance_max, value2_tolerance_min, value2_tolerance_max, value3_tolerance_min, value3_tolerance_max, tofill_column, value1_column, value2_column, value3_column, sym_mean, debug, debug_file_name, debug_start_year) ) { ++*no_gaps_filled_count; continue; } - - /* compute quality */ - gf_rows[i].quality = (gf_rows[i].method > 0) + - ((gf_rows[i].method == 1 && gf_rows[i].time_window > 14) || (gf_rows[i].method == 2 && gf_rows[i].time_window > 14) || (gf_rows[i].method == 3 && gf_rows[i].time_window > 1)) + - ((gf_rows[i].method == 1 && gf_rows[i].time_window > 56) || (gf_rows[i].method == 2 && gf_rows[i].time_window > 28) || (gf_rows[i].method == 3 && gf_rows[i].time_window > 5)); - } - - /* ok */ - return gf_rows; -} - -/* */ -GF_ROW *gf_mds(PREC *values, const int struct_size, const int rows_count, const int columns_count, const int hourly_dataset, - PREC value1_tolerance_min, - PREC value1_tolerance_max, - PREC value2_tolerance, - PREC value3_tolerance, - const int tofill_column, - const int value1_column, - const int value2_column, - const int value3_column, - const int values_min, - const int compute_hat, - int *no_gaps_filled_count) { - return gf_mds_with_bounds( values, - struct_size, - rows_count, - columns_count, - hourly_dataset, - value1_tolerance_min, - value1_tolerance_max, - value2_tolerance, - value3_tolerance, - tofill_column, - value1_column, - value2_column, - value3_column, - -1, - -1, - -1, - INVALID_VALUE, - values_min, - compute_hat, - -1, - -1, - no_gaps_filled_count - ); -} - -/* */ -GF_ROW *gf_mds_with_qc(PREC *values, const int struct_size, const int rows_count, const int columns_count, const int hourly_dataset, - PREC value1_tolerance_min, - PREC value1_tolerance_max, - PREC value2_tolerance, - PREC value3_tolerance, - const int tofill_column, - const int value1_column, - const int value2_column, - const int value3_column, - const int value1_qc_column, - const int value2_qc_column, - const int value3_qc_column, - const int qc_thrs, - const int values_min, - const int compute_hat, - int *no_gaps_filled_count) { - return gf_mds_with_bounds( values, - struct_size, - rows_count, - columns_count, - hourly_dataset, - value1_tolerance_min, - value1_tolerance_max, - value2_tolerance, - value3_tolerance, - tofill_column, - value1_column, - value2_column, - value3_column, - value1_qc_column, - value2_qc_column, - value3_qc_column, - qc_thrs, - values_min, - compute_hat, - -1, - -1, - no_gaps_filled_count - ); -} - -/* temp functions used for G in energy_proc */ -GF_ROW *temp_gf_mds( PREC *values, - const int struct_size, - const int rows_count, - const int columns_count, - const int hourly_dataset, - PREC value1_tolerance_min, - PREC value1_tolerance_max, - PREC value2_tolerance, - PREC value3_tolerance, - const int tofill_column, - const int value1_column, - const int value2_column, - const int value3_column, - const int value1_qc_column, - const int value2_qc_column, - const int value3_qc_column, - const int qc_thrs, - const int values_min, - const int compute_hat, - int *no_gaps_filled_count) { - int i; - int c; - int start_row = -1; - int end_row = -1; - int valids_count; - GF_ROW *gf_rows; - - /* */ - assert(values && rows_count && no_gaps_filled_count); - - /* reset */ - *no_gaps_filled_count = 0; - if ( start_row < 0 ) { - start_row = 0; - } - if ( -1 == end_row ) { - end_row = rows_count; - } else if ( end_row > rows_count ) { - end_row = rows_count; - } - - /* allocate memory */ - gf_rows = malloc(rows_count*sizeof*gf_rows); - if ( !gf_rows ) { - puts(err_out_of_memory); - return NULL; - } - - /* reset */ - for ( i = 0; i < rows_count; i++ ) { - gf_rows[i].mask = 0; - gf_rows[i].similiar = INVALID_VALUE; - gf_rows[i].stddev = INVALID_VALUE; - gf_rows[i].filled = INVALID_VALUE; - gf_rows[i].quality = INVALID_VALUE; - gf_rows[i].time_window = 0; - gf_rows[i].samples_count = 0; - gf_rows[i].method = 0; - } - - /* update mask and count valids TO FILL */ - valids_count = 0; - for ( i = start_row; i < end_row; i++ ) { - for ( c = 0; c < columns_count; c++ ) { - if ( !IS_INVALID_VALUE(((PREC *)(((char *)values)+i*struct_size))[c]) ) { - if ( tofill_column == c ) { - gf_rows[i].mask |= GF_TOFILL_VALID; - } else if ( value1_column == c ) { - gf_rows[i].mask |= GF_VALUE1_VALID; - } else if ( value2_column == c ) { - gf_rows[i].mask |= GF_VALUE2_VALID; - } else if ( value3_column == c ) { - gf_rows[i].mask |= GF_VALUE3_VALID; + } + } } } } - /* check for QC */ - if ( !IS_INVALID_VALUE(qc_thrs) && - (value1_qc_column != -1) && - !IS_INVALID_VALUE(((PREC *)(((char *)values)+i*struct_size))[value1_qc_column]) ) { - if ( ((PREC *)(((char *)values)+i*struct_size))[value1_qc_column] > qc_thrs ) { - gf_rows[i].mask &= ~GF_VALUE1_VALID; - } - } - - if ( !IS_INVALID_VALUE(qc_thrs) && - (value2_qc_column != -1) && - !IS_INVALID_VALUE(((PREC *)(((char *)values)+i*struct_size))[value2_qc_column]) ) { - if ( ((PREC *)(((char *)values)+i*struct_size))[value2_qc_column] > qc_thrs ) { - gf_rows[i].mask &= ~GF_VALUE2_VALID; - } - } - - if ( !IS_INVALID_VALUE(qc_thrs) && - (value3_qc_column != -1) && - !IS_INVALID_VALUE(((PREC *)(((char *)values)+i*struct_size))[value3_qc_column]) ) { - if ( ((PREC *)(((char *)values)+i*struct_size))[value3_qc_column] > qc_thrs ) { - gf_rows[i].mask &= ~GF_VALUE3_VALID; - } - } - - if ( IS_FLAG_SET(gf_rows[i].mask, GF_TOFILL_VALID) ) { - ++valids_count; - } - } - - if ( valids_count < values_min ) { - puts(err_gf_too_less_values); - free(gf_rows); - return NULL; - } - - /* */ - if ( IS_INVALID_VALUE(value1_tolerance_min) && IS_INVALID_VALUE(value1_tolerance_max) ) { - value1_tolerance_min = GF_SW_IN_TOLERANCE_MIN; - value1_tolerance_max = GF_SW_IN_TOLERANCE_MAX; - } - - /* */ - if ( IS_INVALID_VALUE(value2_tolerance) ) { - value2_tolerance = GF_TA_TOLERANCE; - } - - /* */ - if ( IS_INVALID_VALUE(value3_tolerance) ) { - value3_tolerance = GF_VPD_TOLERANCE; - } - - /* loop for each row */ - for ( i = start_row; i < end_row; i++ ) { - /* copy value from TOFILL to FILLED */ - gf_rows[i].filled = ((PREC *)(((char *)values)+i*struct_size))[tofill_column]; - - /* compute hat ? */ - if ( !IS_INVALID_VALUE(gf_rows[i].filled) && !compute_hat ) { - continue; - } - - /* fill - Added 20140422: if a gap is impossible to fill, e.g. if with MDV there are no data in the whole dataset acquired in a range +/- one hour, - the data point is not filled and the qc is set to -9999 - */ - if ( !gapfill(values, struct_size, gf_rows, start_row, end_row, i, 7, 14, 7, GF_ALL_METHOD, hourly_dataset, value1_tolerance_min, value1_tolerance_max, value2_tolerance, value3_tolerance, tofill_column, value1_column, value2_column, value3_column) ) - if ( !gapfill(values, struct_size, gf_rows, start_row, end_row, i, 7, 7, 7, GF_VALUE1_METHOD, hourly_dataset, value1_tolerance_min, value1_tolerance_max, value2_tolerance, value3_tolerance, tofill_column, value1_column, value2_column, value3_column) ) - if ( !gapfill(values, struct_size, gf_rows, start_row, end_row, i, 0, 2, 1, GF_TOFILL_METHOD, hourly_dataset, value1_tolerance_min, value1_tolerance_max, value2_tolerance, value3_tolerance, tofill_column, value1_column, value2_column, value3_column) ) - if ( !gapfill(values, struct_size, gf_rows, start_row, end_row, i, 21, 77, 7, GF_ALL_METHOD, hourly_dataset, value1_tolerance_min, value1_tolerance_max, value2_tolerance, value3_tolerance, tofill_column, value1_column, value2_column, value3_column) ) - if ( !gapfill(values, struct_size, gf_rows, start_row, end_row, i, 14, 77, 7, GF_VALUE1_METHOD, hourly_dataset, value1_tolerance_min, value1_tolerance_max, value2_tolerance, value3_tolerance, tofill_column, value1_column, value2_column, value3_column) ) - if ( !gapfill(values, struct_size, gf_rows, start_row, end_row, i, 3, 91, 3, GF_TOFILL_METHOD, hourly_dataset, value1_tolerance_min, value1_tolerance_max, value2_tolerance, value3_tolerance, tofill_column, value1_column, value2_column, value3_column) ) { - ++*no_gaps_filled_count; - continue; - } - /* compute quality */ gf_rows[i].quality = (gf_rows[i].method > 0) + ((gf_rows[i].method == 1 && gf_rows[i].time_window > 14) || (gf_rows[i].method == 2 && gf_rows[i].time_window > 14) || (gf_rows[i].method == 3 && gf_rows[i].time_window > 1)) + @@ -3116,7 +2857,7 @@ GF_ROW *temp_gf_mds( PREC *values, return gf_rows; } -/* private function for parse_dd */ +/* static function for parse_dd */ static int parse_time_zone(DD *const dd, char *const string) { int i; PREC v; @@ -3204,7 +2945,7 @@ static int parse_time_zone(DD *const dd, char *const string) { return !(check & 1); } -/* private function for parse_dd */ +/* static function for parse_dd */ static int parse_htower(DD *const dd, char *const string) { int i; PREC h; @@ -3260,7 +3001,7 @@ static int parse_htower(DD *const dd, char *const string) { return !(check & 1); } -/* private function for parse_dd */ +/* static function for parse_dd */ static int parse_sc_negles(DD *const dd, char *const string) { int i; int flag; @@ -3345,7 +3086,7 @@ static int parse_sc_negles(DD *const dd, char *const string) { } -/* private function for parse_dd */ +/* static function for parse_dd */ static int parse_timeres(DD *const dd, const char *const string) { int i; @@ -4492,8 +4233,8 @@ void check_memory_leak(void) { _CrtSetReportMode(_CRT_ERROR, _CRTDBG_MODE_DEBUG); _CrtSetReportMode(_CRT_ASSERT, _CRTDBG_MODE_DEBUG); _CrtDumpMemoryLeaks(); -#else /* _WIN32 */ - assert(0 && "no memory leak available for this platform"); +//#else /* _WIN32 */ + // no memory leak checker available for this platform! #endif #endif /* _DEBUG */ } diff --git a/oneflux_steps/common/common.h b/oneflux_steps/common/common.h index b531ffdd..01058d77 100644 --- a/oneflux_steps/common/common.h +++ b/oneflux_steps/common/common.h @@ -111,10 +111,10 @@ enum { /* gf */ enum { - GF_TOFILL_VALID = 1, - GF_VALUE1_VALID = 2, /* was GF_SWIN_VALID */ - GF_VALUE2_VALID = 4, /* was GF_TA_VALID */ - GF_VALUE3_VALID = 8, /* was GF_VPD_VALID */ + GF_TOFILL_VALID = 1 << 0 , + GF_VALUE1_VALID = 1 << 1 , + GF_VALUE2_VALID = 1 << 2 , + GF_VALUE3_VALID = 1 << 3 , GF_ALL_VALID = GF_TOFILL_VALID|GF_VALUE1_VALID|GF_VALUE2_VALID|GF_VALUE3_VALID }; @@ -169,14 +169,16 @@ enum { #define YEAR_LEN 5 /* including null-terminating char */ /* constants for gapfilling */ -#define GF_SW_IN_TOLERANCE_MIN 20.0 -#define GF_SW_IN_TOLERANCE_MAX 50.0 -#define GF_TA_TOLERANCE 2.5 -#define GF_VPD_TOLERANCE 5.0 -#define GF_TOKEN_LENGTH_MAX 32 -#define GF_ROWS_MIN_MIN 0 -#define GF_ROWS_MIN 0 -#define GF_ROWS_MIN_MAX 10000 +#define GF_DRIVER_1_TOLERANCE_MIN 20.0 +#define GF_DRIVER_1_TOLERANCE_MAX 50.0 +#define GF_DRIVER_2A_TOLERANCE_MIN 2.5 +#define GF_DRIVER_2A_TOLERANCE_MAX INVALID_VALUE +#define GF_DRIVER_2B_TOLERANCE_MIN 5.0 +#define GF_DRIVER_2B_TOLERANCE_MAX INVALID_VALUE +#define GF_TOKEN_LENGTH_MAX 32 +#define GF_ROWS_MIN_MIN 0 +#define GF_ROWS_MIN 0 +#define GF_ROWS_MIN_MAX 10000 /* */ #define TIMESTAMP_STRING "TIMESTAMP" @@ -238,6 +240,16 @@ typedef struct { int time_window; int samples_count; int method; + PREC value1_sym_mean; + PREC value1_w_sym_mean; + PREC similiar_sym_mean; + PREC filled_sym_mean; + PREC mean_above; + PREC mean_below; + int n_above; + int n_below; + + int index; /* internal use, debug */ } GF_ROW; /* structure for timezone */ @@ -304,7 +316,7 @@ char *string_copy(const char *const string); char *string_tokenizer(char *string, const char *delimiters, char **p); char *get_current_directory(void); int add_char_to_string(char *const string, char c, const int size); -int mystrcat(char *const string, const char *const string_to_add, const int size); +int string_concat(char *const string, const char *const string_to_add, const int size); int file_exists(const char *const file); int path_exists(const char *const path); int compare_prec(const void * a, const void * b); @@ -321,115 +333,60 @@ char *info_get_part_by_number(const INFO *const info, const int index); void info_free(INFO *info); TIMESTAMP *get_timestamp(const char *const string); -int get_row_by_timestamp(const TIMESTAMP *const t, const int hourly_dataset); +int get_row_by_timestamp(const TIMESTAMP *const t, const int timeres); int get_year_from_timestamp_string(const char *const string); char *get_filename_ext(const char *const filename); -/* */ -#define timestamp_start_by_row(r,y,h) timestamp_get_by_row((r),(y),(h),1) -#define timestamp_end_by_row(r,y,h) timestamp_get_by_row((r),(y),(h),0) -TIMESTAMP *timestamp_get_by_row(int row, int yy, const int hourly_dataset, const int start); -#define timestamp_start_by_row_s(r,y,h) timestamp_get_by_row_s((r),(y),(h),1) -#define timestamp_end_by_row_s(r,y,h) timestamp_get_by_row_s((r),(y),(h),0) -char *timestamp_get_by_row_s(int row, int yy, const int hourly_dataset, const int start); +#define timestamp_start_by_row(r,y,t) timestamp_get_by_row((r),(y),(t),1) +#define timestamp_end_by_row(r,y,t) timestamp_get_by_row((r),(y),(t),0) +TIMESTAMP *timestamp_get_by_row(int row, int yy, const int timeres, const int start); +#define timestamp_start_by_row_s(r,y,t) timestamp_get_by_row_s((r),(y),(t),1) +#define timestamp_end_by_row_s(r,y,t) timestamp_get_by_row_s((r),(y),(t),0) +char *timestamp_get_by_row_s(int row, int yy, const int timeres, const int start); #define timestamp_start_ww_by_row(r,y,h) timestamp_ww_get_by_row((r),(y),(h),1) #define timestamp_end_ww_by_row(r,y,h) timestamp_ww_get_by_row((r),(y),(h),0) -TIMESTAMP *timestamp_ww_get_by_row(int row, int yy, const int hourly_dataset, int start); +TIMESTAMP *timestamp_ww_get_by_row(int row, int yy, const int timeres, int start); #define timestamp_start_ww_by_row_s(r,y,h) timestamp_get_by_row_s((r),(y),(h),1) #define timestamp_end_ww_by_row_s(r,y,h) timestamp_get_by_row_s((r),(y),(h),0) -char *timestamp_ww_get_by_row_s(int row, int yy, const int hourly_dataset, const int start); +char *timestamp_ww_get_by_row_s(int row, int yy, const int timeres, const int start); /* gf */ -GF_ROW *gf_mds( PREC *values, - const int struct_size, - const int rows_count, - const int columns_count, - const int hourly_dataset, - PREC value1_tolerance_min, - PREC value1_tolerance_max, - PREC value2_tolerance, - PREC value3_tolerance, - const int tofill_column, - const int value1_column, - const int value2_column, - const int value3_column, - const int values_min, - const int compute_hat, - int *no_gaps_filled_count -); - -GF_ROW *gf_mds_with_qc( PREC *values, - const int struct_size, - const int rows_count, - const int columns_count, - const int hourly_dataset, - PREC value1_tolerance_min, - PREC value1_tolerance_max, - PREC value2_tolerance, - PREC value3_tolerance, - const int tofill_column, - const int value1_column, - const int value2_column, - const int value3_column, - const int value1_qc_column, - const int value2_qc_column, - const int value3_qc_column, - const int qc_thrs, - const int values_min, - const int compute_hat, - int *no_gaps_filled_count -); - -GF_ROW *gf_mds_with_bounds( PREC *values, - const int struct_size, - const int rows_count, - const int columns_count, - const int hourly_dataset, - PREC value1_tolerance_min, - PREC value1_tolerance_max, - PREC value2_tolerance, - PREC value3_tolerance, - const int tofill_column, - const int value1_column, - const int value2_column, - const int value3_column, - const int value1_qc_column, - const int value2_qc_column, - const int value3_qc_column, - const int qc_thrs, - const int values_min, - const int compute_hat, - int start_row, - int end_row, - int *no_gaps_filled_count +GF_ROW *gf_mds(PREC *values, + const int struct_size, + const int rows_count, + const int columns_count, + const int hourly_dataset, + PREC value1_tolerance_min, + PREC value1_tolerance_max, + PREC value2_tolerance_min, + PREC value2_tolerance_max, + PREC value3_tolerance_min, + PREC value3_tolerance_max, + const int tofill_column, + const int value1_column, + const int value2_column, + const int value3_column, + const int value1_qc_column, + const int value2_qc_column, + const int value3_qc_column, + const int qc_thrs1, + const int qc_thrs2, + const int qc_thrs3, + const int values_min, + const int compute_hat, + int start_row, + int end_row, + int *no_gaps_filled_count, + int sym_mean, + int max_mdv_win, + int debug, + const char* debug_file_name, + int debug_start_year ); PREC gf_get_similiar_standard_deviation(const GF_ROW *const gf_rows, const int rows_count); PREC gf_get_similiar_median(const GF_ROW *const gf_rows, const int rows_count, int *const error); -/* temp functions used for G in energy_proc */ -GF_ROW *temp_gf_mds( PREC *values, - const int struct_size, - const int rows_count, - const int columns_count, - const int hourly_dataset, - PREC value1_tolerance_min, - PREC value1_tolerance_max, - PREC value2_tolerance, - PREC value3_tolerance, - const int tofill_column, - const int value1_column, - const int value2_column, - const int value3_column, - const int value1_qc_column, - const int value2_qc_column, - const int value3_qc_column, - const int qc_thrs, - const int values_min, - const int compute_hat, - int *no_gaps_filled_count -); - /* dataset details */ DD *alloc_dd(void); void zero_dd(DD *const dd); @@ -437,6 +394,7 @@ DD *parse_dd(FILE *const f); int write_dd(const DD *const dd, FILE *const f, const char *const notes_to_add); int write_dds(const DD **const dd, const int count, FILE *const f, const char *const notes_to_add); void free_dd(DD *dd); +int get_rows_count_by_timeres(const int timeres, const int year); int get_rows_count_by_dd(const DD *const dd); int get_rows_per_day_by_dd(const DD *const dd); @@ -453,6 +411,9 @@ PREC get_dtime_by_row(const int row, const int hourly); PREC *get_rpot(DD *const details); PREC *get_rpot_with_solar_noon(DD *const details, const int s_n_month, const int s_n_day, int *const solar_noon); +int check_timestamp(const TIMESTAMP* const p); +int timestamp_difference_in_seconds(const TIMESTAMP* const p1, const TIMESTAMP* const p2); + /* */ void check_memory_leak(void); diff --git a/oneflux_steps/common/common_pi.c b/oneflux_steps/common/common_pi.c deleted file mode 100644 index f32f7525..00000000 --- a/oneflux_steps/common/common_pi.c +++ /dev/null @@ -1,4530 +0,0 @@ -/* - common.c - - author: Alessio Ribeca - owner: DIBAF - University of Tuscia, Viterbo, Italy - - scientific contact: Dario Papale -*/ - -/* includes */ -#include -#include -#include -#include -#include -#include -#include -#include "common.h" - -/* os dependant */ -#if defined (_WIN32) -#ifndef STRICT -#define STRICT -#endif /* STRICT */ -#ifndef WIN32_MEAN_AND_LEAN -#define WIN32_MEAN_AND_LEAN -#endif /* WIN32_MEAN_AND_LEAN */ -#include -/* for memory leak */ -#ifdef _DEBUG -#define _CRTDBG_MAP_ALLOC -#include -#include -#endif /* _DEBUG */ -#pragma comment(lib, "kernel32.lib") -static WIN32_FIND_DATA wfd; -static HANDLE handle; -#elif defined (linux) || defined (__linux) || defined (__linux__) || defined (__APPLE__) -#include -#include -#include -#include -#include -#include -static DIR *dir; -static struct dirent *dit; -#endif - -/* - note on August 17, 2011 removed space in delimiters 'cause path with - spaces e.g.: c:\documents and settings .... will not be handled correctly -*/ -static const char comma_delimiter[] = ","; -static const char plus_delimiter[] = "+"; -static const char filter[] = "*.*"; -static const char dd_field_delimiter[] = ",\r\n"; -static const char *dds[DETAILS_SIZE] = { "site", "year", "lat", "lon", "timezone", "htower", "timeres", "sc_negl", "notes" }; -static const char *timeress[TIMERES_SIZE] = { "spot", "quaterhourly", "halfhourly", "hourly", "daily", "monthly" }; - -/* error strings */ -static const char err_unable_open_path1[] = "unable to open path 1: %s\n\n"; -static const char err_unable_open_path2[] = "unable to open path 2: %s\n\n"; -static const char err_unable_open_path3[] = "unable to open path 3: %s\n\n"; -static const char err_unable_open_path4[] = "unable to open path 4: %s\n\n"; -static const char err_unable_open_path5[] = "unable to open path 5: %s\n\n"; -static const char err_unable_open_path6[] = "unable to open path 6: %s\n\n"; -static const char err_unable_open_file[] = "unable to open file: %s\n\n"; -static const char err_path_too_big[] = "specified path \"%s\" is too big.\n\n"; -static const char err_filename_too_big[] = "filename \"%s\" is too big.\n\n"; -static const char err_empty_argument[] = "empty argument\n"; -static const char err_unknown_argument[] = "unknown argument: \"%s\"\n\n"; -static const char err_gf_too_less_values[] = "too few valid values to apply gapfilling\n"; -static const char err_wildcards_with_no_extension_used[] = "wildcards with no extension used\n"; - -/* external strings */ -const char err_out_of_memory[] = "out of memory"; - -/* stolen to http://www.gnu.org/software/libtool/manual/autoconf/Function-Portability.html */ -int isnan_f (float x) { return x != x; } -int isnan_d (double x) { return x != x; } -int isnan_ld (long double x) { return x != x; } -int isinf_f (float x) { return !isnan (x) && isnan (x - x); } -int isinf_d (double x) { return !isnan (x) && isnan (x - x); } -int isinf_ld (long double x) { return !isnan (x) && isnan (x - x); } - -/* stolen to http://www.codeproject.com/Articles/10606/Folder-Utility-Create-Path-Remove-Folder-Remove-Al */ -int create_dir(char *Path) { -#if defined (_WIN32) - char DirName[256]; - char* p = Path; - char* q = DirName; - - while ( *p ) { - if ( ('\\' == *p) || ('/' == *p) ) { - if ( ':' != *(p-1) ) { - if ( !path_exists(DirName) ) { - if ( !CreateDirectory(DirName, NULL) ) { - return 0; - } - } - } - } - *q++ = *p++; - *q = '\0'; - } - if ( !path_exists(DirName)) { - return CreateDirectory(DirName, NULL); - } else { - return 1; - } -#elif defined (linux) || defined (__linux) || defined (__linux__) || defined (__APPLE__) - if ( -1 == mkdir(Path,0777) ) { - return 0; - } else { - return 1; - } -#else - return 0; -#endif -} - -char *get_filename_ext(const char *const filename) { - char *p; - - p = strrchr(filename, '.'); - if ( p ) ++p; - return p; -} - -/* */ -static int scan_path(const char *const path) { -#if defined (_WIN32) - handle = FindFirstFile(path, &wfd); - if ( INVALID_HANDLE_VALUE == handle ) { - return 0; - } -#elif defined (linux) || defined (__linux) || defined (__linux__) || defined (__APPLE__) - dir = opendir(path); - if ( !dir ) { - return 0; - } -#endif - /* ok */ - return 1; -} - -/* */ -static int get_files_from_path(const char *const path, FILES **files, int *const count, const int grouped) { - int i; - FILES *files_no_leak; - LIST *list_no_leak; -#if defined (_WIN32) - do { - if ( !IS_FLAG_SET(wfd.dwFileAttributes, FILE_ATTRIBUTE_DIRECTORY) ) { - /* check length */ - i = strlen(wfd.cFileName); - if ( i > FILENAME_SIZE ) { - printf(err_filename_too_big, wfd.cFileName); - free_files(*files, *count); - return 0; - } - - if ( !grouped ) { - /* alloc memory */ - files_no_leak = realloc(*files, (++*count)*sizeof*files_no_leak); - if ( !files_no_leak ) { - puts(err_out_of_memory); - free_files(*files, *count-1); - return 0; - } - - /* assign pointer */ - *files = files_no_leak; - (*files)[*count-1].list = NULL; - (*files)[*count-1].count = 1; - } - - /* allocate memory for list */ - list_no_leak = realloc((*files)[*count-1].list, (grouped ? ++(*files)[*count-1].count : 1) * sizeof*list_no_leak); - if ( !list_no_leak ) { - puts(err_out_of_memory); - free_files(*files, *count); - return 0; - } - (*files)[*count-1].list = list_no_leak; - - /* assign evalues */ - strncpy((*files)[*count-1].list[(*files)[*count-1].count-1].name, wfd.cFileName, i); - (*files)[*count-1].list[(*files)[*count-1].count-1].name[i] = '\0'; - - strcpy((*files)[*count-1].list[(*files)[*count-1].count-1].path, path); - - strcpy((*files)[*count-1].list[(*files)[*count-1].count-1].fullpath, path); - if ( !mystrcat((*files)[*count-1].list[(*files)[*count-1].count-1].fullpath, wfd.cFileName, PATH_SIZE) ) { - printf(err_filename_too_big, wfd.cFileName); - free_files(*files, *count); - return 0; - } - } - } while ( FindNextFile(handle, &wfd) ); - - /* close handle */ - FindClose(handle); -#elif defined (linux) || defined (__linux) || defined (__linux__) || defined (__APPLE__) - for ( ; ; ) { - dit = readdir(dir); - if ( !dit ) { - closedir(dir); - return 1; - } - - if ( dit->d_type == DT_REG ) { - /* check length */ - i = strlen(dit->d_name); - if ( i >= FILENAME_SIZE ) { - printf(err_filename_too_big, dit->d_name); - free_files(*files, *count); - return 0; - } - - if ( !grouped ) { - /* alloc memory */ - files_no_leak = realloc(*files, (++*count)*sizeof*files_no_leak); - if ( !files_no_leak ) { - puts(err_out_of_memory); - free_files(*files, *count-1); - return 0; - } - - /* assign pointer */ - *files = files_no_leak; - (*files)[*count-1].list = NULL; - (*files)[*count-1].count = 1; - } - - /* allocate memory for list */ - list_no_leak = realloc((*files)[*count-1].list, (grouped ? ++(*files)[*count-1].count : 1) * sizeof*list_no_leak); - if ( !list_no_leak ) { - puts(err_out_of_memory); - free_files(*files, *count); - return 0; - } - (*files)[*count-1].list = list_no_leak; - - /* assign evalues */ - strncpy((*files)[*count-1].list[(*files)[*count-1].count-1].name, dit->d_name, i); - (*files)[*count-1].list[(*files)[*count-1].count-1].name[i] = '\0'; - - strcpy((*files)[*count-1].list[(*files)[*count-1].count-1].path, path); - - strcpy((*files)[*count-1].list[(*files)[*count-1].count-1].fullpath, path); - if ( !mystrcat((*files)[*count-1].list[(*files)[*count-1].count-1].fullpath, dit->d_name, PATH_SIZE) ) { - printf(err_filename_too_big, dit->d_name); - free_files(*files, *count); - return 0; - } - } - } - /* close handle */ - closedir(dir); -#endif - - /* ok */ - return 1; -} - -/* - free_files -*/ -void free_files(FILES *files, const int count) { - if ( files ) { - int i; - for ( i = 0 ; i < count; i++ ) { - free(files[i].list); - } - free(files); - } -} - -/* - CHECK: on ubuntu fopen erroneously open a path (maybe a bug on NTFS partition driver ?) -*/ -FILES *get_files(const char *const program_path, char *string, int *const count, int *const error) { - int i; - int y; - int plusses_count; - int token_length; - char *token_by_comma; - char *token_by_plus; - char *p; - char *p2; - char *p3; - - FILE *f; - FILES *files; - FILES *files_no_leak; - LIST *list_no_leak; - - /* check parameters */ - assert(string && count && error); - - /* reset */ - files = NULL; - *count = 0; - *error = 0; - - /* loop for each commas */ - for ( token_by_comma = string_tokenizer(string, comma_delimiter, &p); token_by_comma; token_by_comma = string_tokenizer(NULL, comma_delimiter, &p) ) { - /* get token length */ - for ( token_length = 0; token_by_comma[token_length]; token_length++ ); - - /* if length is 0 skip to next token */ - if ( !token_length ) { - continue; - } - - /* scan for plusses */ - plusses_count = 0; - for ( y = 0; y < token_length; y++ ) { - if ( plus_delimiter[0] == token_by_comma[y] ) { - /* check if next char is a plus too */ - if ( y < token_length-1 ) { - if ( plus_delimiter[0] == token_by_comma[y+1] ) { - ++y; - continue; - } - } - - /* plus found! */ - ++plusses_count; - } - } - - /* no grouping */ - if ( !plusses_count ) { - /* token is a path ? */ - if ( token_by_comma[token_length-1] == FOLDER_DELIMITER ) { - #if defined (_WIN32) - /* add length of filter */ - for ( i = 0; filter[i]; i++ ); - token_length += i; - #endif - - /* add null terminating char */ - ++token_length; - - /* alloc memory */ - p2 = malloc(token_length*sizeof*p2); - if ( !p2 ) { - puts(err_out_of_memory); - *error = 1; - free_files(files, *count); - return NULL; - } - - /* copy token */ - strcpy(p2, token_by_comma); - - #if defined (_WIN32) - /* add filter at end */ - strcat(p2, filter); - #endif - - /* scan path */ - if ( !scan_path(p2) ) { - printf(err_unable_open_path1, p2); - *error = 1; - free(p2); - free_files(files, *count); - return NULL; - } - - /* get files */ - if ( !get_files_from_path(token_by_comma, &files, count, 0) ) { - printf(err_unable_open_path2, token_by_comma); - *error = 1; - free(p2); - free_files(files, *count); - return NULL; - } - - /* free memory */ - free(p2); - } else { - /* check for wildcard */ - if ( '*' == token_by_comma[0] ) { - if ( token_length < 2 ) { - puts(err_wildcards_with_no_extension_used); - *error = 1; - free_files(files, *count); - return NULL; - } - - /* add length of filter */ - for ( i = 0; program_path[i]; i++ ); - token_length += i; - - /* add null terminating char */ - ++token_length; - - /* alloc memory */ - p2 = malloc(token_length*sizeof*p2); - if ( !p2 ) { - puts(err_out_of_memory); - *error = 1; - free_files(files, *count); - return NULL; - } - - /* copy token */ - strcpy(p2, program_path); - - #if defined (_WIN32) - /* add filter at end */ - strcat(p2, token_by_comma); - #endif - - /* scan path */ - if ( !scan_path(p2) ) { - printf(err_unable_open_path3, p2); - *error = 1; - free(p2); - free_files(files, *count); - return NULL; - } - - /* get files */ - if ( !get_files_from_path(program_path, &files, count, 0) ) { - printf(err_unable_open_path4, token_by_comma); - *error = 1; - free(p2); - free_files(files, *count); - return NULL; - } - - /* free memory */ - free(p2); - } else { - /* check if we can simply open token_by_comma */ - f = fopen(token_by_comma, "r"); - if ( !f ) { - printf(err_unable_open_file, token_by_comma); - *error = 1; - free_files(files, *count); - return NULL; - } - - /* close file */ - fclose(f); - - /* allocate memory */ - files_no_leak = realloc(files, (++*count)*sizeof*files_no_leak); - if ( !files_no_leak ) { - puts(err_out_of_memory); - free_files(files, *count-1); - *error = 1; - return NULL; - } - - /* assign memory */ - files = files_no_leak; - - /* allocate memory for 1 file */ - files[*count-1].count = 1; - files[*count-1].list = malloc(sizeof*files[*count-1].list); - if ( !files[*count-1].list ) { - puts(err_out_of_memory); - *error = 1; - free_files(files, *count); - return NULL; - } - - /* check if token has a FOLDER_DELIMITER */ - p2 = strrchr(token_by_comma, FOLDER_DELIMITER); - if ( p2 ) { - /* skip FOLDER_DELIMITER */ - ++p2; - /* get length */ - y = strlen(p2); - - /* check filename length */ - if ( y > FILENAME_SIZE ) { - printf(err_filename_too_big, p2); - *error = 1; - free_files(files, *count); - return NULL; - } - - /* assign values */ - strncpy(files[*count-1].list->name, p2, y); - files[*count-1].list->name[y] = '\0'; - - strcpy(files[*count-1].list->fullpath, token_by_comma); - *p2 = '\0'; - - strcpy(files[*count-1].list->path, token_by_comma); - } else { - /* assign values */ - strcpy(files[*count-1].list->name, token_by_comma); - if ( program_path ) { - strcpy(files[*count-1].list->path, program_path); - strcpy(files[*count-1].list->fullpath, program_path); - if ( !mystrcat(files[*count-1].list->fullpath, token_by_comma, PATH_SIZE) ) { - printf(err_filename_too_big, token_by_comma); - free_files(files, *count); - return 0; - } - } else { - strcpy(files[*count-1].list->path, token_by_comma); - strcpy(files[*count-1].list->fullpath, token_by_comma); - } - } - } - } - } else { - /* alloc memory */ - files_no_leak = realloc(files, (++*count)*sizeof*files_no_leak); - if ( !files_no_leak ) { - puts(err_out_of_memory); - free_files(files, *count-1); - return 0; - } - - /* assign pointer */ - files = files_no_leak; - files[*count-1].list = NULL; - files[*count-1].count = 0; - - /* loop for each plus */ - for ( token_by_plus = string_tokenizer(token_by_comma, plus_delimiter, &p2); token_by_plus; token_by_plus = string_tokenizer(NULL, plus_delimiter, &p2) ) { - /* get token length */ - i = strlen(token_by_plus); - /* token is a path ? */ - /* Peter Isaac thinks this is a bug, token_length should be replaced by i */ - /* if ( token_by_plus[token_length-1] == FOLDER_DELIMITER ) { */ - if ( token_by_plus[i-1] == FOLDER_DELIMITER ) { - /* add length of filter */ - token_length += strlen(filter); - - /* add null terminating char */ - ++token_length; - - /* alloc memory */ - p3 = malloc(i*sizeof*p3); - if ( !p3 ) { - puts(err_out_of_memory); - *error = 1; - free_files(files, *count); - return NULL; - } - - /* copy token */ - strcpy(p3, token_by_plus); - - /* add filter at end */ - strcat(p3, filter); - - /* scan path */ - if ( !scan_path(p3) ) { - printf(err_unable_open_path5, p3); - *error = 1; - free(p3); - free_files(files, *count); - return NULL; - } - - /* get files */ - if ( !get_files_from_path(token_by_plus, &files, count, 1) ) { - printf(err_unable_open_path6, token_by_plus); - *error = 1; - free(p3); - free_files(files, *count); - return NULL; - } - - /* free memory */ - free(p3); - } else { - /* check if we can simply open path */ - f = fopen(token_by_plus, "r"); - if ( !f ) { - printf(err_unable_open_file, token_by_plus); - *error = 1; - free_files(files, *count); - return NULL; - } - - /* close file */ - fclose(f); - - /* check length */ - if ( token_length >= PATH_SIZE ) { - printf(err_path_too_big, token_by_plus); - *error = 1; - free_files(files, *count); - return NULL; - } - - /* allocate memory */ - ++files[*count-1].count; - list_no_leak = realloc(files[*count-1].list, files[*count-1].count*sizeof*list_no_leak); - if ( !list_no_leak ) { - puts(err_out_of_memory); - *error = 1; - free_files(files, *count); - return NULL; - } - - /* assign pointer */ - files[*count-1].list = list_no_leak; - - /* check if token has a FOLDER_DELIMITER */ - p3 = strrchr(token_by_comma, FOLDER_DELIMITER); - if ( p3 ) { - /* skip FOLDER_DELIMITER */ - ++p3; - - /* get length */ - y = strlen(p3); - - /* check filename length */ - if ( y > FILENAME_SIZE ) { - printf(err_filename_too_big, p3); - *error = 1; - free_files(files, *count); - return NULL; - } - - /* assign values */ - strncpy(files[*count-1].list[files[*count-1].count-1].name, p3, y); - files[*count-1].list->name[y] = '\0'; - - strcpy(files[*count-1].list[files[*count-1].count-1].fullpath, token_by_plus); - *p3 = '\0'; - - strcpy(files[*count-1].list[files[*count-1].count-1].path, token_by_plus); - - } else { - /* check length */ - if ( i > FILENAME_SIZE ) { - printf(err_filename_too_big, token_by_plus); - *error = 1; - free_files(files, *count); - return NULL; - } - - /* assign values */ - strcpy(files[*count-1].list[files[*count-1].count-1].name, token_by_plus); - if ( program_path ) { - strcpy(files[*count-1].list[files[*count-1].count-1].path, program_path); - strcpy(files[*count-1].list[files[*count-1].count-1].fullpath, program_path); - if ( !mystrcat(files[*count-1].list[files[*count-1].count-1].fullpath, token_by_plus, PATH_SIZE) ) { - printf(err_filename_too_big, token_by_plus); - free_files(files, *count); - return 0; - } - } else { - strcpy(files[*count-1].list[files[*count-1].count-1].path, token_by_plus); - strcpy(files[*count-1].list[files[*count-1].count-1].fullpath, token_by_plus); - } - } - } - } - } - } - - /* ok */ - return files; -} - -/* */ -FILES *get_files_again(const char *const program_path, char *string, FILES **files, int *const count, int *const error) { - int i; - int temp_files_count; - FILES *temp_files; - FILES *files_no_leak; - - /* get files */ - temp_files = get_files(program_path, string, &temp_files_count, error); - if ( !temp_files_count ) { - return *files; - } - - /* */ - files_no_leak = realloc(*files, (*count+temp_files_count)*sizeof*files_no_leak); - if ( !files_no_leak ) { - puts(err_out_of_memory); - *error = 1; - free_files(temp_files, temp_files_count); - return *files; - } - *files = files_no_leak; - - /* */ - for ( i = 0; i < temp_files_count; i++ ) { - (*files)[i+*count].count = temp_files[i].count; - (*files)[i+*count].list = temp_files[i].list; - } - *count += temp_files_count; - - free(temp_files); - - /* ok */ - return *files; -} - -/* */ -PREC convert_string_to_prec(const char *const string, int *const error) { - PREC value; - char *p; - - /* reset */ - *error = 0; - - if ( !string ) { - *error = 1; - return 0.0; - } - - errno = 0; - - value = (PREC)STRTOD(string, &p); - STRTOD(p, NULL); - if ( string == p || *p || errno ) { - *error = 1; - } - - return value; -} - -/* */ -void init_random_seed(void) { - srand((unsigned int)time(NULL)); -} - -/* */ -int get_random_number(int max) { - /* taken from http://c-faq.com/lib/randrange.html */ - return (rand() / (RAND_MAX / max + 1)); -} - -/* */ -static int check_for_argument(const char *const string , const char *const pattern, char **param) { - char *pptr = NULL; - char *sptr = NULL; - char *start = NULL; - - /* reset */ - *param = NULL; - - for ( start = (char *)string; *start; start++ ) { - /* find start of pattern in string */ - for ( ; (*start && (toupper(*start) != toupper(*pattern))); start++) - ; - if (start != string+1 ) - return 0; - - pptr = (char *)pattern; - sptr = (char *)start; - - while (toupper(*sptr) == toupper(*pptr)) { - sptr++; - pptr++; - - /* if end of pattern then pattern was found */ - if ( !*pptr ) { - if ( !*sptr ) { - return 1; - } else - /* check for next char to be an '=' */ - if ( *sptr == '=' ) { - if ( *(++sptr) ) { - *param = sptr; - } - return 1; - } - - return 0; - } - } - } - - return 0; -} - -/* */ -int parse_arguments(int argc, char *argv[], const ARGUMENT *const args, const int arg_count) { - int i; - int ok; - char *param; - - /* */ - while ( argc > 1 ) { - /* check for arguments */ - if ( ( '-' != argv[1][0]) && ( '/' != argv[1][0]) ) { - if ( '\0' == argv[1][0] ) { - puts(err_empty_argument); - } else { - printf(err_unknown_argument, argv[1]); - } - return 0; - } - - /* */ - ok = 0; - for ( i = 0; i < arg_count; i++ ) { - if ( check_for_argument(argv[1], args[i].name, ¶m) ) { - ok = 1; - - /* check if function is present */ - assert(args[i].f); - - /* call function */ - if ( !args[i].f(args[i].name, param, args[i].p) ) { - return 0; - } - - break; - } - } - - /* */ - if ( !ok ) { - printf(err_unknown_argument, argv[1]+1); - return 0; - } - - /* */ - ++argv; - --argc; - } - - /* ok */ - return 1; -} - -/* */ -int string_compare_i(const char *str1, const char *str2) { - register signed char __res; - - /* added on April 23, 2013 */ - if ( (NULL == str1) && (NULL == str2 ) ) { - return 0; - } - if ( NULL == str1 ) { - return 1; - } - if ( NULL == str2 ) { - return -1; - } - - /* */ - while ( 1 ) { - if ( (__res = toupper( *str1 ) - toupper( *str2++ )) != 0 || !*str1++ ) { - break; - } - } - - /* returns an integer greater than, equal to, or less than zero */ - return __res; -} - -/* */ -int string_n_compare_i(const char *str1, const char *str2, const int len) { - int i; - register signed char __res; - - if ( len <= 0 ) { - return -1; - } - - /* added on April 23, 2013 */ - if ( (NULL == str1) && (NULL == str2 ) ) { - return 0; - } - if ( NULL == str1 ) { - return 1; - } - if ( NULL == str2 ) { - return -1; - } - - /* */ - i = 0; - while ( 1 ) { - if ( (__res = toupper( *str1 ) - toupper( *str2++ )) != 0 || !*str1++ ) { - break; - } - if ( ++i >= len ) { - break; - } - } - - /* returns an integer greater than, equal to, or less than zero */ - return __res; -} - -/* */ -char *string_copy(const char *const string) { - int i; - int len; - char *p; - - /* check for null pointer */ - if ( ! string ) { - return NULL; - } - - /* get length of string */ - for ( len = 0; string[len]; len++ ); - - /* allocate memory */ - p = malloc(len+1); - if ( ! p ) { - return NULL; - } - - /* copy ! */ - for ( i = 0; i < len; i++ ) { - p[i] = string[i]; - } - p[len] = '\0'; - - return p; -} - -/* stolen to wikipedia */ -char *string_tokenizer(char *string, const char *delimiters, char **p) { - char *sbegin; - char *send; - - sbegin = string ? string : *p; - sbegin += strspn(sbegin, delimiters); - if ( *sbegin == '\0') { - *p = ""; - return NULL; - } - - send = sbegin + strcspn(sbegin, delimiters); - if ( *send != '\0') { - *send++ = '\0'; - } - *p = send; - - return sbegin; -} - -/* */ -int convert_string_to_int(const char *const string, int *const error) { - int value = 0; - char *p = NULL; - - /* reset */ - *error = 0; - - if ( !string ) { - *error = 1; - return 0; - } - - errno = 0; - value = (int)strtod(string, &p); - strtod(p, NULL); - if ( string == p || *p || errno ) { - *error = 1; - } - - return value; -} - -/* */ -char *get_current_directory(void) { - char *p; -#if defined (_WIN32) - p = malloc((MAX_PATH+1)*sizeof *p); - if ( !p ) { - return NULL; - } - if ( !GetModuleFileName(NULL, p, MAX_PATH) ) { - free(p); - return NULL; - } - p[(strrchr(p, '\\')-p)+1] = '\0'; - return p; -#elif defined (linux) || defined (__linux) || defined (__linux__) || defined (__APPLE__) - int len; - p = malloc((MAXPATHLEN+1)*sizeof *p); - if ( !p ) { - return NULL; - } - if ( !getcwd(p, MAXPATHLEN) ) { - free(p); - return NULL; - } - /* check if last chars is a FOLDER_DELIMITER */ - len = strlen(p); - if ( !len ) { - free(p); - return NULL; - } - if ( p[len-1] != FOLDER_DELIMITER ) { - if ( !add_char_to_string(p, FOLDER_DELIMITER, MAXPATHLEN) ) { - free(p); - return NULL; - } - } - return p; -#else - return NULL; -#endif -} - -/* */ -int add_char_to_string(char *const string, char c, const int size) { - int i; - - /* check for null pointer */ - if ( !string ) { - return 0; - } - - /* compute length */ - for ( i = 0; string[i]; i++ ); - - /* check length */ - if ( i >= size-1 ) { - return 0; - } - - /* add char */ - string[i] = c; - string[i+1] = '\0'; - - /* */ - return 1; -} - -/* */ -int mystrcat(char *const string, const char *const string_to_add, const int size) { - int i; - int y; - - /* check for null pointer */ - if ( !string || !string_to_add ) { - return 0; - } - - /* compute lenghts */ - for ( i = 0; string[i]; i++ ); - for ( y = 0; string_to_add[y]; y++ ); - - /* check length */ - if ( i >= size-y-1 ) { - return 0; - } - - strcat(string, string_to_add); - - /* */ - return 1; -} - -/* */ -int path_exists(const char *const path) { -#if defined (_WIN32) - DWORD dwResult; -#endif - if ( !path ) { - return 0; - } -#if defined (_WIN32) - dwResult = GetFileAttributes(path); - if (dwResult != INVALID_FILE_ATTRIBUTES && (dwResult & FILE_ATTRIBUTE_DIRECTORY)) { - return 1; - } -#elif defined (linux) || defined (__linux) || defined (__linux__) || defined (__APPLE__) - if ( !access(path, W_OK) ) { - return 1; - } -#endif - return 0; -} - -/* */ -int file_exists(const char *const file) { -#if defined (_WIN32) - DWORD dwResult; -#endif - if ( !file ) { - return 0; - } -#if defined (_WIN32) - dwResult = GetFileAttributes(file); - if (dwResult != INVALID_FILE_ATTRIBUTES && !(dwResult & FILE_ATTRIBUTE_NORMAL)) { - return 1; - } -#elif defined (linux) || defined (__linux) || defined (__linux__) || defined (__APPLE__) - if ( !access(file, W_OK) ) { - return 1; - } -#endif - return 0; -} - -/* */ -PREC get_mean(const PREC *const values, const int count) { - int i; - PREC mean; - - /* check for null pointer */ - assert(values && count); - - /* */ - if ( 1 == count ) { - return values[0]; - } - - /* compute mean */ - mean = 0.0; - for ( i = 0; i < count; i++ ) { - if ( IS_INVALID_VALUE(values[i]) ) { - return INVALID_VALUE; - } - mean += values[i]; - } - mean /= count; - - /* check for NAN */ - if ( mean != mean ) { - mean = INVALID_VALUE; - } - - /* ok */ - return mean; -} - -/* */ -PREC get_standard_deviation(const PREC *const values, const int count) { - int i; - PREC mean; - PREC sum; - PREC sum2; - - /* check for null pointer */ - assert(values && count); - - /* */ - if ( 1 == count ) { - return INVALID_VALUE; - } - - /* get mean */ - mean = get_mean(values, count); - if ( IS_INVALID_VALUE(mean) ) { - return INVALID_VALUE; - } - - /* compute stddev */ - sum = 0.0; - sum2 = 0.0; - for ( i = 0; i < count; i++ ) { - sum = (values[i] - mean); - sum *= sum; - sum2 += sum; - } - sum2 /= count-1; - sum2 = (PREC)SQRT(sum2); - - /* check for NAN */ - if ( sum2 != sum2 ) { - sum2 = INVALID_VALUE; - } - - /* ok */ - return sum2; -} - -/* */ -PREC get_median(const PREC *const values, const int count, int *const error) { - int i; - int median_count; - PREC *p_median; - PREC *median_no_leak; - PREC result; - - /* check for null pointer */ - assert(values); - - /* reset */ - *error = 0; - - /* get valid values */ - p_median = NULL; - median_count = 0; - for ( i = 0; i < count; i++ ) { - if ( !IS_INVALID_VALUE(values[i]) ) { - median_no_leak = realloc(p_median, ++median_count*sizeof *median_no_leak); - if ( !median_no_leak ) { - *error = 1; - free(p_median); - return 0; - } - - p_median = median_no_leak; - p_median[median_count-1] = values[i]; - } - } - - if ( !median_count ) { - return INVALID_VALUE; - } else if ( 1 == median_count ) { - result = p_median[0]; - free(p_median); - return result; - } - - /* sort values */ - qsort(p_median, median_count, sizeof *p_median, compare_prec); - - /* get median */ - if ( median_count & 1 ) { - result = p_median[((median_count+1)/2)-1]; - } else { - result = ((p_median[(median_count/2)-1] + p_median[median_count/2]) / 2); - } - - /* free memory */ - free(p_median); - - /* check for NAN */ - if ( result != result ) { - result = INVALID_VALUE; - } - - /* */ - return result; -} - -/* todo : implement a better comparison for equality */ -int compare_prec(const void * a, const void * b) { - if ( *(PREC *)a < *(PREC *)b ) { - return -1; - } else if ( *(PREC *)a > *(PREC *)b ) { - return 1; - } else { - return 0; - } -} - -/* */ -int compare_int(const void *a, const void *b) { - return ( *(int *)a - *(int *)b ); -} - -/* */ -int timestampSort(const void *a, const void *b) { - TIMESTAMP *ta; - TIMESTAMP *tb; - - ta = (TIMESTAMP *)a; - tb = (TIMESTAMP *)b; - - if ( ta->YYYY == tb->YYYY ) - if ( ta->MM == tb->MM ) - if ( ta->DD == tb->DD ) - if ( ta->hh == tb->hh ) - if ( ta->mm == tb->mm ) - if ( ta->ss == tb->ss ) return 0; - else return (ta->ss > tb->ss) ? 1 : -1; - else return (ta->mm > tb->mm) ? 1 : -1; - else return (ta->hh > tb->hh) ? 1 : -1; - else return (ta->DD > tb->DD) ? 1 : -1; - else return (ta->MM > tb->MM) ? 1 : -1; - else return (ta->YYYY > tb->YYYY) ? 1 : -1; -} - -/* */ -static int compare_time_zones(const void *a, const void *b) { - return timestampSort(&((TIME_ZONE *)a)->timestamp, &((TIME_ZONE *)b)->timestamp); -} - -/* */ -static int compare_htower(const void *a, const void *b) { - return timestampSort(&((HEIGHT *)a)->timestamp, &((HEIGHT *)b)->timestamp); -} - -/* */ -static int compare_sc_negles(const void *a, const void *b) { - return timestampSort(&((SC_NEGL *)a)->timestamp, &((SC_NEGL *)b)->timestamp); -} - -/* */ -char *tokenizer(char *string, char *delimiter, char **p) { - int i; - int j; - int c; - char *_p; - - /* */ - if ( string ) { - *p = string; - _p = string; - } else { - _p = *p; - if ( !_p ) { - return NULL; - } - } - - /* */ - c = 0; - for (i = 0; _p[i]; i++) { - for (j = 0; delimiter[j]; j++) { - if ( _p[i] == delimiter[j] ) { - *p += c + 1; - _p[i] = '\0'; - return _p; - } else { - ++c; - } - } - } - - /* */ - *p = NULL; - return _p; -} - -/* */ -PREC get_percentile(const PREC *values, const int n, const float percentile, int *const error) { - int i; - int y; - int index; - PREC r; - PREC *v; - - /* check parameters */ - assert(values && error); - - /* reset */ - *error = 0; - - /* */ - if ( !n ) { - return 0.0; - } else if ( 1 == n ) { - if ( IS_INVALID_VALUE(values[0]) ) { - *error = 1; - return 0.0; - } else { - return values[0]; - } - } - - /* percentile MUST be a value between 0 and 100*/ - if ( percentile < 0.0 || percentile > 100.0 ) { - *error = 1; - return 0.0; - } - - /* */ - v = malloc(n*sizeof*v); - if ( !v ) { - *error = 1; - return 0.0; - } - - /* */ - y = 0; - for ( i = 0; i < n; i++ ) { - if ( !IS_INVALID_VALUE(values[i]) ) { - v[y++] = values[i]; - } - } - if ( !y ) { - free(v); - *error = 1; - return 0.0; - } - - /* */ - qsort(v, y, sizeof *v, compare_prec); - - /* - - changed on May 20, 2013 - FROM index = ROUND((percentile / 100.0) * y + 0.5); - TO index = (percentile / 100) * y; - - - changed again Nov 4, 2013 - added ROUND where 0.5 is added to perc*y and then truncated to the integer. - E.g.: perc 50 of 5 values is INTEGER[((50/100)*5)+0.5] = 3 - */ - index = ROUND((percentile / 100) * y); - if ( --index < 0 ) { - index = 0; - } - - if ( index >= y ) { - return v[y-1]; - } - - /* */ - r = v[index]; - - /* */ - free(v); - - /* */ - return r; -} - -/* */ -PREC *get_percentiles(const PREC *values, const int n, const float *const percentiles, const int percentiles_count) { - int i; - int y; - int index; - PREC *v; - PREC *arr; - - /* check parameters */ - assert(values && n && percentiles && percentiles_count); - - /* check percentiles */ - for ( i = 0; i < percentiles_count; i++ ) { - /* percentile MUST be a value between 0 and 100*/ - if ( percentiles[i] < 0.0 || percentiles[i] > 100.0 ) { - return NULL; - } - } - - /* alloc memory */ - arr = malloc(percentiles_count*sizeof*arr); - if ( !arr ) { - return 0; - } - - /* 1 row of dataset ?*/ - if ( 1 == n ) { - if ( IS_INVALID_VALUE(values[0]) ) { - free(arr); - return NULL; - } - for ( i = 0; i < percentiles_count; i++ ) { - arr[i] = values[0]; - } - return arr; - } - - /* */ - v = malloc(n*sizeof*v); - if ( !v ) { - free(arr); - return NULL; - } - - /* build-up dataset without invalid_values */ - y = 0; - for ( i = 0; i < n; i++ ) { - if ( !IS_INVALID_VALUE(values[i]) ) { - v[y++] = values[i]; - } - } - if ( !y ) { - free(v); - free(arr); - return NULL; - } - - /* sort array */ - qsort(v, y, sizeof *v, compare_prec); - - /* loop on each percentile */ - for ( i = 0; i < percentiles_count; i++ ) { - /* - - changed on May 20, 2013 - FROM index = ROUND((percentile / 100.0) * y + 0.5); - TO index = (percentile / 100) * y; - - - changed again Nov 4, 2013 - added ROUND where 0.5 is added to perc*y and then truncated to the integer. - E.g.: perc 50 of 5 values is INTEGER[((50/100)*5)+0.5] = 3 - */ - index = ROUND((percentiles[i] / 100) * y); - if ( --index < 0 ) { - index = 0; - } - - if ( index >= y ) { - arr[i] = v[y-1]; - } else { - arr[i] = v[index]; - } - } - - /* free memory */ - free(v); - - /* */ - return arr; -} - -/* -* -* INFO STUFF -* -*/ - -typedef enum { - INFO_NO_ERROR = 0, - INFO_UNBALANCED_OPEN_TOKEN, - INFO_UNBALANCED_CLOSE_TOKEN, - INFO_OUT_OF_MEMORY, - INFO_NO_PARTS_FOUND - -} INFO_ERROR; - -/* */ -static INFO_ERROR info_parse(INFO *const info) { - int i; - int token_open; - PART *parts_no_leak; - - assert(info); - - token_open = 0; - for ( i = 0; info->text[i]; i++ ) { - switch ( info->text[i] ) { - case TOKEN_OPEN: - if ( token_open ) { - return INFO_UNBALANCED_OPEN_TOKEN; - } - parts_no_leak = realloc(info->parts, ++info->parts_count*sizeof*parts_no_leak); - if ( ! parts_no_leak ) { - --info->parts_count; - return INFO_OUT_OF_MEMORY; - } - info->text[i] = '\0'; - info->parts = parts_no_leak; - info->parts[info->parts_count-1].name = &info->text[i+1]; - token_open = 1; - break; - - case TOKEN_CLOSE: - if ( ! token_open ) { - return INFO_UNBALANCED_OPEN_TOKEN; - } - info->text[i] = '\0'; - info->parts[info->parts_count-1].text = &info->text[i+1]; - if ( '\r' == info->parts[info->parts_count-1].text[0] ) { - ++info->parts[info->parts_count-1].text; - } - if ( '\n' == info->parts[info->parts_count-1].text[0] ) { - ++info->parts[info->parts_count-1].text; - } - token_open = 0; - break; - - } - } - - if ( token_open ) { - return INFO_UNBALANCED_CLOSE_TOKEN; - } - - if ( ! info->parts_count ) { - return INFO_NO_PARTS_FOUND; - } - - return INFO_NO_ERROR; -} - -/* */ -INFO *info_import(const char *const filename) { - FILE *f; - INFO *info; - INFO_ERROR err; - - if ( !filename || !filename[0] ) { - return NULL; - } - - info = malloc(sizeof*info); - if ( ! info ) { - printf("out of memory during parse of %s\n", filename); - return NULL; - } - info->text = NULL; - info->parts = NULL; - info->parts_count = 0; - - f = fopen(filename, "rb"); - if ( !f ) { - printf("unable to open %s\n", filename); - info_free(info); - return NULL; - } - fseek(f, 0, SEEK_END); - info->size = ftell(f); - fseek(f, 0, SEEK_SET); - info->text = malloc(info->size+1); - if ( !info->text ) { - printf("%s\n", err_out_of_memory); - fclose(f); - info_free(info); - return NULL; - } - - if ( info->size != fread(info->text, sizeof*info->text, info->size, f) ) { - printf("unable to read file: %s\n", filename); - fclose(f); - info_free(info); - return NULL; - } - fclose(f); - info->text[info->size] = '\0'; - - /* parse info */ - err = info_parse(info); - switch ( err ) { - case INFO_NO_ERROR: - /* do nothing */ - break; - - case INFO_UNBALANCED_OPEN_TOKEN: - case INFO_UNBALANCED_CLOSE_TOKEN: - printf("unbalanced %s token for %s\n", (INFO_UNBALANCED_OPEN_TOKEN == err) ? "open" : "close", filename); - break; - - case INFO_OUT_OF_MEMORY: - printf("out of memory during parse of %s\n", filename); - break; - - case INFO_NO_PARTS_FOUND: - printf("no parts found on %s\n", filename); - break; - } - - if ( INFO_NO_ERROR != err ) { - info_free(info); - info = NULL; - } - - return info; -} - -/* */ -INFO *info_get(const char *const string) { - INFO *info; - INFO_ERROR err; - - assert(string); - - info = malloc(sizeof*info); - if ( ! info ) { - puts("out of memory during parse of info"); - return NULL; - } - info->text = NULL; - info->parts = NULL; - info->parts_count = 0; - - /* copy string */ - info->text = string_copy(string); - if ( ! info->text ) { - puts("out of memory during parse of info"); - info_free(info); - return NULL; - } - - /* get string length */ - for ( info->size = 0; info->text[info->size]; info->size++ ); - - /* parse info */ - err = info_parse(info); - switch ( err ) { - case INFO_NO_ERROR: - /* do nothing */ - break; - - case INFO_UNBALANCED_OPEN_TOKEN: - case INFO_UNBALANCED_CLOSE_TOKEN: - printf("unbalanced %s token for info\n", (INFO_UNBALANCED_OPEN_TOKEN == err) ? "open" : "close"); - break; - - case INFO_OUT_OF_MEMORY: - puts("out of memory during parse of info"); - break; - - case INFO_NO_PARTS_FOUND: - puts("no parts found on info"); - break; - } - - if ( INFO_NO_ERROR != err ) { - info_free(info); - info = NULL; - } - - return info; -} - -/* */ -char *info_get_part_name(const INFO *const info, const int index) { - if ( (index >= 0) && (index < info->parts_count) ) { - return info->parts[index].name; - } else { - return NULL; - } -} - -/* */ -char *info_get_part_by_name(const INFO *const info, const char *const name) { - int i; - - for ( i = 0; i < info->parts_count; i++ ) { - if ( ! string_compare_i(info->parts[i].name, name) ) { - return info->parts[i].text; - } - } - return NULL; -} - -/* */ -char *info_get_part_by_number(const INFO *const info, const int index) { - if ( (index >= 0) && (index < info->parts_count) ) { - return info->parts[index].text; - } else { - return NULL; - } -} - -/* */ -void info_free(INFO *info) { - if ( info ) { - free(info->parts); - free(info->text); - free(info); - } -} - -/* -* -* END INFO STUFF -* -*/ - -/* */ -TIMESTAMP *get_timestamp(const char *const string) { - int i; - int j; - int index; - int error; - char *p; - char token[5]; - TIMESTAMP *t; - const char *field[] = { "year", "month", "day", "hour", "minute", "second" }; - const int field_size[] = { 4, 2, 2, 2, 2, 2 }; - - t = malloc(sizeof*t); - if ( ! t ) { - puts(err_out_of_memory); - return NULL; - } - t->YYYY = 0; - t->MM = 0; - t->DD = 0; - t->hh = 0; - t->mm = 0; - t->ss = 0; - - for ( i = 0; string[i]; i++ ); - if ( (i < 0) || (i > 14) || (i & 1) ) { - printf("bad length for %s\n", TIMESTAMP_STRING); - return NULL; - } - - p = (char *)string; - index = 0; - while ( 1 ) { - i = 0; - while ( i < field_size[index] ) { - token[i++] = *p++; - } - if ( i != field_size[index] ) { - printf("bad field '%s' on %s\n\n", field[index], string); - free(t); - return NULL; - } - token[field_size[index]] = '\0'; - j = convert_string_to_int(token, &error); - if ( error ) { - printf("bad value '%s' for field '%s' on %s\n\n", token, field[index], string); - free(t); - return NULL; - } - - switch ( index ) { - case 0: /* year */ - t->YYYY = j; - break; - - case 1: /* month */ - t->MM = j; - break; - - case 2: /* day */ - t->DD = j; - break; - - case 3: /* hour */ - t->hh = j; - break; - - case 4: /* minute */ - t->mm = j; - break; - - case 5: /* second */ - t->ss = j; - break; - } - ++index; - if ( ! *p ) { - return t; - } - } -} - -/* */ -int get_row_by_timestamp(const TIMESTAMP *const t, const int hourly_dataset) { - int i; - int y; - int rows_per_day; - int rows_per_hour; - const int days_in_month[] = { 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 }; - - /* */ - if ( ! t ) { - return -1; - } - - /* */ - rows_per_day = 48; - if ( hourly_dataset ) { - rows_per_day = 24; - } - rows_per_hour = 2; - if ( hourly_dataset ) { - rows_per_hour = 1; - } - - /* check for last row */ - if ( (1 == t->DD) && - (1 == t->MM) && - (0 == t->hh) && - (0 == t->mm) ) { - if ( IS_LEAP_YEAR(t->YYYY-1) ) { - i = LEAP_YEAR_ROWS; - } else { - i = YEAR_ROWS; - } - if ( hourly_dataset ) { - i /= 2; - } - } else { - i = 0; - for (y = 0; y < t->MM - 1; y++ ) { - i += days_in_month[y]; - } - - /* leap year ? */ - if ( IS_LEAP_YEAR(t->YYYY) && ((t->MM - 1) > 1) ) { - ++i; - } - - /* */ - i += t->DD - 1; - i *= rows_per_day; - i += t->hh * rows_per_hour; - if ( t->mm > 0 ) { - ++i; - } - } - - /* return zero based index */ - return --i; -} - -/* */ -int get_year_from_timestamp_string(const char *const string) { - int year; - TIMESTAMP *t; - - if ( ! string || ! string[0] ) { - return -1; - } - - t = get_timestamp(string); - if ( ! t ) { - return -1; - } - - year = t->YYYY; - free(t); - - return year; - -} - -/* */ -TIMESTAMP *timestamp_get_by_row(int row, int yy, const int hourly_dataset, const int start) { - int i; - int is_leap; - int rows_per_day; - int days_per_month[12] = { 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 }; - static TIMESTAMP t = { 0 }; - - /* reset */ - t.YYYY = yy; - t.ss = 0; - - /* leap year ? */ - is_leap = IS_LEAP_YEAR(yy); - if ( is_leap ) { - ++days_per_month[1]; - } - - /* inc row...we used 1 based index */ - if ( ! start ) { - ++row; - } - - /* compute rows per day */ - rows_per_day = hourly_dataset ? 24: 48; - - /* get day and month */ - t.DD = row / rows_per_day; - for ( i = 0, t.MM = 0; t.MM < 12; ++t.MM ) { - i += days_per_month[t.MM]; - if ( t.DD <= i ) { - t.DD -= i - days_per_month[t.MM]; - break; - } - } - if ( ++t.DD > days_per_month[t.MM] ) { - t.DD = 1; - if ( ++t.MM > 11 ) { - t.MM = 0; - ++t.YYYY; - } - } - ++t.MM; - - /* get hour */ - if ( hourly_dataset ) { - t.hh = row % rows_per_day; - t.mm = 0; - } else { - t.hh = (row % rows_per_day) / 2; - /* even row ? */ - if ( row & 1 ) { - t.mm = 30; - } else { - t.mm = 0; - } - } - - /* ok */ - return &t; -} - -/* */ -char *timestamp_get_by_row_s(int row, int yy, const int hourly_dataset, const int start) { - TIMESTAMP *t; - static char buffer[12+1] = { 0 }; - - t = timestamp_get_by_row(row, yy, hourly_dataset, start); - sprintf(buffer, "%04d%02d%02d%02d%02d", t->YYYY, t->MM, t->DD, t->hh, t->mm); - return buffer; -} - -/* */ -TIMESTAMP *timestamp_ww_get_by_row(int row, int year, const int hourly_dataset, int start) { - int i; - int last; - - /* */ - assert((row >= 0) &&(row < 52)); - - /* */ - last = (52-1 == row); - i = 7 * (hourly_dataset ? 24 : 48); - row *= i; - - /* */ - if ( ! start ) { - if ( last ) { - row = IS_LEAP_YEAR(year) ? 17568 : 17520; - if ( hourly_dataset ) { - row /= 2; - } - } else { - row += i; - } - start = 1; - --row; - } - - /* */ - return timestamp_get_by_row(row, year, hourly_dataset, start); -} - -/* */ -char *timestamp_ww_get_by_row_s(int row, int yy, const int hourly_dataset, const int start) { - TIMESTAMP *t; - static char buffer[8+1] = { 0 }; - - t = timestamp_ww_get_by_row(row, yy, hourly_dataset, start); - sprintf(buffer, "%04d%02d%02d", t->YYYY, t->MM, t->DD); - return buffer; -} - -/* private function for gapfilling */ -static PREC gf_get_similiar_mean(const GF_ROW *const gf_rows, const int rows_count) { - int i; - PREC mean; - - /* check parameter */ - assert(gf_rows); - - /* get mean */ - mean = 0.0; - for ( i = 0; i < rows_count; i++ ) { - mean += gf_rows[i].similiar; - } - mean /= rows_count; - - /* check for NAN */ - if ( mean != mean ) { - mean = INVALID_VALUE; - } - - /* */ - return mean; -} - -/* gapfilling */ -PREC gf_get_similiar_standard_deviation(const GF_ROW *const gf_rows, const int rows_count) { - int i; - PREC mean; - PREC sum; - PREC sum2; - - /* check parameter */ - assert(gf_rows); - - /* get mean */ - mean = gf_get_similiar_mean(gf_rows, rows_count); - if ( IS_INVALID_VALUE(mean) ) { - return INVALID_VALUE; - } - - /* compute standard deviation */ - sum = 0.0; - sum2 = 0.0; - for ( i = 0; i < rows_count; i++ ) { - sum = (gf_rows[i].similiar - mean); - sum *= sum; - sum2 += sum; - } - sum2 /= rows_count-1; - sum2 = (PREC)SQRT(sum2); - - /* check for NAN */ - if ( sum2 != sum2 ) { - sum2 = INVALID_VALUE; - } - - /* */ - return sum2; -} - -/* gapfilling */ -PREC gf_get_similiar_median(const GF_ROW *const gf_rows, const int rows_count, int *const error) { - int i; - PREC *p_median; - PREC result; - - /* check for null pointer */ - assert(gf_rows); - - /* reset */ - *error = 0; - - if ( !rows_count ) { - return INVALID_VALUE; - } else if ( 1 == rows_count ) { - return gf_rows[0].similiar; - } - - /* get valid values */ - p_median = malloc(rows_count*sizeof*p_median); - if ( !p_median ) { - *error = 1; - return INVALID_VALUE; - } - for ( i = 0; i < rows_count; i++ ) { - p_median[i] = gf_rows[i].similiar; - } - - /* sort values */ - qsort(p_median, rows_count, sizeof *p_median, compare_prec); - - /* get median */ - if ( rows_count & 1 ) { - result = p_median[((rows_count+1)/2)-1]; - } else { - result = ((p_median[(rows_count/2)-1] + p_median[rows_count/2]) / 2); - } - - /* free memory */ - free(p_median); - - /* check for NAN */ - if ( result != result ) { - result = INVALID_VALUE; - } - - /* */ - return result; -} - -/* private function for gapfilling */ -static int gapfill( PREC *values, - const int struct_size, - GF_ROW *const gf_rows, - const int start_window, - const int end_window, - const int current_row, - const int start, - const int end, - const int step, - const int method, - const int hourly_dataset, - const PREC value1_tolerance_min, - const PREC value1_tolerance_max, - const PREC value2_tolerance, - const PREC value3_tolerance, - const int tofill_column, - const int value1_column, - const int value2_column, - const int value3_column) { - int i; - int y; - int j; - int z; - int window; - int window_start; - int window_end; - int window_current; - int samples_count; - PREC value1_tolerance; - PREC *window_current_values; - PREC *row_current_values; - - /* check parameter */ - assert(values && gf_rows && (method >=0 && method < GF_METHODS)); - - /* reset */ - window = 0; - window_start = 0; - window_end = 0; - window_current = 0; - samples_count = 0; - value1_tolerance = 0.0; - - /* j is and index checker for hourly method */ - if ( hourly_dataset ) { - j = 3; - } else { - j = 5; - } - - /* */ - i = start; - if ( GF_TOFILL_METHOD == method ) { - z = hourly_dataset ? 24 : 48; - } else { - z = 1; - } - while ( i <= end ) { - /* reset */ - samples_count = 0; - - /* compute window */ - window = 48 * i; - if ( hourly_dataset ) { - window /= 2; - } - - /* get window start index */ - window_start = current_row - window; - if ( GF_TOFILL_METHOD == method ) { - if ( hourly_dataset ) { - window_start -= 1; - } else { - window_start -= 2; - } - } - - if ( GF_TOFILL_METHOD != method ) { - /* fix for recreate markus code */ - ++window_start; - } - - /* get window end index */ - window_end = current_row + window; - if (GF_TOFILL_METHOD == method ) { - if ( hourly_dataset ) { - window_end += 2; - } else { - window_end += 3; - } - } - - /* fix bounds for first two methods - cause in hour method (NEE_METHOD) a window start at -32 and window end at 69, - it will be fixed to window start at 0 and this is an error... - */ - if ( GF_TOFILL_METHOD != method ) { - if ( window_start < 0 ) { - window_start = 0; - } - - if ( window_end > end_window ) { - window_end = end_window; - } - - /* modified on June 25, 2013 */ - /* compute tolerance for value1 */ - if ( IS_INVALID_VALUE(value1_tolerance_min) ) { - value1_tolerance = value1_tolerance_max; - } else if ( IS_INVALID_VALUE(value1_tolerance_max) ) { - value1_tolerance = value1_tolerance_min; - } else { - value1_tolerance = ((PREC *)(((char *)values)+current_row*struct_size))[value1_column]; - if ( value1_tolerance < value1_tolerance_min ) { - value1_tolerance = value1_tolerance_min; - } else if ( value1_tolerance > value1_tolerance_max ) { - value1_tolerance = value1_tolerance_max; - } - } - } - - /* loop through window */ - for ( window_current = window_start; window_current < window_end; window_current += z ) { - window_current_values = ((PREC *)(((char *)values)+window_current*struct_size)); - row_current_values = ((PREC *)(((char *)values)+current_row*struct_size)); - - switch ( method ) { - case GF_ALL_METHOD: - if ( IS_FLAG_SET(gf_rows[window_current].mask, GF_ALL_VALID) ) { - if ( - (FABS(window_current_values[value2_column]-row_current_values[value2_column]) < value2_tolerance) && - (FABS(window_current_values[value1_column]-row_current_values[value1_column]) < value1_tolerance) && - (FABS(window_current_values[value3_column]-row_current_values[value3_column]) < value3_tolerance) - ) { - gf_rows[samples_count++].similiar = window_current_values[tofill_column]; - } - } - break; - - case GF_VALUE1_METHOD: - if ( IS_FLAG_SET(gf_rows[window_current].mask, (GF_TOFILL_VALID|GF_VALUE1_VALID)) ) { - if ( FABS(window_current_values[value1_column]-row_current_values[value1_column]) < value1_tolerance ) { - gf_rows[samples_count++].similiar = window_current_values[tofill_column]; - } - } - break; - - case GF_TOFILL_METHOD: - for ( y = 0; y < j; y++ ) { - if ( ((window_current+y) < 0) || (window_current+y) >= end_window ) { - continue; - } - if ( IS_FLAG_SET(gf_rows[window_current+y].mask, GF_TOFILL_VALID) ) { - gf_rows[samples_count++].similiar = ((PREC *)(((char *)values)+((window_current+y)*struct_size)))[tofill_column]; - } - } - break; - } - } - - if ( samples_count > 1 ) { - /* set mean */ - gf_rows[current_row].filled = gf_get_similiar_mean(gf_rows, samples_count); - - /* set standard deviation */ - gf_rows[current_row].stddev = gf_get_similiar_standard_deviation(gf_rows, samples_count); - - /* set method */ - gf_rows[current_row].method = method + 1; - - /* set time-window */ - gf_rows[current_row].time_window = i * 2; - - /* fix hour method timewindow */ - if ( GF_TOFILL_METHOD == method ) { - ++gf_rows[current_row].time_window; - } - - /* set samples */ - gf_rows[current_row].samples_count = samples_count; - - /* ok */ - return 1; - } - - /* inc loop */ - i += step; - - /* break if window bigger than */ - if ( (window_start < start_window) && (window_end > end_window) ) { - break; - } - } - - /* */ - return 0; -} - -/* DEV version */ -static int dev_mds_gf( PREC *values, - const int struct_size, - GF_ROW *const gf_rows, - const int start_window, - const int end_window, - const int current_row, - const int hourly_dataset, - const PREC value1_tolerance_min, - const PREC value1_tolerance_max, - const PREC value2_tolerance, - const PREC value3_tolerance, - const int tofill_column, - const int value1_column, - const int value2_column, - const int value3_column) { - int i; - int y; - int j; - int z; - int window; - int window_start; - int window_end; - int window_current; - int samples_count; - int current_method; - int start; - int end; - int step; - int method; - PREC value1_tolerance; - PREC *window_current_values; - PREC *row_current_values; - - struct { - int start; - int end; - int step; - int method; - } mds[] = { - { 7, 14, 7, GF_ALL_METHOD } - , { 7, 7, 7, GF_VALUE1_METHOD } - , { 0, 2, 1, GF_TOFILL_METHOD } - , { 21, 77, 7, GF_ALL_METHOD } - , { 14, 77, 7, GF_VALUE1_METHOD } - , { 3, 0, 3, GF_TOFILL_METHOD } - }; - - /* check parameter */ - assert(values && gf_rows); - - /* reset */ - window = 0; - window_start = 0; - window_end = 0; - window_current = 0; - samples_count = 0; - value1_tolerance = 0.0; - - /* fix last method */ - mds[5].end = end_window + 1; - - /* j is and index checker for hourly method */ - if ( hourly_dataset ) { - j = 3; - } else { - j = 5; - } - - /* */ - current_method = 0; - while ( current_method < SIZEOF_ARRAY(mds) ) { - start = mds[current_method].start; - end = mds[current_method].end; - step = mds[current_method].step; - method = mds[current_method].method; - - i = start; - if ( GF_TOFILL_METHOD == method ) { - z = hourly_dataset ? 24 : 48; - } else { - z = 1; - } - while ( i <= end ) { - /* reset */ - samples_count = 0; - - /* compute window */ - window = 48 * i; - if ( hourly_dataset ) { - window /= 2; - } - - /* get window start index */ - window_start = current_row - window; - if ( GF_TOFILL_METHOD == method ) { - if ( hourly_dataset ) { - window_start -= 1; - } else { - window_start -= 2; - } - } - - if ( GF_TOFILL_METHOD != method ) { - /* fix for recreate markus code */ - ++window_start; - } - - /* get window end index */ - window_end = current_row + window; - if (GF_TOFILL_METHOD == method ) { - if ( hourly_dataset ) { - window_end += 2; - } else { - window_end += 3; - } - } - - /* fix bounds for first two methods - cause in hour method (NEE_METHOD) a window start at -32 and window end at 69, - it will be fixed to window start at 0 and this is an error... - */ - if ( GF_TOFILL_METHOD != method ) { - if ( window_start < 0 ) { - window_start = 0; - } - - if ( window_end > end_window ) { - window_end = end_window; - } - - /* modified on June 25, 2013 */ - /* compute tolerance for value1 */ - if ( IS_INVALID_VALUE(value1_tolerance_min) ) { - value1_tolerance = value1_tolerance_max; - } else if ( IS_INVALID_VALUE(value1_tolerance_max) ) { - value1_tolerance = value1_tolerance_min; - } else { - value1_tolerance = ((PREC *)(((char *)values)+current_row*struct_size))[value1_column]; - if ( value1_tolerance < value1_tolerance_min ) { - value1_tolerance = value1_tolerance_min; - } else if ( value1_tolerance > value1_tolerance_max ) { - value1_tolerance = value1_tolerance_max; - } - } - } - - /* loop through window */ - for ( window_current = window_start; window_current < window_end; window_current += z ) { - window_current_values = ((PREC *)(((char *)values)+window_current*struct_size)); - row_current_values = ((PREC *)(((char *)values)+current_row*struct_size)); - - switch ( method ) { - case GF_ALL_METHOD: - if ( IS_FLAG_SET(gf_rows[window_current].mask, GF_ALL_VALID) ) { - if ( - (FABS(window_current_values[value2_column]-row_current_values[value2_column]) < value2_tolerance) && - (FABS(window_current_values[value1_column]-row_current_values[value1_column]) < value1_tolerance) && - (FABS(window_current_values[value3_column]-row_current_values[value3_column]) < value3_tolerance) - ) { - gf_rows[samples_count++].similiar = window_current_values[tofill_column]; - } - } - break; - - case GF_VALUE1_METHOD: - if ( IS_FLAG_SET(gf_rows[window_current].mask, (GF_TOFILL_VALID|GF_VALUE1_VALID)) ) { - if ( FABS(window_current_values[value1_column]-row_current_values[value1_column]) < value1_tolerance ) { - gf_rows[samples_count++].similiar = window_current_values[tofill_column]; - } - } - break; - - case GF_TOFILL_METHOD: - for ( y = 0; y < j; y++ ) { - if ( ((window_current+y) < 0) || (window_current+y) >= end_window ) { - continue; - } - if ( IS_FLAG_SET(gf_rows[window_current+y].mask, GF_TOFILL_VALID) ) { - gf_rows[samples_count++].similiar = ((PREC *)(((char *)values)+((window_current+y)*struct_size)))[tofill_column]; - } - } - break; - } - } - - if ( samples_count > 1 ) { - /* set mean */ - gf_rows[current_row].filled = gf_get_similiar_mean(gf_rows, samples_count); - - /* set standard deviation */ - gf_rows[current_row].stddev = gf_get_similiar_standard_deviation(gf_rows, samples_count); - - /* set method */ - gf_rows[current_row].method = method + 1; - - /* set time-window */ - gf_rows[current_row].time_window = i * 2; - - /* fix hour method timewindow */ - if ( GF_TOFILL_METHOD == method ) { - ++gf_rows[current_row].time_window; - } - - /* set samples */ - gf_rows[current_row].samples_count = samples_count; - - /* ok */ - return 1; - } - - /* inc loop */ - i += step; - - /* break if window bigger than */ - if ( (window_start < start_window) && (window_end > end_window) ) { - break; - } - } - ++current_method; - } - - /* */ - return 0; -} - -/* */ -GF_ROW *dev_gf_mds_with_bounds( PREC *values, - const int struct_size, - const int rows_count, - const int columns_count, - const int hourly_dataset, - PREC value1_tolerance_min, - PREC value1_tolerance_max, - PREC value2_tolerance, - PREC value3_tolerance, - const int tofill_column, - const int value1_column, - const int value2_column, - const int value3_column, - const int value1_qc_column, - const int value2_qc_column, - const int value3_qc_column, - const int qc_thrs, - const int values_min, - const int compute_hat, - int start_row, - int end_row, - int *no_gaps_filled_count) { - int i; - int c; - int valids_count; - GF_ROW *gf_rows; - - /* */ - assert(values && rows_count && no_gaps_filled_count); - - /* reset */ - *no_gaps_filled_count = 0; - if ( start_row < 0 ) { - start_row = 0; - } - if ( -1 == end_row ) { - end_row = rows_count; - } else if ( end_row > rows_count ) { - end_row = rows_count; - } - - /* allocate memory */ - gf_rows = malloc(rows_count*sizeof*gf_rows); - if ( !gf_rows ) { - puts(err_out_of_memory); - return NULL; - } - - /* reset */ - for ( i = 0; i < rows_count; i++ ) { - gf_rows[i].mask = 0; - gf_rows[i].similiar = INVALID_VALUE; - gf_rows[i].stddev = INVALID_VALUE; - gf_rows[i].filled = INVALID_VALUE; - gf_rows[i].quality = INVALID_VALUE; - gf_rows[i].time_window = 0; - gf_rows[i].samples_count = 0; - gf_rows[i].method = 0; - } - - /* update mask and count valids TO FILL */ - valids_count = 0; - for ( i = start_row; i < end_row; i++ ) { - for ( c = 0; c < columns_count; c++ ) { - if ( !IS_INVALID_VALUE(((PREC *)(((char *)values)+i*struct_size))[c]) ) { - if ( tofill_column == c ) { - gf_rows[i].mask |= GF_TOFILL_VALID; - } else if ( value1_column == c ) { - gf_rows[i].mask |= GF_VALUE1_VALID; - } else if ( value2_column == c ) { - gf_rows[i].mask |= GF_VALUE2_VALID; - } else if ( value3_column == c ) { - gf_rows[i].mask |= GF_VALUE3_VALID; - } - } - } - - /* check for QC */ - if ( !IS_INVALID_VALUE(qc_thrs) && - (value1_qc_column != -1) && - !IS_INVALID_VALUE(((PREC *)(((char *)values)+i*struct_size))[value1_qc_column]) ) { - if ( ((PREC *)(((char *)values)+i*struct_size))[value1_qc_column] > qc_thrs ) { - gf_rows[i].mask &= ~GF_VALUE1_VALID; - } - } - - if ( !IS_INVALID_VALUE(qc_thrs) && - (value2_qc_column != -1) && - !IS_INVALID_VALUE(((PREC *)(((char *)values)+i*struct_size))[value2_qc_column]) ) { - if ( ((PREC *)(((char *)values)+i*struct_size))[value2_qc_column] > qc_thrs ) { - gf_rows[i].mask &= ~GF_VALUE2_VALID; - } - } - - if ( !IS_INVALID_VALUE(qc_thrs) && - (value3_qc_column != -1) && - !IS_INVALID_VALUE(((PREC *)(((char *)values)+i*struct_size))[value3_qc_column]) ) { - if ( ((PREC *)(((char *)values)+i*struct_size))[value3_qc_column] > qc_thrs ) { - gf_rows[i].mask &= ~GF_VALUE3_VALID; - } - } - - if ( IS_FLAG_SET(gf_rows[i].mask, GF_TOFILL_VALID) ) { - ++valids_count; - } - } - - if ( valids_count < values_min ) { - puts(err_gf_too_less_values); - free(gf_rows); - return NULL; - } - - /* */ - if ( IS_INVALID_VALUE(value1_tolerance_min) && IS_INVALID_VALUE(value1_tolerance_max) ) { - value1_tolerance_min = GF_SW_IN_TOLERANCE_MIN; - value1_tolerance_max = GF_SW_IN_TOLERANCE_MAX; - } - - /* */ - if ( IS_INVALID_VALUE(value2_tolerance) ) { - value2_tolerance = GF_TA_TOLERANCE; - } - - /* */ - if ( IS_INVALID_VALUE(value3_tolerance) ) { - value3_tolerance = GF_VPD_TOLERANCE; - } - - /* loop for each row */ - for ( i = start_row; i < end_row; i++ ) { - /* copy value from TOFILL to FILLED */ - gf_rows[i].filled = ((PREC *)(((char *)values)+i*struct_size))[tofill_column]; - - /* compute hat ? */ - if ( !IS_INVALID_VALUE(gf_rows[i].filled) && !compute_hat ) { - continue; - } - - /* fill - Added 20140422: if a gap is impossible to fill, e.g. if with MDV there are no data in the whole dataset acquired in a range +/- one hour, - the data point is not filled and the qc is set to -9999 - */ - if ( ! dev_mds_gf(values, struct_size, gf_rows, start_row, end_row, i, hourly_dataset, value1_tolerance_min, value1_tolerance_max, value2_tolerance, value3_tolerance, tofill_column, value1_column, value2_column, value3_column) ) { - ++*no_gaps_filled_count; - continue; - - } - - /* compute quality */ - gf_rows[i].quality = (gf_rows[i].method > 0) + - ((gf_rows[i].method == 1 && gf_rows[i].time_window > 14) || (gf_rows[i].method == 2 && gf_rows[i].time_window > 14) || (gf_rows[i].method == 3 && gf_rows[i].time_window > 1)) + - ((gf_rows[i].method == 1 && gf_rows[i].time_window > 56) || (gf_rows[i].method == 2 && gf_rows[i].time_window > 28) || (gf_rows[i].method == 3 && gf_rows[i].time_window > 5)); - } - - /* ok */ - return gf_rows; -} - -/* */ -GF_ROW *gf_mds_with_bounds( PREC *values, - const int struct_size, - const int rows_count, - const int columns_count, - const int hourly_dataset, - PREC value1_tolerance_min, - PREC value1_tolerance_max, - PREC value2_tolerance, - PREC value3_tolerance, - const int tofill_column, - const int value1_column, - const int value2_column, - const int value3_column, - const int value1_qc_column, - const int value2_qc_column, - const int value3_qc_column, - const int qc_thrs, - const int values_min, - const int compute_hat, - int start_row, - int end_row, - int *no_gaps_filled_count) { - int i; - int c; - int valids_count; - GF_ROW *gf_rows; - - /* */ - assert(values && rows_count && no_gaps_filled_count); - - /* reset */ - *no_gaps_filled_count = 0; - if ( start_row < 0 ) { - start_row = 0; - } - if ( -1 == end_row ) { - end_row = rows_count; - } else if ( end_row > rows_count ) { - end_row = rows_count; - } - - /* allocate memory */ - gf_rows = malloc(rows_count*sizeof*gf_rows); - if ( !gf_rows ) { - puts(err_out_of_memory); - return NULL; - } - - /* reset */ - for ( i = 0; i < rows_count; i++ ) { - gf_rows[i].mask = 0; - gf_rows[i].similiar = INVALID_VALUE; - gf_rows[i].stddev = INVALID_VALUE; - gf_rows[i].filled = INVALID_VALUE; - gf_rows[i].quality = INVALID_VALUE; - gf_rows[i].time_window = 0; - gf_rows[i].samples_count = 0; - gf_rows[i].method = 0; - } - - /* update mask and count valids TO FILL */ - valids_count = 0; - for ( i = start_row; i < end_row; i++ ) { - for ( c = 0; c < columns_count; c++ ) { - if ( !IS_INVALID_VALUE(((PREC *)(((char *)values)+i*struct_size))[c]) ) { - if ( tofill_column == c ) { - gf_rows[i].mask |= GF_TOFILL_VALID; - } else if ( value1_column == c ) { - gf_rows[i].mask |= GF_VALUE1_VALID; - } else if ( value2_column == c ) { - gf_rows[i].mask |= GF_VALUE2_VALID; - } else if ( value3_column == c ) { - gf_rows[i].mask |= GF_VALUE3_VALID; - } - } - } - - /* check for QC */ - if ( !IS_INVALID_VALUE(qc_thrs) && - (value1_qc_column != -1) && - !IS_INVALID_VALUE(((PREC *)(((char *)values)+i*struct_size))[value1_qc_column]) ) { - if ( ((PREC *)(((char *)values)+i*struct_size))[value1_qc_column] > qc_thrs ) { - gf_rows[i].mask &= ~GF_VALUE1_VALID; - } - } - - if ( !IS_INVALID_VALUE(qc_thrs) && - (value2_qc_column != -1) && - !IS_INVALID_VALUE(((PREC *)(((char *)values)+i*struct_size))[value2_qc_column]) ) { - if ( ((PREC *)(((char *)values)+i*struct_size))[value2_qc_column] > qc_thrs ) { - gf_rows[i].mask &= ~GF_VALUE2_VALID; - } - } - - if ( !IS_INVALID_VALUE(qc_thrs) && - (value3_qc_column != -1) && - !IS_INVALID_VALUE(((PREC *)(((char *)values)+i*struct_size))[value3_qc_column]) ) { - if ( ((PREC *)(((char *)values)+i*struct_size))[value3_qc_column] > qc_thrs ) { - gf_rows[i].mask &= ~GF_VALUE3_VALID; - } - } - - if ( IS_FLAG_SET(gf_rows[i].mask, GF_TOFILL_VALID) ) { - ++valids_count; - } - } - - if ( valids_count < values_min ) { - puts(err_gf_too_less_values); - free(gf_rows); - return NULL; - } - - /* */ - if ( IS_INVALID_VALUE(value1_tolerance_min) && IS_INVALID_VALUE(value1_tolerance_max) ) { - value1_tolerance_min = GF_SW_IN_TOLERANCE_MIN; - value1_tolerance_max = GF_SW_IN_TOLERANCE_MAX; - } - - /* */ - if ( IS_INVALID_VALUE(value2_tolerance) ) { - value2_tolerance = GF_TA_TOLERANCE; - } - - /* */ - if ( IS_INVALID_VALUE(value3_tolerance) ) { - value3_tolerance = GF_VPD_TOLERANCE; - } - - /* loop for each row */ - for ( i = start_row; i < end_row; i++ ) { - /* copy value from TOFILL to FILLED */ - gf_rows[i].filled = ((PREC *)(((char *)values)+i*struct_size))[tofill_column]; - - /* compute hat ? */ - if ( !IS_INVALID_VALUE(gf_rows[i].filled) && !compute_hat ) { - continue; - } - - /* fill - Added 20140422: if a gap is impossible to fill, e.g. if with MDV there are no data in the whole dataset acquired in a range +/- one hour, - the data point is not filled and the qc is set to -9999 - */ - if ( !gapfill(values, struct_size, gf_rows, start_row, end_row, i, 7, 14, 7, GF_ALL_METHOD, hourly_dataset, value1_tolerance_min, value1_tolerance_max, value2_tolerance, value3_tolerance, tofill_column, value1_column, value2_column, value3_column) ) - if ( !gapfill(values, struct_size, gf_rows, start_row, end_row, i, 7, 7, 7, GF_VALUE1_METHOD, hourly_dataset, value1_tolerance_min, value1_tolerance_max, value2_tolerance, value3_tolerance, tofill_column, value1_column, value2_column, value3_column) ) - if ( !gapfill(values, struct_size, gf_rows, start_row, end_row, i, 0, 2, 1, GF_TOFILL_METHOD, hourly_dataset, value1_tolerance_min, value1_tolerance_max, value2_tolerance, value3_tolerance, tofill_column, value1_column, value2_column, value3_column) ) - if ( !gapfill(values, struct_size, gf_rows, start_row, end_row, i, 21, 77, 7, GF_ALL_METHOD, hourly_dataset, value1_tolerance_min, value1_tolerance_max, value2_tolerance, value3_tolerance, tofill_column, value1_column, value2_column, value3_column) ) - if ( !gapfill(values, struct_size, gf_rows, start_row, end_row, i, 14, 77, 7, GF_VALUE1_METHOD, hourly_dataset, value1_tolerance_min, value1_tolerance_max, value2_tolerance, value3_tolerance, tofill_column, value1_column, value2_column, value3_column) ) - if ( !gapfill(values, struct_size, gf_rows, start_row, end_row, i, 3, end_row + 1, 3, GF_TOFILL_METHOD, hourly_dataset, value1_tolerance_min, value1_tolerance_max, value2_tolerance, value3_tolerance, tofill_column, value1_column, value2_column, value3_column) ) { - ++*no_gaps_filled_count; - continue; - } - - /* compute quality */ - gf_rows[i].quality = (gf_rows[i].method > 0) + - ((gf_rows[i].method == 1 && gf_rows[i].time_window > 14) || (gf_rows[i].method == 2 && gf_rows[i].time_window > 14) || (gf_rows[i].method == 3 && gf_rows[i].time_window > 1)) + - ((gf_rows[i].method == 1 && gf_rows[i].time_window > 56) || (gf_rows[i].method == 2 && gf_rows[i].time_window > 28) || (gf_rows[i].method == 3 && gf_rows[i].time_window > 5)); - } - - /* ok */ - return gf_rows; -} - -/* */ -GF_ROW *gf_mds(PREC *values, const int struct_size, const int rows_count, const int columns_count, const int hourly_dataset, - PREC value1_tolerance_min, - PREC value1_tolerance_max, - PREC value2_tolerance, - PREC value3_tolerance, - const int tofill_column, - const int value1_column, - const int value2_column, - const int value3_column, - const int values_min, - const int compute_hat, - int *no_gaps_filled_count) { - return gf_mds_with_bounds( values, - struct_size, - rows_count, - columns_count, - hourly_dataset, - value1_tolerance_min, - value1_tolerance_max, - value2_tolerance, - value3_tolerance, - tofill_column, - value1_column, - value2_column, - value3_column, - -1, - -1, - -1, - INVALID_VALUE, - values_min, - compute_hat, - -1, - -1, - no_gaps_filled_count - ); -} - -/* */ -GF_ROW *gf_mds_with_qc(PREC *values, const int struct_size, const int rows_count, const int columns_count, const int hourly_dataset, - PREC value1_tolerance_min, - PREC value1_tolerance_max, - PREC value2_tolerance, - PREC value3_tolerance, - const int tofill_column, - const int value1_column, - const int value2_column, - const int value3_column, - const int value1_qc_column, - const int value2_qc_column, - const int value3_qc_column, - const int qc_thrs, - const int values_min, - const int compute_hat, - int *no_gaps_filled_count) { - return gf_mds_with_bounds( values, - struct_size, - rows_count, - columns_count, - hourly_dataset, - value1_tolerance_min, - value1_tolerance_max, - value2_tolerance, - value3_tolerance, - tofill_column, - value1_column, - value2_column, - value3_column, - value1_qc_column, - value2_qc_column, - value3_qc_column, - qc_thrs, - values_min, - compute_hat, - -1, - -1, - no_gaps_filled_count - ); -} - -/* temp functions used for G in energy_proc */ -GF_ROW *temp_gf_mds( PREC *values, - const int struct_size, - const int rows_count, - const int columns_count, - const int hourly_dataset, - PREC value1_tolerance_min, - PREC value1_tolerance_max, - PREC value2_tolerance, - PREC value3_tolerance, - const int tofill_column, - const int value1_column, - const int value2_column, - const int value3_column, - const int value1_qc_column, - const int value2_qc_column, - const int value3_qc_column, - const int qc_thrs, - const int values_min, - const int compute_hat, - int *no_gaps_filled_count) { - int i; - int c; - int start_row = -1; - int end_row = -1; - int valids_count; - GF_ROW *gf_rows; - - /* */ - assert(values && rows_count && no_gaps_filled_count); - - /* reset */ - *no_gaps_filled_count = 0; - if ( start_row < 0 ) { - start_row = 0; - } - if ( -1 == end_row ) { - end_row = rows_count; - } else if ( end_row > rows_count ) { - end_row = rows_count; - } - - /* allocate memory */ - gf_rows = malloc(rows_count*sizeof*gf_rows); - if ( !gf_rows ) { - puts(err_out_of_memory); - return NULL; - } - - /* reset */ - for ( i = 0; i < rows_count; i++ ) { - gf_rows[i].mask = 0; - gf_rows[i].similiar = INVALID_VALUE; - gf_rows[i].stddev = INVALID_VALUE; - gf_rows[i].filled = INVALID_VALUE; - gf_rows[i].quality = INVALID_VALUE; - gf_rows[i].time_window = 0; - gf_rows[i].samples_count = 0; - gf_rows[i].method = 0; - } - - /* update mask and count valids TO FILL */ - valids_count = 0; - for ( i = start_row; i < end_row; i++ ) { - for ( c = 0; c < columns_count; c++ ) { - if ( !IS_INVALID_VALUE(((PREC *)(((char *)values)+i*struct_size))[c]) ) { - if ( tofill_column == c ) { - gf_rows[i].mask |= GF_TOFILL_VALID; - } else if ( value1_column == c ) { - gf_rows[i].mask |= GF_VALUE1_VALID; - } else if ( value2_column == c ) { - gf_rows[i].mask |= GF_VALUE2_VALID; - } else if ( value3_column == c ) { - gf_rows[i].mask |= GF_VALUE3_VALID; - } - } - } - - /* check for QC */ - if ( !IS_INVALID_VALUE(qc_thrs) && - (value1_qc_column != -1) && - !IS_INVALID_VALUE(((PREC *)(((char *)values)+i*struct_size))[value1_qc_column]) ) { - if ( ((PREC *)(((char *)values)+i*struct_size))[value1_qc_column] > qc_thrs ) { - gf_rows[i].mask &= ~GF_VALUE1_VALID; - } - } - - if ( !IS_INVALID_VALUE(qc_thrs) && - (value2_qc_column != -1) && - !IS_INVALID_VALUE(((PREC *)(((char *)values)+i*struct_size))[value2_qc_column]) ) { - if ( ((PREC *)(((char *)values)+i*struct_size))[value2_qc_column] > qc_thrs ) { - gf_rows[i].mask &= ~GF_VALUE2_VALID; - } - } - - if ( !IS_INVALID_VALUE(qc_thrs) && - (value3_qc_column != -1) && - !IS_INVALID_VALUE(((PREC *)(((char *)values)+i*struct_size))[value3_qc_column]) ) { - if ( ((PREC *)(((char *)values)+i*struct_size))[value3_qc_column] > qc_thrs ) { - gf_rows[i].mask &= ~GF_VALUE3_VALID; - } - } - - if ( IS_FLAG_SET(gf_rows[i].mask, GF_TOFILL_VALID) ) { - ++valids_count; - } - } - - if ( valids_count < values_min ) { - puts(err_gf_too_less_values); - free(gf_rows); - return NULL; - } - - /* */ - if ( IS_INVALID_VALUE(value1_tolerance_min) && IS_INVALID_VALUE(value1_tolerance_max) ) { - value1_tolerance_min = GF_SW_IN_TOLERANCE_MIN; - value1_tolerance_max = GF_SW_IN_TOLERANCE_MAX; - } - - /* */ - if ( IS_INVALID_VALUE(value2_tolerance) ) { - value2_tolerance = GF_TA_TOLERANCE; - } - - /* */ - if ( IS_INVALID_VALUE(value3_tolerance) ) { - value3_tolerance = GF_VPD_TOLERANCE; - } - - /* loop for each row */ - for ( i = start_row; i < end_row; i++ ) { - /* copy value from TOFILL to FILLED */ - gf_rows[i].filled = ((PREC *)(((char *)values)+i*struct_size))[tofill_column]; - - /* compute hat ? */ - if ( !IS_INVALID_VALUE(gf_rows[i].filled) && !compute_hat ) { - continue; - } - - /* fill - Added 20140422: if a gap is impossible to fill, e.g. if with MDV there are no data in the whole dataset acquired in a range +/- one hour, - the data point is not filled and the qc is set to -9999 - */ - if ( !gapfill(values, struct_size, gf_rows, start_row, end_row, i, 7, 14, 7, GF_ALL_METHOD, hourly_dataset, value1_tolerance_min, value1_tolerance_max, value2_tolerance, value3_tolerance, tofill_column, value1_column, value2_column, value3_column) ) - if ( !gapfill(values, struct_size, gf_rows, start_row, end_row, i, 7, 7, 7, GF_VALUE1_METHOD, hourly_dataset, value1_tolerance_min, value1_tolerance_max, value2_tolerance, value3_tolerance, tofill_column, value1_column, value2_column, value3_column) ) - if ( !gapfill(values, struct_size, gf_rows, start_row, end_row, i, 0, 2, 1, GF_TOFILL_METHOD, hourly_dataset, value1_tolerance_min, value1_tolerance_max, value2_tolerance, value3_tolerance, tofill_column, value1_column, value2_column, value3_column) ) - if ( !gapfill(values, struct_size, gf_rows, start_row, end_row, i, 21, 77, 7, GF_ALL_METHOD, hourly_dataset, value1_tolerance_min, value1_tolerance_max, value2_tolerance, value3_tolerance, tofill_column, value1_column, value2_column, value3_column) ) - if ( !gapfill(values, struct_size, gf_rows, start_row, end_row, i, 14, 77, 7, GF_VALUE1_METHOD, hourly_dataset, value1_tolerance_min, value1_tolerance_max, value2_tolerance, value3_tolerance, tofill_column, value1_column, value2_column, value3_column) ) - if ( !gapfill(values, struct_size, gf_rows, start_row, end_row, i, 3, 91, 3, GF_TOFILL_METHOD, hourly_dataset, value1_tolerance_min, value1_tolerance_max, value2_tolerance, value3_tolerance, tofill_column, value1_column, value2_column, value3_column) ) { - ++*no_gaps_filled_count; - continue; - } - - /* compute quality */ - gf_rows[i].quality = (gf_rows[i].method > 0) + - ((gf_rows[i].method == 1 && gf_rows[i].time_window > 14) || (gf_rows[i].method == 2 && gf_rows[i].time_window > 14) || (gf_rows[i].method == 3 && gf_rows[i].time_window > 1)) + - ((gf_rows[i].method == 1 && gf_rows[i].time_window > 56) || (gf_rows[i].method == 2 && gf_rows[i].time_window > 28) || (gf_rows[i].method == 3 && gf_rows[i].time_window > 5)); - } - - /* ok */ - return gf_rows; -} - -/* private function for parse_dd */ -static int parse_time_zone(DD *const dd, char *const string) { - int i; - PREC v; - int check; - int error; - char *token; - char *p; - char *temp; - TIME_ZONE *time_zone; - TIMESTAMP *t; - - /* timezone can be specified without timestamp, so we check if delimiter are present */ - temp = string_copy(string); - if ( !temp ) { - puts(err_out_of_memory); - return 0; - } - token = string_tokenizer(temp, dd_field_delimiter, &p); - if ( '\0' == p[0] ) { - v = convert_string_to_prec(token, &error); - if ( error ) { - return 0; - } - dd->time_zones = malloc(sizeof*dd->time_zones); - if ( !dd->time_zones ) { - puts(err_out_of_memory); - return 0; - } - dd->time_zones[0].timestamp.YYYY = dd->year; - dd->time_zones[0].timestamp.MM = 1; - dd->time_zones[0].timestamp.DD = 1; - dd->time_zones[0].timestamp.hh = (HOURLY_TIMERES == dd->timeres) ? 1 : 0; - dd->time_zones[0].timestamp.mm = (HOURLY_TIMERES == dd->timeres) ? 0 : 30; - dd->time_zones[0].timestamp.ss = 0; - dd->time_zones[0].v = v; - dd->time_zones_count = 1; - check = 0; - } else { - check = 0; - t = NULL; - for ( i = 0, token = string_tokenizer(string, dd_field_delimiter, &p); token; token = string_tokenizer(NULL, dd_field_delimiter, &p), ++i ) { - if ( i & 1 ) { - v = convert_string_to_prec(token, &error); - if ( error ) { - free(t); - return 0; - } - time_zone = realloc(dd->time_zones, ++dd->time_zones_count*sizeof*time_zone); - if ( !time_zone ) { - puts(err_out_of_memory); - --dd->time_zones_count; - free(t); - return 0; - } - dd->time_zones = time_zone; - dd->time_zones[dd->time_zones_count-1].v = v; - dd->time_zones[dd->time_zones_count-1].timestamp.YYYY = t->YYYY; - dd->time_zones[dd->time_zones_count-1].timestamp.MM = t->MM; - dd->time_zones[dd->time_zones_count-1].timestamp.DD = t->DD; - dd->time_zones[dd->time_zones_count-1].timestamp.hh = t->hh; - dd->time_zones[dd->time_zones_count-1].timestamp.mm = t->mm; - dd->time_zones[dd->time_zones_count-1].timestamp.ss = t->ss; - free(t); - t = NULL; - } else { - t = get_timestamp(token); - if ( ! t ) { - return 0; - } - } - ++check; - } - free(t); - } - - /* */ - free(temp); - - /* */ - if( ! dd->time_zones_count ) { - return 0; - } - - /* */ - return !(check & 1); -} - -/* private function for parse_dd */ -static int parse_htower(DD *const dd, char *const string) { - int i; - PREC h; - int check; - int error; - char *token; - char *p; - HEIGHT *htower; - TIMESTAMP *t; - - check = 0; - t = NULL; - for ( i = 0, token = string_tokenizer(string, dd_field_delimiter, &p); token; token = string_tokenizer(NULL, dd_field_delimiter, &p), ++i ) { - if ( i & 1 ) { - h = convert_string_to_prec(token, &error); - if ( error ) { - free(t); - return 0; - } - htower = realloc(dd->htower, ++dd->htower_count*sizeof*htower); - if ( !htower ) { - puts(err_out_of_memory); - --dd->htower_count; - free(t); - return 0; - } - dd->htower = htower; - dd->htower[dd->htower_count-1].h = h; - dd->htower[dd->htower_count-1].timestamp.YYYY = t->YYYY; - dd->htower[dd->htower_count-1].timestamp.MM = t->MM; - dd->htower[dd->htower_count-1].timestamp.DD = t->DD; - dd->htower[dd->htower_count-1].timestamp.hh = t->hh; - dd->htower[dd->htower_count-1].timestamp.mm = t->mm; - dd->htower[dd->htower_count-1].timestamp.ss = t->ss; - free(t); - t = NULL; - } else { - t = get_timestamp(token); - if ( ! t ) { - return 0; - } - } - ++check; - } - free(t); - - /* */ - if( !dd->htower_count ) { - return 0; - } - - /* */ - return !(check & 1); -} - -/* private function for parse_dd */ -static int parse_sc_negles(DD *const dd, char *const string) { - int i; - int flag; - int check; - int error; - char *token; - char *p; - SC_NEGL *sc_negles; - TIMESTAMP *t; - - /* Sc_negl can be specified without timestamp, so we check if delimiter are present */ - token = string_tokenizer(string, dd_field_delimiter, &p); - if ( '\0' == p[0] ) { - flag = convert_string_to_int(token, &error); - if ( error ) { - return 0; - } - if ( (flag < 0) || (flag > 1) ) { - printf("bad Sc_negl specified: %d\n", flag); - return 0; - } - sc_negles = malloc(sizeof*sc_negles); - if ( !sc_negles ) { - puts(err_out_of_memory); - return 0; - } - dd->sc_negles = sc_negles; - dd->sc_negles[0].flag = flag; - dd->sc_negles[0].timestamp.YYYY = dd->year; - dd->sc_negles[0].timestamp.MM = 1; - dd->sc_negles[0].timestamp.DD = 1; - dd->sc_negles[0].timestamp.hh = (HOURLY_TIMERES == dd->timeres) ? 1 : 0; - dd->sc_negles[0].timestamp.mm = (HOURLY_TIMERES == dd->timeres) ? 0 : 30; - dd->sc_negles[0].timestamp.ss = 0; - dd->sc_negles_count = 1; - check = 0; - } else { - check = 0; - t = NULL; - for ( i = 0, token = string_tokenizer(string, dd_field_delimiter, &p); token; token = string_tokenizer(NULL, dd_field_delimiter, &p), ++i ) { - if ( i & 1 ) { - flag = convert_string_to_int(token, &error); - if ( error ) { - free(t); - return 0; - } - sc_negles = realloc(dd->sc_negles, ++dd->sc_negles_count*sizeof*sc_negles); - if ( !sc_negles ) { - puts(err_out_of_memory); - --dd->sc_negles_count; - free(t); - return 0; - } - dd->sc_negles = sc_negles; - dd->sc_negles[dd->sc_negles_count-1].flag = flag; - dd->sc_negles[dd->sc_negles_count-1].timestamp.YYYY = t->YYYY; - dd->sc_negles[dd->sc_negles_count-1].timestamp.MM = t->MM; - dd->sc_negles[dd->sc_negles_count-1].timestamp.DD = t->DD; - dd->sc_negles[dd->sc_negles_count-1].timestamp.hh = t->hh; - dd->sc_negles[dd->sc_negles_count-1].timestamp.mm = t->mm; - dd->sc_negles[dd->sc_negles_count-1].timestamp.ss = t->ss; - free(t); - t = NULL; - } else { - t = get_timestamp(token); - if ( ! t ) { - return 0; - } - } - ++check; - } - free(t); - } - - /* */ - if( !dd->sc_negles_count ) { - return 0; - } - - /* */ - return !(check & 1); -} - - -/* private function for parse_dd */ -static int parse_timeres(DD *const dd, const char *const string) { - int i; - - for ( i = 0; i < TIMERES_SIZE; i++ ) { - if ( ! string_compare_i(string, timeress[i]) ) { - dd->timeres = i; - return 1; - } - } - - /* error */ - return 0; -} - -/* */ -void zero_dd(DD *const dd) { - if ( dd ) { - dd->site[0] = '\0'; - dd->year = -1; - dd->lat = .0; - dd->lon = .0; - dd->time_zones = NULL; - dd->time_zones_count = 0; - dd->timeres = SPOT_TIMERES; - dd->htower = NULL; - dd->htower_count = 0; - dd->sc_negles = NULL; - dd->sc_negles_count = 0; - dd->notes = NULL; - dd->notes_count = 0; - } -} - -/* */ -DD *alloc_dd(void) { - DD *details; - - details = malloc(sizeof*details); - if ( !details ) { - puts(err_out_of_memory); - } else { - zero_dd(details); - } - return details; -} - - -/* */ -void free_dd(DD *dd) { - int i; - - if ( dd ) { - free(dd->sc_negles); - free(dd->htower); - free(dd->time_zones); - for ( i = 0; i < dd->notes_count; i++ ) { - free(dd->notes[i]); - } - free(dd->notes); - free(dd); - } -} - -/* */ -DD *parse_dd(FILE *const f) { - int i; - int error; - int index; - double v; - char *buffer; - char *token; - char *p; - char **ppchar_no_leak; - DD *dd; - - /* */ - assert(f); - - /* */ - buffer = malloc(HUGE_BUFFER_SIZE*sizeof*buffer); - if ( !buffer ) { - puts(err_out_of_memory); - return NULL; - } - - /* */ - dd = malloc(sizeof*dd); - if ( !dd ) { - puts(err_out_of_memory); - free(buffer); - return NULL; - } - zero_dd(dd); - - /* parse dataset details */ - index = 0; - while ( get_valid_line_from_file(f, buffer, HUGE_BUFFER_SIZE) ) { - token = string_tokenizer(buffer, dd_field_delimiter, &p); - /* - if ( (SC_NEGL_DETAIL == index) && mystricmp(token, dds[index]) ) { - ++index; - } - */ - if ( string_compare_i(token, dds[index]) ) { - if ( (NOTES_DETAIL == index ) && ( dd->notes_count >= 1) ) { - break; - } - printf("no '%s' keyword found.\n", dds[index]); - free(buffer); - free_dd(dd); - return NULL; - } - - /* */ - switch ( index ) { - case SITE_DETAIL: - token = string_tokenizer(NULL, dd_field_delimiter, &p); - for ( i = 0; token[i]; i++ ); - if ( (i != SITE_LEN-1) || token[2] != '-' ) { - printf("bad '%s' specified\n", dds[index]); - free(buffer); - free_dd(dd); - return 0; - } - strcpy(dd->site, token); - dd->site[SITE_LEN-1] = '\0'; - break; - - case YEAR_DETAIL: - token = string_tokenizer(NULL, dd_field_delimiter, &p); - i = convert_string_to_int(token, &error); - if ( error ) { - printf("bad '%s' specified\n", dds[index]); - free(buffer); - free_dd(dd); - return NULL; - } - dd->year = i; - break; - - case LAT_DETAIL: - token = string_tokenizer(NULL, dd_field_delimiter, &p); - v = convert_string_to_prec(token, &error); - if ( error ) { - printf("bad '%s' specified\n", dds[index]); - free(buffer); - free_dd(dd); - return NULL; - } - dd->lat = v; - break; - - case LON_DETAIL: - token = string_tokenizer(NULL, dd_field_delimiter, &p); - v = convert_string_to_prec(token, &error); - if ( error ) { - printf("bad '%s' specified\n", dds[index]); - free(buffer); - free_dd(dd); - return NULL; - } - dd->lon = v; - break; - - case TIMEZONE_DETAIL: - if ( !parse_time_zone(dd, p) ) { - printf("bad '%s' specified\n", dds[index]); - free(buffer); - free_dd(dd); - return NULL; - } - break; - - case HTOWER_DETAIL: - if ( !parse_htower(dd, p) ) { - printf("bad '%s' specified\n", dds[index]); - free(buffer); - free_dd(dd); - return NULL; - } - break; - - case TIMERES_DETAIL: - token = string_tokenizer(NULL, dd_field_delimiter, &p); - if ( !parse_timeres(dd, token) ) { - printf("bad '%s' specified\n", dds[index]); - free(buffer); - free_dd(dd); - return NULL; - } - break; - - case SC_NEGL_DETAIL: - if ( ! parse_sc_negles(dd, p) ) { - printf("bad '%s' specified\n", dds[index]); - free(buffer); - free_dd(dd); - return NULL; - } - break; - - case NOTES_DETAIL: - token = string_tokenizer(NULL, dd_field_delimiter, &p); - ppchar_no_leak = realloc(dd->notes, ++dd->notes_count*sizeof*dd->notes); - if ( !ppchar_no_leak ) { - puts(err_out_of_memory); - --dd->notes_count; - free(buffer); - free_dd(dd); - return NULL; - } - dd->notes = ppchar_no_leak; - dd->notes[dd->notes_count-1] = string_copy(token); - if ( !dd->notes[dd->notes_count-1] ) { - puts(err_out_of_memory); - --dd->notes_count; - free(buffer); - free_dd(dd); - return NULL; - } - break; - } - - if ( index < NOTES_DETAIL ) { - ++index; - } - } - - /* rewind file */ - fseek(f, 0, SEEK_SET); - - /* */ - index = DETAILS_SIZE - 1 + dd->notes_count; - for ( i = 0; i < index; i++ ) { - get_valid_line_from_file(f, buffer, HUGE_BUFFER_SIZE); - } - - /* free memory */ - free(buffer); - - /* sort timestamps */ - qsort(dd->time_zones, dd->time_zones_count, sizeof *dd->time_zones, compare_time_zones); - qsort(dd->htower, dd->htower_count, sizeof *dd->htower, compare_htower); - qsort(dd->sc_negles, dd->sc_negles_count, sizeof *dd->sc_negles, compare_sc_negles); - - /* ok */ - return dd; -} - -int write_dd(const DD *const dd, FILE *const f, const char *const notes_to_add) { - int i; - - /* */ - assert(dd && f); - - /* */ - fprintf(f, "%s,%s\n", dds[SITE_DETAIL], dd->site); - fprintf(f, "%s,%d\n", dds[YEAR_DETAIL], dd->year); - fprintf(f, "%s,%g\n", dds[LAT_DETAIL], dd->lat); - fprintf(f, "%s,%g\n", dds[LON_DETAIL], dd->lon); - fprintf(f, "%s", dds[TIMEZONE_DETAIL]); - for ( i = 0; i < dd->time_zones_count; i++ ) { - fprintf(f, ",%04d%02d%02d%02d%02d,%g", dd->time_zones[i].timestamp.YYYY, - dd->time_zones[i].timestamp.MM, - dd->time_zones[i].timestamp.DD, - dd->time_zones[i].timestamp.hh, - dd->time_zones[i].timestamp.mm, - dd->time_zones[i].v - ); - } - fputs("\n", f); - fprintf(f, "%s", dds[HTOWER_DETAIL]); - for ( i = 0; i < dd->htower_count; i++ ) { - fprintf(f, ",%04d%02d%02d%02d%02d,%g", dd->htower[i].timestamp.YYYY, - dd->htower[i].timestamp.MM, - dd->htower[i].timestamp.DD, - dd->htower[i].timestamp.hh, - dd->htower[i].timestamp.mm, - dd->htower[i].h - ); - } - fputs("\n", f); - fprintf(f, "%s,%s\n", dds[TIMERES_DETAIL], timeress[dd->timeres]); - - fprintf(f, "%s,%d", dds[SC_NEGL_DETAIL], dd->sc_negles[0].flag); - /* - for ( i = 0; i < dd->sc_negles_count; i++ ) { - fprintf(f, ",%04d%02d%02d%02d%02d,%d", dd->sc_negles[i].timestamp.YYYY, - dd->sc_negles[i].timestamp.MM, - dd->sc_negles[i].timestamp.DD, - dd->sc_negles[i].timestamp.hh, - dd->sc_negles[i].timestamp.mm, - dd->sc_negles[i].flag - ); - } - */ - fputs("\n", f); - - if ( notes_to_add ) { - fprintf(f, "%s,%s\n", dds[NOTES_DETAIL], notes_to_add); - } - for ( i = 0; i < dd->notes_count; i++ ) { - fprintf(f, "%s,%s\n", dds[NOTES_DETAIL], dd->notes[i]); - } - - /* ok */ - return 1; -} - -/* */ -int write_dds(const DD **const dd, const int count, FILE *const f, const char *const notes_to_add) { - int i; - int y; - - /* */ - assert(dd && f); - - /* */ - fputs(dds[SITE_DETAIL], f); - for ( y = 0; y < count; y++ ) { - if ( dd[y]->site ) { - fprintf(f, ",%s", dd[y]->site); - } - } - fputs("\n", f); - - fputs(dds[YEAR_DETAIL], f); - for ( y = 0; y < count; y++ ) { - if ( dd[y]->site ) { - fprintf(f, ",%d", dd[y]->year); - } - } - fputs("\n", f); - - fputs(dds[LAT_DETAIL], f); - for ( y = 0; y < count; y++ ) { - if ( dd[y]->site ) { - fprintf(f, ",%g", dd[y]->lat); - } - } - fputs("\n", f); - - fputs(dds[LON_DETAIL], f); - for ( y = 0; y < count; y++ ) { - if ( dd[y]->site ) { - fprintf(f, ",%g", dd[y]->lon); - } - } - fputs("\n", f); - - fputs(dds[TIMEZONE_DETAIL], f); - for ( y = 0; y < count; y++ ) { - if ( dd[y]->site ) { - for ( i = 0; i < dd[y]->time_zones_count; i++ ) { - fprintf(f, ",%04d%02d%02d%02d%02d,%g", dd[y]->time_zones[i].timestamp.YYYY, - dd[y]->time_zones[i].timestamp.MM, - dd[y]->time_zones[i].timestamp.DD, - dd[y]->time_zones[i].timestamp.hh, - dd[y]->time_zones[i].timestamp.mm, - dd[y]->time_zones[i].v - ); - } - } - } - fputs("\n", f); - - fputs(dds[HTOWER_DETAIL], f); - for ( y = 0; y < count; y++ ) { - if ( dd[y]->site ) { - for ( i = 0; i < dd[y]->htower_count; i++ ) { - fprintf(f, ",%04d%02d%02d%02d%02d,%g", dd[y]->htower[i].timestamp.YYYY, - dd[y]->htower[i].timestamp.MM, - dd[y]->htower[i].timestamp.DD, - dd[y]->htower[i].timestamp.hh, - dd[y]->htower[i].timestamp.mm, - dd[y]->htower[i].h - ); - } - } - } - fputs("\n", f); - - fputs(dds[TIMERES_DETAIL], f); - for ( y = 0; y < count; y++ ) { - if ( dd[y]->site ) { - fprintf(f, ",%s", timeress[dd[y]->timeres]); - } - } - fputs("\n", f); - - fputs(dds[SC_NEGL_DETAIL], f); - for ( y = 0; y < count; y++ ) { - if ( dd[y]->site ) { - for ( i = 0; i < dd[y]->sc_negles_count; i++ ) { - fprintf(f, ",%04d%02d%02d%02d%02d,%d", dd[y]->sc_negles[i].timestamp.YYYY, - dd[y]->sc_negles[i].timestamp.MM, - dd[y]->sc_negles[i].timestamp.DD, - dd[y]->sc_negles[i].timestamp.hh, - dd[y]->sc_negles[i].timestamp.mm, - dd[y]->sc_negles[i].flag - ); - } - } - } - fputs("\n", f); - - /* write notes to add */ - if ( notes_to_add ) { - fprintf(f, "notes,%s\n", notes_to_add); - } - - /* write notes */ - for ( y = 0; y < count; y++ ) { - if ( dd[y]->site ) { - for ( i = 0; i < dd[y]->notes_count; i++ ) { - fprintf(f, "notes,%s\n", dd[y]->notes[i]); - } - } - } - - /* ok */ - return 1; -} - -/* */ -char *get_datetime_in_timestamp_format(void) { - const char timestamp_format[] = "%04d%02d%02d%02d%02d%02d"; - static char buffer[14 + 1]; -#if defined (_WIN32) - SYSTEMTIME st; - - GetLocalTime(&st); - sprintf(buffer, timestamp_format, st.wYear, - st.wMonth, - st.wDay, - st.wHour, - st.wMinute, - st.wSecond - ); -#elif defined (linux) || defined (__linux) || defined (__linux__) || defined (__APPLE__) - struct timeval tval; - struct tm *time; - if (gettimeofday(&tval, NULL) != 0) { - fprintf(stderr, "Fail of gettimeofday\n"); - return buffer; - } - time = localtime(&tval.tv_sec); - if (time == NULL) { - fprintf(stderr, "Fail of localtime\n"); - return buffer; - } - - sprintf(buffer, timestamp_format, - time->tm_year + 1900, - time->tm_mon, - time->tm_mday, - time->tm_hour, - time->tm_min, - time->tm_sec - ); -#endif - return buffer; -} - -/* */ -char *get_timeres_in_string(int timeres) { - if ( (timeres < 0) || (timeres >= TIMERES_SIZE) ) { - return NULL; - } - - switch ( timeres ) { - case SPOT_TIMERES: return "spot"; - case QUATERHOURLY_TIMERES: return "quaterhourly"; - case HALFHOURLY_TIMERES: return "halfhourly"; - case HOURLY_TIMERES: return "hourly"; - case DAILY_TIMERES: return "daily"; - case MONTHLY_TIMERES: return "monthly"; - default: return NULL; - } -} - -/* */ -int get_valid_line_from_file(FILE *const f, char *buffer, const int size) { - int i; - - if ( !f || !buffer || !size ) { - return 0; - } - - buffer[0] = '\0'; - while ( 1 ) { - if ( ! fgets(buffer, size, f) ) { - return 0; - } - - /* remove carriage return and newline */ - for ( i = 0; buffer[i]; i++ ) { - if ( ('\r' == buffer[i]) || ('\n' == buffer[i]) ) { - buffer[i] = '\0'; - break; - } - } - - /* skip empty line */ - if ( buffer[0] != '\0' ) { - break; - } - } - return 1; -} - -/* */ -int get_rows_count_from_file( FILE *const f) { - int i; - int rows_count; - char buffer[HUGE_BUFFER_SIZE]; - - if ( !f ) { - return -1; - } - - rows_count = 0; - while ( 1 ) { - if ( !fgets(buffer, HUGE_BUFFER_SIZE, f) ) { - break; - } - - /* remove carriage return and newline */ - for ( i = 0; buffer[i]; i++ ) { - if ( ('\r' == buffer[i]) || ('\n' == buffer[i]) ) { - buffer[i] = '\0'; - break; - } - } - - /* skip empty line */ - if ( '\0' == buffer[0] ) { - continue; - } - - ++rows_count; - } - return rows_count; -} - -/* */ -int get_rows_count_by_dd(const DD *const dd) { - int leap_year; - - assert(dd); - - leap_year = IS_LEAP_YEAR(dd->year); - switch ( dd->timeres ) { - case QUATERHOURLY_TIMERES: return 24*4*(365+leap_year); - case HALFHOURLY_TIMERES: return 24*2*(365+leap_year); - case HOURLY_TIMERES: return 24*(365+leap_year); - case DAILY_TIMERES: return 365+leap_year; - case MONTHLY_TIMERES: return 12; - default: return 0; - } -} - -/* */ -int get_rows_per_day_by_dd(const DD *const dd) { - assert(dd); - - switch ( dd->timeres ) { - case QUATERHOURLY_TIMERES: return 24*4; - case HALFHOURLY_TIMERES: return 24*2; - case HOURLY_TIMERES: return 24; - case DAILY_TIMERES: return 1; - default: return 0; - } -} - -/* row is zero based index */ -PREC get_dtime_by_row(const int row, const int hourly) { - int i; - PREC d; - int row_per_day; - int row_max; - const PREC decimal_part_dtime[] = { - 0.02083, - 0.04167, - 0.0625, - 0.08333, - 0.10417, - 0.125, - 0.14583, - 0.16667, - 0.1875, - 0.20833, - 0.22917, - 0.25, - 0.27083, - 0.29167, - 0.3125, - 0.33333, - 0.35417, - 0.375, - 0.39583, - 0.41667, - 0.4375, - 0.45833, - 0.47917, - 0.5, - 0.52083, - 0.54167, - 0.5625, - 0.58333, - 0.60417, - 0.625, - 0.64583, - 0.66667, - 0.6875, - 0.70833, - 0.72917, - 0.75, - 0.77083, - 0.79167, - 0.8125, - 0.83333, - 0.85417, - 0.875, - 0.89583, - 0.91667, - 0.9375, - 0.95833, - 0.97917, - 1, - }; - - row_max = LEAP_YEAR_ROWS; - if ( hourly ) { - row_max /= 2; - } - - assert((row >= 0) && ( row < row_max)); - - row_per_day = hourly ? 24 : 48; - - d = (int)row / row_per_day + 1; - i = row % row_per_day; - if ( hourly ) { - i *= 2; - ++i; - } - - return d + decimal_part_dtime[i]; -} - -/* - - NOON COMPUTATION BLOCK - contains functions from IDL code by Ankur Desai... - original notes reports: - - ;Sunrise, Sunset, Solar noon calculator - ;Source: http://www.srrb.noaa.gov/highlights/sunrise/sunrise.html - ;Converted to IDL by Ankur Desai, 7 April 2003 - -*/ - -/* */ -static double calcJD(int year, int month, int day) { - double A; - double B; - double JD; - - if ( month <= 2 ) { - year -= 1; - month += 12; - } - - A = floor((double)year/100); - B = 2 - A + floor((double)A/4); - JD = floor(365.25*(year + 4716)) + floor(30.6001*(month+1)) + day + B - 1524.5; - - /* */ - return JD; -} - -/* convert radian angle to degrees */ -static double radToDeg(const double angleRad) { - return (180.0 * angleRad / M_PI); -} - -/* convert degree angle to radians */ -static double degToRad(const double angleDeg) { - return (M_PI * angleDeg / 180.0); -} - -/* */ -static double calcTimeJulianCent(const double jd) { - return (jd - 2451545.0) / 36525.0; -} - -/* */ -static double calcJDFromJulianCent(const double t) { - return t * 36525.0 + 2451545.0; -} - -/* */ -static double calcGeomMeanLongSun(const double t) { - double L0; - - L0 = 280.46646 + t * (36000.76983 + 0.0003032 * t); - while ( L0 > 360.0 ) { - L0 -= 360.0; - } - - while ( L0 < 0.0 ) { - L0 += 360.0; - } - - return L0; /* in degrees */ -} - -/* */ -static double calcGeomMeanAnomalySun(const double t) { - return 357.52911 + t * (35999.05029 - 0.0001537 * t); /* in degrees */ -} - -/* */ -static double calcEccentricityEarthOrbit(const double t) { - return 0.016708634 - t * (0.000042037 + 0.0000001267 * t); /* unitless */ -} - -/* */ -static double calcMeanObliquityOfEcliptic(const double t) { - double seconds; - double e0; - - seconds = 21.448 - t*(46.8150 + t*(0.00059 - t*(0.001813))); - e0 = 23.0 + (26.0 + (seconds/60.0))/60.0; - - return e0; /* in degrees */ -} - -/* */ -static double calcObliquityCorrection(const double t) { - double e0; - double omega; - double e; - - e0 = calcMeanObliquityOfEcliptic(t); - omega = 125.04 - 1934.136 * t; - e = e0 + 0.00256 * cos(degToRad(omega)); - - return e; /* in degrees */ -} - -/* */ -static double calcEquationOfTime(const double t) { - double epsilon; - double l0; - double e; - double m; - double y; - double sin2l0; - double sinm; - double cos2l0; - double sin4l0; - double sin2m; - double Etime; - - epsilon = calcObliquityCorrection(t); - l0 = calcGeomMeanLongSun(t); - e = calcEccentricityEarthOrbit(t); - m = calcGeomMeanAnomalySun(t); - y = tan(degToRad(epsilon)/2.0); - y *= y; - - sin2l0 = sin(2.0 * degToRad(l0)); - sinm = sin(degToRad(m)); - cos2l0 = cos(2.0 * degToRad(l0)); - sin4l0 = sin(4.0 * degToRad(l0)); - sin2m = sin(2.0 * degToRad(m)); - - Etime = y * sin2l0 - 2.0 * e * sinm + 4.0 * e * y * sinm * cos2l0 - - 0.5 * y * y * sin4l0 - 1.25 * e * e * sin2m; - - return radToDeg(Etime)*4.0; /* in minutes of time */ -} - -/* */ -static double calcSolNoonUTC(const double t, const double longitude) { - double tnoon; - double eqTime; - double solNoonUTC; - double newt; - - /* First pass uses approximate solar noon to calculate eqtime */ - tnoon = calcTimeJulianCent(calcJDFromJulianCent(t) + longitude/360.0); - eqTime = calcEquationOfTime(tnoon); - solNoonUTC = 720 + (longitude * 4) - eqTime; /* min */ - - newt = calcTimeJulianCent(calcJDFromJulianCent(t) -0.5 + solNoonUTC/1440.0); - - eqTime = calcEquationOfTime(newt); - solNoonUTC = 720 + (longitude * 4) - eqTime; /* min */ - - return solNoonUTC; -} - -/* */ -static void get_solar_noon(const int year, const int month, const int day, const double longitude, const double zone, const int day_saving, int *const hour, int *const minute, int *const second) { - double floatHour; - double floatMinute; - double floatSec; - double jd; - double t; - double noon; - - jd = calcJD(year, month, day); - t = calcTimeJulianCent(jd); - - noon = calcSolNoonUTC(t, longitude) - (60 * zone) + (60 * day_saving); - - floatHour = noon / 60.0; - *hour = (int)floor(floatHour); - floatMinute = 60.0 * (floatHour - floor(floatHour)); - *minute = (int)floor(floatMinute); - floatSec = 60.0 * (floatMinute - floor(floatMinute)); - *second = (int)floor(floatSec + 0.5); - if ( *second > 59 ) { - *second = 0; - *minute += 1; - } -} - -/* - - END OF NOON COMPUTATION BLOCK - -*/ - - -/* - - RPOT COMPUTATION BLOCK - -*/ - -/* */ -static void get_month_and_day_by_row(const int row, const int year, int *const month, int *const day) { - int monthdays; - int is_leap; - int days_in_month[] = { 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 }; - - /* reset */ - *day = 0; - *month = 0; - monthdays = 0; - - is_leap = IS_LEAP_YEAR(year); - - /* check */ - if ( (row <= 0) || (row > 365 + is_leap) ) { - return; - } - - /* */ - if ( is_leap ) { - ++days_in_month[1]; - } - - /* */ - while ( row > monthdays ) { - monthdays += days_in_month[(*month)++]; - } - *day = row - monthdays + days_in_month[*month-1]; -} - -/* */ -static void shift(double *rpots_daily, const int hroz) { - int i; - int y; - int row; - - if ( 720 == hroz ) { - return; - } - - row = hroz - 720; - - if ( row > 0 ) { - for ( i = 0; i < row; i++ ) { - for ( y = 1440-1; y > 0; y-- ) { - rpots_daily[y] = rpots_daily[y-1]; - } - rpots_daily[y] = 0; - } - } else { - row *= -1; - for ( i = 0; i < row; i++ ) { - for ( y = 1; y < 1440; y++ ) { - rpots_daily[y-1] = rpots_daily[y]; - } - rpots_daily[y-1] = 0; - } - } -} - -/* */ -static int shift_by_2(double *rpots_daily, const int rpots_daily_count, const int hroz, const int hroz2, const int new_tz_index) { - int i; - int y; - int row; - double *shift2; - - /* alloc memory */ - shift2 = malloc(rpots_daily_count*sizeof*shift2); - if ( !shift2 ) { - puts(err_out_of_memory); - return 0; - } - - /* copy values */ - for ( i = 0; i < rpots_daily_count; i++ ) { - shift2[i] = rpots_daily[i]; - } - - if ( 720 != hroz ) { - row = hroz - 720; - - if ( row > 0 ) { - for ( i = 0; i < row; i++ ) { - for ( y = 1440-1; y > 0; y-- ) { - rpots_daily[y] = rpots_daily[y-1]; - } - rpots_daily[y] = 0; - } - } else { - row *= -1; - for ( i = 0; i < row; i++ ) { - for ( y = 1; y < 1440; y++ ) { - rpots_daily[y-1] = rpots_daily[y]; - } - rpots_daily[y-1] = 0; - } - } - } - - if ( 720 != hroz2 ) { - row = hroz2 - 720; - - if ( row > 0 ) { - for ( i = 0; i < row; i++ ) { - for ( y = 1440-1; y > 0; y-- ) { - shift2[y] = shift2[y-1]; - } - shift2[y] = 0; - } - } else { - row *= -1; - for ( i = 0; i < row; i++ ) { - for ( y = 1; y < 1440; y++ ) { - shift2[y-1] = shift2[y]; - } - shift2[y-1] = 0; - } - } - } - - /* aggregate */ - for ( i = new_tz_index; i < rpots_daily_count; i++ ) { - rpots_daily[i] = shift2[i]; - } - - /* free memory */ - free(shift2); - - /* ok */ - return 1; -} - -/* based on rg_pot function on ECOFRUNC.PRO (markus code ?) */ -static double get_daily_rpot(const double latitude, const double longitude, const int d_, const double t_) { - double localstandardtime; - double localapparentsolartime; - double tthet; - double signedLAS; - double omega; - double decl_rad; - double lat_rad; - double theta_rad; - double Rpot; - double Rpot_h; - - const double pi = 3.141592654; - const double solarconst = 1376.; - - localstandardtime = t_; - tthet = 2.*pi*(d_-1.) / 365.; - - localapparentsolartime = localstandardtime; - signedLAS = 12.-localapparentsolartime; - signedLAS = fabs(signedLAS); - - omega = -15.*signedLAS; - decl_rad = 0.006918-0.399912*cos(tthet)+0.070257*sin(tthet)-0.006758*cos(2*tthet)+0.000907*sin(2*tthet)-0.002697*cos(3*tthet)+0.00148*sin(3*tthet); - lat_rad = latitude*pi/180.; - - theta_rad = acos(sin(decl_rad)*sin(lat_rad)+cos(decl_rad)*cos(lat_rad)*cos(omega*pi/180.)); - - Rpot = solarconst*(1.00011+0.034221*cos(tthet)+0.00128*sin(tthet)+0.000719*cos(2*tthet)+0.000077*sin(2*tthet)); - Rpot_h = Rpot*cos(theta_rad); - - return (Rpot_h > 0) ? Rpot_h : 0.0; -} - -/* */ -PREC *get_rpot_with_solar_noon(DD *const details, const int s_n_month, const int s_n_day, int *const solar_noon) { - int i; - int y; - int row; - int hour; - int minute; - int second; - int day; - int month; - int year; - int rows_per_day; - int rows_count; - int rpot_rows_count; - int aggr_rows; - int hroz; - int hroz2; - int day_saving; - int time_zone_next_index; - float time_zone; - double longitude; - double doy; - double hrs; - double mean; - double *rpots_daily; - double *rpots; - - /* */ - assert(details); - - /* reset */ - if ( solar_noon ) { - *solar_noon = INVALID_VALUE; - } - - /* 60 * 24...rpot is computed for each minute */ - rows_per_day = 1440; - - year = details->year; - time_zone = details->time_zones[0].v * -1; - longitude = details->lon * -1; - day_saving = 0; - - rows_count = IS_LEAP_YEAR(details->year) ? 366 : 365; - aggr_rows = (HOURLY_TIMERES == details->timeres) ? 60: 30; - rpot_rows_count = rows_count * rows_per_day / aggr_rows; - - /* */ - rpots = malloc(rpot_rows_count*sizeof*rpots); - if ( !rpots ) { - puts(err_out_of_memory); - return NULL; - } - - /* */ - rpots_daily = malloc(sizeof*rpots_daily*rows_per_day); - if ( !rpots_daily ) { - puts(err_out_of_memory); - free(rpots); - return NULL; - } - - /* */ - time_zone_next_index = (details->time_zones_count > 1) ? 1: 0; - for ( row = 0; row < rows_count; row++ ) { - for ( i = 0; i < rows_per_day; i++ ) { - doy = ((double)(row*rows_per_day+i) / rows_per_day) + 1; - hrs = (double)((row*rows_per_day+i) % rows_per_day) / 60.; - rpots_daily[i] = get_daily_rpot(details->lat, longitude, doy, hrs); - } - - get_month_and_day_by_row(row+1, year, &month, &day); - - if ( time_zone_next_index && - (details->time_zones[time_zone_next_index].timestamp.MM == month) && - (details->time_zones[time_zone_next_index].timestamp.DD == day) ) { - get_solar_noon(year, month, day, longitude, time_zone, day_saving, &hour, &minute, &second); - hroz = 60*hour+minute; - time_zone = details->time_zones[time_zone_next_index].v * -1; - i = 60*details->time_zones[time_zone_next_index].timestamp.hh+details->time_zones[time_zone_next_index].timestamp.mm; - if ( ++time_zone_next_index >= details->time_zones_count ) { - time_zone_next_index = 0; - } - get_solar_noon(year, month, day, longitude, time_zone, day_saving, &hour, &minute, &second); - if ( solar_noon && (s_n_month == month) && (s_n_day == day) ) { - *solar_noon = hour * 10000 + minute * 100 + second; - } - hroz2 = 60*hour+minute; - shift_by_2(rpots_daily, rows_per_day, hroz, hroz2, i); - } else { - get_solar_noon(year, month, day, longitude, time_zone, day_saving, &hour, &minute, &second); - if ( solar_noon && (s_n_month == month) && (s_n_day == day) ) { - *solar_noon = hour * 10000 + minute * 100 + second; - } - hroz = 60*hour+minute; - shift(rpots_daily, hroz); - } - - /* aggregated */ - for ( i = 0; i < rows_per_day / aggr_rows; i++ ) { - mean = 0.; - for ( y = 0; y < aggr_rows; y++ ) { - mean += rpots_daily[i*aggr_rows+y]; - } - mean /= aggr_rows; - rpots[row*(rows_per_day / aggr_rows)+i] = mean; - } - } - - /* free memory */ - free(rpots_daily); - - /* ok */ - return rpots; -} - -/* */ -PREC *get_rpot(DD *const details) { - return get_rpot_with_solar_noon(details, 0, 0, NULL); -} - -/* - - END OF RPOT COMPUTATION BLOCK - -*/ - -void check_memory_leak(void) { -#ifdef _DEBUG -#if _WIN32 - _CrtSetReportMode(_CRT_WARN, _CRTDBG_MODE_DEBUG); - _CrtSetReportMode(_CRT_ERROR, _CRTDBG_MODE_DEBUG); - _CrtSetReportMode(_CRT_ASSERT, _CRTDBG_MODE_DEBUG); - _CrtDumpMemoryLeaks(); -#else /* _WIN32 */ - assert(0 && "no memory leak available for this platform"); -#endif -#endif /* _DEBUG */ -} - -/* return -2 on error, -1 if not found */ -int get_column_of(const char *const buffer, const char *const delimiter, const char * const string) { - char *buffer_copy; - char *p; - char *token; - int i; - int ret; - - buffer_copy = string_copy(buffer); - if ( ! buffer_copy) { - return -2; - } - - ret = -1; - for ( i = 0, token = string_tokenizer(buffer_copy, delimiter, &p); token; token = string_tokenizer(NULL, delimiter, &p), ++i ) { - if ( ! string_compare_i(token, string) ) { - ret = i; - break; - } - } - free(buffer_copy); - return ret; -} diff --git a/oneflux_steps/energy_proc/src/dataset.c b/oneflux_steps/energy_proc/src/dataset.c index 86a97d4e..6dd1a392 100644 --- a/oneflux_steps/energy_proc/src/dataset.c +++ b/oneflux_steps/energy_proc/src/dataset.c @@ -455,10 +455,10 @@ int save_output_diff(const DATASET *const dataset) { } for ( row = 0; row < y; row++ ) { /* TIMESTAMP_START */ - p = timestamp_start_by_row_s(row, dataset->years[i].year, dataset->hourly); + p = timestamp_start_by_row_s(row, dataset->years[i].year, dataset->details->timeres); fputs(p, f); /* TIMESTAMP_END */ - p = timestamp_end_by_row_s(row, dataset->years[i].year, dataset->hourly); + p = timestamp_end_by_row_s(row, dataset->years[i].year, dataset->details->timeres); fprintf(f, ",%s,", p); /* write values */ fprintf(f, output_format_diff, get_dtime_by_row(row, dataset->hourly), @@ -530,10 +530,10 @@ int save_output_midhourly(const DATASET *const dataset) { } for ( row = 0; row < y; row++ ) { /* TIMESTAMP_START */ - p = timestamp_start_by_row_s(row, dataset->years[i].year, dataset->hourly); + p = timestamp_start_by_row_s(row, dataset->years[i].year, dataset->details->timeres); fputs(p, f); /* TIMESTAMP_END */ - p = timestamp_end_by_row_s(row, dataset->years[i].year, dataset->hourly); + p = timestamp_end_by_row_s(row, dataset->years[i].year, dataset->details->timeres); fprintf(f, ",%s,", p); /* write values */ if ( no_rand_unc ) { @@ -659,7 +659,7 @@ int save_output_daily(const DATASET *const dataset) { for ( i = 0; i < dataset->years_count; i++ ) { y = IS_LEAP_YEAR(dataset->years[i].year) ? 366 : 365; for ( row = 0; row < y; row++ ) { - t = timestamp_end_by_row(row*(dataset->hourly ? 24 : 48), dataset->years[i].year, dataset->hourly); + t = timestamp_end_by_row(row*(dataset->hourly ? 24 : 48), dataset->years[i].year, dataset->details->timeres); /* write values */ if ( no_rand_unc ) { fprintf(f, output_format_dd_no_rand_unc, t->YYYY, @@ -778,10 +778,10 @@ int save_output_weekly(const DATASET *const dataset) { for ( i = 0; i < dataset->years_count; i++ ) { for ( row = 0; row < 52; row++ ) { /* write timestamp_start */ - p = timestamp_ww_get_by_row_s(row, dataset->years[i].year, dataset->hourly, 1); + p = timestamp_ww_get_by_row_s(row, dataset->years[i].year, dataset->details->timeres, 1); fprintf(f, "%s,", p); /* write timestamp_end */ - p = timestamp_ww_get_by_row_s(row, dataset->years[i].year, dataset->hourly, 0); + p = timestamp_ww_get_by_row_s(row, dataset->years[i].year, dataset->details->timeres, 0); fprintf(f, "%s,", p); /* write values */ if ( no_rand_unc ) { @@ -1029,10 +1029,10 @@ int debug_save_original(const DATASET *const dataset) { } for ( row = 0; row < y; row++ ) { /* TIMESTAMP_START */ - p = timestamp_start_by_row_s(row, dataset->years[i].year, dataset->hourly); + p = timestamp_start_by_row_s(row, dataset->years[i].year, dataset->details->timeres); fputs(p, f); /* TIMESTAMP_END */ - p = timestamp_end_by_row_s(row, dataset->years[i].year, dataset->hourly); + p = timestamp_end_by_row_s(row, dataset->years[i].year, dataset->details->timeres); fprintf(f, ",%s,", p); /* write values */ fprintf(f, output_format_debug_original, get_dtime_by_row(row, dataset->hourly), @@ -1086,10 +1086,10 @@ int debug_save_gapfilled(const DATASET *const dataset) { } for ( row = 0; row < y; row++ ) { /* TIMESTAMP_START */ - p = timestamp_start_by_row_s(row, dataset->years[i].year, dataset->hourly); + p = timestamp_start_by_row_s(row, dataset->years[i].year, dataset->details->timeres); fputs(p, f); /* TIMESTAMP_END */ - p = timestamp_end_by_row_s(row, dataset->years[i].year, dataset->hourly); + p = timestamp_end_by_row_s(row, dataset->years[i].year, dataset->details->timeres); fprintf(f, ",%s,", p); /* write values */ fprintf(f, output_format_debug_gapfilled, get_dtime_by_row(row, dataset->hourly), @@ -1146,10 +1146,10 @@ int debug_save_ecbcf_hh(const DATASET *const dataset, const PREC *const EBCcfs, } for ( row = 0; row < y; row++ ) { /* TIMESTAMP_START */ - p = timestamp_start_by_row_s(row, dataset->years[i].year, dataset->hourly); + p = timestamp_start_by_row_s(row, dataset->years[i].year, dataset->details->timeres); fputs(p, f); /* TIMESTAMP_END */ - p = timestamp_end_by_row_s(row, dataset->years[i].year, dataset->hourly); + p = timestamp_end_by_row_s(row, dataset->years[i].year, dataset->details->timeres); fprintf(f, ",%s,", p); /* write values */ fprintf(f, output_format_debug_ecbcf_hh, get_dtime_by_row(row, dataset->hourly), @@ -1193,7 +1193,7 @@ int debug_save_aggr_by_day(const DATASET *const dataset, const PREC *const ECBcf for ( i = 0; i < dataset->years_count; i++ ) { y = IS_LEAP_YEAR(dataset->years[i].year) ? 366 : 365; for ( row = 0; row < y; row++ ) { - t = timestamp_end_by_row(row*(dataset->hourly ? 24: 48), dataset->years[i].year, dataset->hourly); + t = timestamp_end_by_row(row*(dataset->hourly ? 24: 48), dataset->years[i].year, dataset->details->timeres); /* write values */ fprintf(f, output_format_debug_aggr_by_day, t->YYYY, t->MM, @@ -1425,13 +1425,13 @@ static int get_meteo(DATASET *const dataset) { for ( token = string_tokenizer(buffer, dataset_delimiter, &p), i = 0; token; token = string_tokenizer(NULL, dataset_delimiter, &p), ++i ) { if ( ! string_compare_i(token, "TA_m") ) { TA = i; - } else if ( ! string_compare_i(token, "SWIN_m") ) { + } else if ( ! string_compare_i(token, "SW_IN_m") ) { SWIN = i; } else if ( ! string_compare_i(token, "VPD_m") ) { VPD = i; } else if ( ! string_compare_i(token, "TA_mqc") ) { TA_QC = i; - } else if ( ! string_compare_i(token, "SWIN_mqc") ) { + } else if ( ! string_compare_i(token, "SW_IN_mqc") ) { SWIN_QC = i; } else if ( ! string_compare_i(token, "VPD_mqc") ) { VPD_QC = i; @@ -1497,7 +1497,7 @@ static int get_meteo(DATASET *const dataset) { return 0; } } - puts("founded"); + puts("found"); /* get values */ index = 0; @@ -1515,6 +1515,9 @@ static int get_meteo(DATASET *const dataset) { while ( fgets(buffer, BUFFER_SIZE, f) ) { for ( token = string_tokenizer(buffer, dataset_delimiter, &p), i = 0; token; token = string_tokenizer(NULL, dataset_delimiter, &p), i++ ) { if ( !i ) { + // skip timestamp start + continue; + } else if ( 1 == i ) { t = get_timestamp(token); if ( ! t ) { fclose(f); @@ -1537,7 +1540,7 @@ static int get_meteo(DATASET *const dataset) { } } - current_row = get_row_by_timestamp(t, (HOURLY_TIMERES==dataset->details->timeres)); + current_row = get_row_by_timestamp(t, dataset->details->timeres); free(t); if ( element != current_row ) { printf("bad timestamp: %s", token); @@ -1827,51 +1830,38 @@ int compute_datasets(DATASET *const datasets, const int datasets_count) { /* gapfilling LE */ printf("gapfilling LE..."); - current_dataset->gf_rows[LE_INDEX] = gf_mds_with_qc( current_dataset->rows->value, - sizeof(ROW), - current_dataset->rows_count, - DATASET_VALUES, - current_dataset->hourly, - GF_SW_IN_TOLERANCE_MIN, - GF_SW_IN_TOLERANCE_MAX, - GF_TA_TOLERANCE, - GF_VPD_TOLERANCE, - LE, - SWIN, - TA, - VPD, - SWIN_QC_GF, - TA_QC_GF, - VPD_QC_GF, - qc_gf_threshold, - GF_ROWS_MIN, - 1, - &no_gaps_filled_count); + current_dataset->gf_rows[LE_INDEX] = gf_mds(current_dataset->rows->value, + sizeof(ROW), + current_dataset->rows_count, + DATASET_VALUES, + current_dataset->hourly ? HOURLY_TIMERES : HALFHOURLY_TIMERES, + GF_DRIVER_1_TOLERANCE_MIN, + GF_DRIVER_1_TOLERANCE_MAX, + GF_DRIVER_2A_TOLERANCE_MIN, + GF_DRIVER_2A_TOLERANCE_MAX, + GF_DRIVER_2B_TOLERANCE_MIN, + GF_DRIVER_2B_TOLERANCE_MAX, + LE, + SWIN, + TA, + VPD, + SWIN_QC_GF, + TA_QC_GF, + VPD_QC_GF, + qc_gf_threshold, + qc_gf_threshold, + qc_gf_threshold, + GF_ROWS_MIN, + 1, + -1, + -1, + &no_gaps_filled_count, + 0, + 0, + 0, + NULL, + 0); - /* - current_dataset->gf_rows[LE_INDEX] = dev_gf_mds_with_bounds(current_dataset->rows->value, - sizeof(ROW), - current_dataset->rows_count, - DATASET_VALUES, - current_dataset->hourly, - GF_SW_IN_TOLERANCE_MIN, - GF_SW_IN_TOLERANCE_MAX, - GF_TA_TOLERANCE, - GF_VPD_TOLERANCE, - LE, - SWIN, - TA, - VPD, - SWIN_QC_GF, - TA_QC_GF, - VPD_QC_GF, - qc_gf_threshold, - GF_ROWS_MIN, - 1, - -1, - -1, - &no_gaps_filled_count); - */ if ( ! current_dataset->gf_rows[LE_INDEX] ) { /* free memory */ free(temp); @@ -1910,50 +1900,37 @@ int compute_datasets(DATASET *const datasets, const int datasets_count) { /* gapfilling H */ printf("gapfilling H..."); - current_dataset->gf_rows[H_INDEX] = gf_mds_with_qc( current_dataset->rows->value, - sizeof(ROW), - current_dataset->rows_count, - DATASET_VALUES, - current_dataset->hourly, - GF_SW_IN_TOLERANCE_MIN, - GF_SW_IN_TOLERANCE_MAX, - GF_TA_TOLERANCE, - GF_VPD_TOLERANCE, - H, - SWIN, - TA, - VPD, - SWIN_QC_GF, - TA_QC_GF, - VPD_QC_GF, - qc_gf_threshold, - GF_ROWS_MIN, - 1, - &no_gaps_filled_count); - /* - current_dataset->gf_rows[H_INDEX] = dev_gf_mds_with_bounds( current_dataset->rows->value, - sizeof(ROW), - current_dataset->rows_count, - DATASET_VALUES, - current_dataset->hourly, - GF_SW_IN_TOLERANCE_MIN, - GF_SW_IN_TOLERANCE_MAX, - GF_TA_TOLERANCE, - GF_VPD_TOLERANCE, - H, - SWIN, - TA, - VPD, - SWIN_QC_GF, - TA_QC_GF, - VPD_QC_GF, - qc_gf_threshold, - GF_ROWS_MIN, - 1, - -1, - -1, - &no_gaps_filled_count); - */ + current_dataset->gf_rows[H_INDEX] = gf_mds( current_dataset->rows->value, + sizeof(ROW), + current_dataset->rows_count, + DATASET_VALUES, + current_dataset->hourly ? HOURLY_TIMERES : HALFHOURLY_TIMERES, + GF_DRIVER_1_TOLERANCE_MIN, + GF_DRIVER_1_TOLERANCE_MAX, + GF_DRIVER_2A_TOLERANCE_MIN, + GF_DRIVER_2A_TOLERANCE_MAX, + GF_DRIVER_2B_TOLERANCE_MIN, + GF_DRIVER_2B_TOLERANCE_MAX, + H, + SWIN, + TA, + VPD, + SWIN_QC_GF, + TA_QC_GF, + VPD_QC_GF, + qc_gf_threshold, + qc_gf_threshold, + qc_gf_threshold, + GF_ROWS_MIN, + 1, + -1, + -1, + &no_gaps_filled_count, + 0, + 0, + 0, + NULL, + 0); if ( ! current_dataset->gf_rows[H_INDEX] ) { /* free memory */ free(temp); @@ -1993,15 +1970,17 @@ int compute_datasets(DATASET *const datasets, const int datasets_count) { /* gapfilling G */ if ( g_valid_rows_count ) { printf("gapfilling G..."); - current_dataset->gf_rows[G_INDEX] = temp_gf_mds( current_dataset->rows->value, + current_dataset->gf_rows[G_INDEX] = gf_mds( current_dataset->rows->value, sizeof(ROW), current_dataset->rows_count, DATASET_VALUES, - current_dataset->hourly, - GF_SW_IN_TOLERANCE_MIN, - GF_SW_IN_TOLERANCE_MAX, - GF_TA_TOLERANCE, - GF_VPD_TOLERANCE, + current_dataset->hourly ? HOURLY_TIMERES : HALFHOURLY_TIMERES, + GF_DRIVER_1_TOLERANCE_MIN, + GF_DRIVER_1_TOLERANCE_MAX, + GF_DRIVER_2A_TOLERANCE_MIN, + INVALID_VALUE, + GF_DRIVER_2B_TOLERANCE_MIN, + INVALID_VALUE, G, SWIN, TA, @@ -2010,35 +1989,18 @@ int compute_datasets(DATASET *const datasets, const int datasets_count) { TA_QC_GF, VPD_QC_GF, qc_gf_threshold, - GF_ROWS_MIN, - 0, /* do not compute hat for G */ - &no_gaps_filled_count); - - - /* - current_dataset->gf_rows[G_INDEX] = dev_gf_mds_with_bounds( current_dataset->rows->value, - sizeof(ROW), - current_dataset->rows_count, - DATASET_VALUES, - current_dataset->hourly, - GF_SW_IN_TOLERANCE_MIN, - GF_SW_IN_TOLERANCE_MAX, - GF_TA_TOLERANCE, - GF_VPD_TOLERANCE, - G, - SWIN, - TA, - VPD, - SWIN_QC_GF, - TA_QC_GF, - VPD_QC_GF, + qc_gf_threshold, qc_gf_threshold, GF_ROWS_MIN, - 0, /* do not compute hat for G * + 0, /* do not compute hat for G */ -1, -1, - &no_gaps_filled_count); - */ + &no_gaps_filled_count, + 0, + 45, + 0, + NULL, + 0); } else { /* FIX: alloc an empty array */ printf("gapfilling G..."); @@ -2741,14 +2703,14 @@ DATASET *get_datasets(const char *const path, const char *ext, int *const datase int y; int gap; int assigned; - int files_founded_count; + int files_found_count; int error; int file_index; YEAR *years_no_leak; DD *details; DATASET *datasets; DATASET *datasets_no_leak; - FILES *files_founded; + FILES *files_found; FILE *f; /* check parameters */ @@ -2759,31 +2721,31 @@ DATASET *get_datasets(const char *const path, const char *ext, int *const datase *datasets_count = 0; /* scan path */ - files_founded = get_files(path, ext, &files_founded_count, &error); - if ( error || !files_founded_count ) { - puts("no files founded!"); + files_found = get_files(path, ext, &files_found_count, &error); + if ( error || !files_found_count ) { + puts("no files found!"); return NULL; } - /* loop on each files founded */ - for ( file_index = 0; file_index < files_founded_count; file_index++ ) { + /* loop on each files found */ + for ( file_index = 0; file_index < files_found_count; file_index++ ) { /* check filename */ - if ( !is_valid_filename(files_founded[file_index].list[0].name) ) { + if ( !is_valid_filename(files_found[file_index].list[0].name) ) { continue; } /* open file */ - printf("processing %s...", files_founded[file_index].list[0].name); - f = fopen(files_founded[file_index].list[0].fullpath, "r"); + printf("processing %s...", files_found[file_index].list[0].name); + f = fopen(files_found[file_index].list[0].fullpath, "r"); if ( !f ) { - printf("unable to open %s\n", files_founded[file_index].list[0].fullpath); + printf("unable to open %s\n", files_found[file_index].list[0].fullpath); continue; } /* get details */ details = parse_dd(f); if ( !details ) { - free_files(files_founded, files_founded_count); + free_files(files_found, files_found_count); free_datasets(datasets, *datasets_count); return NULL; } @@ -2805,7 +2767,7 @@ DATASET *get_datasets(const char *const path, const char *ext, int *const datase datasets_no_leak = realloc(datasets, ++*datasets_count*sizeof*datasets_no_leak); if ( !datasets_no_leak ) { puts(err_out_of_memory); - free_files(files_founded, files_founded_count); + free_files(files_found, files_found_count); free_datasets(datasets, *datasets_count); return NULL; @@ -2838,7 +2800,7 @@ DATASET *get_datasets(const char *const path, const char *ext, int *const datase for ( y = 0; y < datasets[i].years_count; y++ ) { if ( details->year == datasets[i].years[y].year ) { puts(err_out_of_memory); - free_files(files_founded, files_founded_count); + free_files(files_found, files_found_count); free_datasets(datasets, *datasets_count); return NULL; } @@ -2848,7 +2810,7 @@ DATASET *get_datasets(const char *const path, const char *ext, int *const datase years_no_leak = realloc(datasets[i].years, ++datasets[i].years_count*sizeof*years_no_leak); if ( !years_no_leak ) { puts(err_out_of_memory); - free_files(files_founded, files_founded_count); + free_files(files_found, files_found_count); free_datasets(datasets, *datasets_count); return NULL; } @@ -2858,7 +2820,7 @@ DATASET *get_datasets(const char *const path, const char *ext, int *const datase datasets[i].years[datasets[i].years_count-1].year = details->year; datasets[i].years[datasets[i].years_count-1].exist = 1; datasets[i].years[datasets[i].years_count-1].hle = 0; - strcpy(datasets[i].years[datasets[i].years_count-1].filename, files_founded[file_index].list[0].fullpath); + strcpy(datasets[i].years[datasets[i].years_count-1].filename, files_found[file_index].list[0].fullpath); datasets[i].rows_count += ((IS_LEAP_YEAR(details->year) ? LEAP_YEAR_ROWS : YEAR_ROWS) / (datasets[i].hourly ? 2 : 1)); /* free memory */ @@ -2870,7 +2832,7 @@ DATASET *get_datasets(const char *const path, const char *ext, int *const datase } /* free memory */ - free_files(files_founded, files_founded_count); + free_files(files_found, files_found_count); /* sort per year */ for ( i = 0 ; i < *datasets_count; i++ ) { diff --git a/oneflux_steps/energy_proc/src/main.c b/oneflux_steps/energy_proc/src/main.c index 771eaf5d..ee11198e 100644 --- a/oneflux_steps/energy_proc/src/main.c +++ b/oneflux_steps/energy_proc/src/main.c @@ -1,7 +1,12 @@ /* main.c - this file is part of energy_proc + This file is part of the energy_proc step of processing. + It is responsible for the gapfilling of the sensible heat + and latent heat fluxes and the calculation of the energy + balance closure to create a version of H and LE with the + closure forced (bowen ration method). It also performs the + temporal aggregation at different time resolutions. author: Alessio Ribeca owner: DIBAF - University of Tuscia, Viterbo, Italy @@ -20,7 +25,7 @@ #include "../../compiler.h" /* constants */ -#define PROGRAM_VERSION "v1.0" +#define PROGRAM_VERSION "v1.01" #define BUFFER_SIZE 1024 /* global variables */ diff --git a/oneflux_steps/energy_proc/src/randunc.c b/oneflux_steps/energy_proc/src/randunc.c index 2db3ab69..87e129f5 100644 --- a/oneflux_steps/energy_proc/src/randunc.c +++ b/oneflux_steps/energy_proc/src/randunc.c @@ -62,10 +62,10 @@ void random_method_1(DATASET *const dataset, const int index) { /* compute tolerance for SWin */ swin_tolerance = dataset->rows[i].value[SWIN]; - if ( swin_tolerance < GF_SW_IN_TOLERANCE_MIN ) { - swin_tolerance = GF_SW_IN_TOLERANCE_MIN; - } else if ( swin_tolerance > GF_SW_IN_TOLERANCE_MAX ) { - swin_tolerance = GF_SW_IN_TOLERANCE_MAX; + if ( swin_tolerance < GF_DRIVER_1_TOLERANCE_MIN ) { + swin_tolerance = GF_DRIVER_1_TOLERANCE_MIN; + } else if ( swin_tolerance > GF_DRIVER_1_TOLERANCE_MAX ) { + swin_tolerance = GF_DRIVER_1_TOLERANCE_MAX; } /* loop through window */ @@ -77,9 +77,9 @@ void random_method_1(DATASET *const dataset, const int index) { if ( (!dataset->gf_rows[index][window_current+y].quality) && IS_FLAG_SET(dataset->gf_rows[index][window_current+y].mask, GF_ALL_VALID) && - (FABS(dataset->rows[window_current+y].value[TA]-dataset->rows[i].value[TA]) < GF_TA_TOLERANCE) && + (FABS(dataset->rows[window_current+y].value[TA]-dataset->rows[i].value[TA]) < GF_DRIVER_2A_TOLERANCE_MIN) && (FABS(dataset->rows[window_current+y].value[SWIN]-dataset->rows[i].value[SWIN]) < swin_tolerance ) && - (FABS(dataset->rows[window_current+y].value[VPD]-dataset->rows[i].value[VPD]) < GF_VPD_TOLERANCE) ) { + (FABS(dataset->rows[window_current+y].value[VPD]-dataset->rows[i].value[VPD]) < GF_DRIVER_2B_TOLERANCE_MIN) ) { dataset->gf_rows[index][samples_count++].similiar = dataset->rows[window_current+y].value[index]; } } diff --git a/oneflux_steps/meteo_proc/src/dataset.c b/oneflux_steps/meteo_proc/src/dataset.c index 4a0af751..5770df46 100644 --- a/oneflux_steps/meteo_proc/src/dataset.c +++ b/oneflux_steps/meteo_proc/src/dataset.c @@ -193,9 +193,9 @@ static const char output_format_yy[] = "%04d," "%.3f,%g"; static const char output_stat_file[] = "%s%s_meteo_%s_info.txt"; -static const char err_no_files_founded[] = "no files founded."; -static const char err_no_meteo_files_founded[] = "no meteo files founded."; -static const char err_no_era_files_founded[] = "no era files founded."; +static const char err_no_files_found[] = "no files found."; +static const char err_no_meteo_files_found[] = "no meteo files found."; +static const char err_no_era_files_found[] = "no era files found."; /* */ static int is_valid_era_filename(const char *const filename) { @@ -785,7 +785,7 @@ static int import_meteo_values(DATASET *const dataset) { int rows_count; int error; int year; - int columns_founded_count; + int columns_found_count; int columns_index[MET_VALUES]; int profile; int *int_no_leak; @@ -870,7 +870,7 @@ static int import_meteo_values(DATASET *const dataset) { /* check for same profile on header */ for ( j = 0; j < ts_profiles_count; j++ ) { if ( profile == ts_profiles[j] ) { - printf("profile %d for var TS already founded\n", profile); + printf("profile %d for var TS already found\n", profile); free(ts_profiles); free(swc_profiles); free(swc_columns); @@ -947,7 +947,7 @@ static int import_meteo_values(DATASET *const dataset) { /* check for same profile on header */ for ( j = 0; j < swc_profiles_count; j++ ) { if ( profile == swc_profiles[j] ) { - printf("profile %d for var SWC already founded\n", profile); + printf("profile %d for var SWC already found\n", profile); free(ts_profiles); free(swc_profiles); free(swc_columns); @@ -1337,15 +1337,15 @@ static int import_meteo_values(DATASET *const dataset) { } } - /* check founded columns */ - columns_founded_count = swc_columns_count+ts_columns_count; + /* check found columns */ + columns_found_count = swc_columns_count+ts_columns_count; for ( i = 0; i < MET_VALUES; i++ ) { if ( columns_index[i] != -1 ) { - ++columns_founded_count; + ++columns_found_count; } } - if ( !columns_founded_count ) { + if ( !columns_found_count ) { puts("no columns found!"); fclose(f); return 0; @@ -1510,8 +1510,8 @@ static int import_meteo_values(DATASET *const dataset) { } /* check assigned */ - if ( assigned != columns_founded_count ) { - printf("imported %d value not %d\n", assigned, columns_founded_count); + if ( assigned != columns_found_count ) { + printf("imported %d value not %d\n", assigned, columns_found_count); free(swc_columns); free(ts_columns); fclose(f); @@ -2990,10 +2990,10 @@ static int save_dataset_hh(const DATASET *const dataset) { } for ( row = 0; row < y; row++ ) { /* TIMESTAMP_START */ - p = timestamp_start_by_row_s(row, dataset->years[i].year, dataset->hourly); + p = timestamp_start_by_row_s(row, dataset->years[i].year, dataset->timeres); fputs(p, f); /* TIMESTAMP_END */ - p = timestamp_end_by_row_s(row, dataset->years[i].year, dataset->hourly); + p = timestamp_end_by_row_s(row, dataset->years[i].year, dataset->timeres); fprintf(f, ",%s,", p); /* write values */ fprintf(f, output_format_hh, @@ -3146,7 +3146,7 @@ static int save_dataset_dd(const DATASET *const dataset) { } y /= (dataset->hourly ? 24 : 48); for ( row = 0; row < y; row++ ) { - t = timestamp_end_by_row(row*(dataset->hourly ? 24 : 48), dataset->years[i].year, dataset->hourly); + t = timestamp_end_by_row(row*(dataset->hourly ? 24 : 48), dataset->years[i].year, dataset->timeres); /* write values */ fprintf(f, output_format_dd, t->YYYY, @@ -3312,10 +3312,10 @@ static int save_dataset_ww(const DATASET *const dataset) { year = dataset->years[i].year; for ( row = 0; row < 52; row++ ) { /* write timestamp_start */ - p = timestamp_ww_get_by_row_s(row, year, dataset->hourly, 1); + p = timestamp_ww_get_by_row_s(row, year, dataset->timeres, 1); fprintf(f, "%s,", p); /* write timestamp_end */ - p = timestamp_ww_get_by_row_s(row, year, dataset->hourly, 0); + p = timestamp_ww_get_by_row_s(row, year, dataset->timeres, 0); fprintf(f, "%s,", p); /* write values */ fprintf(f, output_format_ww, @@ -3874,7 +3874,7 @@ DATASET *get_datasets(int *const datasets_count) { if ( error ) { return NULL; } else if ( !files_temp_count ) { - puts(err_no_meteo_files_founded); + puts(err_no_meteo_files_found); return NULL; } @@ -3886,7 +3886,7 @@ DATASET *get_datasets(int *const datasets_count) { return NULL; } - /* loop on each files founded */ + /* loop on each files found */ for ( file = 0; file < files_temp_count; file++ ) { /* check filename */ if ( is_valid_era_filename(files_temp[file].list[0].name) ) { @@ -3912,13 +3912,13 @@ DATASET *get_datasets(int *const datasets_count) { /* free memory */ free_files(files_temp, files_temp_count); - /* check for non files founded */ + /* check for non files found */ if ( !files_count ) { - puts(err_no_files_founded); + puts(err_no_files_found); return NULL; } - /* loop on each files founded */ + /* loop on each files found */ for ( file = 0; file < files_count; file++ ) { type = files[file].type; if ( ERA_FILES == type ) { @@ -4131,7 +4131,7 @@ int compute_datasets(DATASET *const datasets, const int datasets_count) { /* check if has any met files */ if ( !check_met_files(current_dataset) ) { - puts("MET files not founded."); + puts("MET files not found."); continue; } @@ -4266,28 +4266,38 @@ int compute_datasets(DATASET *const datasets, const int datasets_count) { end_row = current_dataset->rows_count; } - gf_rows = gf_mds_with_bounds( current_dataset->rows->value, - sizeof(ROW), - current_dataset->rows_count, - VALUES, - current_dataset->hourly, - GF_SW_IN_TOLERANCE_MIN, - GF_SW_IN_TOLERANCE_MAX, - GF_TA_TOLERANCE, - GF_VPD_TOLERANCE, - TA_MET, - SW_IN_MET, - TA_MET, - VPD_MET, - -1, - -1, - -1, - INVALID_VALUE, - GF_ROWS_MIN, - 0, - start_row, - end_row, - ¬_gf_count + gf_rows = gf_mds( current_dataset->rows->value, + sizeof(ROW), + current_dataset->rows_count, + VALUES, + current_dataset->hourly ? HOURLY_TIMERES : HALFHOURLY_TIMERES, + GF_DRIVER_1_TOLERANCE_MIN, + GF_DRIVER_1_TOLERANCE_MAX, + GF_DRIVER_2A_TOLERANCE_MIN, + GF_DRIVER_2A_TOLERANCE_MAX, + GF_DRIVER_2B_TOLERANCE_MIN, + GF_DRIVER_2B_TOLERANCE_MAX, + TA_MET, + SW_IN_MET, + TA_MET, + VPD_MET, + -1, + -1, + -1, + INVALID_VALUE, + INVALID_VALUE, + INVALID_VALUE, + GF_ROWS_MIN, + 0, + start_row, + end_row, + ¬_gf_count, + 0, + 0, + 0, + NULL, + 0 + ); if ( !gf_rows ) { @@ -4360,28 +4370,37 @@ int compute_datasets(DATASET *const datasets, const int datasets_count) { end_row = current_dataset->rows_count; } - gf_rows = gf_mds_with_bounds( current_dataset->rows->value, - sizeof(ROW), - current_dataset->rows_count, - VALUES, - current_dataset->hourly, - GF_SW_IN_TOLERANCE_MIN, - GF_SW_IN_TOLERANCE_MAX, - GF_TA_TOLERANCE, - GF_VPD_TOLERANCE, - SW_IN_MET, - SW_IN_MET, - TA_MET, - VPD_MET, - -1, - -1, - -1, - INVALID_VALUE, - GF_ROWS_MIN, - 0, - start_row, - end_row, - ¬_gf_count + gf_rows = gf_mds( current_dataset->rows->value, + sizeof(ROW), + current_dataset->rows_count, + VALUES, + current_dataset->hourly ? HOURLY_TIMERES : HALFHOURLY_TIMERES, + GF_DRIVER_1_TOLERANCE_MIN, + GF_DRIVER_1_TOLERANCE_MAX, + GF_DRIVER_2A_TOLERANCE_MIN, + GF_DRIVER_2A_TOLERANCE_MAX, + GF_DRIVER_2B_TOLERANCE_MIN, + GF_DRIVER_2B_TOLERANCE_MAX, + SW_IN_MET, + SW_IN_MET, + TA_MET, + VPD_MET, + -1, + -1, + -1, + INVALID_VALUE, + INVALID_VALUE, + INVALID_VALUE, + GF_ROWS_MIN, + 0, + start_row, + end_row, + ¬_gf_count, + 0, + 0, + 0, + NULL, + 0 ); if ( !gf_rows ) { @@ -4454,28 +4473,37 @@ int compute_datasets(DATASET *const datasets, const int datasets_count) { end_row = current_dataset->rows_count; } - gf_rows = gf_mds_with_bounds( current_dataset->rows->value, - sizeof(ROW), - current_dataset->rows_count, - VALUES, - current_dataset->hourly, - GF_SW_IN_TOLERANCE_MIN, - GF_SW_IN_TOLERANCE_MAX, - GF_TA_TOLERANCE, - GF_VPD_TOLERANCE, - LW_IN_MET, - SW_IN_MET, - TA_MET, - VPD_MET, - -1, - -1, - -1, - INVALID_VALUE, - GF_ROWS_MIN, - 0, - start_row, - end_row, - ¬_gf_count + gf_rows = gf_mds( current_dataset->rows->value, + sizeof(ROW), + current_dataset->rows_count, + VALUES, + current_dataset->hourly ? HOURLY_TIMERES : HALFHOURLY_TIMERES, + GF_DRIVER_1_TOLERANCE_MIN, + GF_DRIVER_1_TOLERANCE_MAX, + GF_DRIVER_2A_TOLERANCE_MIN, + GF_DRIVER_2A_TOLERANCE_MAX, + GF_DRIVER_2B_TOLERANCE_MIN, + GF_DRIVER_2B_TOLERANCE_MAX, + LW_IN_MET, + SW_IN_MET, + TA_MET, + VPD_MET, + -1, + -1, + -1, + INVALID_VALUE, + INVALID_VALUE, + INVALID_VALUE, + GF_ROWS_MIN, + 0, + start_row, + end_row, + ¬_gf_count, + 0, + 0, + 0, + NULL, + 0 ); if ( !gf_rows ) { @@ -4548,28 +4576,37 @@ int compute_datasets(DATASET *const datasets, const int datasets_count) { end_row = current_dataset->rows_count; } - gf_rows = gf_mds_with_bounds( current_dataset->rows->value, - sizeof(ROW), - current_dataset->rows_count, - VALUES, - current_dataset->hourly, - GF_SW_IN_TOLERANCE_MIN, - GF_SW_IN_TOLERANCE_MAX, - GF_TA_TOLERANCE, - GF_VPD_TOLERANCE, - VPD_MET, - SW_IN_MET, - TA_MET, - VPD_MET, - -1, - -1, - -1, - INVALID_VALUE, - GF_ROWS_MIN, - 0, - start_row, - end_row, - ¬_gf_count + gf_rows = gf_mds( current_dataset->rows->value, + sizeof(ROW), + current_dataset->rows_count, + VALUES, + current_dataset->hourly ? HOURLY_TIMERES : HALFHOURLY_TIMERES, + GF_DRIVER_1_TOLERANCE_MIN, + GF_DRIVER_1_TOLERANCE_MAX, + GF_DRIVER_2A_TOLERANCE_MIN, + GF_DRIVER_2A_TOLERANCE_MAX, + GF_DRIVER_2B_TOLERANCE_MIN, + GF_DRIVER_2B_TOLERANCE_MAX, + VPD_MET, + SW_IN_MET, + TA_MET, + VPD_MET, + -1, + -1, + -1, + INVALID_VALUE, + INVALID_VALUE, + INVALID_VALUE, + GF_ROWS_MIN, + 0, + start_row, + end_row, + ¬_gf_count, + 0, + 0, + 0, + NULL, + 0 ); if ( !gf_rows ) { @@ -4642,28 +4679,37 @@ int compute_datasets(DATASET *const datasets, const int datasets_count) { end_row = current_dataset->rows_count; } - gf_rows = gf_mds_with_bounds( current_dataset->rows->value, - sizeof(ROW), - current_dataset->rows_count, - VALUES, - current_dataset->hourly, - GF_SW_IN_TOLERANCE_MIN, - GF_SW_IN_TOLERANCE_MAX, - GF_TA_TOLERANCE, - GF_VPD_TOLERANCE, - CO2_MET, - SW_IN_MET, - TA_MET, - VPD_MET, - -1, - -1, - -1, - INVALID_VALUE, - GF_ROWS_MIN, - 0, - start_row, - end_row, - ¬_gf_count + gf_rows = gf_mds( current_dataset->rows->value, + sizeof(ROW), + current_dataset->rows_count, + VALUES, + current_dataset->hourly ? HOURLY_TIMERES : HALFHOURLY_TIMERES, + GF_DRIVER_1_TOLERANCE_MIN, + GF_DRIVER_1_TOLERANCE_MAX, + GF_DRIVER_2A_TOLERANCE_MIN, + GF_DRIVER_2A_TOLERANCE_MAX, + GF_DRIVER_2B_TOLERANCE_MIN, + GF_DRIVER_2B_TOLERANCE_MAX, + CO2_MET, + SW_IN_MET, + TA_MET, + VPD_MET, + -1, + -1, + -1, + INVALID_VALUE, + INVALID_VALUE, + INVALID_VALUE, + GF_ROWS_MIN, + 0, + start_row, + end_row, + ¬_gf_count, + 0, + 0, + 0, + NULL, + 0 ); if ( !gf_rows ) { @@ -4739,28 +4785,37 @@ int compute_datasets(DATASET *const datasets, const int datasets_count) { } /* do gf */ - gf_rows = gf_mds_with_bounds( current_dataset->rows->value, - sizeof(ROW), - current_dataset->rows_count, - VALUES, - current_dataset->hourly, - GF_SW_IN_TOLERANCE_MIN, - GF_SW_IN_TOLERANCE_MAX, - GF_TA_TOLERANCE, - GF_VPD_TOLERANCE, - TEMP, - SW_IN_MET, - TA_MET, - VPD_MET, - -1, - -1, - -1, - INVALID_VALUE, - GF_ROWS_MIN, - 0, - start_row, - end_row, - ¬_gf_count + gf_rows = gf_mds( current_dataset->rows->value, + sizeof(ROW), + current_dataset->rows_count, + VALUES, + current_dataset->hourly ? HOURLY_TIMERES : HALFHOURLY_TIMERES, + GF_DRIVER_1_TOLERANCE_MIN, + GF_DRIVER_1_TOLERANCE_MAX, + GF_DRIVER_2A_TOLERANCE_MIN, + GF_DRIVER_2A_TOLERANCE_MAX, + GF_DRIVER_2B_TOLERANCE_MIN, + GF_DRIVER_2B_TOLERANCE_MAX, + TEMP, + SW_IN_MET, + TA_MET, + VPD_MET, + -1, + -1, + -1, + INVALID_VALUE, + INVALID_VALUE, + INVALID_VALUE, + GF_ROWS_MIN, + 0, + start_row, + end_row, + ¬_gf_count, + 0, + 0, + 0, + NULL, + 0 ); if ( !gf_rows ) { @@ -4837,28 +4892,38 @@ int compute_datasets(DATASET *const datasets, const int datasets_count) { } /* do gf */ - gf_rows = gf_mds_with_bounds( current_dataset->rows->value, - sizeof(ROW), - current_dataset->rows_count, - VALUES, - current_dataset->hourly, - GF_SW_IN_TOLERANCE_MIN, - GF_SW_IN_TOLERANCE_MAX, - GF_TA_TOLERANCE, - GF_VPD_TOLERANCE, - TEMP, - SW_IN_MET, - TA_MET, - VPD_MET, - -1, - -1, - -1, - INVALID_VALUE, - GF_ROWS_MIN, - 0, - start_row, - end_row, - ¬_gf_count + gf_rows = gf_mds( current_dataset->rows->value, + sizeof(ROW), + current_dataset->rows_count, + VALUES, + current_dataset->hourly ? HOURLY_TIMERES : HALFHOURLY_TIMERES, + GF_DRIVER_1_TOLERANCE_MIN, + GF_DRIVER_1_TOLERANCE_MAX, + GF_DRIVER_2A_TOLERANCE_MIN, + GF_DRIVER_2A_TOLERANCE_MAX, + GF_DRIVER_2B_TOLERANCE_MIN, + GF_DRIVER_2B_TOLERANCE_MAX, + TEMP, + SW_IN_MET, + TA_MET, + VPD_MET, + -1, + -1, + -1, + INVALID_VALUE, + INVALID_VALUE, + INVALID_VALUE, + GF_ROWS_MIN, + 0, + start_row, + end_row, + ¬_gf_count, + 0, + 0, + 0, + NULL, + 0 + ); if ( !gf_rows ) { @@ -4914,7 +4979,7 @@ int compute_datasets(DATASET *const datasets, const int datasets_count) { New option added 20160616 to take into account the possibility that ERA is not present in a given year (recent years). In this case the data filled using MDS are used for the _M and the QC is set as 3 - AGGIUNGI COME SOPRA, SE LW_IN_CALC_M è ancora -9999 metter LW_IN_CALC e QC = 3 + if LW_IN_CALC_M is still INVALID_VALUE, LW_IN_CALC and LW_IN_CALC_M_QC are set to 3 */ if ( IS_INVALID_VALUE(current_dataset->rows[i].value[LW_IN_CALC_M]) ) { current_dataset->rows[i].value[LW_IN_CALC_M] = current_dataset->rows[i].value[LW_IN_CALC]; diff --git a/oneflux_steps/meteo_proc/src/main.c b/oneflux_steps/meteo_proc/src/main.c index 859b30ca..d2281d09 100644 --- a/oneflux_steps/meteo_proc/src/main.c +++ b/oneflux_steps/meteo_proc/src/main.c @@ -1,7 +1,10 @@ /* main.c - this file is part of meteo_proc + This file is part of the meteo_proc step of processing. + It is responsible for the meteorological data gapfilling + including the merging with the downscaled meteo data and + the temporal aggregation at different time resolutions. author: Alessio Ribeca owner: DIBAF - University of Tuscia, Viterbo, Italy @@ -20,7 +23,7 @@ #include "../../compiler.h" /* constants */ -#define PROGRAM_VERSION "v1.0" +#define PROGRAM_VERSION "v1.01" #define BUFFER_SIZE 1024 #define QC_AUTO_PATH "qc_auto" #define ERA_PATH "era" @@ -32,10 +35,10 @@ char *era_files_path = NULL; /* mandatory */ char *output_files_path = NULL; /* mandatory */ char folder_delimiter_str[2]; /* used to get folder delimiter string from FOLDER_DELIMITER char in common.h*/ -PREC swin_tolerance_min = GF_SW_IN_TOLERANCE_MIN; /* see common.h */ -PREC swin_tolerance_max = GF_SW_IN_TOLERANCE_MAX; /* see common.h */ -PREC ta_tolerance = GF_TA_TOLERANCE; /* see common.h */ -PREC vpd_tolerance = GF_VPD_TOLERANCE; /* see common.h */ +PREC swin_tolerance_min = GF_DRIVER_1_TOLERANCE_MIN; /* see common.h */ +PREC swin_tolerance_max = GF_DRIVER_1_TOLERANCE_MAX; /* see common.h */ +PREC ta_tolerance = GF_DRIVER_2A_TOLERANCE_MIN; /* see common.h */ +PREC vpd_tolerance = GF_DRIVER_2B_TOLERANCE_MIN; /* see common.h */ /* strings */ static const char banner[] = "\nmeteo_proc "PROGRAM_VERSION"\n" diff --git a/oneflux_steps/nee_proc/src/dataset.c b/oneflux_steps/nee_proc/src/dataset.c index 156a0369..7375cb35 100644 --- a/oneflux_steps/nee_proc/src/dataset.c +++ b/oneflux_steps/nee_proc/src/dataset.c @@ -421,13 +421,13 @@ static int get_meteo(DATASET *const dataset) { for ( token = string_tokenizer(buffer, dataset_delimiter, &p), i = 0; token; token = string_tokenizer(NULL, dataset_delimiter, &p), ++i ) { if ( ! string_compare_i(token, "TA_m") ) { _TA = i; - } else if ( ! string_compare_i(token, "SWIN_m") ) { + } else if ( ! string_compare_i(token, "SW_IN_m") ) { _SWIN = i; } else if ( ! string_compare_i(token, "VPD_m") ) { _VPD = i; } else if ( ! string_compare_i(token, "TA_mqc") ) { TA_QC = i; - } else if ( ! string_compare_i(token, "SWIN_mqc") ) { + } else if ( ! string_compare_i(token, "SW_IN_mqc") ) { SWIN_QC = i; } else if ( ! string_compare_i(token, "VPD_mqc") ) { VPD_QC = i; @@ -493,7 +493,7 @@ static int get_meteo(DATASET *const dataset) { return 0; } } - puts("founded"); + puts("found"); /* get values */ index = 0; @@ -510,7 +510,11 @@ static int get_meteo(DATASET *const dataset) { } while ( fgets(buffer, BUFFER_SIZE, f) ) { for ( token = string_tokenizer(buffer, dataset_delimiter, &p), i = 0; token; token = string_tokenizer(NULL, dataset_delimiter, &p), i++ ) { - if ( !i ) { + if ( ! i ) { + // skip timestamp start + continue; + } else if ( 1 == i ) { + // get timestamp t = get_timestamp(token); if ( ! t ) { fclose(f); @@ -533,7 +537,7 @@ static int get_meteo(DATASET *const dataset) { } } - current_row = get_row_by_timestamp(t, (HOURLY_TIMERES==dataset->details->timeres)); + current_row = get_row_by_timestamp(t, dataset->details->timeres); free(t); if ( element != current_row ) { printf("bad timestamp: %s", token); @@ -1137,24 +1141,39 @@ static int is_valid_filename(const char *const filename) { return 1; } +/* */ +void clear_dataset(DATASET* dataset) { + if ( dataset ) { + if ( dataset->umna_count ) { + free(dataset->umna); + dataset->umna = NULL; + } + if ( dataset->details ) { + free_dd(dataset->details); + dataset->details = NULL; + } + if ( dataset->gf_rows ) { + free(dataset->gf_rows); + dataset->gf_rows = NULL; + } + if ( dataset->rows ) { + free(dataset->rows); + dataset->rows = NULL; + } + if ( dataset->years ) { + free(dataset->years); + dataset->years = NULL; + } + } +} + /* */ void free_datasets(DATASET *datasets, const int datasets_count) { int i; /* */ for ( i = 0; i < datasets_count; i++ ) { - if ( datasets[i].umna_count ) { - free(datasets[i].umna); - } - free_dd(datasets[i].details); - free(datasets[i].gf_rows); - free(datasets[i].rows); - free(datasets[i].years); - datasets[i].umna = NULL; - datasets[i].details = NULL; - datasets[i].rows = NULL; - datasets[i].gf_rows = NULL; - datasets[i].years = NULL; + clear_dataset(&datasets[i]); } free(datasets); } @@ -2435,22 +2454,22 @@ int save_nee_matrix(const NEE_MATRIX *const m, const DATASET *const d, int type) for ( row = 0; row < y; row++ ) { switch ( type ) { case HH_TR: - t = timestamp_start_by_row(row, d->years[i].year, (HOURLY_TIMERES == d->details->timeres)); + t = timestamp_start_by_row(row, d->years[i].year, d->details->timeres); fprintf(f, "%04d%02d%02d%02d%02d,", t->YYYY, t->MM, t->DD, t->hh, t->mm); - t = timestamp_end_by_row(row, d->years[i].year, (HOURLY_TIMERES == d->details->timeres)); + t = timestamp_end_by_row(row, d->years[i].year, d->details->timeres); fprintf(f, "%04d%02d%02d%02d%02d,", t->YYYY, t->MM, t->DD, t->hh, t->mm); break; case DD_TR: - t = timestamp_start_by_row(row*rows_per_day, d->years[i].year, (HOURLY_TIMERES == d->details->timeres)); + t = timestamp_start_by_row(row*rows_per_day, d->years[i].year, d->details->timeres); fprintf(f, "%04d%02d%02d,", t->YYYY, t->MM, t->DD); break; case WW_TR: /* timestamp_start */ - fprintf(f, "%s,", timestamp_ww_get_by_row_s(row, d->years[i].year, (HOURLY_TIMERES == d->details->timeres), 1)); + fprintf(f, "%s,", timestamp_ww_get_by_row_s(row, d->years[i].year, d->details->timeres, 1)); /* timestamp_end */ - fprintf(f, "%s,", timestamp_ww_get_by_row_s(row, d->years[i].year, (HOURLY_TIMERES == d->details->timeres), 0)); + fprintf(f, "%s,", timestamp_ww_get_by_row_s(row, d->years[i].year, d->details->timeres, 0)); break; case MM_TR: @@ -3283,7 +3302,7 @@ int compute_datasets(DATASET *const datasets, const int datasets_count) { int is_leap; int exists; int columns_index[INPUT_VALUES]; - int columns_founded_count; + int columns_found_count; int rows_per_day; int no_gaps_filled_count; int on_error; @@ -3684,7 +3703,7 @@ int compute_datasets(DATASET *const datasets, const int datasets_count) { } /* parse header */ - columns_founded_count = 0; + columns_found_count = 0; for ( token = string_tokenizer(buffer, dataset_delimiter, &p), i = 0; token; token = string_tokenizer(NULL, dataset_delimiter, &p), ++i ) { for ( y = 0; y < INPUT_VALUES; y++ ) { /* create itp name */ @@ -3715,7 +3734,7 @@ int compute_datasets(DATASET *const datasets, const int datasets_count) { break; } else { columns_index[y] = i; - ++columns_founded_count; + ++columns_found_count; // do not skip, continue searching for redundant columns } } @@ -3730,7 +3749,7 @@ int compute_datasets(DATASET *const datasets, const int datasets_count) { } /* check imported values */ - if ( columns_founded_count != INPUT_VALUES ) { + if ( columns_found_count != INPUT_VALUES ) { for ( i = 0; i < INPUT_VALUES; i++ ) { if ( -1 == columns_index[i] ) { /* VPD can be missing */ @@ -3859,8 +3878,8 @@ int compute_datasets(DATASET *const datasets, const int datasets_count) { } /* check assigned */ - if ( assigned != columns_founded_count ) { - printf("expected %d columns not %d\n", columns_founded_count, assigned); + if ( assigned != columns_found_count ) { + printf("expected %d columns not %d\n", columns_found_count, assigned); fclose(f); if ( compute_nee_flags ) { if ( datasets[dataset].years_count >= 3 ) { @@ -4063,26 +4082,37 @@ int compute_datasets(DATASET *const datasets, const int datasets_count) { /* gapfilling */ printf(" -> gf..."); - datasets[dataset].gf_rows = gf_mds_with_qc( datasets[dataset].rows->value, - sizeof(ROW), - datasets[dataset].rows_count, - REQUIRED_DATASET_VALUES, - HOURLY_TIMERES == datasets[dataset].details->timeres, - GF_SW_IN_TOLERANCE_MIN, - GF_SW_IN_TOLERANCE_MAX, - GF_TA_TOLERANCE, - GF_VPD_TOLERANCE, - NEE_VALUE, - SWIN_VALUE, - TA_VALUE, - VPD_VALUE, - SWIN_QC_VALUE, - TA_QC_VALUE, - VPD_QC_VALUE, - qc_gf_threshold, - GF_ROWS_MIN, - 1, - &no_gaps_filled_count); + datasets[dataset].gf_rows = gf_mds( datasets[dataset].rows->value, + sizeof(ROW), + datasets[dataset].rows_count, + REQUIRED_DATASET_VALUES, + (HOURLY_TIMERES == datasets[dataset].details->timeres) ? HOURLY_TIMERES : HALFHOURLY_TIMERES, + GF_DRIVER_1_TOLERANCE_MIN, + GF_DRIVER_1_TOLERANCE_MAX, + GF_DRIVER_2A_TOLERANCE_MIN, + GF_DRIVER_2A_TOLERANCE_MAX, + GF_DRIVER_2B_TOLERANCE_MIN, + GF_DRIVER_2B_TOLERANCE_MAX, + NEE_VALUE, + SWIN_VALUE, + TA_VALUE, + VPD_VALUE, + SWIN_QC_VALUE, + TA_QC_VALUE, + VPD_QC_VALUE, + qc_gf_threshold, + qc_gf_threshold, + qc_gf_threshold, + GF_ROWS_MIN, + 1, + -1, + -1, + &no_gaps_filled_count, + 0, + 0, + 0, + NULL, + 0); if ( !datasets[dataset].gf_rows ) { if ( compute_nee_flags ) { if ( datasets[dataset].years_count >= 3 ) { @@ -4289,15 +4319,17 @@ int compute_datasets(DATASET *const datasets, const int datasets_count) { /* gapfilling */ printf(" gf..."); free(datasets[dataset].gf_rows); - datasets[dataset].gf_rows = gf_mds_with_qc( datasets[dataset].rows->value, + datasets[dataset].gf_rows = gf_mds( datasets[dataset].rows->value, sizeof(ROW), datasets[dataset].rows_count, REQUIRED_DATASET_VALUES, - HOURLY_TIMERES == datasets[dataset].details->timeres, - GF_SW_IN_TOLERANCE_MIN, - GF_SW_IN_TOLERANCE_MAX, - GF_TA_TOLERANCE, - GF_VPD_TOLERANCE, + (HOURLY_TIMERES == datasets[dataset].details->timeres) ? HOURLY_TIMERES : HALFHOURLY_TIMERES, + GF_DRIVER_1_TOLERANCE_MIN, + GF_DRIVER_1_TOLERANCE_MAX, + GF_DRIVER_2A_TOLERANCE_MIN, + GF_DRIVER_2A_TOLERANCE_MAX, + GF_DRIVER_2B_TOLERANCE_MIN, + GF_DRIVER_2B_TOLERANCE_MAX, NEE_VALUE, SWIN_VALUE, TA_VALUE, @@ -4306,9 +4338,18 @@ int compute_datasets(DATASET *const datasets, const int datasets_count) { TA_QC_VALUE, VPD_QC_VALUE, qc_gf_threshold, + qc_gf_threshold, + qc_gf_threshold, GF_ROWS_MIN, 1, - &no_gaps_filled_count); + -1, + -1, + &no_gaps_filled_count, + 0, + 0, + 0, + NULL, + 0); if ( !datasets[dataset].gf_rows ) { if ( compute_nee_flags ) { if ( datasets[dataset].years_count >= 3 ) { @@ -4523,9 +4564,9 @@ int compute_datasets(DATASET *const datasets, const int datasets_count) { } for ( row = 0; row < y; row++ ) { - t = timestamp_start_by_row(row, datasets[dataset].years[i].year, (HOURLY_TIMERES == datasets[dataset].details->timeres)); + t = timestamp_start_by_row(row, datasets[dataset].years[i].year, datasets[dataset].details->timeres); fprintf(f, "%04d%02d%02d%02d%02d,", t->YYYY, t->MM, t->DD, t->hh, t->mm); - t = timestamp_end_by_row(row, datasets[dataset].years[i].year, (HOURLY_TIMERES == datasets[dataset].details->timeres)); + t = timestamp_end_by_row(row, datasets[dataset].years[i].year, datasets[dataset].details->timeres); fprintf(f, "%04d%02d%02d%02d%02d,", t->YYYY, t->MM, t->DD, t->hh, t->mm); fprintf(f, "%d,%d", nee_flags_y[j+row].value[ref_y], nee_flags_y[j+row].value[PERCENTILES_COUNT_2-1]); if ( datasets[dataset].years_count >= 3 ) { @@ -4632,10 +4673,10 @@ int compute_datasets(DATASET *const datasets, const int datasets_count) { exists = datasets[dataset].years[i].exist; for ( row = 0; row < y; row++ ) { /* timestamp start */ - t = timestamp_start_by_row(row, datasets[dataset].years[i].year, (HOURLY_TIMERES == datasets[dataset].details->timeres)); + t = timestamp_start_by_row(row, datasets[dataset].years[i].year, datasets[dataset].details->timeres); fprintf(f, "%04d%02d%02d%02d%02d,", t->YYYY, t->MM, t->DD, t->hh, t->mm); /* timestamp end */ - t = timestamp_end_by_row(row, datasets[dataset].years[i].year, (HOURLY_TIMERES == datasets[dataset].details->timeres)); + t = timestamp_end_by_row(row, datasets[dataset].years[i].year, datasets[dataset].details->timeres); fprintf(f, "%04d%02d%02d%02d%02d,", t->YYYY, t->MM, t->DD, t->hh, t->mm); /* dtime */ fprintf(f, "%g,", get_dtime_by_row(row, (HOURLY_TIMERES == datasets[dataset].details->timeres))); @@ -5122,7 +5163,7 @@ int compute_datasets(DATASET *const datasets, const int datasets_count) { /* */ exists = datasets[dataset].years[i].exist; for ( row = 0; row < y; row++ ) { - t = timestamp_end_by_row(row*((HOURLY_TIMERES == datasets[dataset].details->timeres) ? 24 : 48), datasets[dataset].years[i].year, (HOURLY_TIMERES == datasets[dataset].details->timeres)); + t = timestamp_end_by_row(row*((HOURLY_TIMERES == datasets[dataset].details->timeres) ? 24 : 48), datasets[dataset].years[i].year, datasets[dataset].details->timeres); fprintf(f, "%04d%02d%02d,%d,", t->YYYY, t->MM, t->DD, @@ -5956,7 +5997,7 @@ int compute_datasets(DATASET *const datasets, const int datasets_count) { y /= rows_per_day; for ( row = 0; row < y; row++ ) { t = timestamp_end_by_row(row*((HOURLY_TIMERES == datasets[dataset].details->timeres) ? 24 : 48) - , datasets[dataset].years[i].year, (HOURLY_TIMERES == datasets[dataset].details->timeres)); + , datasets[dataset].years[i].year, datasets[dataset].details->timeres); fprintf(f, "%04d%02d%02d", t->YYYY, t->MM, t->DD); for ( z = 0; z < PERCENTILES_COUNT_2; ++z ) { fprintf(f, ",%g,%g", rows_night_daily[j+row].night_columns_y[z] @@ -5999,7 +6040,7 @@ int compute_datasets(DATASET *const datasets, const int datasets_count) { y /= rows_per_day; for ( row = 0; row < y; row++ ) { t = timestamp_end_by_row(row*((HOURLY_TIMERES == datasets[dataset].details->timeres) ? 24 : 48) - , datasets[dataset].years[i].year, (HOURLY_TIMERES == datasets[dataset].details->timeres)); + , datasets[dataset].years[i].year, datasets[dataset].details->timeres); fprintf(f, "%04d%02d%02d", t->YYYY, t->MM, t->DD); for ( z = 0; z < PERCENTILES_COUNT_2; ++z ) { fprintf(f, ",%g,%g", rows_night_daily[j+row].day_columns_y[z] @@ -6042,7 +6083,7 @@ int compute_datasets(DATASET *const datasets, const int datasets_count) { y /= rows_per_day; for ( row = 0; row < y; row++ ) { t = timestamp_end_by_row(row*((HOURLY_TIMERES == datasets[dataset].details->timeres) ? 24 : 48) - , datasets[dataset].years[i].year, (HOURLY_TIMERES == datasets[dataset].details->timeres)); + , datasets[dataset].years[i].year, datasets[dataset].details->timeres); fprintf(f, "%04d%02d%02d", t->YYYY, t->MM, t->DD); for ( z = 0; z < PERCENTILES_COUNT_2; ++z ) { fprintf(f, ",%g,%g", rows_night_daily[j+row].night_columns_c[z] @@ -6085,7 +6126,7 @@ int compute_datasets(DATASET *const datasets, const int datasets_count) { y /= rows_per_day; for ( row = 0; row < y; row++ ) { t = timestamp_end_by_row(row*((HOURLY_TIMERES == datasets[dataset].details->timeres) ? 24 : 48) - , datasets[dataset].years[i].year, (HOURLY_TIMERES == datasets[dataset].details->timeres)); + , datasets[dataset].years[i].year, datasets[dataset].details->timeres); fprintf(f, "%04d%02d%02d", t->YYYY, t->MM, t->DD); for ( z = 0; z < PERCENTILES_COUNT_2; ++z ) { fprintf(f, ",%g,%g", rows_night_daily[j+row].day_columns_c[z] @@ -6459,8 +6500,8 @@ int compute_datasets(DATASET *const datasets, const int datasets_count) { fputs("\n", f); for ( i = 0; i < datasets[dataset].years_count; i++ ) { for ( row = 0; row < y; row++ ) { - fprintf(f, "%s,", timestamp_ww_get_by_row_s(row, datasets[dataset].years[i].year, (HOURLY_TIMERES == datasets[dataset].details->timeres), 1)); - fprintf(f, "%s,", timestamp_ww_get_by_row_s(row, datasets[dataset].years[i].year, (HOURLY_TIMERES == datasets[dataset].details->timeres), 0)); + fprintf(f, "%s,", timestamp_ww_get_by_row_s(row, datasets[dataset].years[i].year, datasets[dataset].details->timeres, 1)); + fprintf(f, "%s,", timestamp_ww_get_by_row_s(row, datasets[dataset].years[i].year, datasets[dataset].details->timeres, 0)); for ( z = 0; z < PERCENTILES_COUNT_2; ++z ) { fprintf(f, ",%g,%g", rows_night_weekly[j+row].night_columns_y[z] , rows_night_weekly[j+row].night_qc_columns_y[z]); @@ -6496,8 +6537,8 @@ int compute_datasets(DATASET *const datasets, const int datasets_count) { fputs("\n", f); for ( i = 0; i < datasets[dataset].years_count; i++ ) { for ( row = 0; row < y; row++ ) { - fprintf(f, "%s,", timestamp_ww_get_by_row_s(row, datasets[dataset].years[i].year, (HOURLY_TIMERES == datasets[dataset].details->timeres), 1)); - fprintf(f, "%s,", timestamp_ww_get_by_row_s(row, datasets[dataset].years[i].year, (HOURLY_TIMERES == datasets[dataset].details->timeres), 0)); + fprintf(f, "%s,", timestamp_ww_get_by_row_s(row, datasets[dataset].years[i].year, datasets[dataset].details->timeres, 1)); + fprintf(f, "%s,", timestamp_ww_get_by_row_s(row, datasets[dataset].years[i].year, datasets[dataset].details->timeres, 0)); for ( z = 0; z < PERCENTILES_COUNT_2; ++z ) { fprintf(f, ",%g,%g", rows_night_weekly[j+row].day_columns_y[z] , rows_night_weekly[j+row].day_qc_columns_y[z]); @@ -6533,8 +6574,8 @@ int compute_datasets(DATASET *const datasets, const int datasets_count) { fputs("\n", f); for ( i = 0; i < datasets[dataset].years_count; i++ ) { for ( row = 0; row < y; row++ ) { - fprintf(f, "%s,", timestamp_ww_get_by_row_s(row, datasets[dataset].years[i].year, (HOURLY_TIMERES == datasets[dataset].details->timeres), 1)); - fprintf(f, "%s,", timestamp_ww_get_by_row_s(row, datasets[dataset].years[i].year, (HOURLY_TIMERES == datasets[dataset].details->timeres), 0)); + fprintf(f, "%s,", timestamp_ww_get_by_row_s(row, datasets[dataset].years[i].year, datasets[dataset].details->timeres, 1)); + fprintf(f, "%s,", timestamp_ww_get_by_row_s(row, datasets[dataset].years[i].year, datasets[dataset].details->timeres, 0)); for ( z = 0; z < PERCENTILES_COUNT_2; ++z ) { fprintf(f, ",%g,%g", rows_night_weekly[j+row].night_columns_c[z] , rows_night_weekly[j+row].night_qc_columns_c[z]); @@ -6570,8 +6611,8 @@ int compute_datasets(DATASET *const datasets, const int datasets_count) { fputs("\n", f); for ( i = 0; i < datasets[dataset].years_count; i++ ) { for ( row = 0; row < y; row++ ) { - fprintf(f, "%s,", timestamp_ww_get_by_row_s(row, datasets[dataset].years[i].year, (HOURLY_TIMERES == datasets[dataset].details->timeres), 1)); - fprintf(f, "%s,", timestamp_ww_get_by_row_s(row, datasets[dataset].years[i].year, (HOURLY_TIMERES == datasets[dataset].details->timeres), 0)); + fprintf(f, "%s,", timestamp_ww_get_by_row_s(row, datasets[dataset].years[i].year, datasets[dataset].details->timeres, 1)); + fprintf(f, "%s,", timestamp_ww_get_by_row_s(row, datasets[dataset].years[i].year, datasets[dataset].details->timeres, 0)); for ( z = 0; z < PERCENTILES_COUNT_2; ++z ) { fprintf(f, ",%g,%g", rows_night_weekly[j+row].day_columns_c[z] , rows_night_weekly[j+row].day_qc_columns_c[z]); @@ -6661,9 +6702,9 @@ int compute_datasets(DATASET *const datasets, const int datasets_count) { exists = datasets[dataset].years[i].exist; for ( row = 0; row < 52; row++ ) { /* timestamp_start */ - fprintf(f, "%s,", timestamp_ww_get_by_row_s(row, datasets[dataset].years[i].year, (HOURLY_TIMERES == datasets[dataset].details->timeres), 1)); + fprintf(f, "%s,", timestamp_ww_get_by_row_s(row, datasets[dataset].years[i].year, datasets[dataset].details->timeres, 1)); /* timestamp_end */ - fprintf(f, "%s,", timestamp_ww_get_by_row_s(row, datasets[dataset].years[i].year, (HOURLY_TIMERES == datasets[dataset].details->timeres), 0)); + fprintf(f, "%s,", timestamp_ww_get_by_row_s(row, datasets[dataset].years[i].year, datasets[dataset].details->timeres, 0)); /* week */ fprintf(f, "%d,", row+1); if ( no_rand_unc ) { @@ -9295,6 +9336,8 @@ int compute_datasets(DATASET *const datasets, const int datasets_count) { free(nee_matrix_c_daily); free(nee_matrix_c); } + + clear_dataset(&datasets[dataset]); } /* free memory */ @@ -9313,9 +9356,9 @@ DATASET *get_datasets(const char *const path, int *const datasets_count) { int error; int assigned; int file_index; - int files_founded_count; + int files_found_count; FILE *f; - FILES *files_founded; + FILES *files_found; DD *details; YEAR *years_no_leak; DATASET *datasets; @@ -9329,32 +9372,32 @@ DATASET *get_datasets(const char *const path, int *const datasets_count) { *datasets_count = 0; /* scan path */ - files_founded = get_files(path, "*.csv", &files_founded_count, &error); - if ( error || !files_founded_count ) { - puts("no files founded!"); + files_found = get_files(path, "*.csv", &files_found_count, &error); + if ( error || !files_found_count ) { + puts("no files found!"); return NULL; } - /* loop on each files founded */ + /* loop on each files found */ skipped = 0; - for ( file_index = 0; file_index < files_founded_count; file_index++ ) { + for ( file_index = 0; file_index < files_found_count; file_index++ ) { /* check filename */ - if ( !is_valid_filename(files_founded[file_index].list[0].name) ) { + if ( !is_valid_filename(files_found[file_index].list[0].name) ) { ++skipped; continue; } /* open file */ - f = fopen(files_founded[file_index].list[0].fullpath, "r"); + f = fopen(files_found[file_index].list[0].fullpath, "r"); if ( !f ) { - printf("unable to open %s\n", files_founded[file_index].list[0].fullpath); + printf("unable to open %s\n", files_found[file_index].list[0].fullpath); continue; } /* get details */ details = parse_dd(f); if ( !details ) { - free_files(files_founded, files_founded_count); + free_files(files_found, files_found_count); free_datasets(datasets, *datasets_count); return NULL; } @@ -9395,7 +9438,7 @@ DATASET *get_datasets(const char *const path, int *const datasets_count) { if ( datasets[*datasets_count-1].years_count > 1 ) { if ( datasets[*datasets_count-1].details->timeres != details->timeres ) { puts("different time resolution between years!"); - free_files(files_founded, files_founded_count); + free_files(files_found, files_found_count); free_datasets(datasets, *datasets_count); return NULL; } @@ -9409,7 +9452,7 @@ DATASET *get_datasets(const char *const path, int *const datasets_count) { for ( y = 0; y < datasets[i].years_count; y++ ) { if ( details->year == datasets[i].years[y].year ) { puts(err_out_of_memory); - free_files(files_founded, files_founded_count); + free_files(files_found, files_found_count); free_datasets(datasets, *datasets_count); return NULL; } @@ -9436,12 +9479,12 @@ DATASET *get_datasets(const char *const path, int *const datasets_count) { } /* free memory */ - free_files(files_founded, files_founded_count); - files_founded_count -= skipped; + free_files(files_found, files_found_count); + files_found_count -= skipped; /* check imported files */ - if ( ! files_founded_count ) { - printf("no files founded!"); + if ( ! files_found_count ) { + printf("no files found!"); if ( skipped ) printf(" (%d file%s skipped)", skipped, (skipped > 1) ? "s" : ""); puts(""); diff --git a/oneflux_steps/nee_proc/src/main.c b/oneflux_steps/nee_proc/src/main.c index c7a648ae..f9b084be 100644 --- a/oneflux_steps/nee_proc/src/main.c +++ b/oneflux_steps/nee_proc/src/main.c @@ -1,7 +1,11 @@ /* main.c - this file is part of nee_proc + This file is part of the nee_proc step of processing. + It is responsible for the ustar filtering and gapfilling + of the nee using an ensamble of ustar thresholds. It also + performs the temporal aggregation at different time + resolutions. author: Alessio Ribeca owner: DIBAF - University of Tuscia, Viterbo, Italy @@ -20,7 +24,7 @@ #include "../../compiler.h" /* constants */ -#define PROGRAM_VERSION "v1.0" +#define PROGRAM_VERSION "v1.01" #define BUFFER_SIZE 1024 #define QC_AUTO_PATH "qc_auto" #define USTAR_MP_PATH "ustar_mp" diff --git a/oneflux_steps/nee_proc/src/randunc.c b/oneflux_steps/nee_proc/src/randunc.c index 23ebed89..254ce78e 100644 --- a/oneflux_steps/nee_proc/src/randunc.c +++ b/oneflux_steps/nee_proc/src/randunc.c @@ -172,10 +172,10 @@ void random_method_1(RAND_UNC_ROW *const unc_rows, const int unc_rows_count, con /* compute tolerance for SWin */ swin_tolerance = unc_rows[i].value[SWIN_UNC]; - if ( swin_tolerance < GF_SW_IN_TOLERANCE_MIN ) { - swin_tolerance = GF_SW_IN_TOLERANCE_MIN; - } else if ( swin_tolerance > GF_SW_IN_TOLERANCE_MAX ) { - swin_tolerance = GF_SW_IN_TOLERANCE_MAX; + if ( swin_tolerance < GF_DRIVER_1_TOLERANCE_MIN ) { + swin_tolerance = GF_DRIVER_1_TOLERANCE_MIN; + } else if ( swin_tolerance > GF_DRIVER_1_TOLERANCE_MAX ) { + swin_tolerance = GF_DRIVER_1_TOLERANCE_MAX; } /* loop through window */ @@ -187,9 +187,9 @@ void random_method_1(RAND_UNC_ROW *const unc_rows, const int unc_rows_count, con if ( (0 == unc_rows[window_current+y].qc[to_rand_index]) && IS_FLAG_SET(unc_rows[window_current+y].mask, GF_ALL_VALID) && - (FABS(unc_rows[window_current+y].value[TA_UNC]-unc_rows[i].value[TA_UNC]) < GF_TA_TOLERANCE) && + (FABS(unc_rows[window_current+y].value[TA_UNC]-unc_rows[i].value[TA_UNC]) < GF_DRIVER_2A_TOLERANCE_MIN) && (FABS(unc_rows[window_current+y].value[SWIN_UNC]-unc_rows[i].value[SWIN_UNC]) < swin_tolerance ) && - (FABS(unc_rows[window_current+y].value[VPD_UNC]-unc_rows[i].value[VPD_UNC]) < GF_VPD_TOLERANCE) ) { + (FABS(unc_rows[window_current+y].value[VPD_UNC]-unc_rows[i].value[VPD_UNC]) < GF_DRIVER_2B_TOLERANCE_MIN) ) { unc_rows[samples_count++].similiar = unc_rows[window_current+y].value[to_rand_index]; } } diff --git a/oneflux_steps/qc_auto/src/dataset.c b/oneflux_steps/qc_auto/src/dataset.c index 9e9eee04..41e69f19 100644 --- a/oneflux_steps/qc_auto/src/dataset.c +++ b/oneflux_steps/qc_auto/src/dataset.c @@ -109,7 +109,7 @@ const char *const var_flag_names[] = { "NEE", /* error strings */ static const char err_unable_open_file[] = "unable to open file."; -static const char err_redundancy[] = "redundancy: var \"%s\" already founded at column %d.\n"; +static const char err_redundancy[] = "redundancy: var \"%s\" already found at column %d.\n"; static const char err_unable_find_column[] = "unable to find column for \"%s\" var.\n"; static const char err_conversion[] = "error during conversion of \"%s\" value at row %d, column %d.\n"; static const char err_too_many_rows[] = "too many rows."; @@ -219,7 +219,7 @@ static int parse_header(DATASET *const dataset, char *header, const char *const if ( dataset->header[i].index ) { printf("_%d", dataset->header[i].index); } - printf(" founded at row %d and %d", i+1, j+1); + printf(" found at row %d and %d", i+1, j+1); return 0; } } diff --git a/oneflux_steps/qc_auto/src/main.c b/oneflux_steps/qc_auto/src/main.c index b9178a62..4e20b8a9 100644 --- a/oneflux_steps/qc_auto/src/main.c +++ b/oneflux_steps/qc_auto/src/main.c @@ -1,7 +1,10 @@ /* main.c - this file is part of qc_auto + This file is part of the qc_auto step of processing. + It is responsible for a number of basic quality check test + and the creation of the input files for the following + processing steps author: Alessio Ribeca owner: DIBAF - University of Tuscia, Viterbo, Italy @@ -23,7 +26,7 @@ #include "../../compiler.h" /* constants */ -#define PROGRAM_VERSION "v1.0" +#define PROGRAM_VERSION "v1.01" const int days_per_month[MONTHS] = { 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 }; /* enum */ @@ -93,7 +96,7 @@ int spike_check_3_return = SPIKE_CHECK_3_RETURN; /* see common.h */ PREC spike_threshold_nee = SPIKE_THRESHOLD_NEE; /* see common.h */ PREC spike_threshold_le = SPIKE_THRESHOLD_LE; /* see common.h */ PREC spike_threshold_h = SPIKE_THRESHOLD_H; /* see common.h */ -int files_founded_count; +int files_found_count; PREC height; static int doy; static int qc2_filter; /* default is off */ @@ -110,7 +113,7 @@ static int solar_output; /* default is off */ static int one_timestamp; /* default is off */ /* static global variables */ -static FILES *files_founded; +static FILES *files_found; /* strings */ static const char banner[] = "\nqc_auto "PROGRAM_VERSION"\n" @@ -134,7 +137,7 @@ static const char msg_dataset_path[] = "dataset path = %s\n"; static const char msg_output_path[] = "output path = %s\n\n"; static const char msg_processing[] = " - found %s, %d...ok\n"; static const char msg_ok[] = "ok"; -static const char msg_summary[] = "\n%d file%s founded: %d processed, %d skipped.\n\n"; +static const char msg_summary[] = "\n%d file%s found: %d processed, %d skipped.\n\n"; static const char msg_usage[] = "usage: qc_auto parameters output_formats\n\n" "parameters:\n\n" "-input_path=filename or path to be processed (optional)\n" @@ -221,8 +224,8 @@ static void clean_up(void) { if ( program_path ) { free(program_path); } - if ( files_founded ) { - free_files(files_founded, files_founded_count); + if ( files_found ) { + free_files(files_found, files_found_count); } check_memory_leak(); } @@ -1008,11 +1011,11 @@ static void set_invalid_by_flag(DATASET *const dataset, const char *const var, c } /* */ -static char *timestamp_for_sr(int row, int yy, const int hourly) { +static char *timestamp_for_sr(int row, int yy, const int timeres) { TIMESTAMP *t; static char buffer[16+1] = { 0 }; - t = timestamp_end_by_row(row, yy, hourly); + t = timestamp_end_by_row(row, yy, timeres); sprintf(buffer, "%02d,%02d,%04d,%02d,%02d", t->MM, t->DD, t->YYYY, t->hh, t->mm); /* */ @@ -1189,10 +1192,10 @@ static int save_db_file(DATASET *const dataset) { for ( i = 0; i < dataset->rows_count; i++ ) { /* ...timestamp_start.... */ if ( ! one_timestamp ) { - fprintf(f, "%s,", timestamp_start_by_row_s(i, dataset->details->year, (HOURLY_TIMERES == dataset->details->timeres))); + fprintf(f, "%s,", timestamp_start_by_row_s(i, dataset->details->year, dataset->details->timeres)); } /* ...timestamp_end.... */ - fprintf(f, "%s", timestamp_end_by_row_s(i, dataset->details->year, (HOURLY_TIMERES == dataset->details->timeres))); + fprintf(f, "%s", timestamp_end_by_row_s(i, dataset->details->year, dataset->details->timeres)); /* ...values... */ for ( y = 0; y < SIZEOF_ARRAY(vars_index); y++ ) { var = get_var_index(dataset, var_names[vars_index[y]]); @@ -1797,10 +1800,10 @@ static int save_graph_file(DATASET *const dataset) { for ( i = 0; i < dataset->rows_count; i++ ) { /* ...timestamp_start.... */ if ( ! one_timestamp ) { - fprintf(f, "%s,", timestamp_start_by_row_s(i, dataset->details->year, (HOURLY_TIMERES == dataset->details->timeres))); + fprintf(f, "%s,", timestamp_start_by_row_s(i, dataset->details->year, dataset->details->timeres)); } /* ...timestamp_end.... */ - fprintf(f, "%s", timestamp_end_by_row_s(i, dataset->details->year, (HOURLY_TIMERES == dataset->details->timeres))); + fprintf(f, "%s", timestamp_end_by_row_s(i, dataset->details->year, dataset->details->timeres)); /* ...values... */ for ( y = 0; y < SIZEOF_ARRAY(vars_index); y++ ) { var = get_var_index(dataset, var_names[vars_index[y]]); @@ -1917,10 +1920,10 @@ static int save_nee_file(DATASET *const dataset) { for ( i = 0; i < dataset->rows_count; i++ ) { /* ...timestamp_start.... */ if ( ! one_timestamp ) { - fprintf(f, "%s,", timestamp_start_by_row_s(i, dataset->details->year, (HOURLY_TIMERES == dataset->details->timeres))); + fprintf(f, "%s,", timestamp_start_by_row_s(i, dataset->details->year, dataset->details->timeres)); } /* ...timestamp_end.... */ - fprintf(f, "%s", timestamp_end_by_row_s(i, dataset->details->year, (HOURLY_TIMERES == dataset->details->timeres))); + fprintf(f, "%s", timestamp_end_by_row_s(i, dataset->details->year, dataset->details->timeres)); /* ...values... */ for ( y = 0; y < SIZEOF_ARRAY(vars_index); y++ ) { var = get_var_index(dataset, var_names[vars_index[y]]); @@ -2014,10 +2017,10 @@ static int save_energy_file(DATASET *const dataset) { for ( i = 0; i < dataset->rows_count; i++ ) { /* ...timestamp_start.... */ if ( ! one_timestamp ) { - fprintf(f, "%s,", timestamp_start_by_row_s(i, dataset->details->year, (HOURLY_TIMERES == dataset->details->timeres))); + fprintf(f, "%s,", timestamp_start_by_row_s(i, dataset->details->year, dataset->details->timeres)); } /* ...timestamp_end.... */ - fprintf(f, "%s", timestamp_end_by_row_s(i, dataset->details->year, (HOURLY_TIMERES == dataset->details->timeres))); + fprintf(f, "%s", timestamp_end_by_row_s(i, dataset->details->year, dataset->details->timeres)); /* ...values... */ for ( y = 0; y < SIZEOF_ARRAY(vars_index); y++ ) { var = get_var_index(dataset, var_names[vars_index[y]]); @@ -2093,10 +2096,10 @@ static int save_ustar_file(DATASET *const dataset) { for ( i = 0; i < dataset->rows_count; i++ ) { /* ...timestamp_start.... */ if ( ! one_timestamp ) { - fprintf(f, "%s,", timestamp_start_by_row_s(i, dataset->details->year, (HOURLY_TIMERES == dataset->details->timeres))); + fprintf(f, "%s,", timestamp_start_by_row_s(i, dataset->details->year, dataset->details->timeres)); } /* ...timestamp_end.... */ - fprintf(f, "%s", timestamp_end_by_row_s(i, dataset->details->year, (HOURLY_TIMERES == dataset->details->timeres))); + fprintf(f, "%s", timestamp_end_by_row_s(i, dataset->details->year, dataset->details->timeres)); /* ...values... */ for ( y = 0; y < SIZEOF_ARRAY(vars_index); y++ ) { var = get_var_index(dataset, var_names[vars_index[y]]); @@ -2973,10 +2976,10 @@ static int save_meteo_file(DATASET *const dataset) { for ( i = 0; i < dataset->rows_count; i++ ) { /* ...timestamp_start.... */ if ( ! one_timestamp ) { - fprintf(f, "%s,", timestamp_start_by_row_s(i, dataset->details->year, (HOURLY_TIMERES == dataset->details->timeres))); + fprintf(f, "%s,", timestamp_start_by_row_s(i, dataset->details->year, dataset->details->timeres)); } /* ...timestamp_end.... */ - fprintf(f, "%s", timestamp_end_by_row_s(i, dataset->details->year, (HOURLY_TIMERES == dataset->details->timeres))); + fprintf(f, "%s", timestamp_end_by_row_s(i, dataset->details->year, dataset->details->timeres)); /* ...values... */ if ( dataset->rows_count != dataset->missings[CO2] ) fprintf(f, ",%g", dataset->rows[i].value[CO2]); if ( dataset->rows_count != dataset->missings[P] ) fprintf(f, ",%g", dataset->rows[i].value[P]); @@ -3163,14 +3166,14 @@ static int save_sr_file(DATASET *const dataset) { /* write values */ for ( i = 0; i < dataset->rows_count; i++ ) { /* ...timestamp.... */ - fprintf(f, "%s,", timestamp_for_sr(i, dataset->details->year, (HOURLY_TIMERES == dataset->details->timeres))); + fprintf(f, "%s,", timestamp_for_sr(i, dataset->details->year, dataset->details->timeres)); /* ...values... */ fprintf(f, "%g,", dataset->rows[i].value[FC]); fprintf(f, "%g,", dataset->rows[i].value[SC]); fprintf(f, "%g,", dataset->rows[i].value[USTAR]); fprintf(f, "%g,", dataset->rows[i].value[SWIN]); fprintf(f, "%g,", dataset->rows[i].value[TS]); - fprintf(f, "%s,", timestamp_end_by_row_s(i, dataset->details->year, (HOURLY_TIMERES == dataset->details->timeres))); + fprintf(f, "%s,", timestamp_end_by_row_s(i, dataset->details->year, dataset->details->timeres)); fprintf(f, "%g,", dataset->rows[i].value[TA]); fprintf(f, "%g\n", dataset->rows[i].value[VPD]); } @@ -3199,7 +3202,7 @@ static int set_sc_negl(DATASET *const dataset) { } for ( i = 0; i < dataset->details->sc_negles_count; i++ ) { - z = get_row_by_timestamp(&dataset->details->sc_negles[i].timestamp, (HOURLY_TIMERES == dataset->details->timeres)); + z = get_row_by_timestamp(&dataset->details->sc_negles[i].timestamp, dataset->details->timeres); for ( y = z; y < dataset->rows_count; y++ ) { dataset->rows[y].value[SC_NEGL] = dataset->details->sc_negles[i].flag; } @@ -3374,7 +3377,7 @@ int main(int argc, char *argv[]) { printf(msg_output_path, output_path); /* get files */ - files_founded = get_files(program_path, input_path, &files_founded_count, &error); + files_found = get_files(program_path, input_path, &files_found_count, &error); if ( error ) { return 1; } @@ -3385,13 +3388,13 @@ int main(int argc, char *argv[]) { total_files_count = 0; /* loop for searching file */ - for ( z = 0; z < files_founded_count; z++) { + for ( z = 0; z < files_found_count; z++) { /* inc */ ++total_files_count; /* import dataset */ - printf("processing %s...", files_founded[z].list[0].name); - dataset = import_dataset(files_founded[z].list[0].fullpath); + printf("processing %s...", files_found[z].list[0].name); + dataset = import_dataset(files_found[z].list[0].fullpath); if ( !dataset ) { puts("nothing found."); ++files_not_processed_count; diff --git a/oneflux_steps/tools/SW_IN_POT_Calculator/src/main.c b/oneflux_steps/tools/SW_IN_POT_Calculator/src/main.c index d9e4d40d..05e74493 100644 --- a/oneflux_steps/tools/SW_IN_POT_Calculator/src/main.c +++ b/oneflux_steps/tools/SW_IN_POT_Calculator/src/main.c @@ -1,7 +1,10 @@ /* main.c - this file is part of SW_IN_POT_Calculator + This file is part of SW_IN_POT_Calculator that + calculates the extra-atmosphere potential short-wave + incoming radiation based on geographical location + and time. author: Alessio Ribeca owner: DIBAF - University of Tuscia, Viterbo, Italy @@ -16,7 +19,7 @@ #include "../../../common/common.h" /* constants */ -#define PROGRAM_VERSION "1.0" +#define PROGRAM_VERSION "1.01" #define DEFAULT_EXT ".csv" #define MAX_FILENAME_SIZE 64 @@ -220,10 +223,10 @@ int main(int argc, char *argv[]) { char* p; /* TIMESTAMP_START */ - p = timestamp_start_by_row_s(i, details.year, (HOURLY_TIMERES == details.timeres)); + p = timestamp_start_by_row_s(i, details.year, details.timeres); fputs(p, f); /* TIMESTAMP_END */ - p = timestamp_end_by_row_s(i, details.year, (HOURLY_TIMERES == details.timeres)); + p = timestamp_end_by_row_s(i, details.year, details.timeres); fprintf(f, ",%s,%g\n", p, rpot[i]); } diff --git a/oneflux_steps/tools/gf_mds/Makefile b/oneflux_steps/tools/gf_mds/Makefile new file mode 100644 index 00000000..5946c3ff --- /dev/null +++ b/oneflux_steps/tools/gf_mds/Makefile @@ -0,0 +1,49 @@ +# enables most optimization flags +CC := gcc -O3 + +# create directory command +MKDIR = mkdir -p + +# copy file command (verbose, keep file metadata) +COPY = cp -av + +SRCDIR := $(shell pwd)/ +TGTDIR := + +COMMON_DIR := $(join ${SRCDIR}, ../../common/) +COMMON_SRC := common.c +COMMON_OBJ := $(COMMON_SRC:.c=.o) +COMMON_TGT := common + +GF_MDS_DIR := $(join ${SRCDIR}, src) +GF_MDS_SRC := dataset.c main.c +GF_MDS_OBJ := $(GF_MDS_SRC:.c=.o) +GF_MDS_BIN := $(join ${TGTDIR}, gf_mds) + +all: directories ${GF_MDS_BIN} + @echo "\nBuild finished." + +.PHONY: all clean ${COMMON_TGT} directories + +directories: ${TGTDIR} + +${TGTDIR}: + @echo "\nCreating binary target directory..." + ${MKDIR} ${TGTDIR} + +${COMMON_TGT}: + @echo "\nBuilding common objects..." + cd ${COMMON_DIR} ;\ + ${CC} -w -c ${COMMON_SRC} + +${GF_MDS_BIN}: ${COMMON_TGT} + @echo "\nBuilding ${GF_MDS_BIN}..." + cd ${GF_MDS_DIR} ;\ + ${CC} -w -c ${GF_MDS_SRC} ;\ + ${CC} ${GF_MDS_OBJ} $(join ${COMMON_DIR}, ${COMMON_OBJ}) -w -lm -o ../${GF_MDS_BIN} + +clean: + @echo "\nCleaning up..." + cd ${COMMON_DIR} ; rm -vf ${COMMON_OBJ} + cd ${GF_MDS_DIR} ; rm -vf ${GF_MDS_OBJ} ; rm -vf ${GF_MDS_BIN} + @echo "Done cleaning up...\n" diff --git a/oneflux_steps/tools/gf_mds/src/dataset.c b/oneflux_steps/tools/gf_mds/src/dataset.c index 4a8feb67..9efda51e 100644 --- a/oneflux_steps/tools/gf_mds/src/dataset.c +++ b/oneflux_steps/tools/gf_mds/src/dataset.c @@ -20,6 +20,7 @@ typedef struct { ROW *rows; int *columns; + TIMESTAMP *timestamps; int rows_count; int allocated_rows_count; } DATASET; @@ -27,31 +28,28 @@ typedef struct { /* extern variables */ extern int *years; extern int years_count; -extern DD **details_list; -extern int hourly_dataset; -extern int has_dtime; +extern int timeres; extern int custom_tokens[GF_TOKENS]; extern const char def_tokens[GF_TOKENS][GF_TOKEN_LENGTH_MAX+1]; extern char tokens[GF_TOKENS][GF_TOKEN_LENGTH_MAX+1]; -extern const PREC dtime[LEAP_YEAR_ROWS]; /* strings */ static const char delimiter[] = ", "; -static const char Isodate[] = "TIMESTAMP"; -static const char Dtime[] = "Dtime"; /* error strings */ static const char err_redundancy[] = "redundancy: var \"%s\" already founded at column %d.\n"; static const char err_unable_find_column[] = "unable to find column for \"%s\" var.\n"; static const char err_conversion[] = "error during conversion of \"%s\" value at row %d, column %d.\n"; -static const char err_isodate_conversion[] = "error during conversion of TIMESTAMP at row %d.\n"; -static const char err_row_assigned[] = "row %d already assigned! dtime error.\n"; +static const char err_timestamp_conversion[] = "error during conversion of %s at row %d.\n"; +static const char err_row_assigned[] = "row %d already assigned!\n"; static const char err_invalid_index[]= "invalid %s at row %d\n"; static const char err_unable_get_header[] = "unable to get header."; static const char err_unable_to_import_all_values[] = "unable to import all values for row %d\n"; -static const char err_too_many_rows_csv_standard[] = "too many rows imported: %d should be %d. Timeres specified is %s.\n"; -static const char err_too_many_rows[] = "too many rows imported: %d should be %d.\n"; -static const char err_no_isodate[]= "unable to concatenate dataset with %s. please use %s instead.\n"; +static const char err_no_timestamp[]= "unable to concatenate dataset with %s. please use %s instead.\n"; +static const char err_invalid_timestamp[] = "invalid timestamp at row %d: %04d%02d%02d%02d%02d%02d\n"; +static const char err_invalid_freq[] = "invalid timestamp at row %d\n"; +static const char err_unable_to_create_debug_file[] = "unable to create import debug file: %s\n"; +static const char err_too_many_rows[] = "too many rows found! %d instead of %d\n"; /* extern error strings */ extern const char err_empty_file[]; @@ -63,23 +61,14 @@ static void free_datasets(DATASET *datasets, const int count) { int i; for ( i = 0; i < count; i++ ) { + free(datasets[i].timestamps); free(datasets[i].rows); free(datasets[i].columns); } free(datasets); } -/* */ -void free_details_list(DD **details_list, const int count) { - int i; - - for ( i = 0; i < count; i++ ) { - free_dd(details_list[i]); - } - free(details_list); -} - -/* */ +/* updated on January 17, 2018 */ ROW *import_dataset(const LIST *const list, const int list_count, int *const rows_count) { int i; int y; @@ -88,7 +77,7 @@ ROW *import_dataset(const LIST *const list, const int list_count, int *const row int file; int assigned_required_values_count; int error; - int is_csv_standard; + int old_year; char *p; char *token; FILE *f; @@ -96,18 +85,20 @@ ROW *import_dataset(const LIST *const list, const int list_count, int *const row ROW *rows; ROW *rows_no_leak; DATASET *datasets; - TIMESTAMP *t; char buffer[BUFFER_SIZE]; /* check parameters */ assert(list && rows_count); + *rows_count = 0; + /* alloc memory for datasets */ datasets = malloc(list_count*sizeof*datasets); if ( !datasets ) { puts(err_out_of_memory); return NULL; } + memset(datasets, 0, list_count*sizeof*datasets); /* alloc memory for rows and columns in datasets */ for ( i = 0; i < list_count; i++ ) { @@ -125,36 +116,26 @@ ROW *import_dataset(const LIST *const list, const int list_count, int *const row free_datasets(datasets, list_count); return NULL; } - } - /* alloc memory for details list */ - details_list = malloc(sizeof*details_list*list_count); - if ( !details_list ) { - puts(err_out_of_memory); - free_datasets(datasets, list_count); - return NULL; - } - for ( i = 0; i < list_count; i++ ) { - details_list[i] = NULL; + datasets[i].timestamps = malloc(datasets[i].allocated_rows_count*sizeof*datasets[i].timestamps); + if ( !datasets[i].timestamps ) { + puts(err_out_of_memory); + free_datasets(datasets, list_count); + return NULL; + } + memset(datasets[i].timestamps, 0, datasets[i].allocated_rows_count*sizeof*datasets[i].timestamps); } /* alloc memory for years */ years = malloc(list_count*sizeof*years); if ( !years ) { puts(err_out_of_memory); - free_details_list(details_list, list_count); free_datasets(datasets, list_count); return NULL; } - /* has dtime ? */ - if ( has_dtime ) { - if ( ! string_compare_i(tokens[GF_ROW_INDEX], Isodate) ) { - strcpy(tokens[GF_ROW_INDEX], Dtime); - } - } - /* loop for each file */ + //*rows_count = 0; /* we must reset here 'cause we can have concatenated datasets */ for ( file = 0; file < list_count; file++ ) { /* reset datasets */ datasets[file].rows_count = 0; @@ -163,7 +144,6 @@ ROW *import_dataset(const LIST *const list, const int list_count, int *const row f = fopen(list[file].fullpath, "r"); if ( !f ) { puts(err_unable_open_file); - free_details_list(details_list, list_count); free_datasets(datasets, list_count); return 0; } @@ -172,7 +152,6 @@ ROW *import_dataset(const LIST *const list, const int list_count, int *const row if ( !get_valid_line_from_file(f, buffer, BUFFER_SIZE) ) { puts(err_empty_file); fclose(f); - free_details_list(details_list, list_count); free_datasets(datasets, list_count); return 0; } @@ -180,57 +159,30 @@ ROW *import_dataset(const LIST *const list, const int list_count, int *const row /* reset column positions */ for ( i = 0; i < GF_REQUIRED_DATASET_VALUES; i++ ) { datasets[file].columns[i] = -1; - } - - /* no details ? */ - is_csv_standard = 0; - if ( ! string_n_compare_i(buffer, "site", 4) ) { - /* update flag */ - is_csv_standard = 1; - - /* rewind file */ - fseek(f, 0, SEEK_SET); - - /* get details */ - details_list[file] = parse_dd(f); - if ( !details_list[file] ) { - fclose(f); - free_details_list(details_list, list_count); - free_datasets(datasets, list_count); - return 0; - } - - /* hourly ? */ - hourly_dataset = 0; - if ( HOURLY_TIMERES == details_list[file]->timeres ) { - hourly_dataset = 1; - } - - if ( !get_valid_line_from_file(f, buffer, BUFFER_SIZE) ) { - puts(err_unable_get_header); - free_details_list(details_list, list_count); - free_datasets(datasets, list_count); - fclose(f); - return NULL; - } - } + } /* parse header */ for ( i = 0, token = string_tokenizer(buffer, delimiter, &p); token; token = string_tokenizer(NULL, delimiter, &p), ++i ) { for ( y = 0; y < GF_REQUIRED_DATASET_VALUES; y++ ) { - if ( !string_compare_i(token, tokens[y]) ) { + if ( ! string_compare_i(token, tokens[y]) ) { /* check if column was already assigned */ if ( -1 != datasets[file].columns[y] ) { printf(err_redundancy, tokens[y], datasets[file].columns[y]+1); - free_details_list(details_list, list_count); free_datasets(datasets, list_count); fclose(f); return NULL; } else { /* assign column position */ datasets[file].columns[y] = i; - /* do not break loop 'cause we can use var to be filled in methods! */ - /* break */ + + /* use same case as input for tofill var */ + if ( GF_TOFILL == y ) + { + strcpy(tokens[y], token); + } + + /* do not break loop for var 'cause we can use var to be filled in methods! */ + /*break;*/ } } } @@ -240,7 +192,6 @@ ROW *import_dataset(const LIST *const list, const int list_count, int *const row for ( i = 0; i < GF_REQUIRED_DATASET_VALUES; i++ ) { if ( -1 == datasets[file].columns[i] ) { printf(err_unable_find_column, tokens[i]); - free_details_list(details_list, list_count); free_datasets(datasets, list_count); fclose(f); return NULL; @@ -251,15 +202,13 @@ ROW *import_dataset(const LIST *const list, const int list_count, int *const row while ( get_valid_line_from_file(f, buffer, BUFFER_SIZE) ) { /* alloc rows if needed */ if ( datasets[file].rows_count++ == datasets[file].allocated_rows_count ) { - i = LEAP_YEAR_ROWS; - if ( hourly_dataset ) { - i /= 2; - } + /* we allocate leap rows to prevent out of bounds! */ + i = get_rows_count_by_timeres(timeres, 2000); + datasets[file].allocated_rows_count += i; rows_no_leak = realloc(datasets[file].rows, datasets[file].allocated_rows_count*sizeof*rows_no_leak); if ( !rows_no_leak ) { puts(err_out_of_memory); - free_details_list(details_list, list_count); free_datasets(datasets, list_count); fclose(f); return NULL; @@ -267,6 +216,27 @@ ROW *import_dataset(const LIST *const list, const int list_count, int *const row /* assign pointer */ datasets[file].rows = rows_no_leak; + + /* re-alloc memory for timestamps */ + { + int z; + TIMESTAMP* timestamp_no_leak; + + timestamp_no_leak = realloc(datasets[file].timestamps, datasets[file].allocated_rows_count*sizeof*timestamp_no_leak); + if ( !timestamp_no_leak ) { + puts(err_out_of_memory); + free_datasets(datasets, list_count); + fclose(f); + return NULL; + } + + /* assign pointer */ + datasets[file].timestamps = timestamp_no_leak; + + for ( z = i; z < datasets[file].allocated_rows_count; ++z ) { + memset(&datasets[file].timestamps[z], 0, sizeof(TIMESTAMP)); + } + } } /* get values */ @@ -275,24 +245,45 @@ ROW *import_dataset(const LIST *const list, const int list_count, int *const row /* loop for each mandatory values */ for ( y = 0; y < GF_REQUIRED_DATASET_VALUES; y++ ) { if ( datasets[file].columns[y] == i ) { - if ( (GF_ROW_INDEX == y) && !has_dtime ) { + if ( GF_ROW_INDEX == y ) { + TIMESTAMP* t; + t = get_timestamp(token); if ( ! t ) { printf(err_conversion, token, i+1, datasets[file].rows_count); - free_details_list(details_list, list_count); free_datasets(datasets, list_count); fclose(f); return NULL; } + datasets[file].timestamps[datasets[file].rows_count-1] = *t; + /* set year */ if ( !(datasets[file].rows_count-1) ) { years[file] = t->YYYY; + old_year = years[file]; + *rows_count += get_rows_count_by_timeres(timeres, t->YYYY); + } else { + /* + check if year is changed... + we don't know if is a multi-year dataset... + + */ + if ( old_year != t->YYYY ) { + /* check if is last year */ + if ( (t->MM != 1) + || (t->DD != 1) + || (t->hh != 0) + || (t->mm != 0) ) { + *rows_count += get_rows_count_by_timeres(timeres, t->YYYY); + old_year = t->YYYY; + } + } + } - j = get_row_by_timestamp(t, hourly_dataset); + j = get_row_by_timestamp(t, timeres); free(t); if ( -1 == j ) { - printf(err_isodate_conversion, datasets[file].rows_count); - free_details_list(details_list, list_count); + printf(err_timestamp_conversion, tokens[GF_ROW_INDEX], datasets[file].rows_count); free_datasets(datasets, list_count); fclose(f); return NULL; @@ -303,26 +294,10 @@ ROW *import_dataset(const LIST *const list, const int list_count, int *const row value = convert_string_to_prec(token, &error); if ( error ) { printf(err_conversion, token, i+1, datasets[file].rows_count); - free_details_list(details_list, list_count); free_datasets(datasets, list_count); fclose(f); return NULL; } - - /* convert dtime to row */ - if ( (GF_ROW_INDEX == y) && has_dtime ) { - if ( hourly_dataset ) { - j = DTIME_TO_ROW_HOURLY(value); - } else { - j = DTIME_TO_ROW(value); - } - - /* fix to zero based index */ - --j; - - /* assign it */ - value = j; - } } /* check for NAN */ @@ -342,54 +317,128 @@ ROW *import_dataset(const LIST *const list, const int list_count, int *const row /* check if all required values have been imported */ if ( assigned_required_values_count != GF_REQUIRED_DATASET_VALUES ) { printf(err_unable_to_import_all_values, datasets[file].rows_count); - free_details_list(details_list, list_count); free_datasets(datasets, list_count); fclose(f); return NULL; } - - /* set year for old dataset */ - if ( !(datasets[file].rows_count-1) && has_dtime ) { - years[file] = INVALID_VALUE; - } } /* close file */ fclose(f); } + /* save imported file for debugging purposes */ +#ifdef _DEBUG + { + for ( file = 0; file < list_count; file++ ) { + char buf[256]; + FILE* s; + + sprintf(buf, "dataset_debug_%02d.csv", file+1); + s = fopen(buf, "w"); + if ( ! s ) { + printf(err_unable_to_create_debug_file, buf); + free_datasets(datasets, list_count); + fclose(f); + return NULL; + } + fputs("ROW_INDEX,", f); + fprintf(f, "%s,", tokens[GF_TOFILL]); + fprintf(f, "%s,", tokens[GF_DRIVER_1]); + fprintf(f, "%s,", tokens[GF_DRIVER_2A]); + fprintf(f, "%s\n", tokens[GF_DRIVER_2B]); + for ( i = 0; i < datasets[file].rows_count; ++i ) { + fprintf(f, "%g,%g,%g,%g,%g\n" + , datasets[file].rows[i].value[GF_ROW_INDEX] + , datasets[file].rows[i].value[GF_TOFILL] + , datasets[file].rows[i].value[GF_DRIVER_1] + , datasets[file].rows[i].value[GF_DRIVER_2A] + , datasets[file].rows[i].value[GF_DRIVER_2B] + ); + } + fclose(s); + } + } +#endif + /* create clean dataset */ j = 0; rows = NULL; for ( file = 0; file < list_count; file++ ) { - /* csv standard ? */ - if ( details_list[file] ) { - i = get_rows_count_by_dd(details_list[file]); - } else { - /* number of rows we want to allocate */ - i = LEAP_YEAR_ROWS; - - /* check if a full not leap dataset was imported */ - if ( (hourly_dataset ? YEAR_ROWS / 2 : YEAR_ROWS) == datasets[file].rows_count ) { - i = YEAR_ROWS; + /* get number of rows by year */ + i = get_rows_count_by_timeres(timeres, years[file]); + + /* check if imported rows are > than year's rows count */ + if ( datasets[file].rows_count > i ) { + printf(err_too_many_rows, datasets[file].rows_count, i); + free(rows); + free_datasets(datasets, list_count); + return NULL; + } + + /* parse timestamps */ + { + int freq; /* timeres */ + int row; + + for( z = 0; z < datasets[file].rows_count; ++z ) { + if ( ! check_timestamp(&datasets[file].timestamps[z]) ) { + printf(err_invalid_timestamp, z+1 + , datasets[file].timestamps[z].YYYY + , datasets[file].timestamps[z].MM + , datasets[file].timestamps[z].DD + , datasets[file].timestamps[z].hh + , datasets[file].timestamps[z].mm + , datasets[file].timestamps[z].ss + ); + free(rows); + free_datasets(datasets, list_count); + return NULL; + } } - if ( hourly_dataset ) { - i /= 2; + /* get timeres by timestamps differences */ + freq = timestamp_difference_in_seconds(&datasets[file].timestamps[1], &datasets[file].timestamps[0]); + row = 0; + for( z = 1; z < datasets[file].rows_count-1; ++z ) { + int diff; + + diff = timestamp_difference_in_seconds(&datasets[file].timestamps[z+1], &datasets[file].timestamps[z]); + + if ( diff < freq ) { + freq = diff; + row = z; + } } - } - /* check if imported rows are > than i (counts of rows we wants to allocate) */ - if ( datasets[file].rows_count > i ) { - if ( details_list[file] ) { - printf(err_too_many_rows_csv_standard, datasets[file].rows_count, i, get_timeres_in_string(details_list[file]->timeres)); - } else { - printf(err_too_many_rows, datasets[file].rows_count, i); + /* check timeres */ + freq /= 60; + { + int err; + + err = 0; + switch ( timeres ) + { + case QUATERHOURLY_TIMERES: + if ( freq != 15 ) err = 1; + break; + + case HALFHOURLY_TIMERES: + if ( freq != 30 ) err = 1; + break; + + case HOURLY_TIMERES: + if ( freq != 60 ) err = 1; + break; + } + + if ( err ) { + printf(err_invalid_freq, row); + free(rows); + free_datasets(datasets, list_count); + return NULL; + } } - free(rows); - free_details_list(details_list, list_count); - free_datasets(datasets, list_count); - return NULL; } /* alloc memory */ @@ -397,7 +446,6 @@ ROW *import_dataset(const LIST *const list, const int list_count, int *const row if ( !rows_no_leak ) { puts(err_out_of_memory); free(rows); - free_details_list(details_list, list_count); free_datasets(datasets, list_count); return NULL; } @@ -408,70 +456,105 @@ ROW *import_dataset(const LIST *const list, const int list_count, int *const row /* reset newly rows */ for ( y = 0; y < i; y++ ) { for ( z = 0; z < GF_REQUIRED_DATASET_VALUES; z++ ) { - rows[j+y].value[z] = INVALID_VALUE; + if ( GF_ROW_INDEX == z ) { + rows[j+y].value[z] = j+y; + } else { + rows[j+y].value[z] = INVALID_VALUE; + } } rows[j+y].assigned = 0; } /* assign rows */ - z = i; - for ( i = 0; i < datasets[file].rows_count; i++ ) { - y = (int)datasets[file].rows[i].value[GF_ROW_INDEX]; - - /* check row index */ - if ( (y < 0) || (y >= z) ) { - printf(err_invalid_index, has_dtime ? Dtime : Isodate, i+1); - free(rows); - free_details_list(details_list, list_count); - free_datasets(datasets, list_count); - return NULL; - } - - /* check if row was already assigned */ - if ( rows[j+y].assigned ) { - printf(err_row_assigned, i+1); - free(rows); - free_details_list(details_list, list_count); - free_datasets(datasets, list_count); - return NULL; - } + { + int old_year; + int k; + + z = i; + k = 0; + old_year = datasets[file].timestamps[0].YYYY; + for ( i = 0; i < datasets[file].rows_count; i++ ) { + y = (int)datasets[file].rows[i].value[GF_ROW_INDEX]; + + if ( (old_year != datasets[file].timestamps[i].YYYY) + && ( + (datasets[file].timestamps[i].MM != 1) + || (datasets[file].timestamps[i].DD != 1) + || (datasets[file].timestamps[i].hh != 0) + || (datasets[file].timestamps[i].mm != 0) + ) + ) { + old_year = datasets[file].timestamps[i].YYYY; + k += get_rows_count_by_timeres(timeres, old_year); + } + + /* check row index */ + if ( (y < 0) || (y >= z) ) { + printf(err_invalid_index, tokens[GF_ROW_INDEX], i+1); + free(rows); + free_datasets(datasets, list_count); + return NULL; + } - /* assign values */ - rows[j+y].value[GF_TOFILL] = datasets[file].rows[i].value[GF_TOFILL]; - rows[j+y].value[GF_SWIN] = datasets[file].rows[i].value[GF_SWIN]; - rows[j+y].value[GF_TA] = datasets[file].rows[i].value[GF_TA]; - rows[j+y].value[GF_VPD] = datasets[file].rows[i].value[GF_VPD]; - rows[j+y].value[GF_ROW_INDEX] = datasets[file].rows[i].value[GF_ROW_INDEX]; - rows[j+y].assigned = 1; + /* check if row was already assigned */ + if ( rows[j+y+k].assigned ) { + printf(err_row_assigned, i+1); + free(rows); + free_datasets(datasets, list_count); + return NULL; + } - /* inc */ - /* ++*rows_count; */ + /* assign values */ + rows[j+y+k].value[GF_TOFILL] = datasets[file].rows[i].value[GF_TOFILL]; + rows[j+y+k].value[GF_DRIVER_1] = datasets[file].rows[i].value[GF_DRIVER_1]; + rows[j+y+k].value[GF_DRIVER_2A] = datasets[file].rows[i].value[GF_DRIVER_2A]; + rows[j+y+k].value[GF_DRIVER_2B] = datasets[file].rows[i].value[GF_DRIVER_2B]; + rows[j+y+k].value[GF_ROW_INDEX] = datasets[file].rows[i].value[GF_ROW_INDEX]; + rows[j+y+k].assigned = 1; + } } /* keep track of allocated rows */ j += z; } + //assert(*rows_count == j); *rows_count = j; - /* fix years for old dataset type */ - y = 0; - for ( i = 0; i < list_count; i++ ) { - if ( IS_INVALID_VALUE(years[i]) ) { - ++y; + /* save imported file for debugging purposes */ +#ifdef _DEBUG + { + char buf[256]; + FILE* s; + + sprintf(buf, "dataset_debug_clean.csv"); + s = fopen(buf, "w"); + if ( ! s ) { + printf(err_unable_to_create_debug_file, buf); + free_datasets(datasets, list_count); + fclose(f); + return NULL; } + fputs("ROW_INDEX,", f); + fprintf(f, "%s,", tokens[GF_TOFILL]); + fprintf(f, "%s,", tokens[GF_DRIVER_1]); + fprintf(f, "%s,", tokens[GF_DRIVER_2A]); + fprintf(f, "%s\n", tokens[GF_DRIVER_2B]); + for ( i = 0; i < *rows_count; ++i ) { + fprintf(f, "%g,%g,%g,%g,%g\n" + , rows[i].value[GF_ROW_INDEX] + , rows[i].value[GF_TOFILL] + , rows[i].value[GF_DRIVER_1] + , rows[i].value[GF_DRIVER_2A] + , rows[i].value[GF_DRIVER_2B] + ); + } + fclose(s); } +#endif /* free memory */ free_datasets(datasets, list_count); - /* */ - if ( y > 1 ) { - printf(err_no_isodate, Dtime, Isodate); - free(rows); - free_details_list(details_list, list_count); - rows = NULL; - } - /* return pointer */ return rows; } diff --git a/oneflux_steps/tools/gf_mds/src/dataset.h b/oneflux_steps/tools/gf_mds/src/dataset.h index df0ff244..71498d45 100644 --- a/oneflux_steps/tools/gf_mds/src/dataset.h +++ b/oneflux_steps/tools/gf_mds/src/dataset.h @@ -17,6 +17,5 @@ /* prototypes */ ROW *import_dataset(const LIST *const list, const int count, int *const rows_count); -void free_details_list(DD **details_list, const int count); #endif /* DATASET_H */ diff --git a/oneflux_steps/tools/gf_mds/src/main.c b/oneflux_steps/tools/gf_mds/src/main.c index 2bd6a031..e568b5d9 100644 --- a/oneflux_steps/tools/gf_mds/src/main.c +++ b/oneflux_steps/tools/gf_mds/src/main.c @@ -1,7 +1,10 @@ /* main.c - this file is part of gf_mds + This file is part of gf_mds tool that applies an + improved and fully parameterizable version of the + Marginal Distribution Sampling gapfilling method + described in Reichstein et al. 2005 author: Alessio Ribeca owner: DIBAF - University of Tuscia, Viterbo, Italy @@ -18,40 +21,52 @@ #include "../../../compiler.h" /* constants */ -#define PROGRAM_VERSION "1.0" -const char def_tokens[GF_TOKENS][GF_TOKEN_LENGTH_MAX+1] = { - "NEE", - "SW_IN", - "TA", - "VPD", - TIMESTAMP_STRING, +#define PROGRAM_VERSION "3.0" +const char def_tokens[GF_TOKENS][GF_TOKEN_LENGTH_MAX+1] = +{ + "NEE" + , "SW_IN" + , "TA" + , "VPD" + , TIMESTAMP_END_STRING }; /* static global variables */ static FILES *files; static int files_count; -static int rows_min = GF_ROWS_MIN; /* see types.h */ +static int rows_min = GF_ROWS_MIN; /* see types.h */ +/* v2.04b */ +static int sym_mean = 0; +/* v3.0 */ +static int max_mdv_win = 0; +static int driver1_qc_col = -1; +static int driver2a_qc_col = -1; +static int driver2b_qc_col = -1; +static PREC driver1_qc_thrs = INVALID_VALUE; +static PREC driver2a_qc_thrs = INVALID_VALUE; +static PREC driver2b_qc_thrs = INVALID_VALUE; /* global variables */ -char *program_path = NULL; /* required */ -char *input_path = NULL; /* required */ -char *output_path = NULL; /* required */ -int hourly_dataset = 0; /* required */ -int *years = NULL; /* required */ -int years_count = 0; /* required */ -DD **details_list = NULL; /* required */ -int has_dtime = 0; /* required */ +char *program_path = NULL; /* required */ +char *input_path = NULL; /* required */ +char *output_path = NULL; /* required */ +int timeres = HALFHOURLY_TIMERES; /* required */ +int *years = NULL; /* required */ +int years_count = 0; /* required */ +PREC driver1_tolerance_min = GF_DRIVER_1_TOLERANCE_MIN; /* required */ +PREC driver1_tolerance_max = GF_DRIVER_1_TOLERANCE_MAX; /* required */ +PREC driver2a_tolerance_min = GF_DRIVER_2A_TOLERANCE_MIN; /* required */ +PREC driver2a_tolerance_max = GF_DRIVER_2A_TOLERANCE_MAX; /* required */ +PREC driver2b_tolerance_min = GF_DRIVER_2B_TOLERANCE_MIN; /* required */ +PREC driver2b_tolerance_max = GF_DRIVER_2B_TOLERANCE_MAX; /* required */ + int custom_tokens[GF_TOKENS]; -PREC swin_tolerance_min = GF_SW_IN_TOLERANCE_MIN; -PREC swin_tolerance_max = GF_SW_IN_TOLERANCE_MAX; -PREC ta_tolerance = GF_TA_TOLERANCE; -PREC vpd_tolerance = GF_VPD_TOLERANCE; /* strings */ -static const char banner[] = "\ngf_mds "PROGRAM_VERSION"\n" - "by Alessio Ribeca\n\n" - "scientific contact: darpap at unitus dot it\n" - "technical contact: a dot ribeca at unitus dot it\n\n" +static const char banner[] = "\ngf_mds v"PROGRAM_VERSION"\n" + "by A. Ribeca\n\n" + "scientific contact: \n" + "technical contact: \n\n" "DIBAF - University of Tuscia, Viterbo, Italy\n" "(builded on "__DATE__" at "__TIME__ " with "COMPILER")\n" "(use -h parameter for more information)\n"; @@ -60,77 +75,100 @@ static const char notes[] = "processed on %s with gf_mds "PROGRAM_VERSION" compi /* must have same order of eValues in types.h */ char tokens[GF_TOKENS][GF_TOKEN_LENGTH_MAX+1]; static const char gap_file[] = "%s%smds.csv"; -static const char gap_header[] = "TIMESTAMP,%s,FILLED,QC,HAT,SAMPLE,STDDEV,METHOD,QC_HAT,TIMEWINDOW\n"; -static const char gap_dtime_header[] = "DTime,%s,FILLED,QC,HAT,SAMPLE,STDDEV,METHOD,QC_HAT,TIMEWINDOW\n"; +static const char gap_header[] = "%s,%s,FILLED,QC,HAT,SAMPLE,STDDEV,METHOD,QC_HAT,TIMEWINDOW\n"; static const char gap_format[] = "%s,%g,%g,%d,%g,%d,%g,%d,%d,%d\n"; -static const char gap_dtime_format[] = "%g,%g,%g,%d,%g,%d,%g,%d,%d,%d\n"; + +/* v2.04b */ +static const char gap_header_sym_mean[] = "%s,%s,FILLED,QC,HAT,DT,DT_MA,DT_MA_SAMPLE,DT_MB,DT_MB_SAMPLE,SAMPLE,STDDEV,METHOD,QC_HAT,TIMEWINDOW\n"; +static const char gap_format_sym_mean[] = "%s,%g,%g,%d,%g,%g,%g,%d,%g,%d,%d,%g,%d,%d,%d\n"; /* messages */ static const char msg_dataset_not_specified[] = -"dataset not specified." -#if defined (_WIN32) || defined (linux) || defined (__linux) || defined (__linux__) || defined (__APPLE__) -" searching..." -#endif -"\n"; +"dataset not specified. searching...\n"; static const char msg_import_dataset[] = "processing: %s%s"; +static const char msg_input_path[] = "input path = %s\n"; static const char msg_output_path[] = "output path = %s\n\n"; static const char msg_rows_min[] = "rows min = %d\n\n"; -static const char msg_tolerances[] = "%s tolerances = %g, %g\n" - "%s tolerance = %g\n" - "%s tolerance = %g\n\n"; static const char msg_ok[] = "ok"; static const char msg_ok_with_gaps_unfilled[] = "ok with %d gaps unfilled.\n"; -static const char msg_summary[] = "\n%d file%s founded: %d processed, %d skipped.\n\n"; +static const char msg_summary[] = "\n%d file%s found: %d processed, %d skipped.\n\n"; static const char msg_usage[] = "This code applies the gapfilling Marginal Distribution Sampling method\n" - "described in Reichstein et al. 2005 (Global Change Biology).\n\n" + "described in Reichstein et al. 2005 (Global Change Biology).\nThis version " + "allows larger flexibility in the selection of the drivers.\n" + "\n" "Differences respect to the original method are the possibilities to:\n" - "1) define the name you used to specify the different variables\n" - "2) process a multi-years dataset\n" - "3) change the tolerance of the different drivers (see the paper for details)\n\n\n" + "1) define the variable to fill and the drivers to use\n" + "2) change the tolerance of the different drivers (see the paper for details)\n" + "3) process a multi-years dataset\n" + "4) process hourly timeseries\n" + "\n" + "Basic on the use:\n" + "The MDS method uses look-up-tables defined around each single gap, looking for\nthe " + "best compromise between size of the window (as small as possible)\nand number of drivers " + "used.\nThe main driver (driver1) is used when it is not possible\nto fill the gap using all the " + "three drivers (driver1, driver2a and driver2b).\nFor details see the original paper.\n" + "\n" "How to use: gf_mds parameters\n\n" - " -input=filename -> file to be processed (optional)\n" - " (if not specified all the files in the folder are processed)\n" - " use , to separate paths\\files\n" - " (e.g. -dataset=ITRoc2003.txt,ITCpz2003.txt will process only these 2 files)\n" - " use + to concatenate paths\\files\n" - " (e.g. -dataset=ITRo2003.txt+ITRo2004.txt will process the 2 years as one dataset)\n\n" + " -input=filename -> name of the file to be processed and multiple years\n" + " (optional, if not specified all the files in the folder are processed)\n" + " use , to separate paths\\files that will be processed singularly\n" + " (e.g. -input=ITRoc2003.txt,ITCpz2003.txt will process only these 2 files)\n" + " use + to concatenate paths\\files that will me merged in a single file\n" + " IMPORTANT: this option should be used only if the years are all consecutive\n" + " (multiyears timeseries, e.g. -input=ITRo2003.txt+ITRo2004.txt\n " + " will process the 2 years as one dataset of two years)\n\n" " -output=path where result files are created (optional)\n" - " (if not specified the folder with the gf_mds.exe file is used)\n\n" + " (if not specified the folder with the program file is used)\n\n" " -hourly -> specify that your file is not halfhourly but hourly\n\n" - " -tofill=XXXX -> name of the the variable to be filled as reported in the header of the\n" - " input file (max %d chars, default is \"%s\")\n\n" - " -sw_in=XXXX -> name of the incoming radiation (W m-2) as reported in the header of the\n" - " input file (max %d chars, default is \"%s\")\n\n" - " -ta=XXXX -> name of the air temperature (C degree) as reported in the header of the\n" - " input file (max %d chars, default is \"%s\")\n\n" - " -vpd=XXXX -> name of the vapor pressure deficit (hP) as reported in the header of the\n" - " input file (max %d chars, default is \"%s\")\n\n" - " -date=XXXX -> name of the date variable as reported in the header of the\n" - " input file (max %d chars, default is \"%s\")\n\n" - " -sw_int=min,max -> change SW_IN tolerances (default min: %g, default max: %g)\n\n" - " -tat=value -> change TA tolerance (default: %g)\n\n" - " -vpdt=value -> change VPD tolerance (default: %g)\n\n" - " -dtime -> timestamp reported as decimal day of the year\n" - " (e.g. 1.02083 for the first half hour of the year) using as variable name DTIME\n" - " (can be changed using the -date parameter). Default is OFF and ISODATE is expected\n\n" - " -rows_min=value -> set rows min (default: %d)\n\n" + " -tofill=XXXX -> name of the the variable to be filled as reported in\n the header of the " + " input file (max %d chrs, default is \"%s\")\n\n" + " -driver1=XXXX -> name of the name of the main driver (as in the header)\n that is used in case using all the 3 drivers " + "it is not possible to fill\n the gap (max %d chrs, default is \"%s\", Incoming Solar Radiation in Wm-2)\n\n" + " -driver2a=XXXX -> name of the first additional driver as reported in\n the header of the input file\n" + " (max %d chrs, default is \"%s\", Air Temperature in degree C)\n\n" + " -driver2b=XXXX -> name of the second additional driver as reported in\n the header of the input file\n" + " (max %d chrs, default is \"%s\", Vapor Pressure Deficit in hPa)\n\n" + " -date=XXXX -> name of the variable in the header of the input file\n where the timestamp is reported " + "in the format YYYYMMDDHHMM\n (max %d chrs, default is \"%s\")\n\n" + " -tdriver1=min[,max] -> set the tolerance values used to define\n similar conditions related to driver1. If only " + "one values is specified\n it is used for all the driver1 values (e.g. -tdriver1=5).\n If two values are reported " + "(e.g. -tdriver1=4,10)\n the first is used for driver1 values below it, the second for\n driver1 values above it " + "while between the two the driver1 value itself\n is also used as tolerance (in the example a tolerance of 6 " + "would be\n used for driver1=6)\n (default is two values for SW_IN in Wm-2 and are min: %g, max: %g)\n\n" + " -tdriver2a=min[,max] -> set the tolerance values used to define\n similar conditions related to driver2a. " + "See description for tdriver1\n" + " (default is one value for TA in degrees C and is %g)\n\n" + " -tdriver2b=min[,max] -> set the tolerance values used to define\n similar conditions related to driver2b. " + "See description for tdriver1\n" + " (default is one value for VPD in hPa and is %g)\n\n" + " -driver1_qc_col=values -> set column for main driver qc check\n\n" + " -driver2a_qc_col=values -> set column for first additional driver qc check\n\n" + " -driver2b_qc_col=values -> set column for second additional driver qc check\n\n" + " -driver1_qc_thrs=values -> set threshold for main driver qc check\n\n" + " -driver2a_qc_thrs=values -> set threshold for first additional driver qc check\n\n" + " -driver2b_qc_thrs=values -> set threshold for second additional driver qc check\n\n" + " -rows_min=value -> set the minimum number of rows with valid data\n" + " to run the gapfilling (default: %d)\n\n" + " -nohat -> disable hat computing (gapfilling applied to all the records,\n" + " even if not missing, enabled by default)\n\n" + " -sym_mean -> enable symmetric mean method\n\n" + " -max_mdv_win=value -> set max window to be used when MDV is applied as last option\n\n" + " -debug -> save used values to compute gf to file\n\n" " -h -> show this help\n\n" - "Others driver variables can be used instead of Air temperature and VPD,\nspecifying the name and the new tolerances.\n\n" - "For example to use Soil Water Content (name swc, expressed as %% 0-100 and with\na tolerance of 5%) instead of " - "VPD you have to specify:\n\ngf_mds -vpd=swc -vpdt=5\n"; +; + /* errors messages */ extern const char err_out_of_memory[]; const char err_unable_open_file[] = "unable to open file."; const char err_empty_file[] = "empty file ?"; -const char err_not_founded[] = "not founded."; static const char err_unable_get_current_directory[] = "unable to retrieve current directory.\n"; static const char err_unable_to_register_atexit[] = "unable to register clean-up routine.\n"; +static const char err_unable_create_output_path[] = "unable to create output path: %s.\n"; static const char err_unable_to_convert_value_for[] = "unable to convert value \"%s\" for %s\n\n"; static const char err_output_path_no_delimiter[] = "output path must terminating with a \"%c\"\n\n"; static const char err_unable_open_output_path[] = "unable to open output path.\n"; -static const char err_swin_no_min_max_available[] = "no min and max available in \"%s\" for SW_IN tolerances.\n\n"; -static const char err_swin_no_min_tolerance[] = "no min tolerance available for SW_IN\n"; -static const char err_swin_no_max_tolerance[] = "no max tolerance available for SW_IN\n"; +static const char err_tolerances_not_specified[] = "tolerances not specified for %s\n\n"; +static const char err_no_min_tolerance[] = "no min tolerance available for %s\n\n"; static const char err_unable_create_gap_file[] = "unable to create gap file."; static const char err_unable_convert_tolerance[] = "unable to convert tolerance \"%s\" for %s.\n\n"; static const char err_arg_needs_param[] = "%s parameter not specified.\n\n"; @@ -142,12 +180,15 @@ static const char err_rows_min[] = "rows_min must be between %d and %d not %d. d /* */ static void clean_up(void) { - if ( program_path ) { - free(program_path); - } if ( files ) { free_files(files, files_count); } + if ( program_path ) { + free(program_path); + } +#if defined (_WIN32) && defined (_DEBUG) + dump_memory_leaks(); +#endif } /* */ @@ -191,7 +232,7 @@ int set_hourly_dataset(char *arg, char *param, void *p) { return 0; } - hourly_dataset = 1; + timeres = HOURLY_TIMERES; /* ok */ return 1; @@ -218,96 +259,114 @@ static int set_token(char *arg, char *param, void *p) { } /* */ -int set_swin_tolerances(char *arg, char *param, void *p) { +typedef struct { + const char *name; + PREC *min; + PREC *max; +} TOLERANCE; + +int set_driver_tolerances(char *arg, char *param, void *p) { int error; PREC min; PREC max; char *t; + TOLERANCE* tol; + + tol = (TOLERANCE*)p; + assert(tol); if ( !param ) { printf(err_arg_needs_param, arg); return 0; } - /* check for comma */ - t = strrchr(param, ','); - if ( !t ) { - printf(err_swin_no_min_max_available, param); - return 0; - } - - /* get min */ - *t = 0; if ( !param[0] ) { - puts(err_swin_no_min_tolerance); - return 0; - } - min = convert_string_to_prec(param, &error); - if ( error ) { - printf(err_unable_convert_tolerance, param, "SW_IN"); + printf(err_tolerances_not_specified, tol->name); return 0; } - /* get max */ - ++t; - if ( !t[0] ) { - puts(err_swin_no_max_tolerance); - return 0; - } - max = convert_string_to_prec(t, &error); - if ( error ) { - printf(err_unable_convert_tolerance, param, "SW_IN"); - return 0; + /* check for comma */ + t = strrchr(param, ','); + if ( !t ) { + min = convert_string_to_prec(param, &error); + if ( error ) { + printf(err_unable_convert_tolerance, param, tol->name); + return 0; + } + max = INVALID_VALUE; + } else { + /* get min */ + *t = 0; + if ( !param[0] ) { + printf(err_no_min_tolerance, tol->name); + return 0; + } + min = convert_string_to_prec(param, &error); + if ( error ) { + printf(err_unable_convert_tolerance, param, tol->name); + return 0; + } + + /* get max */ + ++t; + if ( !t[0] ) { + max = INVALID_VALUE; + } else { + max = convert_string_to_prec(t, &error); + if ( error ) { + printf(err_unable_convert_tolerance, param, tol->name); + return 0; + } + } } /* */ - swin_tolerance_min = min; - swin_tolerance_max = max; + *tol->min = min; + *tol->max = max; /* ok */ return 1; } /* */ -int set_prec_value(char *arg, char *param, void *p) { +static int set_int_value(char *arg, char *param, void *p) { + int i; int error; - PREC v; if ( !param ) { printf(err_arg_needs_param, arg); return 0; } - - v = convert_string_to_prec(param, &error); + i = convert_string_to_int(param, &error); if ( error ) { printf(err_unable_to_convert_value_for, param, arg); return 0; } - /* */ - *((PREC *)p) = v; + /* set value */ + *((int *)p) = i; /* ok */ return 1; } /* */ -static int set_int_value(char *arg, char *param, void *p) { - int i; +static int set_prec_value(char *arg, char *param, void *p) { + PREC v; int error; if ( !param ) { printf(err_arg_needs_param, arg); return 0; } - i = convert_string_to_int(param, &error); + v = convert_string_to_prec(param, &error); if ( error ) { printf(err_unable_to_convert_value_for, param, arg); return 0; } /* set value */ - *((int *)p) = i; + *((PREC *)p) = v; /* ok */ return 1; @@ -327,6 +386,19 @@ static int set_flag(char *arg, char *param, void *p) { return 1; } +/* */ +static int reverse_flag(char *arg, char *param, void *p) { + if ( param ) { + printf(err_arg_no_needs_param, arg); + return 0; + } + + *((int *)p) = ! *((int *)p); + + /* ok */ + return 1; +} + /* */ int show_help(char *arg, char *param, void *p) { if ( param ) { @@ -338,17 +410,17 @@ int show_help(char *arg, char *param, void *p) { GF_TOKEN_LENGTH_MAX, def_tokens[GF_TOFILL], GF_TOKEN_LENGTH_MAX, - def_tokens[GF_SWIN], + def_tokens[GF_DRIVER_1], GF_TOKEN_LENGTH_MAX, - def_tokens[GF_TA], + def_tokens[GF_DRIVER_2A], GF_TOKEN_LENGTH_MAX, - def_tokens[GF_VPD], + def_tokens[GF_DRIVER_2B], GF_TOKEN_LENGTH_MAX, def_tokens[GF_ROW_INDEX], - swin_tolerance_min, - swin_tolerance_max, - ta_tolerance, - vpd_tolerance, + driver1_tolerance_min, + driver1_tolerance_max, + driver2a_tolerance_min, + driver2b_tolerance_min, rows_min ); @@ -356,6 +428,39 @@ int show_help(char *arg, char *param, void *p) { return 0; } +/* added on January 17, 2018 */ +void show_tolerances(void) { + printf("%s tolerance%s= %g" + , tokens[GF_DRIVER_1] + , IS_INVALID_VALUE(driver1_tolerance_max) ? "" : "s " + , driver1_tolerance_min + ); + if ( ! IS_INVALID_VALUE(driver1_tolerance_max) ) { + printf(", %g", driver1_tolerance_max); + } + puts(""); + + printf("%s tolerance%s= %g" + , tokens[GF_DRIVER_2A] + , IS_INVALID_VALUE(driver2a_tolerance_max) ? "" : "s " + , driver2a_tolerance_min + ); + if ( ! IS_INVALID_VALUE(driver2a_tolerance_max) ) { + printf(", %g", driver2a_tolerance_max); + } + puts(""); + + printf("%s tolerance%s= %g" + , tokens[GF_DRIVER_2B] + , IS_INVALID_VALUE(driver2b_tolerance_max) ? "" : "s " + , driver2b_tolerance_min + ); + if ( ! IS_INVALID_VALUE(driver2b_tolerance_max) ) { + printf(", %g", driver2b_tolerance_max); + } + puts("\n"); +} + /* */ int main(int argc, char *argv[]) { int i; @@ -370,27 +475,47 @@ int main(int argc, char *argv[]) { int files_not_processed_count; int total_files_count; int no_gaps_filled_count; + int hat; + int debug; char buffer[BUFFER_SIZE]; char filename[FILENAME_SIZE]; + char debug_name[FILENAME_SIZE]; char *p; char *string; FILE *f; ROW *rows; GF_ROW *gf_rows; + + TOLERANCE tol1 = { "driver1", &driver1_tolerance_min, &driver1_tolerance_max }; + TOLERANCE tol2a = { "driver2a", &driver2a_tolerance_min, &driver2a_tolerance_max }; + TOLERANCE tol2b = { "driver2b", &driver2b_tolerance_min, &driver2b_tolerance_max }; + const ARGUMENT args[] = { { "input", get_input_path, NULL }, { "output", get_output_path, NULL }, { "hourly", set_hourly_dataset, NULL }, { "tofill", set_token, (void *)GF_TOFILL }, - { "ta", set_token, (void *)GF_TA }, - { "sw_in", set_token, (void *)GF_SWIN }, - { "vpd", set_token, (void *)GF_VPD }, + { "driver1", set_token, (void *)GF_DRIVER_1 }, + { "driver2a", set_token, (void *)GF_DRIVER_2A }, + { "driver2b", set_token, (void *)GF_DRIVER_2B }, { "date", set_token, (void *)GF_ROW_INDEX }, - { "sw_int", set_swin_tolerances, NULL }, - { "tat", set_prec_value, &ta_tolerance }, - { "vpdt", set_prec_value, &vpd_tolerance }, + { "tdriver1", set_driver_tolerances, &tol1 }, + { "tdriver2a", set_driver_tolerances, &tol2a }, + { "tdriver2b", set_driver_tolerances, &tol2b }, + { "rows_min", set_int_value, &rows_min }, + { "nohat", reverse_flag, &hat }, + { "debug", reverse_flag, &debug }, { "rows_min", set_int_value, &rows_min }, - { "dtime", set_flag, &has_dtime }, + + /* v3.0 */ + { "symmean", reverse_flag, &sym_mean }, + { "driver1_qc_col", set_int_value, &driver1_qc_col }, + { "driver2a_qc_col", set_int_value, &driver2a_qc_col }, + { "driver2b_qc_col", set_int_value, &driver2b_qc_col }, + { "driver1_qc_thrs", set_prec_value, &driver1_qc_thrs }, + { "driver2a_qc_thrs", set_prec_value, &driver2a_qc_thrs }, + { "driver2b_qc_thrs", set_prec_value, &driver2b_qc_thrs }, + { "h", show_help, NULL }, { "?", show_help, NULL }, { "help", show_help, NULL }, @@ -410,6 +535,13 @@ int main(int argc, char *argv[]) { custom_tokens[i] = 0; } + /* defaults */ + debug = 0; + hat = 1; + + /* v2.04b */ + sym_mean = 0; + /* parse arguments */ if ( !parse_arguments(argc, argv, args, SIZEOF_ARRAY(args)) ) { return 1; @@ -438,8 +570,10 @@ int main(int argc, char *argv[]) { /* check if output path exists */ if ( !path_exists(output_path) ) { - puts(err_unable_open_output_path); - return 1; + if ( !create_dir(output_path) ) { + printf(err_unable_create_output_path, output_path); + return 1; + } } } else { output_path = program_path; @@ -451,7 +585,8 @@ int main(int argc, char *argv[]) { return 1; } - /* show output path */ + /* show paths */ + printf(msg_input_path, input_path); printf(msg_output_path, output_path); /* show rows min */ @@ -471,15 +606,7 @@ int main(int argc, char *argv[]) { } /* show tolerances */ - printf(msg_tolerances, - tokens[GF_SWIN], - swin_tolerance_min, - swin_tolerance_max, - tokens[GF_TA], - ta_tolerance, - tokens[GF_VPD], - vpd_tolerance - ); + show_tolerances(); /* reset */ files_processed_count = 0; @@ -518,7 +645,7 @@ int main(int argc, char *argv[]) { } /* add to filename */ - if ( !mystrcat(filename, string, FILENAME_MAX) && '\0' == filename[0]) { + if ( !string_concat(filename, string, FILENAME_MAX) && '\0' == filename[0]) { puts(err_unable_create_output_filename); files_not_processed_count += files[z].count; free(string); @@ -533,6 +660,17 @@ int main(int argc, char *argv[]) { continue; } + /* v2.04 */ + if ( sym_mean ) + { + if ( !string_concat(filename, "sym_mean_", FILENAME_MAX) && '\0' == filename[0]) { + puts(err_unable_create_output_filename); + files_not_processed_count += files[z].count; + free(string); + continue; + } + } + /* free memory */ free(string); } @@ -545,8 +683,48 @@ int main(int argc, char *argv[]) { continue; } + if ( debug ) { + int len = strlen(files[z].list->name); + char* p = strrchr(files[z].list->name, '.'); + if ( p ) { + len -= strlen(p); + } + sprintf(debug_name, "%.*s", len, files[z].list->name); + } + /* gf */ - gf_rows = gf_mds(rows->value, sizeof(ROW), rows_count, GF_REQUIRED_DATASET_VALUES, hourly_dataset, swin_tolerance_min, swin_tolerance_max, ta_tolerance, vpd_tolerance, GF_TOFILL, GF_SWIN, GF_TA, GF_VPD, rows_min, 1, &no_gaps_filled_count); + gf_rows = gf_mds( rows->value, + sizeof(ROW), + rows_count, + GF_REQUIRED_DATASET_VALUES, + timeres, + driver1_tolerance_min, + driver1_tolerance_max, + driver2a_tolerance_min, + driver2a_tolerance_max, + driver2b_tolerance_min, + driver2b_tolerance_max, + GF_TOFILL, + GF_DRIVER_1, + GF_DRIVER_2A, + GF_DRIVER_2B, + driver1_qc_col, + driver2a_qc_col, + driver2b_qc_col, + driver1_qc_thrs, + driver2a_qc_thrs, + driver2b_qc_thrs, + rows_min, + hat, + -1, + -1, + &no_gaps_filled_count, + sym_mean, + max_mdv_win, + debug, + debug_name, + years[z] + ); if ( !gf_rows ) { free(years); free(rows); @@ -561,65 +739,55 @@ int main(int argc, char *argv[]) { puts(err_unable_create_gap_file); free(years); free(rows); - free_details_list(details_list, files[z].count); files_not_processed_count += files[z].count; continue; } - /* write details */ - if ( details_list && details_list[0] ) { - sprintf(buffer, notes, get_datetime_in_timestamp_format()); - write_dds(details_list, files[z].count, f, buffer); - } - /* write header */ - if ( !IS_INVALID_VALUE(years[0]) ) { - fprintf(f, gap_header, tokens[GF_TOFILL]); + /* v2.04b */ + if ( sym_mean ) { + fprintf(f, gap_header_sym_mean, tokens[GF_ROW_INDEX], tokens[GF_TOFILL]); } else { - fprintf(f, gap_dtime_header, tokens[GF_TOFILL]); + fprintf(f, gap_header, tokens[GF_ROW_INDEX], tokens[GF_TOFILL]); } /* write values */ for ( y = 0; y < files[z].count; y++ ) { - if ( !IS_INVALID_VALUE(years[y]) ) { - y = 0; - w = 0; - j = IS_LEAP_YEAR(years[y]) ? LEAP_YEAR_ROWS : YEAR_ROWS; - if ( hourly_dataset ) { - j /= 2; + y = 0; + w = 0; + j = get_rows_count_by_timeres(timeres, years[y]); + + /* */ + for ( i = 0; i < rows_count; i++ ) { + if ( i == j ) { + ++y; + k = get_rows_count_by_timeres(timeres, years[y]); + j += k; + k = get_rows_count_by_timeres(timeres,years[y-1]); + w += k; } - /* */ - for ( i = 0; i < rows_count; i++ ) { - if ( i == j ) { - ++y; - k = IS_LEAP_YEAR(years[y]) ? LEAP_YEAR_ROWS : YEAR_ROWS; - if ( hourly_dataset ) { - k /= 2; - } - j += k; - k = IS_LEAP_YEAR(years[y-1]) ? LEAP_YEAR_ROWS : YEAR_ROWS; - if ( hourly_dataset ) { - k /= 2; - } - w += k; - } - fprintf(f, gap_format, - timestamp_end_by_row_s(i-w, years[y], hourly_dataset), + /* v2.04b */ + if ( sym_mean ) { + fprintf(f, gap_format_sym_mean, + timestamp_end_by_row_s(i-w, years[y], timeres), rows[i].value[GF_TOFILL], IS_FLAG_SET(gf_rows[i].mask, GF_TOFILL_VALID) ? rows[i].value[GF_TOFILL] : gf_rows[i].filled, IS_FLAG_SET(gf_rows[i].mask, GF_TOFILL_VALID) ? 0 : gf_rows[i].quality, gf_rows[i].filled, + gf_rows[i].filled_sym_mean, + gf_rows[i].mean_above, + gf_rows[i].n_above, + gf_rows[i].mean_below, + gf_rows[i].n_below, gf_rows[i].samples_count, gf_rows[i].stddev, gf_rows[i].method, gf_rows[i].quality, gf_rows[i].time_window ); - } - } else { - for ( i = 0; i < rows_count; i++ ) { - fprintf(f, gap_dtime_format, - get_dtime_by_row(i, hourly_dataset), + } else { + fprintf(f, gap_format, + timestamp_end_by_row_s(i-w, years[y], timeres), rows[i].value[GF_TOFILL], IS_FLAG_SET(gf_rows[i].mask, GF_TOFILL_VALID) ? rows[i].value[GF_TOFILL] : gf_rows[i].filled, IS_FLAG_SET(gf_rows[i].mask, GF_TOFILL_VALID) ? 0 : gf_rows[i].quality, @@ -641,10 +809,7 @@ int main(int argc, char *argv[]) { free(gf_rows); free(years); free(rows); - if ( details_list ) { - free_details_list(details_list, files[z].count); - } - + /* increment processed files count */ files_processed_count += files[z].count; @@ -664,6 +829,5 @@ int main(int argc, char *argv[]) { files_not_processed_count ); - /* memory freeded on return */ return 0; } diff --git a/oneflux_steps/tools/gf_mds/src/types.h b/oneflux_steps/tools/gf_mds/src/types.h index 52b6a5d8..ef5462a2 100644 --- a/oneflux_steps/tools/gf_mds/src/types.h +++ b/oneflux_steps/tools/gf_mds/src/types.h @@ -15,16 +15,12 @@ /* includes */ #include "../../../common/common.h" -/* defines */ -#define DTIME_TO_ROW(x) (int)(((x*48)-48)+0.5) -#define DTIME_TO_ROW_HOURLY(x) (int)(((x*24)-24)+0.5) - /* enums */ enum { GF_TOFILL = 0, - GF_SWIN, - GF_TA, - GF_VPD, + GF_DRIVER_1, + GF_DRIVER_2A, + GF_DRIVER_2B, GF_ROW_INDEX, GF_REQUIRED_DATASET_VALUES, diff --git a/oneflux_steps/tools/shift_solar_noon/src/main.c b/oneflux_steps/tools/shift_solar_noon/src/main.c index 9d1a6681..27d18fc9 100644 --- a/oneflux_steps/tools/shift_solar_noon/src/main.c +++ b/oneflux_steps/tools/shift_solar_noon/src/main.c @@ -1,7 +1,10 @@ /* main.c - this file is part of shift_solar_noon + This file is part of shift_solar_noon tool. + It calculates a composit maximum incoming radiation + that is compared with the potential radiation in order + to identify possible shifts in the timeseries. author: Alessio Ribeca owner: DIBAF - University of Tuscia, Viterbo, Italy @@ -23,7 +26,7 @@ #include "../../../compiler.h" /* constants */ -#define PROGRAM_VERSION "v1.0" +#define PROGRAM_VERSION "v1.01" const int days_per_month[MONTHS] = { 31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31 }; /* enum */ @@ -1100,9 +1103,9 @@ static int save_db_file(DATASET *const dataset) { /* write... */ for ( i = 0; i < dataset->rows_count; i++ ) { /* ...timestamp_start.... */ - fprintf(f, "%s,", timestamp_start_by_row_s(i, dataset->details->year, (HOURLY_TIMERES == dataset->details->timeres))); + fprintf(f, "%s,", timestamp_start_by_row_s(i, dataset->details->year, dataset->details->timeres)); /* ...timestamp_end.... */ - fprintf(f, "%s", timestamp_end_by_row_s(i, dataset->details->year, (HOURLY_TIMERES == dataset->details->timeres))); + fprintf(f, "%s", timestamp_end_by_row_s(i, dataset->details->year, dataset->details->timeres)); /* ...values... */ for ( y = 0; y < SIZEOF_ARRAY(vars_index); y++ ) { var = get_var_index(dataset, var_names[vars_index[y]]); @@ -1806,9 +1809,9 @@ static int save_nee_file(DATASET *const dataset) { /* write... */ for ( i = 0; i < dataset->rows_count; i++ ) { /* ...timestamp_start.... */ - fprintf(f, "%s,", timestamp_start_by_row_s(i, dataset->details->year, (HOURLY_TIMERES == dataset->details->timeres))); + fprintf(f, "%s,", timestamp_start_by_row_s(i, dataset->details->year, dataset->details->timeres)); /* ...timestamp_end.... */ - fprintf(f, "%s", timestamp_end_by_row_s(i, dataset->details->year, (HOURLY_TIMERES == dataset->details->timeres))); + fprintf(f, "%s", timestamp_end_by_row_s(i, dataset->details->year, dataset->details->timeres)); /* ...values... */ for ( y = 0; y < SIZEOF_ARRAY(vars_index); y++ ) { var = get_var_index(dataset, var_names[vars_index[y]]); @@ -1897,9 +1900,9 @@ static int save_energy_file(DATASET *const dataset) { /* write... */ for ( i = 0; i < dataset->rows_count; i++ ) { /* ...timestamp_start.... */ - fprintf(f, "%s,", timestamp_start_by_row_s(i, dataset->details->year, (HOURLY_TIMERES == dataset->details->timeres))); + fprintf(f, "%s,", timestamp_start_by_row_s(i, dataset->details->year, dataset->details->timeres)); /* ...timestamp_end.... */ - fprintf(f, "%s", timestamp_end_by_row_s(i, dataset->details->year, (HOURLY_TIMERES == dataset->details->timeres))); + fprintf(f, "%s", timestamp_end_by_row_s(i, dataset->details->year, dataset->details->timeres)); /* ...values... */ for ( y = 0; y < SIZEOF_ARRAY(vars_index); y++ ) { var = get_var_index(dataset, var_names[vars_index[y]]); @@ -1970,9 +1973,9 @@ static int save_ustar_file(DATASET *const dataset) { /* write... */ for ( i = 0; i < dataset->rows_count; i++ ) { /* ...timestamp_start.... */ - fprintf(f, "%s,", timestamp_start_by_row_s(i, dataset->details->year, (HOURLY_TIMERES == dataset->details->timeres))); + fprintf(f, "%s,", timestamp_start_by_row_s(i, dataset->details->year, dataset->details->timeres)); /* ...timestamp_end.... */ - fprintf(f, "%s", timestamp_end_by_row_s(i, dataset->details->year, (HOURLY_TIMERES == dataset->details->timeres))); + fprintf(f, "%s", timestamp_end_by_row_s(i, dataset->details->year, dataset->details->timeres)); /* ...values... */ for ( y = 0; y < SIZEOF_ARRAY(vars_index); y++ ) { var = get_var_index(dataset, var_names[vars_index[y]]); @@ -2439,7 +2442,7 @@ static int set_sc_negl(DATASET *const dataset) { } for ( i = 0; i < dataset->details->sc_negles_count; i++ ) { - z = get_row_by_timestamp(&dataset->details->sc_negles[i].timestamp, (HOURLY_TIMERES == dataset->details->timeres)); + z = get_row_by_timestamp(&dataset->details->sc_negles[i].timestamp, dataset->details->timeres); for ( y = z; y < dataset->rows_count; y++ ) { dataset->rows[y].value[SC_NEGL] = dataset->details->sc_negles[i].flag; } diff --git a/oneflux_steps/ure/src/dataset.c b/oneflux_steps/ure/src/dataset.c index fac0c954..44f87d28 100644 --- a/oneflux_steps/ure/src/dataset.c +++ b/oneflux_steps/ure/src/dataset.c @@ -831,7 +831,7 @@ int compute_datasets(DATASET *const datasets, const int datasets_count, const in P_MATRIX *p_matrix_c = NULL; /* mandatory */ TIMESTAMP *t; - const int columns_founded_count = PERCENTILES_COUNT_2+PERCENTILES_COUNT_2; + const int columns_found_count = PERCENTILES_COUNT_2+PERCENTILES_COUNT_2; /* allocate memory for buffer */ buffer = malloc(HUGE_BUFFER_SIZE*sizeof*buffer); @@ -998,8 +998,8 @@ int compute_datasets(DATASET *const datasets, const int datasets_count, const in } /* check assigned */ - if ( assigned != columns_founded_count ) { - printf("expected %d columns not %d\n", columns_founded_count, assigned); + if ( assigned != columns_found_count ) { + printf("expected %d columns not %d\n", columns_found_count, assigned); free(buffer); fclose(f); return 0; @@ -1280,10 +1280,10 @@ int compute_datasets(DATASET *const datasets, const int datasets_count, const in exists = datasets[dataset].years[i].exist; for ( row = 0; row < y; row++ ) { /* TIMESTAMP_START */ - p = timestamp_start_by_row_s(row, datasets[dataset].years[i].year, datasets[dataset].hourly); + p = timestamp_start_by_row_s(row, datasets[dataset].years[i].year, datasets[dataset].details->timeres); fputs(p, f); /* TIMESTAMP_END */ - p = timestamp_end_by_row_s(row, datasets[dataset].years[i].year, datasets[dataset].hourly); + p = timestamp_end_by_row_s(row, datasets[dataset].years[i].year, datasets[dataset].details->timeres); fprintf(f, ",%s,", p); /* dtime */ fprintf(f, "%g,", get_dtime_by_row(row, datasets[dataset].hourly)); @@ -2704,10 +2704,10 @@ int compute_sr_datasets(DATASET *const datasets, const int datasets_count, const } for ( row = 0; row < y; row++ ) { /* TIMESTAMP_START */ - p = timestamp_start_by_row_s(row, datasets[dataset].years[i].year, datasets[dataset].hourly); + p = timestamp_start_by_row_s(row, datasets[dataset].years[i].year, datasets[dataset].details->timeres); fputs(p, f); /* TIMESTAMP_END */ - p = timestamp_end_by_row_s(row, datasets[dataset].years[i].year, datasets[dataset].hourly); + p = timestamp_end_by_row_s(row, datasets[dataset].years[i].year, datasets[dataset].details->timeres); fprintf(f, ",%s,", p); /* values */ fprintf(f, "%g,%g\n", get_dtime_by_row(row, datasets[dataset].hourly), datasets[dataset].srs[j+row].reco); @@ -3128,11 +3128,11 @@ DATASET *get_datasets(const char *const path, const int author_index, const int int error; int assigned; int file_index; - int files_founded_count; + int files_found_count; int year; char year_c[YEAR_LEN]; char buffer[FILENAME_SIZE]; - FILES *files_founded; + FILES *files_found; YEAR *years_no_leak; DATASET *datasets; DATASET *datasets_no_leak; @@ -3149,9 +3149,9 @@ DATASET *get_datasets(const char *const path, const int author_index, const int /* scan path */ sprintf(buffer, "*_%s_%s.csv", authors_suffix[author_index], types_suffix[type_index]); - files_founded = get_files(path, buffer, &files_founded_count, &error); - if ( error || !files_founded_count ) { - puts("no files founded!"); + files_found = get_files(path, buffer, &files_found_count, &error); + if ( error || !files_found_count ) { + puts("no files found!"); return NULL; } @@ -3161,25 +3161,25 @@ DATASET *get_datasets(const char *const path, const int author_index, const int return NULL; } - /* loop on each files founded */ - for ( file_index = 0; file_index < files_founded_count; file_index++ ) { + /* loop on each files found */ + for ( file_index = 0; file_index < files_found_count; file_index++ ) { /* check filename */ - if ( !is_valid_filename(files_founded[file_index].list[0].name, author_index, type_index) ) { + if ( !is_valid_filename(files_found[file_index].list[0].name, author_index, type_index) ) { continue; } /* get site */ - strncpy(details->site, files_founded[file_index].list[0].name, SITE_LEN - 1); + strncpy(details->site, files_found[file_index].list[0].name, SITE_LEN - 1); details->site[SITE_LEN - 1] = '\0'; /* get year */ - strncpy(year_c, files_founded[file_index].list[0].name+SITE_LEN, YEAR_LEN - 1); + strncpy(year_c, files_found[file_index].list[0].name+SITE_LEN, YEAR_LEN - 1); year_c[YEAR_LEN-1] = '\0'; /* convert year string to int */ year = convert_string_to_int(year_c, &error); if ( error ) { - printf("unable to convert year for %s\n\n", files_founded[file_index].list[0].name); + printf("unable to convert year for %s\n\n", files_found[file_index].list[0].name); free_dd(details); free_datasets(datasets, *datasets_count); return NULL; @@ -3187,9 +3187,9 @@ DATASET *get_datasets(const char *const path, const int author_index, const int details->year = year; /* get timeres */ - f = fopen(files_founded[file_index].list[0].fullpath, "r"); + f = fopen(files_found[file_index].list[0].fullpath, "r"); if ( !f ) { - printf("unable to get rows count for %s\n\n", files_founded[file_index].list[0].name); + printf("unable to get rows count for %s\n\n", files_found[file_index].list[0].name); free_dd(details); free_datasets(datasets, *datasets_count); return NULL; @@ -3198,7 +3198,7 @@ DATASET *get_datasets(const char *const path, const int author_index, const int i = get_rows_count_from_file(f); if ( !i ) { - printf("no valid rows founded for %s\n\n", files_founded[file_index].list[0].name); + printf("no valid rows found for %s\n\n", files_found[file_index].list[0].name); fclose(f); free_dd(details); free_datasets(datasets, *datasets_count); @@ -3213,7 +3213,7 @@ DATASET *get_datasets(const char *const path, const int author_index, const int } else if ( (LEAP_YEAR_ROWS/2 == i) || (YEAR_ROWS/2 == i) ) { details->timeres = HOURLY_TIMERES; } else { - printf("no valid timeres founded for %s\n\n", files_founded[file_index].list[0].name); + printf("no valid timeres found for %s\n\n", files_found[file_index].list[0].name); free_dd(details); free_datasets(datasets, *datasets_count); return NULL; @@ -3266,7 +3266,7 @@ DATASET *get_datasets(const char *const path, const int author_index, const int if ( details->timeres != datasets[i].details->timeres ) { puts("Different time resolution between years|"); free_dd(details); - free_files(files_founded, files_founded_count); + free_files(files_found, files_found_count); free_datasets(datasets, *datasets_count); return NULL; } @@ -3277,7 +3277,7 @@ DATASET *get_datasets(const char *const path, const int author_index, const int if ( details->year == datasets[i].years[y].year ) { puts(err_out_of_memory); free_dd(details); - free_files(files_founded, files_founded_count); + free_files(files_found, files_found_count); free_datasets(datasets, *datasets_count); return NULL; } @@ -3303,7 +3303,7 @@ DATASET *get_datasets(const char *const path, const int author_index, const int free_dd(details); /* free memory */ - free_files(files_founded, files_founded_count); + free_files(files_found, files_found_count); /* sort per year */ for ( i = 0 ; i < *datasets_count; i++ ) { diff --git a/oneflux_steps/ure/src/main.c b/oneflux_steps/ure/src/main.c index 8a4548e2..91f0f7bd 100644 --- a/oneflux_steps/ure/src/main.c +++ b/oneflux_steps/ure/src/main.c @@ -1,6 +1,10 @@ /* main.c - this file is part of ure - Uncertainty and References Extraction + + This file is part of the ure step of processing. + It is responsible for calculation and extraction of the NEE, + GPP and RECO uncertainties and reference values starting + from the different realizations created by the nee_proc tool. author: Alessio Ribeca owner: DIBAF - University of Tuscia, Viterbo, Italy @@ -18,7 +22,7 @@ #include "../../compiler.h" /* constants */ -#define PROGRAM_VERSION "v1.0" +#define PROGRAM_VERSION "v1.01" #define BUFFER_SIZE 1024 /* global variables */ diff --git a/oneflux_steps/ustar_mp/src/bootstrapping.c b/oneflux_steps/ustar_mp/src/bootstrapping.c index 53813b11..b0d83d4a 100644 --- a/oneflux_steps/ustar_mp/src/bootstrapping.c +++ b/oneflux_steps/ustar_mp/src/bootstrapping.c @@ -185,7 +185,7 @@ int bootstrapping( FILE *const f, ++i; } - /* if night and valid values founded, compute ustar threshold */ + /* if night and valid values found, compute ustar threshold */ if ( rows_boot_count ) { /* sort rows */ qsort(rows_boot, rows_boot_count, sizeof(ROW), compare_row); diff --git a/oneflux_steps/ustar_mp/src/dataset.c b/oneflux_steps/ustar_mp/src/dataset.c index ea47f393..2f6f1edb 100644 --- a/oneflux_steps/ustar_mp/src/dataset.c +++ b/oneflux_steps/ustar_mp/src/dataset.c @@ -42,7 +42,7 @@ extern const char err_out_of_memory[]; /* errors strings */ static const char err_unable_open_dataset[] = "unable to open dataset."; static const char err_empty_file[] = "empty file ?"; -static const char err_redundancy[] = "redundancy: var \"%s\" already founded at column %d\n"; +static const char err_redundancy[] = "redundancy: var \"%s\" already found at column %d\n"; static const char err_unable_find_column[] = "unable to find column for \"%s\" var.\n"; static const char err_conversion[] = "error during conversion of \"%s\" value at row %d, column %d\n"; static const char err_too_many_rows[] = "too many rows."; @@ -408,7 +408,7 @@ UT *import_dataset(const LIST *const list, const int list_count) { /* fix to zero based index */ --row; - token = timestamp_end_by_row_s(row, 0, 0); + token = timestamp_end_by_row_s(row, 0, HALFHOURLY_TIMERES); if ( !token ) { free_ut(ut); fclose(f); diff --git a/oneflux_steps/ustar_mp/src/main.c b/oneflux_steps/ustar_mp/src/main.c index 53a7f3e2..2740a0cb 100644 --- a/oneflux_steps/ustar_mp/src/main.c +++ b/oneflux_steps/ustar_mp/src/main.c @@ -1,7 +1,10 @@ /* main.c - this file is part of ustar_mp + This file is part of ustar_mp tool. + It calculates the ustar threshold to filter out NEE + using the method described in Reichstein 2005 and + Papale 2006 author: Alessio Ribeca owner: DIBAF - University of Tuscia, Viterbo, Italy @@ -24,7 +27,7 @@ #include "../../compiler.h" /* constants */ -#define PROGRAM_VERSION "v1.0" +#define PROGRAM_VERSION "v1.01" /* static global variables */ static char *input_path; @@ -926,7 +929,7 @@ int main(int argc, char *argv[]) { /* reset */ seasons_group_count = seasons_group_count_bak; - /* inc total files founded */ + /* inc total files found */ total_files_count += files[z].count; /* processing and create output filename */ @@ -947,7 +950,7 @@ int main(int argc, char *argv[]) { } /* add to filename */ - if ( !mystrcat(filename, string, FILENAME_MAX) && '\0' == filename[0]) { + if ( !string_concat(filename, string, FILENAME_MAX) && '\0' == filename[0]) { puts(err_unable_create_output_filename); files_not_processed_count += files[z].count; free(string); @@ -1226,7 +1229,7 @@ int main(int argc, char *argv[]) { } /* summary */ - printf("%d file%s founded: %d processed, %d skipped.\n\n", + printf("%d file%s found: %d processed, %d skipped.\n\n", total_files_count, total_files_count > 1 ? "s" : "", files_processed_count, diff --git a/tests/C/Makefile b/tests/C/Makefile new file mode 100644 index 00000000..8c4ba604 --- /dev/null +++ b/tests/C/Makefile @@ -0,0 +1,16 @@ +# enables most optimization flags +CC := gcc -O3 + +TEST01_SRC := test_01.c +TEST01_OBJ := $(TEST01_SRC:.c=.o) +TEST01_BIN := $(join ${TGTDIR}, test_01) + +${TEST01_BIN}: + @echo "Building ${TEST01_BIN}..." + ${CC} -w -c ${TEST01_SRC} ;\ + ${CC} ${TEST01_OBJ} -w -lm -o ${TEST01_BIN} + +clean: + @echo "Cleaning up..." + rm -vf ${TEST01_OBJ} ; rm -vf ${TEST01_BIN} + @echo "Done cleaning up" \ No newline at end of file diff --git a/tests/C/test_01.c b/tests/C/test_01.c new file mode 100644 index 00000000..bd184d4a --- /dev/null +++ b/tests/C/test_01.c @@ -0,0 +1,70 @@ +/* + test_01.c + + check library output for reference: + + Running suite(s): Meteo + 66%: Checks: 3, Failures: 1, Errors: 0 + test_01.c:13:F:Sanity checks:sanity_check:0: this should fail + test_01.c:18:P:Sanity checks:sanity_check_2:0: Passed + test_01.c:53:P:Dataset files:test_is_valid_era_filename:0: Passed +*/ + +#include "utest_oneflux.h" +#include "../../oneflux_steps/common/common.c" +#include "../../oneflux_steps/meteo_proc/src/dataset.c" + +/* + dataset.c needs some external variables +*/ +char *qc_auto_files_path = NULL; /* mandatory */ +char *era_files_path = NULL; /* mandatory */ +char *output_files_path = NULL; /* mandatory */ + +UTEST(sanity_check, 1) +{ + FAIL_UNLESS(5 == 5, "this should succeed"); + FAIL_UNLESS(6 == 5, "this should fail"); +} + +UTEST(sanity_check, 2) +{ + FAIL_UNLESS(5 == 5, "this should succeed"); +} + +UTEST(test_is_valid_era_filename, 1) +{ + char* filename; + + filename = ""; + + /* CK_ASSERT_MSG: Fails test if supplied condition evaluates to false and displays user provided message. */ + + CK_ASSERT_MSG(!is_valid_era_filename(filename), "False should be returned for this string: '%s'", filename); + + filename = " - _9999.csv"; + CK_ASSERT_MSG(is_valid_era_filename(filename), "True should be returned for this string: %s", filename); + + filename = ""; + CK_ASSERT_MSG(!is_valid_era_filename(filename), "False should be returned for this string: %s", filename); + + // example of a correct string + filename = "US-ARc_2009.csv"; + CK_ASSERT_MSG(is_valid_era_filename(filename), "True should be returned for this string: %s", filename); +} + +/* + PLEASE NOTE: + + we can use 'UTEST_MAIN' instead of call 'UTEST_STATE' and 'main' + but in this way we can do our own stuff like load files and so on... +*/ + +UTEST_STATE(); + +int main(int argc, const char *const argv[]) +{ + /* do your own stuff here! */ + + return utest_main(argc, argv); +} diff --git a/tests/C/utest.h b/tests/C/utest.h new file mode 100644 index 00000000..a6fd773b --- /dev/null +++ b/tests/C/utest.h @@ -0,0 +1,1606 @@ +/* + The latest version of this library is available on GitHub; + https://github.com/sheredom/utest.h +*/ + +/* + This is free and unencumbered software released into the public domain. + + Anyone is free to copy, modify, publish, use, compile, sell, or + distribute this software, either in source code form or as a compiled + binary, for any purpose, commercial or non-commercial, and by any + means. + + In jurisdictions that recognize copyright laws, the author or authors + of this software dedicate any and all copyright interest in the + software to the public domain. We make this dedication for the benefit + of the public at large and to the detriment of our heirs and + successors. We intend this dedication to be an overt act of + relinquishment in perpetuity of all present and future rights to this + software under copyright law. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR + OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, + ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + OTHER DEALINGS IN THE SOFTWARE. + + For more information, please refer to +*/ + +#ifndef SHEREDOM_UTEST_H_INCLUDED +#define SHEREDOM_UTEST_H_INCLUDED + +#ifdef _MSC_VER +/* + Disable warning about not inlining 'inline' functions. +*/ +#pragma warning(disable : 4710) + +/* + Disable warning about inlining functions that are not marked 'inline'. +*/ +#pragma warning(disable : 4711) + +/* + Disable warning for alignment padding added +*/ +#pragma warning(disable : 4820) + +#if _MSC_VER > 1900 +/* + Disable warning about preprocessor macros not being defined in MSVC headers. +*/ +#pragma warning(disable : 4668) + +/* + Disable warning about no function prototype given in MSVC headers. +*/ +#pragma warning(disable : 4255) + +/* + Disable warning about pointer or reference to potentially throwing function. +*/ +#pragma warning(disable : 5039) + +/* + Disable warning about macro expansion producing 'defined' has undefined + behavior. +*/ +#pragma warning(disable : 5105) +#endif + +#if _MSC_VER > 1930 +/* + Disable warning about 'const' variable is not used. +*/ +#pragma warning(disable : 5264) +#endif + +#pragma warning(push, 1) +#endif + +#if defined(_MSC_VER) && (_MSC_VER < 1920) +typedef __int64 utest_int64_t; +typedef unsigned __int64 utest_uint64_t; +typedef unsigned __int32 utest_uint32_t; +#else +#include +typedef int64_t utest_int64_t; +typedef uint64_t utest_uint64_t; +typedef uint32_t utest_uint32_t; +#endif + +#include +#include +#include +#include +#include + +#if defined(__cplusplus) +#if defined(_MSC_VER) && !defined(_CPPUNWIND) +/* We're on MSVC and the compiler is compiling without exception support! */ +#elif !defined(_MSC_VER) && !defined(__EXCEPTIONS) +/* We're on a GCC/Clang compiler that doesn't have exception support! */ +#else +#define UTEST_HAS_EXCEPTIONS 1 +#endif +#endif + +#if defined(UTEST_HAS_EXCEPTIONS) +#include +#endif + +#if defined(_MSC_VER) +#pragma warning(pop) +#endif + +#if defined(__cplusplus) +#define UTEST_C_FUNC extern "C" +#else +#define UTEST_C_FUNC +#endif + +#define UTEST_TEST_PASSED (0) +#define UTEST_TEST_FAILURE (1) +#define UTEST_TEST_SKIPPED (2) + +#if defined(_MSC_VER) || defined(__MINGW64__) || defined(__MINGW32__) + +#if defined(__MINGW64__) || defined(__MINGW32__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpragmas" +#pragma GCC diagnostic ignored "-Wunknown-pragmas" +#endif + +#if defined(_WINDOWS_) || defined(_WINDOWS_H) +typedef LARGE_INTEGER utest_large_integer; +#else +// use old QueryPerformanceCounter definitions (not sure is this needed in some +// edge cases or not) on Win7 with VS2015 these extern declaration cause "second +// C linkage of overloaded function not allowed" error +typedef union { + struct { + unsigned long LowPart; + long HighPart; + } DUMMYSTRUCTNAME; + struct { + unsigned long LowPart; + long HighPart; + } u; + utest_int64_t QuadPart; +} utest_large_integer; + +UTEST_C_FUNC __declspec(dllimport) int __stdcall QueryPerformanceCounter( + utest_large_integer *); +UTEST_C_FUNC __declspec(dllimport) int __stdcall QueryPerformanceFrequency( + utest_large_integer *); + +#if defined(__MINGW64__) || defined(__MINGW32__) +#pragma GCC diagnostic pop +#endif +#endif + +#elif defined(__linux__) || defined(__FreeBSD__) || defined(__OpenBSD__) || \ + defined(__NetBSD__) || defined(__DragonFly__) || defined(__sun__) || \ + defined(__HAIKU__) +/* + slightly obscure include here - we need to include glibc's features.h, but + we don't want to just include a header that might not be defined for other + c libraries like musl. Instead we include limits.h, which we know on all + glibc distributions includes features.h +*/ +#include + +#if defined(__GLIBC__) && defined(__GLIBC_MINOR__) +#include + +#if ((2 < __GLIBC__) || ((2 == __GLIBC__) && (17 <= __GLIBC_MINOR__))) +/* glibc is version 2.17 or above, so we can just use clock_gettime */ +#define UTEST_USE_CLOCKGETTIME +#else +#include +#include +#endif +#else // Other libc implementations +#include +#define UTEST_USE_CLOCKGETTIME +#endif + +#elif defined(__APPLE__) +#include +#endif + +#if defined(_MSC_VER) && (_MSC_VER < 1920) +#define UTEST_PRId64 "I64d" +#define UTEST_PRIu64 "I64u" +#else +#include + +#define UTEST_PRId64 PRId64 +#define UTEST_PRIu64 PRIu64 +#endif + +#if defined(__cplusplus) +#define UTEST_INLINE inline + +#if defined(__clang__) +#define UTEST_INITIALIZER_BEGIN_DISABLE_WARNINGS \ + _Pragma("clang diagnostic push") \ + _Pragma("clang diagnostic ignored \"-Wglobal-constructors\"") + +#define UTEST_INITIALIZER_END_DISABLE_WARNINGS _Pragma("clang diagnostic pop") +#else +#define UTEST_INITIALIZER_BEGIN_DISABLE_WARNINGS +#define UTEST_INITIALIZER_END_DISABLE_WARNINGS +#endif + +#define UTEST_INITIALIZER(f) \ + struct f##_cpp_struct { \ + f##_cpp_struct(); \ + }; \ + UTEST_INITIALIZER_BEGIN_DISABLE_WARNINGS static f##_cpp_struct \ + f##_cpp_global UTEST_INITIALIZER_END_DISABLE_WARNINGS; \ + f##_cpp_struct::f##_cpp_struct() +#elif defined(_MSC_VER) +#define UTEST_INLINE __forceinline + +#if defined(_WIN64) +#define UTEST_SYMBOL_PREFIX +#else +#define UTEST_SYMBOL_PREFIX "_" +#endif + +#if defined(__clang__) +#define UTEST_INITIALIZER_BEGIN_DISABLE_WARNINGS \ + _Pragma("clang diagnostic push") \ + _Pragma("clang diagnostic ignored \"-Wmissing-variable-declarations\"") + +#define UTEST_INITIALIZER_END_DISABLE_WARNINGS _Pragma("clang diagnostic pop") +#else +#define UTEST_INITIALIZER_BEGIN_DISABLE_WARNINGS +#define UTEST_INITIALIZER_END_DISABLE_WARNINGS +#endif + +#pragma section(".CRT$XCU", read) +#define UTEST_INITIALIZER(f) \ + static void __cdecl f(void); \ + UTEST_INITIALIZER_BEGIN_DISABLE_WARNINGS \ + __pragma(comment(linker, "/include:" UTEST_SYMBOL_PREFIX #f "_")) \ + UTEST_C_FUNC __declspec(allocate(".CRT$XCU")) void(__cdecl * \ + f##_)(void) = f; \ + UTEST_INITIALIZER_END_DISABLE_WARNINGS \ + static void __cdecl f(void) +#else +#if defined(__linux__) +#if defined(__clang__) +#if __has_warning("-Wreserved-id-macro") +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wreserved-id-macro" +#endif +#endif + +#define __STDC_FORMAT_MACROS 1 + +#if defined(__clang__) +#if __has_warning("-Wreserved-id-macro") +#pragma clang diagnostic pop +#endif +#endif +#endif + +#define UTEST_INLINE inline + +#define UTEST_INITIALIZER(f) \ + static void f(void) __attribute__((constructor)); \ + static void f(void) +#endif + +#if defined(__cplusplus) +#define UTEST_CAST(type, x) static_cast(x) +#define UTEST_PTR_CAST(type, x) reinterpret_cast(x) +#define UTEST_EXTERN extern "C" +#define UTEST_NULL NULL +#else +#define UTEST_CAST(type, x) ((type)(x)) +#define UTEST_PTR_CAST(type, x) ((type)(x)) +#define UTEST_EXTERN extern +#define UTEST_NULL 0 +#endif + +#ifdef _MSC_VER +/* + io.h contains definitions for some structures with natural padding. This is + uninteresting, but for some reason MSVC's behaviour is to warn about + including this system header. That *is* interesting +*/ +#pragma warning(disable : 4820) +#pragma warning(push, 1) +#include +#pragma warning(pop) +#define UTEST_COLOUR_OUTPUT() (_isatty(_fileno(stdout))) +#else +#if defined(__EMSCRIPTEN__) +#include +#define UTEST_COLOUR_OUTPUT() false +#else +#include +#define UTEST_COLOUR_OUTPUT() (isatty(STDOUT_FILENO)) +#endif +#endif + +static UTEST_INLINE void *utest_realloc(void *const pointer, size_t new_size) { + void *const new_pointer = realloc(pointer, new_size); + + if (UTEST_NULL == new_pointer) { + free(new_pointer); + } + + return new_pointer; +} + +static UTEST_INLINE utest_int64_t utest_ns(void) { +#if defined(_MSC_VER) || defined(__MINGW64__) || defined(__MINGW32__) + utest_large_integer counter; + utest_large_integer frequency; + QueryPerformanceCounter(&counter); + QueryPerformanceFrequency(&frequency); + return UTEST_CAST(utest_int64_t, + (counter.QuadPart * 1000000000) / frequency.QuadPart); +#elif defined(__linux__) && defined(__STRICT_ANSI__) + return UTEST_CAST(utest_int64_t, clock()) * 1000000000 / CLOCKS_PER_SEC; +#elif defined(__linux__) || defined(__FreeBSD__) || defined(__OpenBSD__) || \ + defined(__NetBSD__) || defined(__DragonFly__) || defined(__sun__) || \ + defined(__HAIKU__) + struct timespec ts; +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) && \ + !defined(__HAIKU__) + timespec_get(&ts, TIME_UTC); +#else + const clockid_t cid = CLOCK_REALTIME; +#if defined(UTEST_USE_CLOCKGETTIME) + clock_gettime(cid, &ts); +#else + syscall(SYS_clock_gettime, cid, &ts); +#endif +#endif + return UTEST_CAST(utest_int64_t, ts.tv_sec) * 1000 * 1000 * 1000 + ts.tv_nsec; +#elif __APPLE__ + return UTEST_CAST(utest_int64_t, clock_gettime_nsec_np(CLOCK_UPTIME_RAW)); +#elif __EMSCRIPTEN__ + return emscripten_performance_now() * 1000000.0; +#else +#error Unsupported platform! +#endif +} + +typedef void (*utest_testcase_t)(int *, size_t); + +struct utest_test_state_s { + utest_testcase_t func; + size_t index; + char *name; +}; + +struct utest_state_s { + struct utest_test_state_s *tests; + size_t tests_length; + FILE *output; +}; + +/* extern to the global state utest needs to execute */ +UTEST_EXTERN struct utest_state_s utest_state; + +#if defined(_MSC_VER) +#define UTEST_WEAK __forceinline +#elif defined(__MINGW32__) || defined(__MINGW64__) +#define UTEST_WEAK static __attribute__((used)) +#elif defined(__clang__) || defined(__GNUC__) +#define UTEST_WEAK __attribute__((weak)) +#else +#error Non clang, non gcc, non MSVC compiler found! +#endif + +#if defined(_MSC_VER) +#define UTEST_UNUSED +#else +#define UTEST_UNUSED __attribute__((unused)) +#endif + +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wvariadic-macros" +#pragma clang diagnostic ignored "-Wc++98-compat-pedantic" +#endif +#define UTEST_PRINTF(...) \ + if (utest_state.output) { \ + fprintf(utest_state.output, __VA_ARGS__); \ + } \ + printf(__VA_ARGS__) +#ifdef __clang__ +#pragma clang diagnostic pop +#endif + +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wvariadic-macros" +#pragma clang diagnostic ignored "-Wc++98-compat-pedantic" +#endif + +#ifdef _MSC_VER +#define UTEST_SNPRINTF(BUFFER, N, ...) _snprintf_s(BUFFER, N, N, __VA_ARGS__) +#else +#define UTEST_SNPRINTF(...) snprintf(__VA_ARGS__) +#endif + +#ifdef __clang__ +#pragma clang diagnostic pop +#endif + +#if defined(__cplusplus) +/* if we are using c++ we can use overloaded methods (its in the language) */ +#define UTEST_OVERLOADABLE +#elif defined(__clang__) +/* otherwise, if we are using clang with c - use the overloadable attribute */ +#define UTEST_OVERLOADABLE __attribute__((overloadable)) +#endif + +#if defined(__cplusplus) && (__cplusplus >= 201103L) + +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wc++98-compat-pedantic" +#endif + +#include + +template ::value> +struct utest_type_deducer final { + static void _(const T t); +}; + +template <> struct utest_type_deducer { + static void _(const signed char c) { + UTEST_PRINTF("%d", static_cast(c)); + } +}; + +template <> struct utest_type_deducer { + static void _(const unsigned char c) { + UTEST_PRINTF("%u", static_cast(c)); + } +}; + +template <> struct utest_type_deducer { + static void _(const short s) { UTEST_PRINTF("%d", static_cast(s)); } +}; + +template <> struct utest_type_deducer { + static void _(const unsigned short s) { + UTEST_PRINTF("%u", static_cast(s)); + } +}; + +template <> struct utest_type_deducer { + static void _(const float f) { UTEST_PRINTF("%f", static_cast(f)); } +}; + +template <> struct utest_type_deducer { + static void _(const double d) { UTEST_PRINTF("%f", d); } +}; + +template <> struct utest_type_deducer { + static void _(const long double d) { +#if defined(__MINGW32__) || defined(__MINGW64__) + /* MINGW is weird - doesn't like LF at all?! */ + UTEST_PRINTF("%f", (double)d); +#else + UTEST_PRINTF("%Lf", d); +#endif + } +}; + +template <> struct utest_type_deducer { + static void _(const int i) { UTEST_PRINTF("%d", i); } +}; + +template <> struct utest_type_deducer { + static void _(const unsigned int i) { UTEST_PRINTF("%u", i); } +}; + +template <> struct utest_type_deducer { + static void _(const long i) { UTEST_PRINTF("%ld", i); } +}; + +template <> struct utest_type_deducer { + static void _(const unsigned long i) { UTEST_PRINTF("%lu", i); } +}; + +template <> struct utest_type_deducer { + static void _(const long long i) { UTEST_PRINTF("%lld", i); } +}; + +template <> struct utest_type_deducer { + static void _(const unsigned long long i) { UTEST_PRINTF("%llu", i); } +}; + +template struct utest_type_deducer { + static void _(const T *t) { + UTEST_PRINTF("%p", static_cast(const_cast(t))); + } +}; + +template struct utest_type_deducer { + static void _(T *t) { UTEST_PRINTF("%p", static_cast(t)); } +}; + +template struct utest_type_deducer { + static void _(const T t) { + UTEST_PRINTF("%llu", static_cast(t)); + } +}; + +template +UTEST_WEAK UTEST_OVERLOADABLE void utest_type_printer(const T t) { + utest_type_deducer::_(t); +} + +#ifdef __clang__ +#pragma clang diagnostic pop +#endif + +#elif defined(UTEST_OVERLOADABLE) + +UTEST_WEAK UTEST_OVERLOADABLE void utest_type_printer(signed char c); +UTEST_WEAK UTEST_OVERLOADABLE void utest_type_printer(signed char c) { + UTEST_PRINTF("%d", UTEST_CAST(int, c)); +} + +UTEST_WEAK UTEST_OVERLOADABLE void utest_type_printer(unsigned char c); +UTEST_WEAK UTEST_OVERLOADABLE void utest_type_printer(unsigned char c) { + UTEST_PRINTF("%u", UTEST_CAST(unsigned int, c)); +} + +UTEST_WEAK UTEST_OVERLOADABLE void utest_type_printer(float f); +UTEST_WEAK UTEST_OVERLOADABLE void utest_type_printer(float f) { + UTEST_PRINTF("%f", UTEST_CAST(double, f)); +} + +UTEST_WEAK UTEST_OVERLOADABLE void utest_type_printer(double d); +UTEST_WEAK UTEST_OVERLOADABLE void utest_type_printer(double d) { + UTEST_PRINTF("%f", d); +} + +UTEST_WEAK UTEST_OVERLOADABLE void utest_type_printer(long double d); +UTEST_WEAK UTEST_OVERLOADABLE void utest_type_printer(long double d) { +#if defined(__MINGW32__) || defined(__MINGW64__) + /* MINGW is weird - doesn't like LF at all?! */ + UTEST_PRINTF("%f", (double)d); +#else + UTEST_PRINTF("%Lf", d); +#endif +} + +UTEST_WEAK UTEST_OVERLOADABLE void utest_type_printer(int i); +UTEST_WEAK UTEST_OVERLOADABLE void utest_type_printer(int i) { + UTEST_PRINTF("%d", i); +} + +UTEST_WEAK UTEST_OVERLOADABLE void utest_type_printer(unsigned int i); +UTEST_WEAK UTEST_OVERLOADABLE void utest_type_printer(unsigned int i) { + UTEST_PRINTF("%u", i); +} + +UTEST_WEAK UTEST_OVERLOADABLE void utest_type_printer(long int i); +UTEST_WEAK UTEST_OVERLOADABLE void utest_type_printer(long int i) { + UTEST_PRINTF("%ld", i); +} + +UTEST_WEAK UTEST_OVERLOADABLE void utest_type_printer(long unsigned int i); +UTEST_WEAK UTEST_OVERLOADABLE void utest_type_printer(long unsigned int i) { + UTEST_PRINTF("%lu", i); +} + +UTEST_WEAK UTEST_OVERLOADABLE void utest_type_printer(const void *p); +UTEST_WEAK UTEST_OVERLOADABLE void utest_type_printer(const void *p) { + UTEST_PRINTF("%p", p); +} + +/* + long long is a c++11 extension +*/ +#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) || \ + defined(__cplusplus) && (__cplusplus >= 201103L) || \ + (defined(__MINGW32__) || defined(__MINGW64__)) + +#ifdef __clang__ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wc++98-compat-pedantic" +#endif + +UTEST_WEAK UTEST_OVERLOADABLE void utest_type_printer(long long int i); +UTEST_WEAK UTEST_OVERLOADABLE void utest_type_printer(long long int i) { + UTEST_PRINTF("%lld", i); +} + +UTEST_WEAK UTEST_OVERLOADABLE void utest_type_printer(long long unsigned int i); +UTEST_WEAK UTEST_OVERLOADABLE void +utest_type_printer(long long unsigned int i) { + UTEST_PRINTF("%llu", i); +} + +#ifdef __clang__ +#pragma clang diagnostic pop +#endif + +#endif +#elif defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) && \ + !(defined(__MINGW32__) || defined(__MINGW64__)) +#define utest_type_printer(val) \ + UTEST_PRINTF(_Generic((val), signed char \ + : "%d", unsigned char \ + : "%u", short \ + : "%d", unsigned short \ + : "%u", int \ + : "%d", long \ + : "%ld", long long \ + : "%lld", unsigned \ + : "%u", unsigned long \ + : "%lu", unsigned long long \ + : "%llu", float \ + : "%f", double \ + : "%f", long double \ + : "%Lf", default \ + : _Generic((val - val), ptrdiff_t \ + : "%p", default \ + : "undef")), \ + (val)) +#else +/* + we don't have the ability to print the values we got, so we create a macro + to tell our users we can't do anything fancy +*/ +#define utest_type_printer(...) UTEST_PRINTF("undef") +#endif + +#ifdef _MSC_VER +#define UTEST_SURPRESS_WARNING_BEGIN \ + __pragma(warning(push)) __pragma(warning(disable : 4127)) \ + __pragma(warning(disable : 4571)) __pragma(warning(disable : 4130)) +#define UTEST_SURPRESS_WARNING_END __pragma(warning(pop)) +#else +#define UTEST_SURPRESS_WARNING_BEGIN +#define UTEST_SURPRESS_WARNING_END +#endif + +#if defined(__cplusplus) && (__cplusplus >= 201103L) +#define UTEST_AUTO(x) auto +#elif !defined(__cplusplus) + +#if defined(__clang__) +/* clang-format off */ +/* had to disable clang-format here because it malforms the pragmas */ +#define UTEST_AUTO(x) \ + _Pragma("clang diagnostic push") \ + _Pragma("clang diagnostic ignored \"-Wgnu-auto-type\"") __auto_type \ + _Pragma("clang diagnostic pop") +/* clang-format on */ +#else +#define UTEST_AUTO(x) __typeof__(x + 0) +#endif + +#else +#define UTEST_AUTO(x) typeof(x + 0) +#endif + +#if defined(__clang__) +#define UTEST_STRNCMP(x, y, size) \ + _Pragma("clang diagnostic push") \ + _Pragma("clang diagnostic ignored \"-Wdisabled-macro-expansion\"") \ + strncmp(x, y, size) _Pragma("clang diagnostic pop") +#else +#define UTEST_STRNCMP(x, y, size) strncmp(x, y, size) +#endif + +#define UTEST_SKIP(msg) \ + do { \ + UTEST_PRINTF(" Skipped : '%s'\n", (msg)); \ + *utest_result = UTEST_TEST_SKIPPED; \ + return; \ + } while (0) + +#if defined(__clang__) +#define UTEST_EXPECT(x, y, cond) \ + UTEST_SURPRESS_WARNING_BEGIN do { \ + _Pragma("clang diagnostic push") \ + _Pragma("clang diagnostic ignored \"-Wlanguage-extension-token\"") \ + _Pragma("clang diagnostic ignored \"-Wc++98-compat-pedantic\"") \ + _Pragma("clang diagnostic ignored \"-Wfloat-equal\"") \ + UTEST_AUTO(x) xEval = (x); \ + UTEST_AUTO(y) yEval = (y); \ + if (!((xEval)cond(yEval))) { \ + _Pragma("clang diagnostic pop") \ + UTEST_PRINTF("%s:%i: Failure\n", __FILE__, __LINE__); \ + UTEST_PRINTF(" Expected : ("); \ + UTEST_PRINTF(#x ") " #cond " (" #y); \ + UTEST_PRINTF(")\n"); \ + UTEST_PRINTF(" Actual : "); \ + utest_type_printer(xEval); \ + UTEST_PRINTF(" vs "); \ + utest_type_printer(yEval); \ + UTEST_PRINTF("\n"); \ + *utest_result = UTEST_TEST_FAILURE; \ + } \ + } \ + while (0) \ + UTEST_SURPRESS_WARNING_END +#elif defined(__GNUC__) +#define UTEST_EXPECT(x, y, cond) \ + UTEST_SURPRESS_WARNING_BEGIN do { \ + UTEST_AUTO(x) xEval = (x); \ + UTEST_AUTO(y) yEval = (y); \ + if (!((xEval)cond(yEval))) { \ + UTEST_PRINTF("%s:%i: Failure\n", __FILE__, __LINE__); \ + UTEST_PRINTF(" Expected : ("); \ + UTEST_PRINTF(#x ") " #cond " (" #y); \ + UTEST_PRINTF(")\n"); \ + UTEST_PRINTF(" Actual : "); \ + utest_type_printer(xEval); \ + UTEST_PRINTF(" vs "); \ + utest_type_printer(yEval); \ + UTEST_PRINTF("\n"); \ + *utest_result = UTEST_TEST_FAILURE; \ + } \ + } \ + while (0) \ + UTEST_SURPRESS_WARNING_END +#else +#define UTEST_EXPECT(x, y, cond) \ + UTEST_SURPRESS_WARNING_BEGIN do { \ + if (!((x)cond(y))) { \ + UTEST_PRINTF("%s:%i: Failure (Expected " #cond " Actual)\n", __FILE__, \ + __LINE__); \ + *utest_result = UTEST_TEST_FAILURE; \ + } \ + } \ + while (0) \ + UTEST_SURPRESS_WARNING_END +#endif + +#define EXPECT_TRUE(x) \ + UTEST_SURPRESS_WARNING_BEGIN do { \ + const int xEval = !!(x); \ + if (!(xEval)) { \ + UTEST_PRINTF("%s:%i: Failure\n", __FILE__, __LINE__); \ + UTEST_PRINTF(" Expected : true\n"); \ + UTEST_PRINTF(" Actual : %s\n", (xEval) ? "true" : "false"); \ + *utest_result = UTEST_TEST_FAILURE; \ + } \ + } \ + while (0) \ + UTEST_SURPRESS_WARNING_END + +#define EXPECT_FALSE(x) \ + UTEST_SURPRESS_WARNING_BEGIN do { \ + const int xEval = !!(x); \ + if (xEval) { \ + UTEST_PRINTF("%s:%i: Failure\n", __FILE__, __LINE__); \ + UTEST_PRINTF(" Expected : false\n"); \ + UTEST_PRINTF(" Actual : %s\n", (xEval) ? "true" : "false"); \ + *utest_result = UTEST_TEST_FAILURE; \ + } \ + } \ + while (0) \ + UTEST_SURPRESS_WARNING_END + +#define EXPECT_EQ(x, y) UTEST_EXPECT(x, y, ==) +#define EXPECT_NE(x, y) UTEST_EXPECT(x, y, !=) +#define EXPECT_LT(x, y) UTEST_EXPECT(x, y, <) +#define EXPECT_LE(x, y) UTEST_EXPECT(x, y, <=) +#define EXPECT_GT(x, y) UTEST_EXPECT(x, y, >) +#define EXPECT_GE(x, y) UTEST_EXPECT(x, y, >=) + +#define EXPECT_STREQ(x, y) \ + UTEST_SURPRESS_WARNING_BEGIN do { \ + const char *xEval = (x); \ + const char *yEval = (y); \ + if (UTEST_NULL == xEval || UTEST_NULL == yEval || \ + 0 != strcmp(xEval, yEval)) { \ + UTEST_PRINTF("%s:%i: Failure\n", __FILE__, __LINE__); \ + UTEST_PRINTF(" Expected : \"%s\"\n", xEval); \ + UTEST_PRINTF(" Actual : \"%s\"\n", yEval); \ + *utest_result = UTEST_TEST_FAILURE; \ + } \ + } \ + while (0) \ + UTEST_SURPRESS_WARNING_END + +#define EXPECT_STRNE(x, y) \ + UTEST_SURPRESS_WARNING_BEGIN do { \ + const char *xEval = (x); \ + const char *yEval = (y); \ + if (UTEST_NULL == xEval || UTEST_NULL == yEval || \ + 0 == strcmp(xEval, yEval)) { \ + UTEST_PRINTF("%s:%i: Failure\n", __FILE__, __LINE__); \ + UTEST_PRINTF(" Expected : \"%s\"\n", xEval); \ + UTEST_PRINTF(" Actual : \"%s\"\n", yEval); \ + *utest_result = UTEST_TEST_FAILURE; \ + } \ + } \ + while (0) \ + UTEST_SURPRESS_WARNING_END + +#define EXPECT_STRNEQ(x, y, n) \ + UTEST_SURPRESS_WARNING_BEGIN do { \ + const char *xEval = (x); \ + const char *yEval = (y); \ + const size_t nEval = UTEST_CAST(size_t, n); \ + if (UTEST_NULL == xEval || UTEST_NULL == yEval || \ + 0 != UTEST_STRNCMP(xEval, yEval, nEval)) { \ + UTEST_PRINTF("%s:%i: Failure\n", __FILE__, __LINE__); \ + UTEST_PRINTF(" Expected : \"%.*s\"\n", UTEST_CAST(int, nEval), xEval); \ + UTEST_PRINTF(" Actual : \"%.*s\"\n", UTEST_CAST(int, nEval), yEval); \ + *utest_result = UTEST_TEST_FAILURE; \ + } \ + } \ + while (0) \ + UTEST_SURPRESS_WARNING_END + +#define EXPECT_STRNNE(x, y, n) \ + UTEST_SURPRESS_WARNING_BEGIN do { \ + const char *xEval = (x); \ + const char *yEval = (y); \ + const size_t nEval = UTEST_CAST(size_t, n); \ + if (UTEST_NULL == xEval || UTEST_NULL == yEval || \ + 0 == UTEST_STRNCMP(xEval, yEval, nEval)) { \ + UTEST_PRINTF("%s:%i: Failure\n", __FILE__, __LINE__); \ + UTEST_PRINTF(" Expected : \"%.*s\"\n", UTEST_CAST(int, nEval), xEval); \ + UTEST_PRINTF(" Actual : \"%.*s\"\n", UTEST_CAST(int, nEval), yEval); \ + *utest_result = UTEST_TEST_FAILURE; \ + } \ + } \ + while (0) \ + UTEST_SURPRESS_WARNING_END + +#define EXPECT_NEAR(x, y, epsilon) \ + UTEST_SURPRESS_WARNING_BEGIN do { \ + const double diff = \ + utest_fabs(UTEST_CAST(double, x) - UTEST_CAST(double, y)); \ + if (diff > UTEST_CAST(double, epsilon) || utest_isnan(diff)) { \ + UTEST_PRINTF("%s:%i: Failure\n", __FILE__, __LINE__); \ + UTEST_PRINTF(" Expected : %f\n", UTEST_CAST(double, x)); \ + UTEST_PRINTF(" Actual : %f\n", UTEST_CAST(double, y)); \ + *utest_result = UTEST_TEST_FAILURE; \ + } \ + } \ + while (0) \ + UTEST_SURPRESS_WARNING_END + +#if defined(UTEST_HAS_EXCEPTIONS) +#define EXPECT_EXCEPTION(x, exception_type) \ + UTEST_SURPRESS_WARNING_BEGIN do { \ + int exception_caught = 0; \ + try { \ + x; \ + } catch (const exception_type &) { \ + exception_caught = 1; \ + } catch (...) { \ + exception_caught = 2; \ + } \ + if (exception_caught != 1) { \ + UTEST_PRINTF("%s:%i: Failure\n", __FILE__, __LINE__); \ + UTEST_PRINTF(" Expected : %s exception\n", #exception_type); \ + UTEST_PRINTF(" Actual : %s\n", (exception_caught == 2) \ + ? "Unexpected exception" \ + : "No exception"); \ + *utest_result = UTEST_TEST_FAILURE; \ + } \ + } \ + while (0) \ + UTEST_SURPRESS_WARNING_END +#endif + +#if defined(__clang__) +#define UTEST_ASSERT(x, y, cond) \ + UTEST_SURPRESS_WARNING_BEGIN do { \ + _Pragma("clang diagnostic push") \ + _Pragma("clang diagnostic ignored \"-Wlanguage-extension-token\"") \ + _Pragma("clang diagnostic ignored \"-Wc++98-compat-pedantic\"") \ + _Pragma("clang diagnostic ignored \"-Wfloat-equal\"") \ + UTEST_AUTO(x) xEval = (x); \ + UTEST_AUTO(y) yEval = (y); \ + if (!((xEval)cond(yEval))) { \ + _Pragma("clang diagnostic pop") \ + UTEST_PRINTF("%s:%i: Failure\n", __FILE__, __LINE__); \ + UTEST_PRINTF(" Expected : ("); \ + UTEST_PRINTF(#x ") " #cond " (" #y); \ + UTEST_PRINTF(")\n"); \ + UTEST_PRINTF(" Actual : "); \ + utest_type_printer(xEval); \ + UTEST_PRINTF(" vs "); \ + utest_type_printer(yEval); \ + UTEST_PRINTF("\n"); \ + *utest_result = UTEST_TEST_FAILURE; \ + return; \ + } \ + } \ + while (0) \ + UTEST_SURPRESS_WARNING_END +#elif defined(__GNUC__) +#define UTEST_ASSERT(x, y, cond) \ + UTEST_SURPRESS_WARNING_BEGIN do { \ + UTEST_AUTO(x) xEval = (x); \ + UTEST_AUTO(y) yEval = (y); \ + if (!((xEval)cond(yEval))) { \ + UTEST_PRINTF("%s:%i: Failure\n", __FILE__, __LINE__); \ + UTEST_PRINTF(" Expected : ("); \ + UTEST_PRINTF(#x ") " #cond " (" #y); \ + UTEST_PRINTF(")\n"); \ + UTEST_PRINTF(" Actual : "); \ + utest_type_printer(xEval); \ + UTEST_PRINTF(" vs "); \ + utest_type_printer(yEval); \ + UTEST_PRINTF("\n"); \ + *utest_result = UTEST_TEST_FAILURE; \ + return; \ + } \ + } \ + while (0) \ + UTEST_SURPRESS_WARNING_END +#else +#define UTEST_ASSERT(x, y, cond) \ + UTEST_SURPRESS_WARNING_BEGIN do { \ + if (!((x)cond(y))) { \ + UTEST_PRINTF("%s:%i: Failure (Expected " #cond " Actual)\n", __FILE__, \ + __LINE__); \ + *utest_result = UTEST_TEST_FAILURE; \ + return; \ + } \ + } \ + while (0) \ + UTEST_SURPRESS_WARNING_END +#endif + +#define ASSERT_TRUE(x) \ + UTEST_SURPRESS_WARNING_BEGIN do { \ + const int xEval = !!(x); \ + if (!(xEval)) { \ + UTEST_PRINTF("%s:%i: Failure\n", __FILE__, __LINE__); \ + UTEST_PRINTF(" Expected : true\n"); \ + UTEST_PRINTF(" Actual : %s\n", (xEval) ? "true" : "false"); \ + *utest_result = UTEST_TEST_FAILURE; \ + return; \ + } \ + } \ + while (0) \ + UTEST_SURPRESS_WARNING_END + +#define ASSERT_FALSE(x) \ + UTEST_SURPRESS_WARNING_BEGIN do { \ + const int xEval = !!(x); \ + if (xEval) { \ + UTEST_PRINTF("%s:%i: Failure\n", __FILE__, __LINE__); \ + UTEST_PRINTF(" Expected : false\n"); \ + UTEST_PRINTF(" Actual : %s\n", (xEval) ? "true" : "false"); \ + *utest_result = UTEST_TEST_FAILURE; \ + return; \ + } \ + } \ + while (0) \ + UTEST_SURPRESS_WARNING_END + +#define ASSERT_EQ(x, y) UTEST_ASSERT(x, y, ==) +#define ASSERT_NE(x, y) UTEST_ASSERT(x, y, !=) +#define ASSERT_LT(x, y) UTEST_ASSERT(x, y, <) +#define ASSERT_LE(x, y) UTEST_ASSERT(x, y, <=) +#define ASSERT_GT(x, y) UTEST_ASSERT(x, y, >) +#define ASSERT_GE(x, y) UTEST_ASSERT(x, y, >=) + +#define ASSERT_STREQ(x, y) \ + UTEST_SURPRESS_WARNING_BEGIN do { \ + const char *xEval = (x); \ + const char *yEval = (y); \ + if (UTEST_NULL == xEval || UTEST_NULL == yEval || \ + 0 != strcmp(xEval, yEval)) { \ + UTEST_PRINTF("%s:%i: Failure\n", __FILE__, __LINE__); \ + UTEST_PRINTF(" Expected : \"%s\"\n", xEval); \ + UTEST_PRINTF(" Actual : \"%s\"\n", yEval); \ + *utest_result = UTEST_TEST_FAILURE; \ + return; \ + } \ + } \ + while (0) \ + UTEST_SURPRESS_WARNING_END + +#define ASSERT_STRNE(x, y) \ + UTEST_SURPRESS_WARNING_BEGIN do { \ + const char *xEval = (x); \ + const char *yEval = (y); \ + if (UTEST_NULL == xEval || UTEST_NULL == yEval || \ + 0 == strcmp(xEval, yEval)) { \ + UTEST_PRINTF("%s:%i: Failure\n", __FILE__, __LINE__); \ + UTEST_PRINTF(" Expected : \"%s\"\n", xEval); \ + UTEST_PRINTF(" Actual : \"%s\"\n", yEval); \ + *utest_result = UTEST_TEST_FAILURE; \ + return; \ + } \ + } \ + while (0) \ + UTEST_SURPRESS_WARNING_END + +#define ASSERT_STRNEQ(x, y, n) \ + UTEST_SURPRESS_WARNING_BEGIN do { \ + const char *xEval = (x); \ + const char *yEval = (y); \ + const size_t nEval = UTEST_CAST(size_t, n); \ + if (UTEST_NULL == xEval || UTEST_NULL == yEval || \ + 0 != UTEST_STRNCMP(xEval, yEval, nEval)) { \ + UTEST_PRINTF("%s:%i: Failure\n", __FILE__, __LINE__); \ + UTEST_PRINTF(" Expected : \"%.*s\"\n", UTEST_CAST(int, nEval), xEval); \ + UTEST_PRINTF(" Actual : \"%.*s\"\n", UTEST_CAST(int, nEval), yEval); \ + *utest_result = UTEST_TEST_FAILURE; \ + return; \ + } \ + } \ + while (0) \ + UTEST_SURPRESS_WARNING_END + +#define ASSERT_STRNNE(x, y, n) \ + UTEST_SURPRESS_WARNING_BEGIN do { \ + const char *xEval = (x); \ + const char *yEval = (y); \ + const size_t nEval = UTEST_CAST(size_t, n); \ + if (UTEST_NULL == xEval || UTEST_NULL == yEval || \ + 0 == UTEST_STRNCMP(xEval, yEval, nEval)) { \ + UTEST_PRINTF("%s:%i: Failure\n", __FILE__, __LINE__); \ + UTEST_PRINTF(" Expected : \"%.*s\"\n", UTEST_CAST(int, nEval), xEval); \ + UTEST_PRINTF(" Actual : \"%.*s\"\n", UTEST_CAST(int, nEval), yEval); \ + *utest_result = UTEST_TEST_FAILURE; \ + return; \ + } \ + } \ + while (0) \ + UTEST_SURPRESS_WARNING_END + +#define ASSERT_NEAR(x, y, epsilon) \ + UTEST_SURPRESS_WARNING_BEGIN do { \ + const double diff = \ + utest_fabs(UTEST_CAST(double, x) - UTEST_CAST(double, y)); \ + if (diff > UTEST_CAST(double, epsilon) || utest_isnan(diff)) { \ + UTEST_PRINTF("%s:%i: Failure\n", __FILE__, __LINE__); \ + UTEST_PRINTF(" Expected : %f\n", UTEST_CAST(double, x)); \ + UTEST_PRINTF(" Actual : %f\n", UTEST_CAST(double, y)); \ + *utest_result = UTEST_TEST_FAILURE; \ + return; \ + } \ + } \ + while (0) \ + UTEST_SURPRESS_WARNING_END + +#if defined(UTEST_HAS_EXCEPTIONS) +#define ASSERT_EXCEPTION(x, exception_type) \ + UTEST_SURPRESS_WARNING_BEGIN do { \ + int exception_caught = 0; \ + try { \ + x; \ + } catch (const exception_type &) { \ + exception_caught = 1; \ + } catch (...) { \ + exception_caught = 2; \ + } \ + if (exception_caught != 1) { \ + UTEST_PRINTF("%s:%i: Failure\n", __FILE__, __LINE__); \ + UTEST_PRINTF(" Expected : %s exception\n", #exception_type); \ + UTEST_PRINTF(" Actual : %s\n", (exception_caught == 2) \ + ? "Unexpected exception" \ + : "No exception"); \ + *utest_result = UTEST_TEST_FAILURE; \ + return; \ + } \ + } \ + while (0) \ + UTEST_SURPRESS_WARNING_END +#endif + +#define UTEST(SET, NAME) \ + UTEST_EXTERN struct utest_state_s utest_state; \ + static void utest_run_##SET##_##NAME(int *utest_result); \ + static void utest_##SET##_##NAME(int *utest_result, size_t utest_index) { \ + (void)utest_index; \ + utest_run_##SET##_##NAME(utest_result); \ + } \ + UTEST_INITIALIZER(utest_register_##SET##_##NAME) { \ + const size_t index = utest_state.tests_length++; \ + const char *name_part = #SET "." #NAME; \ + const size_t name_size = strlen(name_part) + 1; \ + char *name = UTEST_PTR_CAST(char *, malloc(name_size)); \ + utest_state.tests = UTEST_PTR_CAST( \ + struct utest_test_state_s *, \ + utest_realloc(UTEST_PTR_CAST(void *, utest_state.tests), \ + sizeof(struct utest_test_state_s) * \ + utest_state.tests_length)); \ + if (utest_state.tests) { \ + utest_state.tests[index].func = &utest_##SET##_##NAME; \ + utest_state.tests[index].name = name; \ + utest_state.tests[index].index = 0; \ + } \ + UTEST_SNPRINTF(name, name_size, "%s", name_part); \ + } \ + void utest_run_##SET##_##NAME(int *utest_result) + +#define UTEST_F_SETUP(FIXTURE) \ + static void utest_f_setup_##FIXTURE(int *utest_result, \ + struct FIXTURE *utest_fixture) + +#define UTEST_F_TEARDOWN(FIXTURE) \ + static void utest_f_teardown_##FIXTURE(int *utest_result, \ + struct FIXTURE *utest_fixture) + +#if defined(__GNUC__) && __GNUC__ >= 8 && defined(__cplusplus) +#define UTEST_FIXTURE_SURPRESS_WARNINGS_BEGIN \ + _Pragma("GCC diagnostic push") \ + _Pragma("GCC diagnostic ignored \"-Wclass-memaccess\"") +#define UTEST_FIXTURE_SURPRESS_WARNINGS_END _Pragma("GCC diagnostic pop") +#else +#define UTEST_FIXTURE_SURPRESS_WARNINGS_BEGIN +#define UTEST_FIXTURE_SURPRESS_WARNINGS_END +#endif + +#define UTEST_F(FIXTURE, NAME) \ + UTEST_FIXTURE_SURPRESS_WARNINGS_BEGIN \ + UTEST_EXTERN struct utest_state_s utest_state; \ + static void utest_f_setup_##FIXTURE(int *, struct FIXTURE *); \ + static void utest_f_teardown_##FIXTURE(int *, struct FIXTURE *); \ + static void utest_run_##FIXTURE##_##NAME(int *, struct FIXTURE *); \ + static void utest_f_##FIXTURE##_##NAME(int *utest_result, \ + size_t utest_index) { \ + struct FIXTURE fixture; \ + (void)utest_index; \ + memset(&fixture, 0, sizeof(fixture)); \ + utest_f_setup_##FIXTURE(utest_result, &fixture); \ + if (UTEST_TEST_PASSED != *utest_result) { \ + return; \ + } \ + utest_run_##FIXTURE##_##NAME(utest_result, &fixture); \ + utest_f_teardown_##FIXTURE(utest_result, &fixture); \ + } \ + UTEST_INITIALIZER(utest_register_##FIXTURE##_##NAME) { \ + const size_t index = utest_state.tests_length++; \ + const char *name_part = #FIXTURE "." #NAME; \ + const size_t name_size = strlen(name_part) + 1; \ + char *name = UTEST_PTR_CAST(char *, malloc(name_size)); \ + utest_state.tests = UTEST_PTR_CAST( \ + struct utest_test_state_s *, \ + utest_realloc(UTEST_PTR_CAST(void *, utest_state.tests), \ + sizeof(struct utest_test_state_s) * \ + utest_state.tests_length)); \ + utest_state.tests[index].func = &utest_f_##FIXTURE##_##NAME; \ + utest_state.tests[index].name = name; \ + UTEST_SNPRINTF(name, name_size, "%s", name_part); \ + } \ + UTEST_FIXTURE_SURPRESS_WARNINGS_END \ + void utest_run_##FIXTURE##_##NAME(int *utest_result, \ + struct FIXTURE *utest_fixture) + +#define UTEST_I_SETUP(FIXTURE) \ + static void utest_i_setup_##FIXTURE( \ + int *utest_result, struct FIXTURE *utest_fixture, size_t utest_index) + +#define UTEST_I_TEARDOWN(FIXTURE) \ + static void utest_i_teardown_##FIXTURE( \ + int *utest_result, struct FIXTURE *utest_fixture, size_t utest_index) + +#define UTEST_I(FIXTURE, NAME, INDEX) \ + UTEST_EXTERN struct utest_state_s utest_state; \ + static void utest_run_##FIXTURE##_##NAME##_##INDEX(int *, struct FIXTURE *); \ + static void utest_i_##FIXTURE##_##NAME##_##INDEX(int *utest_result, \ + size_t index) { \ + struct FIXTURE fixture; \ + memset(&fixture, 0, sizeof(fixture)); \ + utest_i_setup_##FIXTURE(utest_result, &fixture, index); \ + if (UTEST_TEST_PASSED != *utest_result) { \ + return; \ + } \ + utest_run_##FIXTURE##_##NAME##_##INDEX(utest_result, &fixture); \ + utest_i_teardown_##FIXTURE(utest_result, &fixture, index); \ + } \ + UTEST_INITIALIZER(utest_register_##FIXTURE##_##NAME##_##INDEX) { \ + size_t i; \ + utest_uint64_t iUp; \ + for (i = 0; i < (INDEX); i++) { \ + const size_t index = utest_state.tests_length++; \ + const char *name_part = #FIXTURE "." #NAME; \ + const size_t name_size = strlen(name_part) + 32; \ + char *name = UTEST_PTR_CAST(char *, malloc(name_size)); \ + utest_state.tests = UTEST_PTR_CAST( \ + struct utest_test_state_s *, \ + utest_realloc(UTEST_PTR_CAST(void *, utest_state.tests), \ + sizeof(struct utest_test_state_s) * \ + utest_state.tests_length)); \ + utest_state.tests[index].func = &utest_i_##FIXTURE##_##NAME##_##INDEX; \ + utest_state.tests[index].index = i; \ + utest_state.tests[index].name = name; \ + iUp = UTEST_CAST(utest_uint64_t, i); \ + UTEST_SNPRINTF(name, name_size, "%s/%" UTEST_PRIu64, name_part, iUp); \ + } \ + } \ + void utest_run_##FIXTURE##_##NAME##_##INDEX(int *utest_result, \ + struct FIXTURE *utest_fixture) + +UTEST_WEAK +double utest_fabs(double d); +UTEST_WEAK +double utest_fabs(double d) { + union { + double d; + utest_uint64_t u; + } both; + both.d = d; + both.u &= 0x7fffffffffffffffu; + return both.d; +} + +UTEST_WEAK +int utest_isnan(double d); +UTEST_WEAK +int utest_isnan(double d) { + union { + double d; + utest_uint64_t u; + } both; + both.d = d; + both.u &= 0x7fffffffffffffffu; + return both.u > 0x7ff0000000000000u; +} + +UTEST_WEAK +int utest_should_filter_test(const char *filter, const char *testcase); +UTEST_WEAK int utest_should_filter_test(const char *filter, + const char *testcase) { + if (filter) { + const char *filter_cur = filter; + const char *testcase_cur = testcase; + const char *filter_wildcard = UTEST_NULL; + + while (('\0' != *filter_cur) && ('\0' != *testcase_cur)) { + if ('*' == *filter_cur) { + /* store the position of the wildcard */ + filter_wildcard = filter_cur; + + /* skip the wildcard character */ + filter_cur++; + + while (('\0' != *filter_cur) && ('\0' != *testcase_cur)) { + if ('*' == *filter_cur) { + /* + we found another wildcard (filter is something like *foo*) so we + exit the current loop, and return to the parent loop to handle + the wildcard case + */ + break; + } else if (*filter_cur != *testcase_cur) { + /* otherwise our filter didn't match, so reset it */ + filter_cur = filter_wildcard; + } + + /* move testcase along */ + testcase_cur++; + + /* move filter along */ + filter_cur++; + } + + if (('\0' == *filter_cur) && ('\0' == *testcase_cur)) { + return 0; + } + + /* if the testcase has been exhausted, we don't have a match! */ + if ('\0' == *testcase_cur) { + return 1; + } + } else { + if (*testcase_cur != *filter_cur) { + /* test case doesn't match filter */ + return 1; + } else { + /* move our filter and testcase forward */ + testcase_cur++; + filter_cur++; + } + } + } + + if (('\0' != *filter_cur) || + (('\0' != *testcase_cur) && + ((filter == filter_cur) || ('*' != filter_cur[-1])))) { + /* we have a mismatch! */ + return 1; + } + } + + return 0; +} + +static UTEST_INLINE FILE *utest_fopen(const char *filename, const char *mode) { +#ifdef _MSC_VER + FILE *file; + if (0 == fopen_s(&file, filename, mode)) { + return file; + } else { + return UTEST_NULL; + } +#else + return fopen(filename, mode); +#endif +} + +static UTEST_INLINE int utest_main(int argc, const char *const argv[]); +int utest_main(int argc, const char *const argv[]) { + utest_uint64_t failed = 0; + utest_uint64_t skipped = 0; + size_t index = 0; + size_t *failed_testcases = UTEST_NULL; + size_t failed_testcases_length = 0; + size_t *skipped_testcases = UTEST_NULL; + size_t skipped_testcases_length = 0; + const char *filter = UTEST_NULL; + utest_uint64_t ran_tests = 0; + int enable_mixed_units = 0; + int random_order = 0; + utest_uint32_t seed = 0; + + enum colours { RESET, GREEN, RED, YELLOW }; + + const int use_colours = UTEST_COLOUR_OUTPUT(); + const char *colours[] = {"\033[0m", "\033[32m", "\033[31m", "\033[33m"}; + + if (!use_colours) { + for (index = 0; index < sizeof colours / sizeof colours[0]; index++) { + colours[index] = ""; + } + } + /* loop through all arguments looking for our options */ + for (index = 1; index < UTEST_CAST(size_t, argc); index++) { + /* Informational switches */ + const char help_str[] = "--help"; + const char list_str[] = "--list-tests"; + /* Test config switches */ + const char filter_str[] = "--filter="; + const char output_str[] = "--output="; + const char enable_mixed_units_str[] = "--enable-mixed-units"; + const char random_order_str[] = "--random-order"; + const char random_order_with_seed_str[] = "--random-order="; + + if (0 == UTEST_STRNCMP(argv[index], help_str, strlen(help_str))) { + printf("utest.h - the single file unit testing solution for C/C++!\n" + "Command line Options:\n" + " --help Show this message and exit.\n" + " --filter= Filter the test cases to run (EG. " + "MyTest*.a would run MyTestCase.a but not MyTestCase.b).\n" + " --list-tests List testnames, one per line. Output " + "names can be passed to --filter.\n"); + printf(" --output= Output an xunit XML file to the file " + "specified in .\n" + " --enable-mixed-units Enable the per-test output to contain " + "mixed units (s/ms/us/ns).\n" + " --random-order[=] Randomize the order that the tests are " + "ran in. If the optional argument is not provided, then a " + "random starting seed is used.\n"); + goto cleanup; + } else if (0 == + UTEST_STRNCMP(argv[index], filter_str, strlen(filter_str))) { + /* user wants to filter what test cases run! */ + filter = argv[index] + strlen(filter_str); + } else if (0 == + UTEST_STRNCMP(argv[index], output_str, strlen(output_str))) { + utest_state.output = utest_fopen(argv[index] + strlen(output_str), "w+"); + } else if (0 == UTEST_STRNCMP(argv[index], list_str, strlen(list_str))) { + for (index = 0; index < utest_state.tests_length; index++) { + UTEST_PRINTF("%s\n", utest_state.tests[index].name); + } + /* when printing the test list, don't actually run the tests */ + return 0; + } else if (0 == UTEST_STRNCMP(argv[index], enable_mixed_units_str, + strlen(enable_mixed_units_str))) { + enable_mixed_units = 1; + } else if (0 == UTEST_STRNCMP(argv[index], random_order_with_seed_str, + strlen(random_order_with_seed_str))) { + seed = + UTEST_CAST(utest_uint32_t, + strtoul(argv[index] + strlen(random_order_with_seed_str), + UTEST_NULL, 10)); + random_order = 1; + } else if (0 == UTEST_STRNCMP(argv[index], random_order_str, + strlen(random_order_str))) { + const utest_int64_t ns = utest_ns(); + + // Some really poor pseudo-random using the current time. I do this + // because I really want to avoid using C's rand() because that'd mean our + // random would be affected by any srand() usage by the user (which I + // don't want). + seed = UTEST_CAST(utest_uint32_t, ns >> 32) * 31 + + UTEST_CAST(utest_uint32_t, ns & 0xffffffff); + random_order = 1; + } + } + + if (random_order) { + // Use Fisher-Yates with the Durstenfield's version to randomly re-order the + // tests. + for (index = utest_state.tests_length; index > 1; index--) { + // For the random order we'll use PCG. + const utest_uint32_t state = seed; + const utest_uint32_t word = + ((state >> ((state >> 28u) + 4u)) ^ state) * 277803737u; + const utest_uint32_t next = + ((word >> 22u) ^ word) % UTEST_CAST(utest_uint32_t, index); + + // Swap the randomly chosen element into the last location. + const struct utest_test_state_s copy = utest_state.tests[index - 1]; + utest_state.tests[index - 1] = utest_state.tests[next]; + utest_state.tests[next] = copy; + + // Move the seed onwards. + seed = seed * 747796405u + 2891336453u; + } + } + + for (index = 0; index < utest_state.tests_length; index++) { + if (utest_should_filter_test(filter, utest_state.tests[index].name)) { + continue; + } + + ran_tests++; + } + + printf("%s[==========]%s Running %" UTEST_PRIu64 " test cases.\n", + colours[GREEN], colours[RESET], UTEST_CAST(utest_uint64_t, ran_tests)); + + if (utest_state.output) { + fprintf(utest_state.output, "\n"); + fprintf(utest_state.output, + "\n", + UTEST_CAST(utest_uint64_t, ran_tests)); + fprintf(utest_state.output, + "\n", + UTEST_CAST(utest_uint64_t, ran_tests)); + } + + for (index = 0; index < utest_state.tests_length; index++) { + int result = UTEST_TEST_PASSED; + utest_int64_t ns = 0; + + if (utest_should_filter_test(filter, utest_state.tests[index].name)) { + continue; + } + + printf("%s[ RUN ]%s %s\n", colours[GREEN], colours[RESET], + utest_state.tests[index].name); + + if (utest_state.output) { + fprintf(utest_state.output, "", + utest_state.tests[index].name); + } + + ns = utest_ns(); + errno = 0; +#if defined(UTEST_HAS_EXCEPTIONS) + UTEST_SURPRESS_WARNING_BEGIN + try { + utest_state.tests[index].func(&result, utest_state.tests[index].index); + } catch (const std::exception &err) { + printf(" Exception : %s\n", err.what()); + result = UTEST_TEST_FAILURE; + } catch (...) { + printf(" Exception : Unknown\n"); + result = UTEST_TEST_FAILURE; + } + UTEST_SURPRESS_WARNING_END +#else + utest_state.tests[index].func(&result, utest_state.tests[index].index); +#endif + ns = utest_ns() - ns; + + if (utest_state.output) { + fprintf(utest_state.output, "\n"); + } + + // Record the failing test. + if (UTEST_TEST_FAILURE == result) { + const size_t failed_testcase_index = failed_testcases_length++; + failed_testcases = UTEST_PTR_CAST( + size_t *, utest_realloc(UTEST_PTR_CAST(void *, failed_testcases), + sizeof(size_t) * failed_testcases_length)); + if (UTEST_NULL != failed_testcases) { + failed_testcases[failed_testcase_index] = index; + } + failed++; + } else if (UTEST_TEST_SKIPPED == result) { + const size_t skipped_testcase_index = skipped_testcases_length++; + skipped_testcases = UTEST_PTR_CAST( + size_t *, utest_realloc(UTEST_PTR_CAST(void *, skipped_testcases), + sizeof(size_t) * skipped_testcases_length)); + if (UTEST_NULL != skipped_testcases) { + skipped_testcases[skipped_testcase_index] = index; + } + skipped++; + } + + { + const char *const units[] = {"ns", "us", "ms", "s", UTEST_NULL}; + unsigned int unit_index = 0; + utest_int64_t time = ns; + + if (enable_mixed_units) { + for (unit_index = 0; UTEST_NULL != units[unit_index]; unit_index++) { + if (10000 > time) { + break; + } + + time /= 1000; + } + } + + if (UTEST_TEST_FAILURE == result) { + printf("%s[ FAILED ]%s %s (%" UTEST_PRId64 "%s)\n", colours[RED], + colours[RESET], utest_state.tests[index].name, time, + units[unit_index]); + } else if (UTEST_TEST_SKIPPED == result) { + printf("%s[ SKIPPED ]%s %s (%" UTEST_PRId64 "%s)\n", colours[YELLOW], + colours[RESET], utest_state.tests[index].name, time, + units[unit_index]); + } else { + printf("%s[ OK ]%s %s (%" UTEST_PRId64 "%s)\n", colours[GREEN], + colours[RESET], utest_state.tests[index].name, time, + units[unit_index]); + } + } + } + + printf("%s[==========]%s %" UTEST_PRIu64 " test cases ran.\n", colours[GREEN], + colours[RESET], ran_tests); + printf("%s[ PASSED ]%s %" UTEST_PRIu64 " tests.\n", colours[GREEN], + colours[RESET], ran_tests - failed - skipped); + + if (0 != skipped) { + printf("%s[ SKIPPED ]%s %" UTEST_PRIu64 " tests, listed below:\n", + colours[YELLOW], colours[RESET], skipped); + for (index = 0; index < skipped_testcases_length; index++) { + printf("%s[ SKIPPED ]%s %s\n", colours[YELLOW], colours[RESET], + utest_state.tests[skipped_testcases[index]].name); + } + } + + if (0 != failed) { + printf("%s[ FAILED ]%s %" UTEST_PRIu64 " tests, listed below:\n", + colours[RED], colours[RESET], failed); + for (index = 0; index < failed_testcases_length; index++) { + printf("%s[ FAILED ]%s %s\n", colours[RED], colours[RESET], + utest_state.tests[failed_testcases[index]].name); + } + } + + if (utest_state.output) { + fprintf(utest_state.output, "\n\n"); + } + +cleanup: + for (index = 0; index < utest_state.tests_length; index++) { + free(UTEST_PTR_CAST(void *, utest_state.tests[index].name)); + } + + free(UTEST_PTR_CAST(void *, skipped_testcases)); + free(UTEST_PTR_CAST(void *, failed_testcases)); + free(UTEST_PTR_CAST(void *, utest_state.tests)); + + if (utest_state.output) { + fclose(utest_state.output); + } + + return UTEST_CAST(int, failed); +} + +/* + we need, in exactly one source file, define the global struct that will hold + the data we need to run utest. This macro allows the user to declare the + data without having to use the UTEST_MAIN macro, thus allowing them to write + their own main() function. +*/ +#define UTEST_STATE() struct utest_state_s utest_state = {0, 0, 0} + +/* + define a main() function to call into utest.h and start executing tests! A + user can optionally not use this macro, and instead define their own main() + function and manually call utest_main. The user must, in exactly one source + file, use the UTEST_STATE macro to declare a global struct variable that + utest requires. +*/ +#define UTEST_MAIN() \ + UTEST_STATE(); \ + int main(int argc, const char *const argv[]) { \ + return utest_main(argc, argv); \ + } + +#endif /* SHEREDOM_UTEST_H_INCLUDED */ diff --git a/tests/C/utest_oneflux.h b/tests/C/utest_oneflux.h new file mode 100644 index 00000000..0566be3d --- /dev/null +++ b/tests/C/utest_oneflux.h @@ -0,0 +1,19 @@ +/* utest_oneflux.h */ +#include "utest.h" + +#define FAIL_UNLESS(x,...) \ + UTEST_SURPRESS_WARNING_BEGIN do { \ + const int xEval = !!(x); \ + if (!(xEval)) { \ + UTEST_PRINTF("%s:%i: Failure\n", __FILE__, __LINE__); \ + UTEST_PRINTF(" Condition : (%s)\n", #x); \ + UTEST_PRINTF(" Message : %s\n", ## __VA_ARGS__); \ + *utest_result = UTEST_TEST_FAILURE; \ + return; \ + } \ + } \ + while (0) \ + UTEST_SURPRESS_WARNING_END + + +#define CK_ASSERT_MSG FAIL_UNLESS \ No newline at end of file