Skip to content

Commit

Permalink
bugfixes:
Browse files Browse the repository at this point in the history 
- fixed bug on quality flags for gapfilled LE and H. Correctly calculate the QC value in case of missing data
- renamed vars from SW_IN_M, SW_IN_mqc to SW_IN_m, SW_IN_mqc
- fixed: changing the value of the QC to -9999 when it was not possible to calculate the energy balance and create the LE_Corr. It is also a "known issue" in FLUXNET2015 release
- cosmetic fixes
  • Loading branch information
@ARibecaJob
ARibecaJob committed Jun 27, 2022
1 parent 9201beb commit c33b1b3
Show file tree
Hide file tree
Showing 3 changed files with 161 additions and 48 deletions.
116 changes: 92 additions & 24 deletions oneflux_steps/energy_proc/src/aggr.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -182,14 +182,31 @@ int aggr_by_days(DATASET *const dataset, PREC *const ECBcfs, PREC *const ECBcfs_

/* loop on each qc */
for ( y = 0; y < VARS_TO_FILL; y++ ) {
dataset->rows_aggr[day].quality[y] = 0;
/* loop on agg */
for ( j = 0; j < rows_per_day; j++ ) {
if ( dataset->gf_rows[y][i+j].quality < 2 ) {
++dataset->rows_aggr[day].quality[y];
PREC v;

/*
July 25th 2019 - bug on quality flags for gapfilled LE and H fixed in ecbcf.c,
here also fixed with the following four IF statements
in order to correctly calculate the QC value in case of missing data
*/

if ( H_INDEX == y ) v = dataset->rows_aggr[day].value[H];
if ( LE_INDEX == y ) v = dataset->rows_aggr[day].value[LE];
if ( G_INDEX == y ) v = dataset->rows_aggr[day].value[G];

if ( IS_INVALID_VALUE(v) ) {
dataset->rows_aggr[day].quality[y] = INVALID_VALUE;
} else {
dataset->rows_aggr[day].quality[y] = 0;
/* loop on agg */
for ( j = 0; j < rows_per_day; j++ ) {
if ( ! IS_INVALID_VALUE(dataset->gf_rows[y][i+j].quality)
&& (dataset->gf_rows[y][i+j].quality < 2) ) {
++dataset->rows_aggr[day].quality[y];
}
}
dataset->rows_aggr[day].quality[y] /= rows_per_day;
}
dataset->rows_aggr[day].quality[y] /= rows_per_day;
}

/* update values */
Expand Down Expand Up @@ -379,14 +396,31 @@ int aggr_by_weeks(DATASET *const dataset, PREC *const ECBcfs, PREC *const ECBcfs

/* loop on each qc */
for ( i = 0; i < VARS_TO_FILL; i++ ) {
dataset->rows_aggr[week+(year*52)].quality[i] = 0;
/* loop on agg */
for ( j = 0; j < rows_per_day*days_per_week; j++ ) {
if ( dataset->gf_rows[i][index+k+j].quality < 2 ) {
++dataset->rows_aggr[week+(year*52)].quality[i];
PREC v;

/*
July 25th 2019 - bug on quality flags for gapfilled LE and H fixed in ecbcf.c,
here also fixed with the following four IF statements
in order to correctly calculate the QC value in case of missing data
*/

if ( H_INDEX == i ) v = dataset->rows_aggr[week+(year*52)].h_mean;
if ( LE_INDEX == i ) v = dataset->rows_aggr[week+(year*52)].le_mean;
if ( G_INDEX == i ) v = dataset->rows_aggr[week+(year*52)].value[G_FILLED];

if ( IS_INVALID_VALUE(v) ) {
dataset->rows_aggr[week+(year*52)].quality[i] = INVALID_VALUE;
} else {
dataset->rows_aggr[week+(year*52)].quality[i] = 0;
/* loop on agg */
for ( j = 0; j < rows_per_day*days_per_week; j++ ) {
if ( ! IS_INVALID_VALUE(dataset->gf_rows[i][index+k+j].quality)
&& (dataset->gf_rows[i][index+k+j].quality < 2) ) {
++dataset->rows_aggr[week+(year*52)].quality[i];
}
}
dataset->rows_aggr[week+(year*52)].quality[i] /= rows_per_day*days_per_week;
}
dataset->rows_aggr[week+(year*52)].quality[i] /= rows_per_day*days_per_week;
}

k += rows_per_day*days_per_week;
Expand Down Expand Up @@ -672,14 +706,31 @@ int aggr_by_months(DATASET *const dataset, PREC *const ECBcfs, PREC *const ECBcf

/* loop on each qc */
for ( i = 0; i < VARS_TO_FILL; i++ ) {
dataset->rows_aggr[month+(year*12)].quality[i] = 0;
/* loop on agg */
for ( j = 0; j < rows_per_day*days_per_month_count; j++ ) {
if ( dataset->gf_rows[i][index+k+j].quality < 2 ) {
++dataset->rows_aggr[month+(year*12)].quality[i];
PREC v;

/*
July 25th 2019 - bug on quality flags for gapfilled LE and H fixed in ecbcf.c,
here also fixed with the following four IF statements
in order to correctly calculate the QC value in case of missing data
*/

if ( H_INDEX == i ) v = dataset->rows_aggr[month+(year*12)].h_mean;
if ( LE_INDEX == i ) v = dataset->rows_aggr[month+(year*12)].le_mean;
if ( G_INDEX == i ) v = dataset->rows_aggr[month+(year*12)].value[G_FILLED];

if ( IS_INVALID_VALUE(v) ) {
dataset->rows_aggr[month+(year*12)].quality[i] = INVALID_VALUE;
} else {
dataset->rows_aggr[month+(year*12)].quality[i] = 0;
/* loop on agg */
for ( j = 0; j < rows_per_day*days_per_month_count; j++ ) {
if ( ! IS_INVALID_VALUE(dataset->gf_rows[i][index+k+j].quality)
&& (dataset->gf_rows[i][index+k+j].quality < 2) ) {
++dataset->rows_aggr[month+(year*12)].quality[i];
}
}
dataset->rows_aggr[month+(year*12)].quality[i] /= rows_per_day*days_per_month_count;
}
dataset->rows_aggr[month+(year*12)].quality[i] /= rows_per_day*days_per_month_count;
}

k += rows_per_day*days_per_month_count;
Expand Down Expand Up @@ -953,14 +1004,31 @@ int aggr_by_years(DATASET *const dataset, PREC *const ECBcfs, PREC *const ECBcfs

/* loop on each qc */
for ( i = 0; i < VARS_TO_FILL; i++ ) {
dataset->rows_aggr[year].quality[i] = 0;
/* loop on agg */
for ( j = 0; j < rows_per_year; j++ ) {
if ( dataset->gf_rows[i][index+j].quality < 2 ) {
++dataset->rows_aggr[year].quality[i];
PREC v;

/*
July 25th 2019 - bug on quality flags for gapfilled LE and H fixed in ecbcf.c,
here also fixed with the following four IF statements
in order to correctly calculate the QC value in case of missing data
*/

if ( H_INDEX == i ) v = dataset->rows_aggr[year].h_mean;
if ( LE_INDEX == i ) v = dataset->rows_aggr[year].le_mean;
if ( G_INDEX == i ) v = dataset->rows_aggr[year].value[G_FILLED];

if ( IS_INVALID_VALUE(v) ) {
dataset->rows_aggr[year].quality[i] = INVALID_VALUE;
} else {
dataset->rows_aggr[year].quality[i] = 0;
/* loop on agg */
for ( j = 0; j < rows_per_year; j++ ) {
if ( ! IS_INVALID_VALUE(dataset->gf_rows[i][index+j].quality)
&& (dataset->gf_rows[i][index+j].quality < 2) ) {
++dataset->rows_aggr[year].quality[i];
}
}
dataset->rows_aggr[year].quality[i] /= rows_per_year;
}
dataset->rows_aggr[year].quality[i] /= rows_per_year;
}

/* */
Expand Down
13 changes: 9 additions & 4 deletions oneflux_steps/energy_proc/src/dataset.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1425,13 +1425,13 @@ static int get_meteo(DATASET *const dataset) {
for ( token = string_tokenizer(buffer, dataset_delimiter, &p), i = 0; token; token = string_tokenizer(NULL, dataset_delimiter, &p), ++i ) {
if ( ! string_compare_i(token, "TA_m") ) {
TA = i;
} else if ( ! string_compare_i(token, "SWIN_m") ) {
} else if ( ! string_compare_i(token, "SW_IN_m") ) {
SWIN = i;
} else if ( ! string_compare_i(token, "VPD_m") ) {
VPD = i;
} else if ( ! string_compare_i(token, "TA_mqc") ) {
TA_QC = i;
} else if ( ! string_compare_i(token, "SWIN_mqc") ) {
} else if ( ! string_compare_i(token, "SW_IN_mqc") ) {
SWIN_QC = i;
} else if ( ! string_compare_i(token, "VPD_mqc") ) {
VPD_QC = i;
Expand Down Expand Up @@ -1514,7 +1514,12 @@ static int get_meteo(DATASET *const dataset) {
}
while ( fgets(buffer, BUFFER_SIZE, f) ) {
for ( token = string_tokenizer(buffer, dataset_delimiter, &p), i = 0; token; token = string_tokenizer(NULL, dataset_delimiter, &p), i++ ) {
if ( !i ) {
if ( ! i ) {
// skip TIMESTAMP_START
continue;
}
// get TIMESTAMP_END
if ( 1 == i ) {
t = get_timestamp(token);
if ( ! t ) {
fclose(f);
Expand All @@ -1540,7 +1545,7 @@ static int get_meteo(DATASET *const dataset) {
current_row = get_row_by_timestamp(t, (HOURLY_TIMERES==dataset->details->timeres));
free(t);
if ( element != current_row ) {
printf("bad timestamp: %s", token);
printf("bad timestamp: %s\n", token);
fclose(f);
return 0;
}
Expand Down
80 changes: 60 additions & 20 deletions oneflux_steps/energy_proc/src/ecbcf.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -450,15 +450,23 @@ int ecbcf_hh(DATASET *const dataset, const int current_row, PREC *const ECBcfs,
/* no method */
dataset->rows[current_row].ecbcf_method = INVALID_VALUE-1;
dataset->rows[current_row].ecbcf_samples_count = INVALID_VALUE;
dataset->gf_rows[LE_INDEX][current_row].quality = INVALID_VALUE;
dataset->gf_rows[H_INDEX][current_row].quality = INVALID_VALUE;

// commented out on July 25, 2019. It was a bug, changing the value of the QC to -9999 when it was
// not possible to calculate the energy balance and create the LE_Corr. It is also a "known issue" in
// FLUXNET2015 release
//dataset->gf_rows[LE_INDEX][current_row].quality = INVALID_VALUE;
//dataset->gf_rows[H_INDEX][current_row].quality = INVALID_VALUE;
}
} else {
/* no method */
dataset->rows[current_row].ecbcf_method = INVALID_VALUE-1;
dataset->rows[current_row].ecbcf_samples_count = INVALID_VALUE;
dataset->gf_rows[LE_INDEX][current_row].quality = INVALID_VALUE;
dataset->gf_rows[H_INDEX][current_row].quality = INVALID_VALUE;

// commented out on July 25, 2019. It was a bug, changing the value of the QC to -9999 when it was
// not possible to calculate the energy balance and create the LE_Corr. It is also a "known issue" in
// FLUXNET2015 release
//dataset->gf_rows[LE_INDEX][current_row].quality = INVALID_VALUE;
//dataset->gf_rows[H_INDEX][current_row].quality = INVALID_VALUE;
}
} else if ( SECOND == dataset->rows[current_row].ecbcf_method ) {
window = (windows_size_alt / 2) * rows_per_day;
Expand Down Expand Up @@ -887,15 +895,23 @@ int ecbcf_dd(DATASET *const dataset, const int current_row, PREC *const ECBcfs,
/* no method */
dataset->rows_aggr[current_row].ecbcf_method = INVALID_VALUE-1;
dataset->rows_aggr[current_row].ecbcf_samples_count = INVALID_VALUE;
dataset->rows_aggr[current_row].quality[LE_INDEX] = INVALID_VALUE;
dataset->rows_aggr[current_row].quality[H_INDEX] = INVALID_VALUE;

// commented out on July 25, 2019. It was a bug, changing the value of the QC to -9999 when it was
// not possible to calculate the energy balance and create the LE_Corr. It is also a "known issue" in
// FLUXNET2015 release
//dataset->rows_aggr[current_row].quality[LE_INDEX] = INVALID_VALUE;
//dataset->rows_aggr[current_row].quality[H_INDEX] = INVALID_VALUE;
}
} else {
/* no method */
dataset->rows_aggr[current_row].ecbcf_method = INVALID_VALUE-1;
dataset->rows_aggr[current_row].ecbcf_samples_count = INVALID_VALUE;
dataset->rows_aggr[current_row].quality[LE_INDEX] = INVALID_VALUE;
dataset->rows_aggr[current_row].quality[H_INDEX] = INVALID_VALUE;

// commented out on July 25, 2019. It was a bug, changing the value of the QC to -9999 when it was
// not possible to calculate the energy balance and create the LE_Corr. It is also a "known issue" in
// FLUXNET2015 release
//dataset->rows_aggr[current_row].quality[LE_INDEX] = INVALID_VALUE;
//dataset->rows_aggr[current_row].quality[H_INDEX] = INVALID_VALUE;
}
} else if ( SECOND == dataset->rows_aggr[current_row].ecbcf_method ) {
window = windows_size_alt / 2;
Expand Down Expand Up @@ -1197,15 +1213,23 @@ int ecbcf_ww(DATASET *const dataset, const int current_row, PREC *const ECBcfs,
/* no method */
dataset->rows_aggr[current_row].ecbcf_method = INVALID_VALUE-1;
dataset->rows_aggr[current_row].ecbcf_samples_count = INVALID_VALUE;
dataset->rows_aggr[current_row].quality[LE_INDEX] = INVALID_VALUE;
dataset->rows_aggr[current_row].quality[H_INDEX] = INVALID_VALUE;

// commented out on July 25, 2019. It was a bug, changing the value of the QC to -9999 when it was
// not possible to calculate the energy balance and create the LE_Corr. It is also a "known issue" in
// FLUXNET2015 release
//dataset->rows_aggr[current_row].quality[LE_INDEX] = INVALID_VALUE;
//dataset->rows_aggr[current_row].quality[H_INDEX] = INVALID_VALUE;
}
} else {
/* no method */
dataset->rows_aggr[current_row].ecbcf_method = INVALID_VALUE-1;
dataset->rows_aggr[current_row].ecbcf_samples_count = INVALID_VALUE;
dataset->rows_aggr[current_row].quality[LE_INDEX] = INVALID_VALUE;
dataset->rows_aggr[current_row].quality[H_INDEX] = INVALID_VALUE;

// commented out on July 25, 2019. It was a bug, changing the value of the QC to -9999 when it was
// not possible to calculate the energy balance and create the LE_Corr. It is also a "known issue" in
// FLUXNET2015 release
//dataset->rows_aggr[current_row].quality[LE_INDEX] = INVALID_VALUE;
//dataset->rows_aggr[current_row].quality[H_INDEX] = INVALID_VALUE;
}
} else if ( SECOND == dataset->rows_aggr[current_row].ecbcf_method ) {
window = window_size / 2;
Expand Down Expand Up @@ -1396,15 +1420,23 @@ int ecbcf_mm(DATASET *const dataset, const int current_row, PREC *const ECBcfs,
/* no method */
dataset->rows_aggr[current_row].ecbcf_method = INVALID_VALUE-1;
dataset->rows_aggr[current_row].ecbcf_samples_count = INVALID_VALUE;
dataset->rows_aggr[current_row].quality[LE_INDEX] = INVALID_VALUE;
dataset->rows_aggr[current_row].quality[H_INDEX] = INVALID_VALUE;

// commented out on July 25, 2019. It was a bug, changing the value of the QC to -9999 when it was
// not possible to calculate the energy balance and create the LE_Corr. It is also a "known issue" in
// FLUXNET2015 release
//dataset->rows_aggr[current_row].quality[LE_INDEX] = INVALID_VALUE;
//dataset->rows_aggr[current_row].quality[H_INDEX] = INVALID_VALUE;
}
} else {
/* no method */
dataset->rows_aggr[current_row].ecbcf_method = INVALID_VALUE-1;
dataset->rows_aggr[current_row].ecbcf_samples_count = INVALID_VALUE;
dataset->rows_aggr[current_row].quality[LE_INDEX] = INVALID_VALUE;
dataset->rows_aggr[current_row].quality[H_INDEX] = INVALID_VALUE;

// commented out on July 25, 2019. It was a bug, changing the value of the QC to -9999 when it was
// not possible to calculate the energy balance and create the LE_Corr. It is also a "known issue" in
// FLUXNET2015 release
//dataset->rows_aggr[current_row].quality[LE_INDEX] = INVALID_VALUE;
//dataset->rows_aggr[current_row].quality[H_INDEX] = INVALID_VALUE;
}
} else if ( SECOND == dataset->rows_aggr[current_row].ecbcf_method ) {
window = window_size / 2;
Expand Down Expand Up @@ -1520,15 +1552,23 @@ int ecbcf_yy(DATASET *const dataset, const int current_row, PREC *const ECBcfs)
/* no method */
dataset->rows_aggr[current_row].ecbcf_method = INVALID_VALUE-1;
dataset->rows_aggr[current_row].ecbcf_samples_count = INVALID_VALUE;
dataset->rows_aggr[current_row].quality[LE_INDEX] = INVALID_VALUE;
dataset->rows_aggr[current_row].quality[H_INDEX] = INVALID_VALUE;

// commented out on July 25, 2019. It was a bug, changing the value of the QC to -9999 when it was
// not possible to calculate the energy balance and create the LE_Corr. It is also a "known issue" in
// FLUXNET2015 release
//dataset->rows_aggr[current_row].quality[LE_INDEX] = INVALID_VALUE;
//dataset->rows_aggr[current_row].quality[H_INDEX] = INVALID_VALUE;
}
} else {
/* no method */
dataset->rows_aggr[current_row].ecbcf_method = INVALID_VALUE-1;
dataset->rows_aggr[current_row].ecbcf_samples_count = INVALID_VALUE;
dataset->rows_aggr[current_row].quality[LE_INDEX] = INVALID_VALUE;
dataset->rows_aggr[current_row].quality[H_INDEX] = INVALID_VALUE;

// commented out on July 25, 2019. It was a bug, changing the value of the QC to -9999 when it was
// not possible to calculate the energy balance and create the LE_Corr. It is also a "known issue" in
// FLUXNET2015 release
//dataset->rows_aggr[current_row].quality[LE_INDEX] = INVALID_VALUE;
//dataset->rows_aggr[current_row].quality[H_INDEX] = INVALID_VALUE;
}
} else if ( SECOND == dataset->rows_aggr[current_row].ecbcf_method ) {
/* how many dataset we have ? */
Expand Down

0 comments on commit c33b1b3

Please sign in to comment.