Merge pull request #96 from appelmar/error_tracking

Error tracking

R/RcppExports.R

@@ -93,8 +93,8 @@ gc_create_view <- function(v) {
     .Call('_gdalcubes_gc_create_view', PACKAGE = 'gdalcubes', v)
 }
 
-gc_create_image_collection_cube <- function(pin, chunk_sizes, mask, v = NULL) {
-    .Call('_gdalcubes_gc_create_image_collection_cube', PACKAGE = 'gdalcubes', pin, chunk_sizes, mask, v)
+gc_create_image_collection_cube <- function(pin, chunk_sizes, mask, strict = TRUE, v = NULL) {
+    .Call('_gdalcubes_gc_create_image_collection_cube', PACKAGE = 'gdalcubes', pin, chunk_sizes, mask, strict, v)
 }
 
 gc_create_ncdf_cube <- function(path, chunk_sizes, auto_unpack) {
@@ -209,8 +209,8 @@ gc_create_stream_cube <- function(pin, cmd) {
     .Call('_gdalcubes_gc_create_stream_cube', PACKAGE = 'gdalcubes', pin, cmd)
 }
 
-gc_create_simple_cube <- function(files, datetime_values, bands, band_names, dx, dy, chunk_sizes) {
-    .Call('_gdalcubes_gc_create_simple_cube', PACKAGE = 'gdalcubes', files, datetime_values, bands, band_names, dx, dy, chunk_sizes)
+gc_create_simple_cube <- function(files, datetime_values, bands, band_names, dx, dy, chunk_sizes, strict = TRUE) {
+    .Call('_gdalcubes_gc_create_simple_cube', PACKAGE = 'gdalcubes', files, datetime_values, bands, band_names, dx, dy, chunk_sizes, strict)
 }
 
 gc_create_fill_time_cube <- function(pin, method) {

R/cube.R

@@ -7,6 +7,7 @@
 #' @param view A data cube view defining the shape (spatiotemporal extent, resolution, and spatial reference), if missing, a default overview is used
 #' @param mask mask pixels of images based on band values, see \code{\link{image_mask}}
 #' @param chunking length-3 vector or a function returning a vector of length 3, defining the size of data cube chunks in the order time, y, x.
+#' @param incomplete_ok logical, if TRUE (the default), chunks will ignore IO failures and simply use as much images as possible, otherwise the result will contain empty chunks if IO errors or similar occur.
 #' @return A proxy data cube object
 #' @details 
 #' The following steps will be performed when the data cube is requested to read data of a chunk:
@@ -38,7 +39,7 @@
 #'  
 #' @note This function returns a proxy object, i.e., it will not start any computations besides deriving the shape of the result.
 #' @export
-raster_cube <- function(image_collection, view, mask=NULL, chunking=.pkgenv$default_chunksize) {
+raster_cube <- function(image_collection, view, mask=NULL, chunking=.pkgenv$default_chunksize, incomplete_ok = TRUE) {
 
   stopifnot(is.image_collection(image_collection))
   if (is.function(chunking)) {
@@ -60,10 +61,10 @@ raster_cube <- function(image_collection, view, mask=NULL, chunking=.pkgenv$defa
   x = NULL
   if (!missing(view)) {
     stopifnot(is.cube_view(view))
-    x = gc_create_image_collection_cube(image_collection, as.integer(chunking), mask, view)
+    x = gc_create_image_collection_cube(image_collection, as.integer(chunking), mask, !incomplete_ok, view)
   }
   else {
-    x = gc_create_image_collection_cube(image_collection, as.integer(chunking), mask)
+    x = gc_create_image_collection_cube(image_collection, as.integer(chunking), mask, !incomplete_ok)
   }
   class(x) <- c("image_collection_cube", "cube", "xptr")
   return(x)
@@ -100,6 +101,7 @@ raster_cube <- function(image_collection, view, mask=NULL, chunking=.pkgenv$defa
 #' @param chunking vector of length 3 defining the size of data cube chunks in the order time, y, x.
 #' @param dx optional target pixel size in x direction, by default (NULL) the original or highest resolution of images is used 
 #' @param dy optional target pixel size in y direction, by default (NULL) the original or highest resolution of images is used 
+#' @param incomplete_ok logical, if TRUE (the default), chunks will ignore IO failures and simply use as much images as possible, otherwise the result will contain empty chunks if IO errors or similar occur.
 #' @return A proxy data cube object
 #' @examples 
 #' # toy example, repeating the same image as a daily time series
@@ -121,7 +123,7 @@ raster_cube <- function(image_collection, view, mask=NULL, chunking=.pkgenv$defa
 #'
 #' @note This function returns a proxy object, i.e., it will not start any computations besides deriving the shape of the result.                     
 #' @export
-stack_cube <- function(x, datetime_values, bands = NULL, band_names = NULL, chunking = c(1, 256, 256), dx=NULL, dy=NULL) {
+stack_cube <- function(x, datetime_values, bands = NULL, band_names = NULL, chunking = c(1, 256, 256), dx=NULL, dy=NULL, incomplete_ok = TRUE) {
 
 
   if (length(datetime_values) != length(x)) {
@@ -157,7 +159,7 @@ stack_cube <- function(x, datetime_values, bands = NULL, band_names = NULL, chun
     dy = -1.0
   }
 
-  x = gc_create_simple_cube(x, datetime_values, bands, band_names, dx, dy, as.integer(chunking))
+  x = gc_create_simple_cube(x, datetime_values, bands, band_names, dx, dy, as.integer(chunking), !incomplete_ok)
   class(x) <- c("simple_cube", "cube", "xptr")
   return(x)
 }

inst/formats/S5P_NO2.json

@@ -0,0 +1,22 @@
+{
+  "description": "Image collection format for Sentinel 5P Level 2 data (NO2 and QA bands only)",
+  "tags": ["Sentinel", "Copernicus", "Planetary-Computer", "S5P"],
+  "pattern": ".*/PRODUCT/(qa_value|nitrogendioxide_tropospheric_column)$",
+  "srs" : "EPSG:4326",
+  "subdatasets" : true,
+  "images": {
+    "pattern": ".*(S5P_.*)\\.nc.*"
+  },
+  "datetime": {
+    "pattern": ".+_(\\d{8}T\\d{6})_\\d{8}T\\d{6}_\\d{5}_\\d{2}_\\d{6}_\\d{8}T\\d{6}\\.nc.*",
+    "format": "%Y%m%dT%H%M%S"
+  },
+  "bands": {
+    "qa_value": {
+      "pattern": ".+\/PRODUCT\/qa_value.*"
+    },
+    "no2": {
+      "pattern": ".+\/PRODUCT\/nitrogendioxide_tropospheric_column.*"
+    }
+  }
+}

src/RcppExports.cpp

@@ -265,16 +265,17 @@ BEGIN_RCPP
 END_RCPP
 }
 // gc_create_image_collection_cube
-SEXP gc_create_image_collection_cube(SEXP pin, Rcpp::IntegerVector chunk_sizes, SEXP mask, SEXP v);
-RcppExport SEXP _gdalcubes_gc_create_image_collection_cube(SEXP pinSEXP, SEXP chunk_sizesSEXP, SEXP maskSEXP, SEXP vSEXP) {
+SEXP gc_create_image_collection_cube(SEXP pin, Rcpp::IntegerVector chunk_sizes, SEXP mask, bool strict, SEXP v);
+RcppExport SEXP _gdalcubes_gc_create_image_collection_cube(SEXP pinSEXP, SEXP chunk_sizesSEXP, SEXP maskSEXP, SEXP strictSEXP, SEXP vSEXP) {
 BEGIN_RCPP
     Rcpp::RObject rcpp_result_gen;
     Rcpp::RNGScope rcpp_rngScope_gen;
     Rcpp::traits::input_parameter< SEXP >::type pin(pinSEXP);
     Rcpp::traits::input_parameter< Rcpp::IntegerVector >::type chunk_sizes(chunk_sizesSEXP);
     Rcpp::traits::input_parameter< SEXP >::type mask(maskSEXP);
+    Rcpp::traits::input_parameter< bool >::type strict(strictSEXP);
     Rcpp::traits::input_parameter< SEXP >::type v(vSEXP);
-    rcpp_result_gen = Rcpp::wrap(gc_create_image_collection_cube(pin, chunk_sizes, mask, v));
+    rcpp_result_gen = Rcpp::wrap(gc_create_image_collection_cube(pin, chunk_sizes, mask, strict, v));
     return rcpp_result_gen;
 END_RCPP
 }
@@ -656,8 +657,8 @@ BEGIN_RCPP
 END_RCPP
 }
 // gc_create_simple_cube
-SEXP gc_create_simple_cube(std::vector<std::string> files, std::vector<std::string> datetime_values, std::vector<std::string> bands, std::vector<std::string> band_names, double dx, double dy, Rcpp::IntegerVector chunk_sizes);
-RcppExport SEXP _gdalcubes_gc_create_simple_cube(SEXP filesSEXP, SEXP datetime_valuesSEXP, SEXP bandsSEXP, SEXP band_namesSEXP, SEXP dxSEXP, SEXP dySEXP, SEXP chunk_sizesSEXP) {
+SEXP gc_create_simple_cube(std::vector<std::string> files, std::vector<std::string> datetime_values, std::vector<std::string> bands, std::vector<std::string> band_names, double dx, double dy, Rcpp::IntegerVector chunk_sizes, bool strict);
+RcppExport SEXP _gdalcubes_gc_create_simple_cube(SEXP filesSEXP, SEXP datetime_valuesSEXP, SEXP bandsSEXP, SEXP band_namesSEXP, SEXP dxSEXP, SEXP dySEXP, SEXP chunk_sizesSEXP, SEXP strictSEXP) {
 BEGIN_RCPP
     Rcpp::RObject rcpp_result_gen;
     Rcpp::RNGScope rcpp_rngScope_gen;
@@ -668,7 +669,8 @@ BEGIN_RCPP
     Rcpp::traits::input_parameter< double >::type dx(dxSEXP);
     Rcpp::traits::input_parameter< double >::type dy(dySEXP);
     Rcpp::traits::input_parameter< Rcpp::IntegerVector >::type chunk_sizes(chunk_sizesSEXP);
-    rcpp_result_gen = Rcpp::wrap(gc_create_simple_cube(files, datetime_values, bands, band_names, dx, dy, chunk_sizes));
+    Rcpp::traits::input_parameter< bool >::type strict(strictSEXP);
+    rcpp_result_gen = Rcpp::wrap(gc_create_simple_cube(files, datetime_values, bands, band_names, dx, dy, chunk_sizes, strict));
     return rcpp_result_gen;
 END_RCPP
 }
@@ -875,7 +877,7 @@ static const R_CallMethodDef CallEntries[] = {
     {"_gdalcubes_gc_add_images", (DL_FUNC) &_gdalcubes_gc_add_images, 4},
     {"_gdalcubes_gc_list_collection_formats", (DL_FUNC) &_gdalcubes_gc_list_collection_formats, 0},
     {"_gdalcubes_gc_create_view", (DL_FUNC) &_gdalcubes_gc_create_view, 1},
-    {"_gdalcubes_gc_create_image_collection_cube", (DL_FUNC) &_gdalcubes_gc_create_image_collection_cube, 4},
+    {"_gdalcubes_gc_create_image_collection_cube", (DL_FUNC) &_gdalcubes_gc_create_image_collection_cube, 5},
     {"_gdalcubes_gc_create_ncdf_cube", (DL_FUNC) &_gdalcubes_gc_create_ncdf_cube, 3},
     {"_gdalcubes_gc_create_dummy_cube", (DL_FUNC) &_gdalcubes_gc_create_dummy_cube, 4},
     {"_gdalcubes_gc_create_empty_cube", (DL_FUNC) &_gdalcubes_gc_create_empty_cube, 3},
@@ -904,7 +906,7 @@ static const R_CallMethodDef CallEntries[] = {
     {"_gdalcubes_gc_write_chunks_ncdf", (DL_FUNC) &_gdalcubes_gc_write_chunks_ncdf, 4},
     {"_gdalcubes_gc_write_tif", (DL_FUNC) &_gdalcubes_gc_write_tif, 8},
     {"_gdalcubes_gc_create_stream_cube", (DL_FUNC) &_gdalcubes_gc_create_stream_cube, 2},
-    {"_gdalcubes_gc_create_simple_cube", (DL_FUNC) &_gdalcubes_gc_create_simple_cube, 7},
+    {"_gdalcubes_gc_create_simple_cube", (DL_FUNC) &_gdalcubes_gc_create_simple_cube, 8},
     {"_gdalcubes_gc_create_fill_time_cube", (DL_FUNC) &_gdalcubes_gc_create_fill_time_cube, 2},
     {"_gdalcubes_gc_create_aggregate_time_cube", (DL_FUNC) &_gdalcubes_gc_create_aggregate_time_cube, 4},
     {"_gdalcubes_gc_create_aggregate_space_cube", (DL_FUNC) &_gdalcubes_gc_create_aggregate_space_cube, 5},

src/gdalcubes.cpp

@@ -844,7 +844,7 @@ SEXP gc_create_view(SEXP v) {
 
 
 // [[Rcpp::export]]
-SEXP gc_create_image_collection_cube(SEXP pin, Rcpp::IntegerVector chunk_sizes, SEXP mask, SEXP v = R_NilValue) {
+SEXP gc_create_image_collection_cube(SEXP pin, Rcpp::IntegerVector chunk_sizes, SEXP mask,bool strict = true, SEXP v = R_NilValue) {
 
   try {
     Rcpp::XPtr<std::shared_ptr<image_collection>> aa = Rcpp::as<Rcpp::XPtr<std::shared_ptr<image_collection>>>(pin);
@@ -858,6 +858,7 @@ SEXP gc_create_image_collection_cube(SEXP pin, Rcpp::IntegerVector chunk_sizes,
       x = new std::shared_ptr<image_collection_cube>( image_collection_cube::create(*aa, cv));
     }
     (*x)->set_chunk_size(chunk_sizes[0], chunk_sizes[1], chunk_sizes[2]);
+    (*x)->set_strict(strict);
 
 
     if (mask != R_NilValue) {
@@ -1464,12 +1465,13 @@ SEXP gc_create_stream_cube(SEXP pin, std::string cmd) {
 // [[Rcpp::export]]
 SEXP gc_create_simple_cube(std::vector<std::string> files,std::vector<std::string> datetime_values, 
                                      std::vector<std::string> bands,  std::vector<std::string> band_names, 
-                                     double dx, double dy,  Rcpp::IntegerVector chunk_sizes) {
+                                     double dx, double dy,  Rcpp::IntegerVector chunk_sizes, bool strict = true) {
   try {
     std::shared_ptr<simple_cube>* x = new std::shared_ptr<simple_cube>( simple_cube::create(files, datetime_values,
                                                                                             bands, band_names,
                                                                                             dx, dy));
     (*x)->set_chunk_size(chunk_sizes[0], chunk_sizes[1], chunk_sizes[2]);
+    (*x)->set_strict(strict);
     Rcpp::XPtr< std::shared_ptr<simple_cube> > p(x, true) ;
     return p;
   }

src/gdalcubes/src/aggregate_space.cpp

@@ -382,6 +382,15 @@ std::shared_ptr<chunk_data> aggregate_space_cube::read_chunk(chunkid_t id) {
             chunkid_t input_chunk_id = _in_cube->chunk_id_from_coords({ccoords[0], ch_y, ch_x});
             std::shared_ptr<chunk_data> in_chunk = _in_cube->read_chunk(input_chunk_id);
 
+            // propagate chunk status
+            if (in_chunk->status() == chunk_data::chunk_status::ERROR) {
+                out->set_status(chunk_data::chunk_status::ERROR);
+            }
+            else if (in_chunk->status() == chunk_data::chunk_status::INCOMPLETE && out->status() != chunk_data::chunk_status::ERROR) {
+                out->set_status(chunk_data::chunk_status::INCOMPLETE);
+            }
+
+
             if (in_chunk->empty()) {
                 continue;
             }

src/gdalcubes/src/aggregate_time.cpp

@@ -397,6 +397,15 @@ std::shared_ptr<chunk_data> aggregate_time_cube::read_chunk(chunkid_t id) {
             }
 
             std::shared_ptr<chunk_data> in_chunk = chunk_cache[cur_in_chunk];
+
+            // propagate chunk status
+            if (in_chunk->status() == chunk_data::chunk_status::ERROR) {
+                out->set_status(chunk_data::chunk_status::ERROR);
+            }
+            else if (in_chunk->status() == chunk_data::chunk_status::INCOMPLETE && out->status() != chunk_data::chunk_status::ERROR) {
+                out->set_status(chunk_data::chunk_status::INCOMPLETE);
+            }
+
             if (in_chunk->empty()) {
                 continue;
             }

src/gdalcubes/src/apply_pixel.cpp

@@ -37,8 +37,9 @@ std::shared_ptr<chunk_data> apply_pixel_cube::read_chunk(chunkid_t id) {
         return std::make_shared<chunk_data>();  // chunk is outside of the view, we don't need to read anything.
 
     std::shared_ptr<chunk_data> out = std::make_shared<chunk_data>();
-
     std::shared_ptr<chunk_data> in = _in_cube->read_chunk(id);
+
+    out->set_status(in->status());  // propagate chunk status
     if (in->empty()) {
         return out;
     }

src/gdalcubes/src/crop.cpp

@@ -58,6 +58,14 @@ std::shared_ptr<chunk_data> crop_cube::read_chunk(chunkid_t id) {
                 chunkid_t input_chunk_id = _in_cube->chunk_id_from_coords({ch_t, ch_y, ch_x});
                 std::shared_ptr<chunk_data> in_chunk = _in_cube->read_chunk(input_chunk_id);
 
+
+                // propagate chunk status
+                if (in_chunk->status() == chunk_data::chunk_status::ERROR) {
+                    out->set_status(chunk_data::chunk_status::ERROR);
+                }
+                else if (in_chunk->status() == chunk_data::chunk_status::INCOMPLETE && out->status() != chunk_data::chunk_status::ERROR) {
+                    out->set_status(chunk_data::chunk_status::INCOMPLETE);
+                }
                 if (in_chunk->empty()) {
                     continue;
                 }