#349 rename FileLayerProvider into LoadCollectionFromAssets

Open-EO · Jan 3, 2025 · 8cd6b37 · 8cd6b37
1 parent 37dcebd
commit 8cd6b37
Show file tree

Hide file tree

Showing 12 changed files with 92 additions and 73 deletions.
diff --git a/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/file/ProbaVPyramidFactory.scala b/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/file/ProbaVPyramidFactory.scala
@@ -8,7 +8,7 @@ import geotrellis.spark.{ContextRDD, MultibandTileLayerRDD}
 import geotrellis.vector._
 import org.apache.spark.SparkContext
 import org.openeo.geotrellis.ProjectedPolygons
-import org.openeo.geotrellis.layers.{FileLayerProvider, ProbaVPathDateExtractor}
+import org.openeo.geotrellis.layers.{LoadCollectionFromAssets, ProbaVPathDateExtractor}
 import org.openeo.geotrelliscommon.DataCubeParameters
 import org.openeo.opensearch.OpenSearchClient
 
@@ -51,7 +51,7 @@ class ProbaVPyramidFactory(openSearchEndpoint: String,
   private def fileLayerProvider(correlationId: String) = {
     val (assetTitles, bandIndices) = _bandNames.map(bandNameToAssetBandIndex).unzip
 
-    FileLayerProvider(
+    LoadCollectionFromAssets(
         OpenSearchClient(openSearchEndpointUrl),
         openSearchCollectionId,
         openSearchLinkTitles = NonEmptyList.of(assetTitles.head, assetTitles.tail: _*),

diff --git a/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/file/PyramidFactory.scala b/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/file/PyramidFactory.scala
@@ -8,7 +8,7 @@ import geotrellis.spark.MultibandTileLayerRDD
 import geotrellis.vector._
 import org.apache.spark.SparkContext
 import org.openeo.geotrellis.ProjectedPolygons
-import org.openeo.geotrellis.layers.{FileLayerProvider, SplitYearMonthDayPathDateExtractor}
+import org.openeo.geotrellis.layers.{LoadCollectionFromAssets, SplitYearMonthDayPathDateExtractor}
 import org.openeo.geotrelliscommon.DataCubeParameters
 import org.openeo.opensearch.OpenSearchClient
 import org.slf4j.LoggerFactory
@@ -67,7 +67,7 @@ class PyramidFactory(openSearchClient: OpenSearchClient,
    */
   private def fileLayerProvider(metadataProperties: Map[String, Any],
                                correlationId: String,
-                               layoutScheme: LayoutScheme = ZoomedLayoutScheme(crs, 256)) = FileLayerProvider(
+                               layoutScheme: LayoutScheme = ZoomedLayoutScheme(crs, 256)) = LoadCollectionFromAssets(
     openSearchClient,
     openSearchCollectionId,
     NonEmptyList.fromListUnsafe(openSearchLinkTitles.asScala.toList),

diff --git a/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/geotiff/PyramidFactory.scala b/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/geotiff/PyramidFactory.scala
@@ -15,7 +15,7 @@ import org.apache.hadoop.conf.Configuration
 import org.apache.hadoop.fs.Path
 import org.apache.spark.SparkContext
 import org.apache.spark.rdd.RDD
-import org.openeo.geotrellis.layers.{FileLayerProvider, MultibandCompositeRasterSource}
+import org.openeo.geotrellis.layers.{LoadCollectionFromAssets, MultibandCompositeRasterSource}
 import org.openeo.geotrellis.{ProjectedPolygons, bucketRegion, s3Client}
 import org.openeo.geotrelliscommon.{DataCubeParameters, DatacubeSupport, OpenEORasterCube, OpenEORasterCubeMetadata}
 import org.slf4j.LoggerFactory
@@ -219,11 +219,11 @@ class PyramidFactory private (rasterSources: => Seq[(RasterSource, ZonedDateTime
 
     val layerMetadata = DatacubeSupport.layerMetadata(theBoundingBox,summary.bounds.get.minKey,summary.bounds.get.maxKey,theZoom,summary.cellType,scheme,summary.cellSize,params.globalExtent)
     logger.info(s"Created user defined datacube with $layerMetadata")
-    val sourceRDD = FileLayerProvider.rasterSourceRDD(rasterSources,layerMetadata,summary.cellSize,"Geotiff collection")
+    val sourceRDD = LoadCollectionFromAssets.rasterSourceRDD(rasterSources,layerMetadata,summary.cellSize,"Geotiff collection")
     val filteredSources: RDD[LayoutTileSource[SpaceTimeKey]] = sourceRDD.filter({ tiledLayoutSource =>
       tiledLayoutSource.source.extent.interiorIntersects(tiledLayoutSource.layout.extent)
     })
-    FileLayerProvider.readMultibandTileLayer(filteredSources, layerMetadata,
+    LoadCollectionFromAssets.readMultibandTileLayer(filteredSources, layerMetadata,
       Array(MultiPolygon(toPolygon(theBoundingBox.extent))), theBoundingBox.crs, sc, retainNoDataTiles = false,
       datacubeParams = Some(params))
 

diff --git a/...eotrellis/src/main/scala/org/openeo/geotrellis/layers/AbstractGlobFileLayerProvider.scala b/...eotrellis/src/main/scala/org/openeo/geotrellis/layers/AbstractGlobFileLayerProvider.scala
@@ -160,7 +160,7 @@ abstract class AbstractGlobFileLayerProvider extends LayerProvider {
     val thePartitioner: Partitioner = if(partitioner.isDefined) {
       partitioner.get
     }else if(datacubeParams.isDefined){
-      FileLayerProvider.createPartitioner(datacubeParams, requiredKeys, tiledLayoutSourceRDD, layerMetadata).get
+      LoadCollectionFromAssets.createPartitioner(datacubeParams, requiredKeys, tiledLayoutSourceRDD, layerMetadata).get
     }else{
       SpacePartitioner(layerMetadata.bounds)
     }

diff --git a/...geotrellis/layers/FileLayerProvider.scala → ...lis/layers/LoadCollectionFromAssets.scala b/...geotrellis/layers/FileLayerProvider.scala → ...lis/layers/LoadCollectionFromAssets.scala
@@ -294,11 +294,9 @@ class MultibandCompositeRasterSource(val sourcesListWithBandIds: NonEmptyList[(R
 
 
 
-object FileLayerProvider {
-
-  private val logger = LoggerFactory.getLogger(classOf[FileLayerProvider])
-
+object LoadCollectionFromAssets {
 
+  private val logger = LoggerFactory.getLogger(classOf[LoadCollectionFromAssets])
 
 
   lazy val sdk = {
@@ -329,7 +327,7 @@ object FileLayerProvider {
   def apply(openSearch: OpenSearchClient, openSearchCollectionId: String, openSearchLinkTitles: NonEmptyList[String], rootPath: String,
             maxSpatialResolution: CellSize, pathDateExtractor: PathDateExtractor, attributeValues: Map[String, Any] = Map(), layoutScheme: LayoutScheme = ZoomedLayoutScheme(WebMercator, 256),
             bandIndices: Seq[Int] = Seq(), correlationId: String = "", experimental: Boolean = false,
-            retainNoDataTiles: Boolean = false, maxSoftErrorsRatio: Double = 0.0): FileLayerProvider = new FileLayerProvider(
+            retainNoDataTiles: Boolean = false, maxSoftErrorsRatio: Double = 0.0): LoadCollectionFromAssets = new LoadCollectionFromAssets(
     openSearch, openSearchCollectionId, openSearchLinkTitles, rootPath, maxSpatialResolution, pathDateExtractor,
     attributeValues, layoutScheme, bandIndices, correlationId, experimental, retainNoDataTiles, maxSoftErrorsRatio,
     disambiguateConstructors = null
@@ -940,14 +938,35 @@ object FileLayerProvider {
       })
 }
 
-class FileLayerProvider private(openSearch: OpenSearchClient, openSearchCollectionId: String, openSearchLinkTitles: NonEmptyList[String], rootPath: String,
-                        maxSpatialResolution: CellSize, pathDateExtractor: PathDateExtractor, attributeValues: Map[String, Any], layoutScheme: LayoutScheme,
-                        bandIndices: Seq[Int], correlationId: String, experimental: Boolean,
-                        retainNoDataTiles: Boolean, maxSoftErrorsRatio: Double,
-                        disambiguateConstructors: Null) extends LayerProvider { // workaround for: constructors have the same type after erasure
+/**
+ * Implements the 'load_collection' process for collections that are built from items and assets, following the [STAC](https://stacspec.org/en/about/stac-spec/)
+ * definition of these terms.
+ *
+ *
+ *
+ * @param openSearch The 'OpenSearch' client, the name is referring to the deprecated opensearch catalog spec, which can be considered somewhat equivalent to the STAC catalog spec.
+ * @param openSearchCollectionId
+ * @param openSearchLinkTitles
+ * @param rootPath
+ * @param maxSpatialResolution
+ * @param pathDateExtractor
+ * @param attributeValues
+ * @param layoutScheme
+ * @param bandIndices
+ * @param correlationId
+ * @param experimental
+ * @param retainNoDataTiles
+ * @param maxSoftErrorsRatio
+ * @param disambiguateConstructors
+ */
+class LoadCollectionFromAssets private(openSearch: OpenSearchClient, openSearchCollectionId: String, openSearchLinkTitles: NonEmptyList[String], rootPath: String,
+                                       maxSpatialResolution: CellSize, pathDateExtractor: PathDateExtractor, attributeValues: Map[String, Any], layoutScheme: LayoutScheme,
+                                       bandIndices: Seq[Int], correlationId: String, experimental: Boolean,
+                                       retainNoDataTiles: Boolean, maxSoftErrorsRatio: Double,
+                                       disambiguateConstructors: Null) extends LayerProvider { // workaround for: constructors have the same type after erasure
 
   import DatacubeSupport._
-  import FileLayerProvider._
+  import LoadCollectionFromAssets._
 
   @deprecated("call a constructor/factory method with flattened bandIndices instead of nested bandIds")
   // TODO: remove this eventually (e.g. after updating geotrellistimeseries)

diff --git a/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/LayerFixtures.scala b/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/LayerFixtures.scala
@@ -14,7 +14,7 @@ import org.apache.commons.io.FileUtils
 import org.apache.spark.SparkContext
 import org.apache.spark.rdd.RDD
 import org.openeo.geotrellis.file.{FixedFeaturesOpenSearchClient, PyramidFactory}
-import org.openeo.geotrellis.layers.{FileLayerProvider, MockOpenSearchFeatures, SplitYearMonthDayPathDateExtractor}
+import org.openeo.geotrellis.layers.{LoadCollectionFromAssets, MockOpenSearchFeatures, SplitYearMonthDayPathDateExtractor}
 import org.openeo.geotrelliscommon.{DataCubeParameters, SparseSpaceTimePartitioner}
 import org.openeo.opensearch.{OpenSearchClient, OpenSearchResponses}
 import org.openeo.opensearch.OpenSearchResponses.{CreoFeatureCollection, FeatureBuilder}
@@ -183,7 +183,7 @@ object LayerFixtures {
 
 
   def sentinel1Sigma0LayerProviderUTM =
-    FileLayerProvider(
+    LoadCollectionFromAssets(
       client,
       "urn:eop:VITO:CGS_S1_GRD_SIGMA0_L1",
       openSearchLinkTitles = NonEmptyList.of("VV"),
@@ -224,7 +224,7 @@ object LayerFixtures {
 
 
   def sentinel2TocLayerProviderUTM =
-    FileLayerProvider(
+    LoadCollectionFromAssets(
       client,
       "urn:eop:VITO:TERRASCOPE_S2_TOC_V2",
       openSearchLinkTitles = NonEmptyList.of("TOC-B04_10M", "TOC-B03_10M", "TOC-B02_10M", "SCENECLASSIFICATION_20M"),
@@ -236,7 +236,7 @@ object LayerFixtures {
     )
 
   def sentinel2TocLayerProviderUTMMultiResolution =
-    FileLayerProvider(
+    LoadCollectionFromAssets(
       client,
       "urn:eop:VITO:TERRASCOPE_S2_TOC_V2",
       openSearchLinkTitles = NonEmptyList.of("TOC-B04_10M", "TOC-B05_20M"),
@@ -248,7 +248,7 @@ object LayerFixtures {
     )
 
   def sentinel2TocLayerProviderUTM20M =
-    FileLayerProvider(
+    LoadCollectionFromAssets(
       client,
       "urn:eop:VITO:TERRASCOPE_S2_TOC_V2",
       openSearchLinkTitles = NonEmptyList.of("TOC-B11_20M", "SCENECLASSIFICATION_20M"),
@@ -422,7 +422,7 @@ object LayerFixtures {
   }
 
   def rgbLayerProvider =
-    FileLayerProvider(
+    LoadCollectionFromAssets(
       openSearch = client,
       openSearchCollectionId = "urn:eop:VITO:TERRASCOPE_S2_TOC_V2",
       openSearchLinkTitles = NonEmptyList.of("TOC-B04_10M", "TOC-B03_10M", "TOC-B02_10M"),

diff --git a/.../layers/AgEra5FileLayerProviderTest.scala → .../AgEra5LoadCollectionFromAssetsTest.scala b/.../layers/AgEra5FileLayerProviderTest.scala → .../AgEra5LoadCollectionFromAssetsTest.scala
@@ -18,21 +18,21 @@ import java.time.{LocalDate, ZoneId}
 import java.util
 import scala.collection.JavaConverters.asScalaBuffer
 
-object AgEra5FileLayerProviderTest extends LocalSparkContextJupyter
+object AgEra5LoadCollectionFromAssetsTest extends LocalSparkContextJupyter
 
-class AgEra5FileLayerProviderTest {
-  import AgEra5FileLayerProviderTest._
+class AgEra5LoadCollectionFromAssetsTest {
+  import AgEra5LoadCollectionFromAssetsTest._
 
   private val bands: util.List[String] = util.Arrays.asList("dewpoint-temperature", "precipitation-flux", "solar-radiation-flux")
 
-  private def layerProvider = FileLayerProvider(new Agera5SearchClient(dataGlob = "/data/MEP/ECMWF/AgERA5/2020/20200424/AgERA5_dewpoint-temperature_*.tif", bands, raw".+_(\d{4})(\d{2})(\d{2})\.tif".r),"",
+  private def layerProvider = LoadCollectionFromAssets(new Agera5SearchClient(dataGlob = "/data/MEP/ECMWF/AgERA5/2020/20200424/AgERA5_dewpoint-temperature_*.tif", bands, raw".+_(\d{4})(\d{2})(\d{2})\.tif".r),"",
     NonEmptyList.fromList(asScalaBuffer(bands).toList).get,
     rootPath = "/data/MEP/ECMWF/AgERA5",
     maxSpatialResolution = CellSize(0.1, 0.1),
     new Sentinel5PPathDateExtractor(maxDepth = 3),
     layoutScheme = FloatingLayoutScheme(256), experimental = false)
 
-  private def layerProvider2 = FileLayerProvider(new Agera5SearchClient(dataGlob = "/data/MEP/ECMWF/AgERA5/2020/20200424/AgERA5_dewpoint-temperature_*.tif", bands, raw".+_(\d{4})(\d{2})(\d{2})\.tif".r),"",
+  private def layerProvider2 = LoadCollectionFromAssets(new Agera5SearchClient(dataGlob = "/data/MEP/ECMWF/AgERA5/2020/20200424/AgERA5_dewpoint-temperature_*.tif", bands, raw".+_(\d{4})(\d{2})(\d{2})\.tif".r),"",
     NonEmptyList.fromList(asScalaBuffer(bands).toList).get,
     rootPath = "/data/MEP/ECMWF/AgERA5",
     maxSpatialResolution = CellSize(3000, 3000),

diff --git a/...s/GlobalNetCdfFileLayerProviderTest.scala → ...lNetCdfLoadCollectionFromAssetsTest.scala b/...s/GlobalNetCdfFileLayerProviderTest.scala → ...lNetCdfLoadCollectionFromAssetsTest.scala
@@ -24,13 +24,13 @@ import org.openeo.opensearch.backends.GlobalNetCDFSearchClient
 import java.util
 import scala.collection.JavaConverters._
 
-object GlobalNetCdfFileLayerProviderTest extends LocalSparkContext {
+object GlobalNetCdfLoadCollectionFromAssetsTest extends LocalSparkContext {
   @AfterClass
   def tearDown(): Unit = GDALWarp.deinit()
 }
 
-class GlobalNetCdfFileLayerProviderTest {
-  import GlobalNetCdfFileLayerProviderTest._
+class GlobalNetCdfLoadCollectionFromAssetsTest {
+  import GlobalNetCdfLoadCollectionFromAssetsTest._
 
   private def layerProvider = new GlobalNetCdfFileLayerProvider(
     dataGlob = "/data/MTDA/BIOPAR/BioPar_LAI300_V1_Global/*/*/*/*.nc",
@@ -40,7 +40,7 @@ class GlobalNetCdfFileLayerProviderTest {
 
   private val bands: util.List[String] = util.Arrays.asList("LAI", "NOBS")
 
-  private def multibandFileLayerProvider = FileLayerProvider(
+  private def multibandFileLayerProvider = LoadCollectionFromAssets(
     new GlobalNetCDFSearchClient(
       dataGlob = "/data/MTDA/BIOPAR/BioPar_LAI300_V1_Global/*/*/*/*.nc",
       bands,
@@ -226,7 +226,7 @@ class GlobalNetCdfFileLayerProviderTest {
     val parameters = new DataCubeParameters()
     parameters.layoutScheme = "FloatingLayoutScheme"
 
-    val layerProvider2WithOneBand = FileLayerProvider(new GlobalNetCDFSearchClient(dataGlob = "/data/MTDA/BIOPAR/BioPar_LAI300_V1_Global/2017/20170110/*/*.nc",bands , raw"_(\d{4})(\d{2})(\d{2})0000_".r.unanchored),"BioPar_LAI300_V1_Global",
+    val layerProvider2WithOneBand = LoadCollectionFromAssets(new GlobalNetCDFSearchClient(dataGlob = "/data/MTDA/BIOPAR/BioPar_LAI300_V1_Global/2017/20170110/*/*.nc",bands , raw"_(\d{4})(\d{2})(\d{2})0000_".r.unanchored),"BioPar_LAI300_V1_Global",
       NonEmptyList.of("LAI"),
       rootPath = "/data/MTDA/BIOPAR/BioPar_LAI300_V1_Global",
       maxSpatialResolution = CellSize(0.002976190476204,0.002976190476190),