From 4db1718ef4d018e5981f8f127df5be65d9c7f329 Mon Sep 17 00:00:00 2001 From: Elien Vandermaesen Date: Tue, 10 Dec 2024 14:20:14 +0100 Subject: [PATCH 01/15] issue #678 support shapely in load collection spatial extent --- openeo/rest/datacube.py | 158 ++++++++++++++------------- tests/rest/datacube/test_datacube.py | 13 +++ 2 files changed, 94 insertions(+), 77 deletions(-) diff --git a/openeo/rest/datacube.py b/openeo/rest/datacube.py index 3e80edbe5..0a7e43186 100644 --- a/openeo/rest/datacube.py +++ b/openeo/rest/datacube.py @@ -143,7 +143,7 @@ def load_collection( cls, collection_id: Union[str, Parameter], connection: Optional[Connection] = None, - spatial_extent: Union[Dict[str, float], Parameter, None] = None, + spatial_extent: Union[Dict[str, float], Parameter, shapely.geometry.base.BaseGeometry, None] = None, temporal_extent: Union[Sequence[InputDate], Parameter, str, None] = None, bands: Union[None, List[str], Parameter] = None, fetch_metadata: bool = True, @@ -187,6 +187,12 @@ def load_collection( "Unexpected parameterized `spatial_extent` in `load_collection`:" f" expected schema with type 'object' but got {spatial_extent.schema!r}." ) + valid_geojson_types = [ + "Polygon", "MultiPolygon", "GeometryCollection", "FeatureCollection" + ] + if spatial_extent and not isinstance(spatial_extent, dict): + spatial_extent = _get_geometry_argument(argument=spatial_extent,valid_geojson_types=valid_geojson_types,connection=connection) + arguments = { 'id': collection_id, # TODO: spatial_extent could also be a "geojson" subtype object, so we might want to allow (and convert) shapely shapes as well here. @@ -628,10 +634,9 @@ def filter_spatial( (which will be loaded client-side to get the geometries as GeoJSON construct). """ valid_geojson_types = [ - "Point", "MultiPoint", "LineString", "MultiLineString", "Polygon", "MultiPolygon", "GeometryCollection", "FeatureCollection" ] - geometries = self._get_geometry_argument(geometries, valid_geojson_types=valid_geojson_types, crs=None) + geometries = _get_geometry_argument(geometries, valid_geojson_types=valid_geojson_types, connection=self.connection, crs=None) return self.process( process_id='filter_spatial', arguments={ @@ -1058,75 +1063,6 @@ def _merge_operator_binary_cubes( } )) - def _get_geometry_argument( - self, - argument: Union[ - shapely.geometry.base.BaseGeometry, - dict, - str, - pathlib.Path, - Parameter, - _FromNodeMixin, - ], - valid_geojson_types: List[str], - crs: Optional[str] = None, - ) -> Union[dict, Parameter, PGNode]: - """ - Convert input to a geometry as "geojson" subtype object or vectorcube. - - :param crs: value that encodes a coordinate reference system. - See :py:func:`openeo.util.normalize_crs` for more details about additional normalization that is applied to this argument. - """ - if isinstance(argument, Parameter): - return argument - elif isinstance(argument, _FromNodeMixin): - return argument.from_node() - - if isinstance(argument, str) and re.match(r"^https?://", argument, flags=re.I): - # Geometry provided as URL: load with `load_url` (with best-effort format guess) - url = urllib.parse.urlparse(argument) - suffix = pathlib.Path(url.path.lower()).suffix - format = { - ".json": "GeoJSON", - ".geojson": "GeoJSON", - ".pq": "Parquet", - ".parquet": "Parquet", - ".geoparquet": "Parquet", - }.get(suffix, suffix.split(".")[-1]) - return self.connection.load_url(url=argument, format=format) - - if ( - isinstance(argument, (str, pathlib.Path)) - and pathlib.Path(argument).is_file() - and pathlib.Path(argument).suffix.lower() in [".json", ".geojson"] - ): - geometry = load_json(argument) - elif isinstance(argument, shapely.geometry.base.BaseGeometry): - geometry = mapping(argument) - elif isinstance(argument, dict): - geometry = argument - else: - raise OpenEoClientException(f"Invalid geometry argument: {argument!r}") - - if geometry.get("type") not in valid_geojson_types: - raise OpenEoClientException("Invalid geometry type {t!r}, must be one of {s}".format( - t=geometry.get("type"), s=valid_geojson_types - )) - if crs: - # TODO: don't warn when the crs is Lon-Lat like EPSG:4326? - warnings.warn(f"Geometry with non-Lon-Lat CRS {crs!r} is only supported by specific back-ends.") - # TODO #204 alternative for non-standard CRS in GeoJSON object? - epsg_code = normalize_crs(crs) - if epsg_code is not None: - # proj did recognize the CRS - crs_name = f"EPSG:{epsg_code}" - else: - # proj did not recognise this CRS - warnings.warn(f"non-Lon-Lat CRS {crs!r} is not known to the proj library and might not be supported.") - crs_name = crs - geometry["crs"] = {"type": "name", "properties": {"name": crs_name}} - return geometry - @openeo_process def aggregate_spatial( self, @@ -1198,7 +1134,7 @@ def aggregate_spatial( "Point", "MultiPoint", "LineString", "MultiLineString", "Polygon", "MultiPolygon", "GeometryCollection", "Feature", "FeatureCollection" ] - geometries = self._get_geometry_argument(geometries, valid_geojson_types=valid_geojson_types, crs=crs) + geometries = _get_geometry_argument(geometries, valid_geojson_types=valid_geojson_types, connection= self.connection, crs=crs) reducer = build_child_callback(reducer, parent_parameters=["data"]) return VectorCube( graph=self._build_pgnode( @@ -1478,8 +1414,8 @@ def chunk_polygon( "Feature", "FeatureCollection", ] - chunks = self._get_geometry_argument( - chunks, valid_geojson_types=valid_geojson_types + chunks = _get_geometry_argument( + chunks, valid_geojson_types=valid_geojson_types, connection=self.connection ) mask_value = float(mask_value) if mask_value is not None else None return self.process( @@ -1568,7 +1504,7 @@ def apply_polygon( process = build_child_callback(process, parent_parameters=["data"], connection=self.connection) valid_geojson_types = ["Polygon", "MultiPolygon", "Feature", "FeatureCollection"] - geometries = self._get_geometry_argument(geometries, valid_geojson_types=valid_geojson_types) + geometries = _get_geometry_argument(geometries, valid_geojson_types=valid_geojson_types, connection=self.connection) mask_value = float(mask_value) if mask_value is not None else None return self.process( process_id="apply_polygon", @@ -2056,7 +1992,7 @@ def mask_polygon( (which will be loaded client-side to get the geometries as GeoJSON construct). """ valid_geojson_types = ["Polygon", "MultiPolygon", "GeometryCollection", "Feature", "FeatureCollection"] - mask = self._get_geometry_argument(mask, valid_geojson_types=valid_geojson_types, crs=srs) + mask = _get_geometry_argument(mask, valid_geojson_types=valid_geojson_types, connection=self.connection, crs=srs) return self.process( process_id="mask_polygon", arguments=dict_no_none( @@ -2860,3 +2796,71 @@ def unflatten_dimension(self, dimension: str, target_dimensions: List[str], labe label_separator=label_separator, ), ) +def _get_geometry_argument( + argument: Union[ + shapely.geometry.base.BaseGeometry, + dict, + str, + pathlib.Path, + Parameter, + _FromNodeMixin, + ], + valid_geojson_types: List[str], + connection: Connection = None, + crs: Optional[str] = None, +) -> Union[dict, Parameter, PGNode]: + """ + Convert input to a geometry as "geojson" subtype object or vectorcube. + + :param crs: value that encodes a coordinate reference system. + See :py:func:`openeo.util.normalize_crs` for more details about additional normalization that is applied to this argument. + """ + if isinstance(argument, Parameter): + return argument + elif isinstance(argument, _FromNodeMixin): + return argument.from_node() + + if isinstance(argument, str) and re.match(r"^https?://", argument, flags=re.I): + # Geometry provided as URL: load with `load_url` (with best-effort format guess) + url = urllib.parse.urlparse(argument) + suffix = pathlib.Path(url.path.lower()).suffix + format = { + ".json": "GeoJSON", + ".geojson": "GeoJSON", + ".pq": "Parquet", + ".parquet": "Parquet", + ".geoparquet": "Parquet", + }.get(suffix, suffix.split(".")[-1]) + return connection.load_url(url=argument, format=format) + # + if ( + isinstance(argument, (str, pathlib.Path)) + and pathlib.Path(argument).is_file() + and pathlib.Path(argument).suffix.lower() in [".json", ".geojson"] + ): + geometry = load_json(argument) + elif isinstance(argument, shapely.geometry.base.BaseGeometry): + geometry = mapping(argument) + elif isinstance(argument, dict): + geometry = argument + else: + raise OpenEoClientException(f"Invalid geometry argument: {argument!r}") + + if geometry.get("type") not in valid_geojson_types: + raise OpenEoClientException("Invalid geometry type {t!r}, must be one of {s}".format( + t=geometry.get("type"), s=valid_geojson_types + )) + if crs: + # TODO: don't warn when the crs is Lon-Lat like EPSG:4326? + warnings.warn(f"Geometry with non-Lon-Lat CRS {crs!r} is only supported by specific back-ends.") + # TODO #204 alternative for non-standard CRS in GeoJSON object? + epsg_code = normalize_crs(crs) + if epsg_code is not None: + # proj did recognize the CRS + crs_name = f"EPSG:{epsg_code}" + else: + # proj did not recognise this CRS + warnings.warn(f"non-Lon-Lat CRS {crs!r} is not known to the proj library and might not be supported.") + crs_name = crs + geometry["crs"] = {"type": "name", "properties": {"name": crs_name}} + return geometry \ No newline at end of file diff --git a/tests/rest/datacube/test_datacube.py b/tests/rest/datacube/test_datacube.py index cd3afbcba..7e8745109 100644 --- a/tests/rest/datacube/test_datacube.py +++ b/tests/rest/datacube/test_datacube.py @@ -136,6 +136,19 @@ def test_load_collection_connectionless_temporal_extent_shortcut(self): } } + def test_load_collection_connectionless_shapely_spatial_extent(self): + polygon = shapely.Polygon(((0.0,1.0),(2.0,1.0),(3.0,2.0),(1.5,0.0),(0.0,1.0))) + cube = DataCube.load_collection("T3", spatial_extent=polygon) + assert cube.flat_graph() == { + "loadcollection1": { + "arguments": {"id": "T3", "spatial_extent": + {'coordinates': (((0.0,1.0),(2.0,1.0),(3.0,2.0),(1.5,0.0),(0.0,1.0)),),'type': 'Polygon'}, + "temporal_extent": None}, + "process_id": "load_collection", + "result": True, + } + } + def test_load_collection_connectionless_save_result(self): cube = DataCube.load_collection("T3").save_result(format="GTiff") assert cube.flat_graph() == { From 146f35e9973129b01482648d7ed4243426ae9867 Mon Sep 17 00:00:00 2001 From: Elien Vandermaesen Date: Tue, 10 Dec 2024 15:00:21 +0100 Subject: [PATCH 02/15] issue #678 support shapely in load collection spatial extent --- openeo/rest/datacube.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openeo/rest/datacube.py b/openeo/rest/datacube.py index 0a7e43186..6f114b369 100644 --- a/openeo/rest/datacube.py +++ b/openeo/rest/datacube.py @@ -190,7 +190,7 @@ def load_collection( valid_geojson_types = [ "Polygon", "MultiPolygon", "GeometryCollection", "FeatureCollection" ] - if spatial_extent and not isinstance(spatial_extent, dict): + if spatial_extent and not (isinstance(spatial_extent, dict) and spatial_extent.keys() & {"west", "east", "north", "south"}): spatial_extent = _get_geometry_argument(argument=spatial_extent,valid_geojson_types=valid_geojson_types,connection=connection) arguments = { From ef50a25dfa817ae6a656650391ebe7780ce35895 Mon Sep 17 00:00:00 2001 From: Elien Vandermaesen Date: Tue, 10 Dec 2024 15:01:18 +0100 Subject: [PATCH 03/15] issue #678 support shapely in load collection spatial extent --- openeo/rest/datacube.py | 1 - 1 file changed, 1 deletion(-) diff --git a/openeo/rest/datacube.py b/openeo/rest/datacube.py index 6f114b369..4a0b26a70 100644 --- a/openeo/rest/datacube.py +++ b/openeo/rest/datacube.py @@ -195,7 +195,6 @@ def load_collection( arguments = { 'id': collection_id, - # TODO: spatial_extent could also be a "geojson" subtype object, so we might want to allow (and convert) shapely shapes as well here. 'spatial_extent': spatial_extent, 'temporal_extent': temporal_extent, } From 8cadb84bda50ec9c2c38ef8511773d835972ae65 Mon Sep 17 00:00:00 2001 From: Elien Vandermaesen Date: Fri, 13 Dec 2024 08:37:26 +0100 Subject: [PATCH 04/15] issue #678 support shapely and local path in load collection spatial extent --- CHANGELOG.md | 1 + openeo/rest/connection.py | 9 +++++++-- openeo/rest/datacube.py | 9 +++++++-- tests/rest/datacube/test_datacube.py | 14 ++++++++++++++ 4 files changed, 29 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe65c0475..241f60fd3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Automatically use `load_url` when providing a URL as geometries to `DataCube.aggregate_spatial()`, `DataCube.mask_polygon()`, etc. ([#104](https://github.com/Open-EO/openeo-python-client/issues/104), [#457](https://github.com/Open-EO/openeo-python-client/issues/457)) +- Argument `spatial_extent` in `load_collection` supports type `shapely` and loading geometry from a local path. ### Changed diff --git a/openeo/rest/connection.py b/openeo/rest/connection.py index 79ee478f8..a1befaa79 100644 --- a/openeo/rest/connection.py +++ b/openeo/rest/connection.py @@ -1230,7 +1230,7 @@ def datacube_from_json(self, src: Union[str, Path], parameters: Optional[dict] = def load_collection( self, collection_id: Union[str, Parameter], - spatial_extent: Union[Dict[str, float], Parameter, None] = None, + spatial_extent: Union[Dict[str, float], Parameter, shapely.geometry.base.BaseGeometry, None] = None, temporal_extent: Union[Sequence[InputDate], Parameter, str, None] = None, bands: Union[None, List[str], Parameter] = None, properties: Union[ @@ -1243,7 +1243,12 @@ def load_collection( Load a DataCube by collection id. :param collection_id: image collection identifier - :param spatial_extent: limit data to specified bounding box or polygons + :param spatial_extent: limit data to specified bounding box or polygons. Can be provided in different ways: + - a shapely geometry + - a GeoJSON-style dictionary, + - a path (:py:class:`str` or :py:class:`~pathlib.Path`) to a local, client-side GeoJSON file, + which will be loaded automatically to get the geometries as GeoJSON construct. + - a :py:class:`~openeo.api.process.Parameter` instance. :param temporal_extent: limit data to specified temporal interval. Typically, just a two-item list or tuple containing start and end date. See :ref:`filtering-on-temporal-extent-section` for more details on temporal extent handling and shorthand notation. diff --git a/openeo/rest/datacube.py b/openeo/rest/datacube.py index 4a0b26a70..4629c37d5 100644 --- a/openeo/rest/datacube.py +++ b/openeo/rest/datacube.py @@ -158,7 +158,12 @@ def load_collection( :param collection_id: image collection identifier :param connection: The backend connection to use. Can be ``None`` to work without connection and collection metadata. - :param spatial_extent: limit data to specified bounding box or polygons + :param spatial_extent: limit data to specified bounding box or polygons. Can be provided in different ways: + - a shapely geometry + - a GeoJSON-style dictionary, + - a path (:py:class:`str` or :py:class:`~pathlib.Path`) to a local, client-side GeoJSON file, + which will be loaded automatically to get the geometries as GeoJSON construct. + - a :py:class:`~openeo.api.process.Parameter` instance. :param temporal_extent: limit data to specified temporal interval. Typically, just a two-item list or tuple containing start and end date. See :ref:`filtering-on-temporal-extent-section` for more details on temporal extent handling and shorthand notation. @@ -188,7 +193,7 @@ def load_collection( f" expected schema with type 'object' but got {spatial_extent.schema!r}." ) valid_geojson_types = [ - "Polygon", "MultiPolygon", "GeometryCollection", "FeatureCollection" + "Polygon", "MultiPolygon", "Feature", "FeatureCollection" ] if spatial_extent and not (isinstance(spatial_extent, dict) and spatial_extent.keys() & {"west", "east", "north", "south"}): spatial_extent = _get_geometry_argument(argument=spatial_extent,valid_geojson_types=valid_geojson_types,connection=connection) diff --git a/tests/rest/datacube/test_datacube.py b/tests/rest/datacube/test_datacube.py index 7e8745109..820f720a4 100644 --- a/tests/rest/datacube/test_datacube.py +++ b/tests/rest/datacube/test_datacube.py @@ -149,6 +149,20 @@ def test_load_collection_connectionless_shapely_spatial_extent(self): } } + @pytest.mark.parametrize("path_factory", [str, pathlib.Path]) + def test_load_collection_connectionless_local_path_spatial_extent(self, path_factory, test_data): + path = path_factory(test_data.get_path("geojson/polygon02.json")) + cube = DataCube.load_collection("T3", spatial_extent=path) + assert cube.flat_graph() == { + "loadcollection1": { + "arguments": {"id": "T3", "spatial_extent": + {"type": "Polygon", "coordinates": [[[3, 50], [4, 50], [4, 51], [3, 50]]]}, + "temporal_extent": None}, + "process_id": "load_collection", + "result": True, + } + } + def test_load_collection_connectionless_save_result(self): cube = DataCube.load_collection("T3").save_result(format="GTiff") assert cube.flat_graph() == { From d2770119a541fdcd8dd7ec200216ffb953e41cb8 Mon Sep 17 00:00:00 2001 From: Elien Vandermaesen Date: Mon, 16 Dec 2024 08:55:39 +0100 Subject: [PATCH 05/15] issue #678 support shapely and local path in load collection spatial extent --- CHANGELOG.md | 2 +- openeo/rest/connection.py | 1 + openeo/rest/datacube.py | 14 +++++++++----- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 241f60fd3..1f4dd5dd3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Automatically use `load_url` when providing a URL as geometries to `DataCube.aggregate_spatial()`, `DataCube.mask_polygon()`, etc. ([#104](https://github.com/Open-EO/openeo-python-client/issues/104), [#457](https://github.com/Open-EO/openeo-python-client/issues/457)) -- Argument `spatial_extent` in `load_collection` supports type `shapely` and loading geometry from a local path. +- Argument `spatial_extent` in `load_collection` supports Shapely objects and loading GeoJSON from a local path. ### Changed diff --git a/openeo/rest/connection.py b/openeo/rest/connection.py index a1befaa79..77d4fc759 100644 --- a/openeo/rest/connection.py +++ b/openeo/rest/connection.py @@ -1249,6 +1249,7 @@ def load_collection( - a path (:py:class:`str` or :py:class:`~pathlib.Path`) to a local, client-side GeoJSON file, which will be loaded automatically to get the geometries as GeoJSON construct. - a :py:class:`~openeo.api.process.Parameter` instance. + - a bounding box dictionary :param temporal_extent: limit data to specified temporal interval. Typically, just a two-item list or tuple containing start and end date. See :ref:`filtering-on-temporal-extent-section` for more details on temporal extent handling and shorthand notation. diff --git a/openeo/rest/datacube.py b/openeo/rest/datacube.py index 4629c37d5..7f12dcfb7 100644 --- a/openeo/rest/datacube.py +++ b/openeo/rest/datacube.py @@ -164,6 +164,7 @@ def load_collection( - a path (:py:class:`str` or :py:class:`~pathlib.Path`) to a local, client-side GeoJSON file, which will be loaded automatically to get the geometries as GeoJSON construct. - a :py:class:`~openeo.api.process.Parameter` instance. + - a bounding box dictionary :param temporal_extent: limit data to specified temporal interval. Typically, just a two-item list or tuple containing start and end date. See :ref:`filtering-on-temporal-extent-section` for more details on temporal extent handling and shorthand notation. @@ -192,11 +193,14 @@ def load_collection( "Unexpected parameterized `spatial_extent` in `load_collection`:" f" expected schema with type 'object' but got {spatial_extent.schema!r}." ) - valid_geojson_types = [ - "Polygon", "MultiPolygon", "Feature", "FeatureCollection" - ] - if spatial_extent and not (isinstance(spatial_extent, dict) and spatial_extent.keys() & {"west", "east", "north", "south"}): - spatial_extent = _get_geometry_argument(argument=spatial_extent,valid_geojson_types=valid_geojson_types,connection=connection) + elif not spatial_extent or (isinstance(spatial_extent, dict) and spatial_extent.keys() & {"west", "east", "north", "south"}): + pass + else: + valid_geojson_types = [ + "Polygon", "MultiPolygon", "Feature", "FeatureCollection" + ] + spatial_extent = _get_geometry_argument(argument=spatial_extent, valid_geojson_types=valid_geojson_types, + connection=connection) arguments = { 'id': collection_id, From 81431c23261b001f89802036116219b9f3401e45 Mon Sep 17 00:00:00 2001 From: Elien Vandermaesen Date: Thu, 19 Dec 2024 08:28:18 +0100 Subject: [PATCH 06/15] issue #678 support shapely and local path in load collection, make changes clear --- openeo/rest/connection.py | 5 ++++- openeo/rest/datacube.py | 11 ++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/openeo/rest/connection.py b/openeo/rest/connection.py index 77d4fc759..44a100746 100644 --- a/openeo/rest/connection.py +++ b/openeo/rest/connection.py @@ -1244,12 +1244,12 @@ def load_collection( :param collection_id: image collection identifier :param spatial_extent: limit data to specified bounding box or polygons. Can be provided in different ways: + - a bounding box dictionary - a shapely geometry - a GeoJSON-style dictionary, - a path (:py:class:`str` or :py:class:`~pathlib.Path`) to a local, client-side GeoJSON file, which will be loaded automatically to get the geometries as GeoJSON construct. - a :py:class:`~openeo.api.process.Parameter` instance. - - a bounding box dictionary :param temporal_extent: limit data to specified temporal interval. Typically, just a two-item list or tuple containing start and end date. See :ref:`filtering-on-temporal-extent-section` for more details on temporal extent handling and shorthand notation. @@ -1268,6 +1268,9 @@ def load_collection( .. versionchanged:: 0.26.0 Add :py:func:`~openeo.rest.graph_building.collection_property` support to ``properties`` argument. + + .. versionchanged:: 0.37.0 + Add support for shapely geometry and local path to GeoJSON file to spatial_extent argument. """ return DataCube.load_collection( collection_id=collection_id, diff --git a/openeo/rest/datacube.py b/openeo/rest/datacube.py index 7f12dcfb7..457b7e61c 100644 --- a/openeo/rest/datacube.py +++ b/openeo/rest/datacube.py @@ -159,12 +159,12 @@ def load_collection( :param connection: The backend connection to use. Can be ``None`` to work without connection and collection metadata. :param spatial_extent: limit data to specified bounding box or polygons. Can be provided in different ways: + - a bounding box dictionary - a shapely geometry - a GeoJSON-style dictionary, - a path (:py:class:`str` or :py:class:`~pathlib.Path`) to a local, client-side GeoJSON file, which will be loaded automatically to get the geometries as GeoJSON construct. - a :py:class:`~openeo.api.process.Parameter` instance. - - a bounding box dictionary :param temporal_extent: limit data to specified temporal interval. Typically, just a two-item list or tuple containing start and end date. See :ref:`filtering-on-temporal-extent-section` for more details on temporal extent handling and shorthand notation. @@ -183,6 +183,9 @@ def load_collection( .. versionchanged:: 0.26.0 Add :py:func:`~openeo.rest.graph_building.collection_property` support to ``properties`` argument. + + .. versionchanged:: 0.37.0 + Add support for shapely geometry and local path to GeoJSON file to spatial_extent argument. """ if temporal_extent: temporal_extent = cls._get_temporal_extent(extent=temporal_extent) @@ -193,7 +196,9 @@ def load_collection( "Unexpected parameterized `spatial_extent` in `load_collection`:" f" expected schema with type 'object' but got {spatial_extent.schema!r}." ) - elif not spatial_extent or (isinstance(spatial_extent, dict) and spatial_extent.keys() & {"west", "east", "north", "south"}): + elif spatial_extent is None or ( + isinstance(spatial_extent, dict) and spatial_extent.keys() & {"west", "east", "north", "south"} + ): pass else: valid_geojson_types = [ @@ -2871,4 +2876,4 @@ def _get_geometry_argument( warnings.warn(f"non-Lon-Lat CRS {crs!r} is not known to the proj library and might not be supported.") crs_name = crs geometry["crs"] = {"type": "name", "properties": {"name": crs_name}} - return geometry \ No newline at end of file + return geometry From a64ce16d6a4489adddde073af0a7ec977f60fcc6 Mon Sep 17 00:00:00 2001 From: Elien Vandermaesen Date: Thu, 2 Jan 2025 09:02:39 +0100 Subject: [PATCH 07/15] issue #678 correct valid types of geojson types in filter_spatial --- openeo/rest/datacube.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/openeo/rest/datacube.py b/openeo/rest/datacube.py index 457b7e61c..2afcf46e7 100644 --- a/openeo/rest/datacube.py +++ b/openeo/rest/datacube.py @@ -647,7 +647,14 @@ def filter_spatial( (which will be loaded client-side to get the geometries as GeoJSON construct). """ valid_geojson_types = [ - "Polygon", "MultiPolygon", "GeometryCollection", "FeatureCollection" + "Point", + "MultiPoint", + "LineString", + "MultiLineString", + "Polygon", + "MultiPolygon", + "GeometryCollection", + "FeatureCollection", ] geometries = _get_geometry_argument(geometries, valid_geojson_types=valid_geojson_types, connection=self.connection, crs=None) return self.process( @@ -2809,6 +2816,8 @@ def unflatten_dimension(self, dimension: str, target_dimensions: List[str], labe label_separator=label_separator, ), ) + + def _get_geometry_argument( argument: Union[ shapely.geometry.base.BaseGeometry, From e9ec44de9ea9071001d672aad0132f57c1e8f3c7 Mon Sep 17 00:00:00 2001 From: Elien Vandermaesen Date: Thu, 9 Jan 2025 08:40:10 +0100 Subject: [PATCH 08/15] issue #678 improve documentation --- openeo/rest/connection.py | 4 ++-- openeo/rest/datacube.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/openeo/rest/connection.py b/openeo/rest/connection.py index 1bddf1c33..14d7c66f3 100644 --- a/openeo/rest/connection.py +++ b/openeo/rest/connection.py @@ -1270,7 +1270,7 @@ def load_collection( :param collection_id: image collection identifier :param spatial_extent: limit data to specified bounding box or polygons. Can be provided in different ways: - a bounding box dictionary - - a shapely geometry + - a Shapely geometry object - a GeoJSON-style dictionary, - a path (:py:class:`str` or :py:class:`~pathlib.Path`) to a local, client-side GeoJSON file, which will be loaded automatically to get the geometries as GeoJSON construct. @@ -1295,7 +1295,7 @@ def load_collection( Add :py:func:`~openeo.rest.graph_building.collection_property` support to ``properties`` argument. .. versionchanged:: 0.37.0 - Add support for shapely geometry and local path to GeoJSON file to spatial_extent argument. + Add support for passing a Shapely geometry or a local path to a GeoJSON file to the ``spatial_extent`` argument. """ return DataCube.load_collection( collection_id=collection_id, diff --git a/openeo/rest/datacube.py b/openeo/rest/datacube.py index 7a83a1ca8..faa966c8a 100644 --- a/openeo/rest/datacube.py +++ b/openeo/rest/datacube.py @@ -160,7 +160,7 @@ def load_collection( Can be ``None`` to work without connection and collection metadata. :param spatial_extent: limit data to specified bounding box or polygons. Can be provided in different ways: - a bounding box dictionary - - a shapely geometry + - a Shapely geometry object - a GeoJSON-style dictionary, - a path (:py:class:`str` or :py:class:`~pathlib.Path`) to a local, client-side GeoJSON file, which will be loaded automatically to get the geometries as GeoJSON construct. @@ -185,7 +185,7 @@ def load_collection( Add :py:func:`~openeo.rest.graph_building.collection_property` support to ``properties`` argument. .. versionchanged:: 0.37.0 - Add support for shapely geometry and local path to GeoJSON file to spatial_extent argument. + Add support for passing a Shapely geometry or a local path to a GeoJSON file to the ``spatial_extent`` argument. """ if temporal_extent: temporal_extent = cls._get_temporal_extent(extent=temporal_extent) From ac11868bda67d0c28da0a8bdd97670232e350345 Mon Sep 17 00:00:00 2001 From: Stefaan Lippens Date: Thu, 16 Jan 2025 15:22:18 +0100 Subject: [PATCH 09/15] Issue #678/#682 further finetuning - doc and typing tweaks - push more functionality to _get_geometry_argument - add support in load_stac as well --- CHANGELOG.md | 2 +- openeo/rest/connection.py | 10 ++-- openeo/rest/datacube.py | 85 +++++++++++++++++++--------- tests/rest/datacube/test_datacube.py | 2 +- 4 files changed, 64 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 35daa8de4..62c95b2e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,12 +9,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Argument `spatial_extent` in `load_collection` supports Shapely objects and loading GeoJSON from a local path. - Added `show_error_logs` argument to `cube.execute_batch()`/`job.start_and_wait()`/... to toggle the automatic printing of error logs on failure ([#505](https://github.com/Open-EO/openeo-python-client/issues/505)) - Added `Connection.web_editor()` to build link to the openEO backend in the openEO Web Editor - Add support for `log_level` in `create_job()` and `execute_job()` ([#704](https://github.com/Open-EO/openeo-python-client/issues/704)) - Add initial support for "geometry" dimension type in `CubeMetadata` ([#705](https://github.com/Open-EO/openeo-python-client/issues/705)) - Add support for parameterized `bands` argument in `load_stac()` +- Argument `spatial_extent` in `load_collection()`/`load_stac()`: add support for Shapely objects and loading GeoJSON from a local path. ([#678](https://github.com/Open-EO/openeo-python-client/issues/678)) ### Changed diff --git a/openeo/rest/connection.py b/openeo/rest/connection.py index 615b13f3d..49067f61b 100644 --- a/openeo/rest/connection.py +++ b/openeo/rest/connection.py @@ -1256,7 +1256,7 @@ def datacube_from_json(self, src: Union[str, Path], parameters: Optional[dict] = def load_collection( self, collection_id: Union[str, Parameter], - spatial_extent: Union[Dict[str, float], Parameter, shapely.geometry.base.BaseGeometry, None] = None, + spatial_extent: Union[dict, Parameter, shapely.geometry.base.BaseGeometry, None] = None, temporal_extent: Union[Sequence[InputDate], Parameter, str, None] = None, bands: Union[Iterable[str], Parameter, str, None] = None, properties: Union[ @@ -1272,8 +1272,8 @@ def load_collection( :param spatial_extent: limit data to specified bounding box or polygons. Can be provided in different ways: - a bounding box dictionary - a Shapely geometry object - - a GeoJSON-style dictionary, - - a path (:py:class:`str` or :py:class:`~pathlib.Path`) to a local, client-side GeoJSON file, + - a GeoJSON-style dictionary + - a path (as :py:class:`str` or :py:class:`~pathlib.Path`) to a local, client-side GeoJSON file, which will be loaded automatically to get the geometries as GeoJSON construct. - a :py:class:`~openeo.api.process.Parameter` instance. :param temporal_extent: limit data to specified temporal interval. @@ -1296,7 +1296,7 @@ def load_collection( Add :py:func:`~openeo.rest.graph_building.collection_property` support to ``properties`` argument. .. versionchanged:: 0.37.0 - Add support for passing a Shapely geometry or a local path to a GeoJSON file to the ``spatial_extent`` argument. + Argument ``spatial_extent``: add support for passing a Shapely geometry or a local path to a GeoJSON file. """ return DataCube.load_collection( collection_id=collection_id, @@ -1562,7 +1562,7 @@ def load_geojson( return VectorCube.load_geojson(connection=self, data=data, properties=properties) @openeo_process - def load_url(self, url: str, format: str, options: Optional[dict] = None): + def load_url(self, url: str, format: str, options: Optional[dict] = None) -> VectorCube: """ Loads a file from a URL diff --git a/openeo/rest/datacube.py b/openeo/rest/datacube.py index 1de00ac3a..166892b29 100644 --- a/openeo/rest/datacube.py +++ b/openeo/rest/datacube.py @@ -143,7 +143,7 @@ def load_collection( cls, collection_id: Union[str, Parameter], connection: Optional[Connection] = None, - spatial_extent: Union[Dict[str, float], Parameter, shapely.geometry.base.BaseGeometry, None] = None, + spatial_extent: Union[dict, Parameter, shapely.geometry.base.BaseGeometry, str, pathlib.Path, None] = None, temporal_extent: Union[Sequence[InputDate], Parameter, str, None] = None, bands: Union[Iterable[str], Parameter, str, None] = None, fetch_metadata: bool = True, @@ -161,8 +161,8 @@ def load_collection( :param spatial_extent: limit data to specified bounding box or polygons. Can be provided in different ways: - a bounding box dictionary - a Shapely geometry object - - a GeoJSON-style dictionary, - - a path (:py:class:`str` or :py:class:`~pathlib.Path`) to a local, client-side GeoJSON file, + - a GeoJSON-style dictionary + - a path (as :py:class:`str` or :py:class:`~pathlib.Path`) to a local, client-side GeoJSON file, which will be loaded automatically to get the geometries as GeoJSON construct. - a :py:class:`~openeo.api.process.Parameter` instance. :param temporal_extent: limit data to specified temporal interval. @@ -185,27 +185,20 @@ def load_collection( Add :py:func:`~openeo.rest.graph_building.collection_property` support to ``properties`` argument. .. versionchanged:: 0.37.0 - Add support for passing a Shapely geometry or a local path to a GeoJSON file to the ``spatial_extent`` argument. + Argument ``spatial_extent``: add support for passing a Shapely geometry or a local path to a GeoJSON file. """ if temporal_extent: temporal_extent = cls._get_temporal_extent(extent=temporal_extent) - - if isinstance(spatial_extent, Parameter): - if not schema_supports(spatial_extent.schema, type="object"): - warnings.warn( - "Unexpected parameterized `spatial_extent` in `load_collection`:" - f" expected schema compatible with type 'object' but got {spatial_extent.schema!r}." - ) - elif spatial_extent is None or ( - isinstance(spatial_extent, dict) and spatial_extent.keys() & {"west", "east", "north", "south"} - ): - pass - else: - valid_geojson_types = [ - "Polygon", "MultiPolygon", "Feature", "FeatureCollection" - ] - spatial_extent = _get_geometry_argument(argument=spatial_extent, valid_geojson_types=valid_geojson_types, - connection=connection) + spatial_extent = _get_geometry_argument( + argument=spatial_extent, + valid_geojson_types=["Polygon", "MultiPolygon", "Feature", "FeatureCollection"], + connection=connection, + allow_none=True, + allow_parameter=True, + allow_bounding_box=True, + argument_name="spatial_extent", + process_id="load_collection", + ) arguments = { 'id': collection_id, @@ -392,9 +385,18 @@ def load_stac( """ arguments = {"url": url} - # TODO #425 more normalization/validation of extent/band parameters if spatial_extent: - arguments["spatial_extent"] = spatial_extent + arguments["spatial_extent"] = _get_geometry_argument( + argument=spatial_extent, + valid_geojson_types=["Polygon", "MultiPolygon", "Feature", "FeatureCollection"], + connection=connection, + allow_none=True, + allow_parameter=True, + allow_bounding_box=True, + argument_name="spatial_extent", + process_id="load_stac", + ) + if temporal_extent: arguments["temporal_extent"] = DataCube._get_temporal_extent(extent=temporal_extent) bands = cls._get_bands(bands, process_id="load_stac") @@ -2892,23 +2894,47 @@ def _get_geometry_argument( Parameter, _FromNodeMixin, ], + *, valid_geojson_types: List[str], connection: Connection = None, crs: Optional[str] = None, -) -> Union[dict, Parameter, PGNode]: + allow_parameter: bool = True, + allow_bounding_box: bool = False, + allow_none: bool = False, + argument_name: str = "n/a", + process_id: str = "n/a", +) -> Union[dict, Parameter, PGNode, _FromNodeMixin, None]: """ - Convert input to a geometry as "geojson" subtype object or vectorcube. + Convert input to a geometry as "geojson" subtype object or vector cube. :param crs: value that encodes a coordinate reference system. See :py:func:`openeo.util.normalize_crs` for more details about additional normalization that is applied to this argument. + :param allow_parameter: allow argument to be a :py:class:`Parameter` instance, and pass-through as such + :param allow_none: allow argument to be ``None`` and pass-through as such + :param allow_bounding_box: allow argument to be a bounding box dictionary and pass-through as such """ - if isinstance(argument, Parameter): + # Some quick exit shortcuts + if allow_parameter and isinstance(argument, Parameter): + if not schema_supports(argument.schema, type="object"): + warnings.warn( + f"Unexpected parameterized `{argument_name}` in `{process_id}`:" + f" expected schema compatible with type 'object' but got {argument.schema!r}." + ) return argument elif isinstance(argument, _FromNodeMixin): + # Typical use case here: VectorCube instance return argument.from_node() + elif allow_none and argument is None: + return argument + elif ( + allow_bounding_box + and isinstance(argument, dict) + and all(k in argument for k in ["west", "south", "east", "north"]) + ): + return argument + # Support URL based geometry references (with `load_url` and best-effort format guess) if isinstance(argument, str) and re.match(r"^https?://", argument, flags=re.I): - # Geometry provided as URL: load with `load_url` (with best-effort format guess) url = urllib.parse.urlparse(argument) suffix = pathlib.Path(url.path.lower()).suffix format = { @@ -2919,7 +2945,8 @@ def _get_geometry_argument( ".geoparquet": "Parquet", }.get(suffix, suffix.split(".")[-1]) return connection.load_url(url=argument, format=format) - # + + # Support loading GeoJSON from local files if ( isinstance(argument, (str, pathlib.Path)) and pathlib.Path(argument).is_file() @@ -2933,6 +2960,8 @@ def _get_geometry_argument( else: raise OpenEoClientException(f"Invalid geometry argument: {argument!r}") + # The assumption at this point is that we are working with a GeoJSON style dictionary + assert isinstance(geometry, dict) if geometry.get("type") not in valid_geojson_types: raise OpenEoClientException("Invalid geometry type {t!r}, must be one of {s}".format( t=geometry.get("type"), s=valid_geojson_types diff --git a/tests/rest/datacube/test_datacube.py b/tests/rest/datacube/test_datacube.py index 48a05444f..67faf2c3f 100644 --- a/tests/rest/datacube/test_datacube.py +++ b/tests/rest/datacube/test_datacube.py @@ -138,7 +138,7 @@ def test_load_collection_connectionless_temporal_extent_shortcut(self): } def test_load_collection_connectionless_shapely_spatial_extent(self): - polygon = shapely.Polygon(((0.0,1.0),(2.0,1.0),(3.0,2.0),(1.5,0.0),(0.0,1.0))) + polygon = shapely.geometry.Polygon(((0.0, 1.0), (2.0, 1.0), (3.0, 2.0), (1.5, 0.0), (0.0, 1.0))) cube = DataCube.load_collection("T3", spatial_extent=polygon) assert cube.flat_graph() == { "loadcollection1": { From adf4f4cb64d912428a30be6eda1b698a21523db2 Mon Sep 17 00:00:00 2001 From: Stefaan Lippens Date: Thu, 16 Jan 2025 16:40:59 +0100 Subject: [PATCH 10/15] fixup! Issue #678/#682 further finetuning --- openeo/api/process.py | 3 +++ tests/api/test_process.py | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/openeo/api/process.py b/openeo/api/process.py index 1e2d840ae..0947a47d4 100644 --- a/openeo/api/process.py +++ b/openeo/api/process.py @@ -467,6 +467,9 @@ def schema_supports(schema: Union[dict, List[dict]], type: str, subtype: Optiona elif isinstance(actual_type, list): if type not in actual_type: return False + elif actual_type is None: + # Without explicit "type", anything is accepted + return True else: raise ValueError(actual_type) if subtype: diff --git a/tests/api/test_process.py b/tests/api/test_process.py index b39b4555c..ddd02899a 100644 --- a/tests/api/test_process.py +++ b/tests/api/test_process.py @@ -291,3 +291,11 @@ def test_schema_supports_list(): assert schema_supports(schema, type="object") is True assert schema_supports(schema, type="object", subtype="datacube") is True assert schema_supports(schema, type="object", subtype="geojson") is False + + +def test_default_parameter_supports_anything(): + parameter = Parameter(name="foo") + assert schema_supports(parameter.schema, type="string") is True + assert schema_supports(parameter.schema, type="number") is True + assert schema_supports(parameter.schema, type="object") is True + assert schema_supports(parameter.schema, type="object", subtype="datacube") is True From a6f1cf9fda4fb1dbb20b43a0af8da60768092410 Mon Sep 17 00:00:00 2001 From: Stefaan Lippens Date: Thu, 16 Jan 2025 16:54:10 +0100 Subject: [PATCH 11/15] fixup! fixup! Issue #678/#682 further finetuning --- openeo/rest/connection.py | 7 +++++-- openeo/rest/datacube.py | 2 ++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/openeo/rest/connection.py b/openeo/rest/connection.py index 49067f61b..6d087c27b 100644 --- a/openeo/rest/connection.py +++ b/openeo/rest/connection.py @@ -1256,7 +1256,7 @@ def datacube_from_json(self, src: Union[str, Path], parameters: Optional[dict] = def load_collection( self, collection_id: Union[str, Parameter], - spatial_extent: Union[dict, Parameter, shapely.geometry.base.BaseGeometry, None] = None, + spatial_extent: Union[dict, Parameter, shapely.geometry.base.BaseGeometry, str, Path, None] = None, temporal_extent: Union[Sequence[InputDate], Parameter, str, None] = None, bands: Union[Iterable[str], Parameter, str, None] = None, properties: Union[ @@ -1355,7 +1355,7 @@ def load_result( def load_stac( self, url: str, - spatial_extent: Union[Dict[str, float], Parameter, None] = None, + spatial_extent: Union[dict, Parameter, shapely.geometry.base.BaseGeometry, str, Path, None] = None, temporal_extent: Union[Sequence[InputDate], Parameter, str, None] = None, bands: Union[Iterable[str], Parameter, str, None] = None, properties: Optional[Dict[str, Union[str, PGNode, Callable]]] = None, @@ -1457,6 +1457,9 @@ def load_stac( .. versionchanged:: 0.23.0 Argument ``temporal_extent``: add support for year/month shorthand notation as discussed at :ref:`date-shorthand-handling`. + + .. versionchanged:: 0.37.0 + Argument ``spatial_extent``: add support for passing a Shapely geometry or a local path to a GeoJSON file. """ return DataCube.load_stac( url=url, diff --git a/openeo/rest/datacube.py b/openeo/rest/datacube.py index 166892b29..7d4d73333 100644 --- a/openeo/rest/datacube.py +++ b/openeo/rest/datacube.py @@ -383,6 +383,8 @@ def load_stac( .. versionadded:: 0.33.0 + .. versionchanged:: 0.37.0 + Argument ``spatial_extent``: add support for passing a Shapely geometry or a local path to a GeoJSON file. """ arguments = {"url": url} if spatial_extent: From 45f021584b50752081ac5dbc9661608a46c1c432 Mon Sep 17 00:00:00 2001 From: Stefaan Lippens Date: Fri, 17 Jan 2025 10:25:34 +0100 Subject: [PATCH 12/15] Issue #678/#682 more test coverage of `spatial_extent` handling --- openeo/rest/_testing.py | 2 +- openeo/rest/connection.py | 1 + openeo/rest/datacube.py | 9 +- tests/rest/test_connection.py | 205 +++++++++++++++++++++++++++++++--- tests/rest/test_udp.py | 2 +- 5 files changed, 197 insertions(+), 22 deletions(-) diff --git a/openeo/rest/_testing.py b/openeo/rest/_testing.py index 7940210d6..63e9460c5 100644 --- a/openeo/rest/_testing.py +++ b/openeo/rest/_testing.py @@ -153,7 +153,7 @@ def setup_collection( json={ "id": collection_id, # define temporal and band dim - "cube:dimensions": {"t": {"type": "temporal"}, "bands": {"type": "bands"}}, + "cube:dimensions": cube_dimensions, }, ) return self diff --git a/openeo/rest/connection.py b/openeo/rest/connection.py index 6d087c27b..f687d5ca1 100644 --- a/openeo/rest/connection.py +++ b/openeo/rest/connection.py @@ -1275,6 +1275,7 @@ def load_collection( - a GeoJSON-style dictionary - a path (as :py:class:`str` or :py:class:`~pathlib.Path`) to a local, client-side GeoJSON file, which will be loaded automatically to get the geometries as GeoJSON construct. + - a URL to a publicly accessible GeoJSON document - a :py:class:`~openeo.api.process.Parameter` instance. :param temporal_extent: limit data to specified temporal interval. Typically, just a two-item list or tuple containing start and end date. diff --git a/openeo/rest/datacube.py b/openeo/rest/datacube.py index 7d4d73333..ec3df056b 100644 --- a/openeo/rest/datacube.py +++ b/openeo/rest/datacube.py @@ -164,6 +164,7 @@ def load_collection( - a GeoJSON-style dictionary - a path (as :py:class:`str` or :py:class:`~pathlib.Path`) to a local, client-side GeoJSON file, which will be loaded automatically to get the geometries as GeoJSON construct. + - a URL to a publicly accessible GeoJSON document - a :py:class:`~openeo.api.process.Parameter` instance. :param temporal_extent: limit data to specified temporal interval. Typically, just a two-item list or tuple containing start and end date. @@ -2907,7 +2908,9 @@ def _get_geometry_argument( process_id: str = "n/a", ) -> Union[dict, Parameter, PGNode, _FromNodeMixin, None]: """ - Convert input to a geometry as "geojson" subtype object or vector cube. + Normalize a user input to a openEO-compatible geometry representation, + like a GeoJSON construct, vector cube reference, bounding box construct, + a parameter reference, ... :param crs: value that encodes a coordinate reference system. See :py:func:`openeo.util.normalize_crs` for more details about additional normalization that is applied to this argument. @@ -2919,8 +2922,8 @@ def _get_geometry_argument( if allow_parameter and isinstance(argument, Parameter): if not schema_supports(argument.schema, type="object"): warnings.warn( - f"Unexpected parameterized `{argument_name}` in `{process_id}`:" - f" expected schema compatible with type 'object' but got {argument.schema!r}." + f"Schema mismatch with parameter given to `{argument_name}` in `{process_id}`:" + f" expected a schema compatible with type 'object' but got {argument.schema!r}." ) return argument elif isinstance(argument, _FromNodeMixin): diff --git a/tests/rest/test_connection.py b/tests/rest/test_connection.py index 20c36987a..ceec1a78e 100644 --- a/tests/rest/test_connection.py +++ b/tests/rest/test_connection.py @@ -2396,25 +2396,196 @@ def test_authenticate_oidc_access_token_wrong_provider(self): connection.authenticate_oidc_access_token(access_token="Th3Tok3n!@#", provider_id="nope") -def test_load_collection_arguments_100(requests_mock): - requests_mock.get(API_URL, json={"api_version": "1.0.0"}) - conn = Connection(API_URL) - requests_mock.get(API_URL + "collections/FOO", json={ - "summaries": {"eo:bands": [{"name": "red"}, {"name": "green"}, {"name": "blue"}]} - }) - spatial_extent = {"west": 1, "south": 2, "east": 3, "north": 4} - temporal_extent = ["2019-01-01", "2019-01-22"] - im = conn.load_collection( - "FOO", spatial_extent=spatial_extent, temporal_extent=temporal_extent, bands=["red", "green"] - ) - assert im._pg.process_id == "load_collection" - assert im._pg.arguments == { - "id": "FOO", - "spatial_extent": spatial_extent, - "temporal_extent": temporal_extent, - "bands": ["red", "green"] +class TestLoadCollection: + def test_load_collection_arguments_basic(self, dummy_backend): + spatial_extent = {"west": 1, "south": 2, "east": 3, "north": 4} + temporal_extent = ["2019-01-01", "2019-01-22"] + cube = dummy_backend.connection.load_collection( + "S2", spatial_extent=spatial_extent, temporal_extent=temporal_extent, bands=["B2", "B3"] + ) + cube.execute() + assert dummy_backend.get_sync_pg() == { + "loadcollection1": { + "process_id": "load_collection", + "arguments": { + "id": "S2", + "spatial_extent": {"east": 3, "north": 4, "south": 2, "west": 1}, + "temporal_extent": ["2019-01-01", "2019-01-22"], + "bands": ["B2", "B3"], + }, + "result": True, + } + } + + def test_load_collection_spatial_extent_bbox(self, dummy_backend): + spatial_extent = {"west": 1, "south": 2, "east": 3, "north": 4} + cube = dummy_backend.connection.load_collection("S2", spatial_extent=spatial_extent) + cube.execute() + assert dummy_backend.get_sync_pg()["loadcollection1"]["arguments"] == { + "id": "S2", + "spatial_extent": {"east": 3, "north": 4, "south": 2, "west": 1}, + "temporal_extent": None, + } + + # TODO: make this more reusable + GEOJSON_POINT_01 = {"type": "Point", "coordinates": [3, 52]} + GEOJSON_LINESTRING_01 = {"type": "LineString", "coordinates": [[3, 50], [4, 51], [5, 53]]} + GEOJSON_POLYGON_01 = { + "type": "Polygon", + "coordinates": [[[3, 51], [4, 51], [4, 52], [3, 52], [3, 51]]], + } + GEOJSON_MULTIPOLYGON_01 = { + "type": "MultiPolygon", + "coordinates": [[[[3, 51], [4, 51], [4, 52], [3, 52], [3, 51]]]], + } + GEOJSON_FEATURE_01 = { + "type": "Feature", + "properties": {}, + "geometry": GEOJSON_POLYGON_01, + } + GEOJSON_FEATURE_02 = { + "type": "Feature", + "properties": {}, + "geometry": GEOJSON_MULTIPOLYGON_01, + } + GEOJSON_FEATURECOLLECTION_01 = { + "type": "FeatureCollection", + "features": [ + GEOJSON_FEATURE_01, + GEOJSON_FEATURE_02, + ], + } + GEOJSON_GEOMETRYCOLLECTION_01 = { + "type": "GeometryCollection", + "geometries": [ + GEOJSON_POINT_01, + GEOJSON_POLYGON_01, + ], } + @pytest.mark.parametrize( + "spatial_extent", + [ + GEOJSON_POLYGON_01, + GEOJSON_MULTIPOLYGON_01, + GEOJSON_FEATURE_01, + GEOJSON_FEATURECOLLECTION_01, + ], + ) + def test_load_collection_spatial_extent_geojson(self, dummy_backend, spatial_extent): + cube = dummy_backend.connection.load_collection("S2", spatial_extent=spatial_extent) + cube.execute() + assert dummy_backend.get_sync_pg()["loadcollection1"]["arguments"] == { + "id": "S2", + "spatial_extent": spatial_extent, + "temporal_extent": None, + } + + @pytest.mark.parametrize( + "spatial_extent", + [GEOJSON_POINT_01, GEOJSON_LINESTRING_01, GEOJSON_GEOMETRYCOLLECTION_01], + ) + def test_load_collection_spatial_extent_geojson_wrong_type(self, con120, spatial_extent): + with pytest.raises(OpenEoClientException, match="Invalid geometry type"): + _ = con120.load_collection("S2", spatial_extent=spatial_extent) + + @pytest.mark.parametrize( + "geojson", + [ + GEOJSON_POLYGON_01, + GEOJSON_MULTIPOLYGON_01, + ], + ) + def test_load_collection_spatial_extent_shapely(self, geojson, dummy_backend): + spatial_extent = shapely.geometry.shape(geojson) + cube = dummy_backend.connection.load_collection("S2", spatial_extent=spatial_extent) + cube.execute() + assert dummy_backend.get_sync_pg()["loadcollection1"]["arguments"] == { + "id": "S2", + "spatial_extent": geojson, + "temporal_extent": None, + } + + @pytest.mark.parametrize( + "geojson", + [ + GEOJSON_POINT_01, + GEOJSON_GEOMETRYCOLLECTION_01, + ], + ) + def test_load_collection_spatial_extent_shapely_wrong_type(self, geojson, dummy_backend): + spatial_extent = shapely.geometry.shape(geojson) + with pytest.raises(OpenEoClientException, match="Invalid geometry type"): + _ = dummy_backend.connection.load_collection("S2", spatial_extent=spatial_extent) + + @pytest.mark.parametrize( + "geojson", + [ + GEOJSON_MULTIPOLYGON_01, + GEOJSON_FEATURECOLLECTION_01, + ], + ) + @pytest.mark.parametrize("path_factory", [str, Path]) + def test_load_collection_spatial_extent_path(self, geojson, dummy_backend, tmp_path, path_factory): + path = tmp_path / "geometry.json" + with path.open("w") as f: + json.dump(geojson, f) + cube = dummy_backend.connection.load_collection("S2", spatial_extent=path_factory(path)) + cube.execute() + assert dummy_backend.get_sync_pg()["loadcollection1"]["arguments"] == { + "id": "S2", + "spatial_extent": geojson, + "temporal_extent": None, + } + + def test_load_collection_spatial_extent_url(self, dummy_backend): + cube = dummy_backend.connection.load_collection("S2", spatial_extent="https://geo.test/geometry.json") + cube.execute() + assert dummy_backend.get_sync_pg() == { + "loadurl1": { + "process_id": "load_url", + "arguments": {"url": "https://geo.test/geometry.json", "format": "GeoJSON"}, + }, + "loadcollection1": { + "process_id": "load_collection", + "arguments": {"id": "S2", "spatial_extent": {"from_node": "loadurl1"}, "temporal_extent": None}, + "result": True, + }, + } + + @pytest.mark.parametrize( + "parameter", + [ + Parameter("zpatial_extent"), + Parameter.spatial_extent("zpatial_extent"), + Parameter.geojson("zpatial_extent"), + ], + ) + def test_load_collection_spatial_extent_parameter(self, dummy_backend, parameter, recwarn): + cube = dummy_backend.connection.load_collection("S2", spatial_extent=parameter) + assert len(recwarn) == 0 + + cube.execute() + assert dummy_backend.get_sync_pg()["loadcollection1"]["arguments"] == { + "id": "S2", + "spatial_extent": {"from_parameter": "zpatial_extent"}, + "temporal_extent": None, + } + + def test_load_collection_spatial_extent_parameter_schema_mismatch(self, dummy_backend, recwarn): + cube = dummy_backend.connection.load_collection( + "S2", spatial_extent=Parameter.number("zpatial_extent", description="foo") + ) + assert [str(w.message) for w in recwarn] == [ + "Schema mismatch with parameter given to `spatial_extent` in `load_collection`: expected a schema compatible with type 'object' but got {'type': 'number'}." + ] + + cube.execute() + assert dummy_backend.get_sync_pg()["loadcollection1"]["arguments"] == { + "id": "S2", + "spatial_extent": {"from_parameter": "zpatial_extent"}, + "temporal_extent": None, + } def test_load_result(requests_mock): requests_mock.get(API_URL, json={"api_version": "1.0.0"}) diff --git a/tests/rest/test_udp.py b/tests/rest/test_udp.py index 8ac23236d..cd203a5e7 100644 --- a/tests/rest/test_udp.py +++ b/tests/rest/test_udp.py @@ -411,7 +411,7 @@ def test_build_parameterized_cube_load_collection_invalid_bbox_schema(con100): bbox = Parameter.string("bbox", description="Spatial extent") with pytest.warns( UserWarning, - match="Unexpected parameterized `spatial_extent` in `load_collection`: expected schema compatible with type 'object' but got {'type': 'string'}.", + match="Schema mismatch with parameter given to `spatial_extent` in `load_collection`: expected a schema compatible with type 'object' but got {'type': 'string'}.", ): cube = con100.load_collection(layer, spatial_extent=bbox, temporal_extent=dates) From 8a3ba030092bd14b281549306215360655a18711 Mon Sep 17 00:00:00 2001 From: Stefaan Lippens Date: Fri, 17 Jan 2025 11:26:08 +0100 Subject: [PATCH 13/15] fixup! Issue #678/#682 more test coverage of `spatial_extent` handling --- tests/rest/test_connection.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/tests/rest/test_connection.py b/tests/rest/test_connection.py index ceec1a78e..767b6bfe1 100644 --- a/tests/rest/test_connection.py +++ b/tests/rest/test_connection.py @@ -2587,6 +2587,29 @@ def test_load_collection_spatial_extent_parameter_schema_mismatch(self, dummy_ba "temporal_extent": None, } + def test_load_collection_spatial_extent_vector_cube(self, dummy_backend): + vector_cube = VectorCube.load_url( + connection=dummy_backend.connection, url="https://geo.test/geometry.json", format="GeoJSON" + ) + cube = dummy_backend.connection.load_collection("S2", spatial_extent=vector_cube) + cube.execute() + assert dummy_backend.get_sync_pg() == { + "loadurl1": { + "process_id": "load_url", + "arguments": {"format": "GeoJSON", "url": "https://geo.test/geometry.json"}, + }, + "loadcollection1": { + "process_id": "load_collection", + "arguments": { + "id": "S2", + "spatial_extent": {"from_node": "loadurl1"}, + "temporal_extent": None, + }, + "result": True, + }, + } + + def test_load_result(requests_mock): requests_mock.get(API_URL, json={"api_version": "1.0.0"}) con = Connection(API_URL) From 0b486a42e7ca573d00b59ebbce51d17609d15b79 Mon Sep 17 00:00:00 2001 From: Stefaan Lippens Date: Fri, 17 Jan 2025 12:06:06 +0100 Subject: [PATCH 14/15] Issue #678/#682 more tests for load_stac spatial_extent handling --- tests/rest/test_connection.py | 257 ++++++++++++++++++++++++++++------ 1 file changed, 217 insertions(+), 40 deletions(-) diff --git a/tests/rest/test_connection.py b/tests/rest/test_connection.py index 767b6bfe1..408d79bd0 100644 --- a/tests/rest/test_connection.py +++ b/tests/rest/test_connection.py @@ -51,7 +51,44 @@ BASIC_ENDPOINTS = [{"path": "/credentials/basic", "methods": ["GET"]}] + +GEOJSON_POINT_01 = {"type": "Point", "coordinates": [3, 52]} +GEOJSON_LINESTRING_01 = {"type": "LineString", "coordinates": [[3, 50], [4, 51], [5, 53]]} +GEOJSON_POLYGON_01 = { + "type": "Polygon", + "coordinates": [[[3, 51], [4, 51], [4, 52], [3, 52], [3, 51]]], +} +GEOJSON_MULTIPOLYGON_01 = { + "type": "MultiPolygon", + "coordinates": [[[[3, 51], [4, 51], [4, 52], [3, 52], [3, 51]]]], +} +GEOJSON_FEATURE_01 = { + "type": "Feature", + "properties": {}, + "geometry": GEOJSON_POLYGON_01, +} +GEOJSON_FEATURE_02 = { + "type": "Feature", + "properties": {}, + "geometry": GEOJSON_MULTIPOLYGON_01, +} +GEOJSON_FEATURECOLLECTION_01 = { + "type": "FeatureCollection", + "features": [ + GEOJSON_FEATURE_01, + GEOJSON_FEATURE_02, + ], +} +GEOJSON_GEOMETRYCOLLECTION_01 = { + "type": "GeometryCollection", + "geometries": [ + GEOJSON_POINT_01, + GEOJSON_POLYGON_01, + ], +} + # Trick to avoid linting/auto-formatting tools to complain about or fix unused imports of these pytest fixtures +# TODO: use proper way to reuse fixtures instead of this hack auth_config = auth_config refresh_token_store = refresh_token_store @@ -2427,42 +2464,6 @@ def test_load_collection_spatial_extent_bbox(self, dummy_backend): "temporal_extent": None, } - # TODO: make this more reusable - GEOJSON_POINT_01 = {"type": "Point", "coordinates": [3, 52]} - GEOJSON_LINESTRING_01 = {"type": "LineString", "coordinates": [[3, 50], [4, 51], [5, 53]]} - GEOJSON_POLYGON_01 = { - "type": "Polygon", - "coordinates": [[[3, 51], [4, 51], [4, 52], [3, 52], [3, 51]]], - } - GEOJSON_MULTIPOLYGON_01 = { - "type": "MultiPolygon", - "coordinates": [[[[3, 51], [4, 51], [4, 52], [3, 52], [3, 51]]]], - } - GEOJSON_FEATURE_01 = { - "type": "Feature", - "properties": {}, - "geometry": GEOJSON_POLYGON_01, - } - GEOJSON_FEATURE_02 = { - "type": "Feature", - "properties": {}, - "geometry": GEOJSON_MULTIPOLYGON_01, - } - GEOJSON_FEATURECOLLECTION_01 = { - "type": "FeatureCollection", - "features": [ - GEOJSON_FEATURE_01, - GEOJSON_FEATURE_02, - ], - } - GEOJSON_GEOMETRYCOLLECTION_01 = { - "type": "GeometryCollection", - "geometries": [ - GEOJSON_POINT_01, - GEOJSON_POLYGON_01, - ], - } - @pytest.mark.parametrize( "spatial_extent", [ @@ -2485,9 +2486,9 @@ def test_load_collection_spatial_extent_geojson(self, dummy_backend, spatial_ext "spatial_extent", [GEOJSON_POINT_01, GEOJSON_LINESTRING_01, GEOJSON_GEOMETRYCOLLECTION_01], ) - def test_load_collection_spatial_extent_geojson_wrong_type(self, con120, spatial_extent): + def test_load_collection_spatial_extent_geojson_wrong_type(self, dummy_backend, spatial_extent): with pytest.raises(OpenEoClientException, match="Invalid geometry type"): - _ = con120.load_collection("S2", spatial_extent=spatial_extent) + _ = dummy_backend.connection.load_collection("S2", spatial_extent=spatial_extent) @pytest.mark.parametrize( "geojson", @@ -2544,11 +2545,18 @@ def test_load_collection_spatial_extent_url(self, dummy_backend): assert dummy_backend.get_sync_pg() == { "loadurl1": { "process_id": "load_url", - "arguments": {"url": "https://geo.test/geometry.json", "format": "GeoJSON"}, + "arguments": { + "url": "https://geo.test/geometry.json", + "format": "GeoJSON", + }, }, "loadcollection1": { "process_id": "load_collection", - "arguments": {"id": "S2", "spatial_extent": {"from_node": "loadurl1"}, "temporal_extent": None}, + "arguments": { + "id": "S2", + "spatial_extent": {"from_node": "loadurl1"}, + "temporal_extent": None, + }, "result": True, }, } @@ -2921,6 +2929,175 @@ def test_bands_parameterized(self, con120): } } + def test_load_stac_spatial_extent_bbox(self, dummy_backend): + spatial_extent = {"west": 1, "south": 2, "east": 3, "north": 4} + # TODO #694 how to avoid request to dummy STAC URL (without mocking, which is overkill for this test) + cube = dummy_backend.connection.load_stac("https://stac.test/data", spatial_extent=spatial_extent) + cube.execute() + assert dummy_backend.get_sync_pg()["loadstac1"]["arguments"] == { + "spatial_extent": {"east": 3, "north": 4, "south": 2, "west": 1}, + "url": "https://stac.test/data", + } + + @pytest.mark.parametrize( + "spatial_extent", + [ + GEOJSON_POLYGON_01, + GEOJSON_MULTIPOLYGON_01, + GEOJSON_FEATURE_01, + GEOJSON_FEATURECOLLECTION_01, + ], + ) + def test_load_stac_spatial_extent_geojson(self, dummy_backend, spatial_extent): + # TODO #694 how to avoid request to dummy STAC URL (without mocking, which is overkill for this test) + cube = dummy_backend.connection.load_stac("https://stac.test/data", spatial_extent=spatial_extent) + cube.execute() + assert dummy_backend.get_sync_pg()["loadstac1"]["arguments"] == { + "spatial_extent": spatial_extent, + "url": "https://stac.test/data", + } + + @pytest.mark.parametrize( + "spatial_extent", + [ + GEOJSON_POINT_01, + GEOJSON_LINESTRING_01, + GEOJSON_GEOMETRYCOLLECTION_01, + ], + ) + def test_load_stac_spatial_extent_geojson_wrong_type(self, dummy_backend, spatial_extent): + # TODO #694 how to avoid request to dummy STAC URL (without mocking, which is overkill for this test) + with pytest.raises(OpenEoClientException, match="Invalid geometry type"): + _ = dummy_backend.connection.load_stac("https://stac.test/data", spatial_extent=spatial_extent) + + @pytest.mark.parametrize( + "geojson", + [ + GEOJSON_POLYGON_01, + GEOJSON_MULTIPOLYGON_01, + ], + ) + def test_load_stac_spatial_extent_shapely(self, dummy_backend, geojson): + spatial_extent = shapely.geometry.shape(geojson) + # TODO #694 how to avoid request to dummy STAC URL (without mocking, which is overkill for this test) + cube = dummy_backend.connection.load_stac("https://stac.test/data", spatial_extent=spatial_extent) + cube.execute() + assert dummy_backend.get_sync_pg()["loadstac1"]["arguments"] == { + "url": "https://stac.test/data", + "spatial_extent": geojson, + } + + @pytest.mark.parametrize( + "geojson", + [ + GEOJSON_POINT_01, + GEOJSON_GEOMETRYCOLLECTION_01, + ], + ) + def test_load_stac_spatial_extent_shapely_wront_type(self, dummy_backend, geojson): + spatial_extent = shapely.geometry.shape(geojson) + # TODO #694 how to avoid request to dummy STAC URL (without mocking, which is overkill for this test) + with pytest.raises(OpenEoClientException, match="Invalid geometry type"): + _ = dummy_backend.connection.load_stac("https://stac.test/data", spatial_extent=spatial_extent) + + @pytest.mark.parametrize( + "geojson", + [ + GEOJSON_MULTIPOLYGON_01, + GEOJSON_FEATURECOLLECTION_01, + ], + ) + @pytest.mark.parametrize("path_factory", [str, Path]) + def test_load_stac_spatial_extent_path(self, geojson, dummy_backend, tmp_path, path_factory): + path = tmp_path / "geometry.json" + with path.open("w") as f: + json.dump(geojson, f) + + # TODO #694 how to avoid request to dummy STAC URL (without mocking, which is overkill for this test) + cube = dummy_backend.connection.load_stac("https://stac.test/data", spatial_extent=path_factory(path)) + cube.execute() + assert dummy_backend.get_sync_pg()["loadstac1"]["arguments"] == { + "url": "https://stac.test/data", + "spatial_extent": geojson, + } + + def test_load_stac_spatial_extent_url(self, dummy_backend): + # TODO #694 how to avoid request to dummy STAC URL (without mocking, which is overkill for this test) + cube = dummy_backend.connection.load_stac( + "https://stac.test/data", spatial_extent="https://geo.test/geometry.json" + ) + cube.execute() + assert dummy_backend.get_sync_pg() == { + "loadurl1": { + "process_id": "load_url", + "arguments": { + "url": "https://geo.test/geometry.json", + "format": "GeoJSON", + }, + }, + "loadstac1": { + "process_id": "load_stac", + "arguments": { + "url": "https://stac.test/data", + "spatial_extent": {"from_node": "loadurl1"}, + }, + "result": True, + }, + } + + @pytest.mark.parametrize( + "parameter", + [ + Parameter("zpatial_extent"), + Parameter.spatial_extent("zpatial_extent"), + Parameter.geojson("zpatial_extent"), + ], + ) + def test_load_stac_spatial_extent_parameter(self, dummy_backend, parameter, recwarn): + cube = dummy_backend.connection.load_stac("https://stac.test/data", spatial_extent=parameter) + assert len(recwarn) == 0 + + cube.execute() + assert dummy_backend.get_sync_pg()["loadstac1"]["arguments"] == { + "url": "https://stac.test/data", + "spatial_extent": {"from_parameter": "zpatial_extent"}, + } + + def test_load_stac_spatial_extent_parameter_schema_mismatch(self, dummy_backend, recwarn): + cube = dummy_backend.connection.load_stac( + "https://stac.test/data", spatial_extent=Parameter.number("zpatial_extent", description="foo") + ) + assert [str(w.message) for w in recwarn] == [ + "Schema mismatch with parameter given to `spatial_extent` in `load_stac`: expected a schema compatible with type 'object' but got {'type': 'number'}." + ] + + cube.execute() + assert dummy_backend.get_sync_pg()["loadstac1"]["arguments"] == { + "url": "https://stac.test/data", + "spatial_extent": {"from_parameter": "zpatial_extent"}, + } + + def test_load_stac_spatial_extent_vector_cube(self, dummy_backend): + vector_cube = VectorCube.load_url( + connection=dummy_backend.connection, url="https://geo.test/geometry.json", format="GeoJSON" + ) + cube = dummy_backend.connection.load_stac("https://stac.test/data", spatial_extent=vector_cube) + cube.execute() + assert dummy_backend.get_sync_pg() == { + "loadurl1": { + "process_id": "load_url", + "arguments": {"format": "GeoJSON", "url": "https://geo.test/geometry.json"}, + }, + "loadstac1": { + "process_id": "load_stac", + "arguments": { + "url": "https://stac.test/data", + "spatial_extent": {"from_node": "loadurl1"}, + }, + "result": True, + }, + } + @pytest.mark.parametrize( "data", From 1cd1cd2425fc131dd24d06fd64976cbd32de9db6 Mon Sep 17 00:00:00 2001 From: Stefaan Lippens Date: Fri, 17 Jan 2025 12:36:01 +0100 Subject: [PATCH 15/15] Issue #678/#682 finetune TestDataCube tests --- openeo/rest/datacube.py | 2 +- tests/rest/datacube/test_datacube.py | 140 +++++++++++++++++++++++---- tests/rest/test_connection.py | 10 +- 3 files changed, 128 insertions(+), 24 deletions(-) diff --git a/openeo/rest/datacube.py b/openeo/rest/datacube.py index ec3df056b..4cbae0db1 100644 --- a/openeo/rest/datacube.py +++ b/openeo/rest/datacube.py @@ -282,7 +282,7 @@ def load_disk_collection(cls, connection: Connection, file_format: str, glob_pat def load_stac( cls, url: str, - spatial_extent: Union[Dict[str, float], Parameter, None] = None, + spatial_extent: Union[dict, Parameter, shapely.geometry.base.BaseGeometry, str, pathlib.Path, None] = None, temporal_extent: Union[Sequence[InputDate], Parameter, str, None] = None, bands: Union[Iterable[str], Parameter, str, None] = None, properties: Optional[Dict[str, Union[str, PGNode, Callable]]] = None, diff --git a/tests/rest/datacube/test_datacube.py b/tests/rest/datacube/test_datacube.py index 67faf2c3f..f353b4d42 100644 --- a/tests/rest/datacube/test_datacube.py +++ b/tests/rest/datacube/test_datacube.py @@ -137,31 +137,70 @@ def test_load_collection_connectionless_temporal_extent_shortcut(self): } } - def test_load_collection_connectionless_shapely_spatial_extent(self): - polygon = shapely.geometry.Polygon(((0.0, 1.0), (2.0, 1.0), (3.0, 2.0), (1.5, 0.0), (0.0, 1.0))) - cube = DataCube.load_collection("T3", spatial_extent=polygon) - assert cube.flat_graph() == { - "loadcollection1": { - "arguments": {"id": "T3", "spatial_extent": - {'coordinates': (((0.0,1.0),(2.0,1.0),(3.0,2.0),(1.5,0.0),(0.0,1.0)),),'type': 'Polygon'}, - "temporal_extent": None}, - "process_id": "load_collection", - "result": True, - } + def test_load_collection_spatial_extent_bbox(self, dummy_backend): + spatial_extent = {"west": 1, "south": 2, "east": 3, "north": 4} + cube = DataCube.load_collection("S2", spatial_extent=spatial_extent, connection=dummy_backend.connection) + cube.execute() + assert dummy_backend.get_sync_pg()["loadcollection1"]["arguments"] == { + "id": "S2", + "spatial_extent": {"west": 1, "south": 2, "east": 3, "north": 4}, + "temporal_extent": None, + } + + def test_load_collection_spatial_extent_shapely(self, dummy_backend): + polygon = shapely.geometry.Polygon([(3, 51), (4, 51), (4, 52), (3, 52)]) + cube = DataCube.load_collection("S2", spatial_extent=polygon, connection=dummy_backend.connection) + cube.execute() + assert dummy_backend.get_sync_pg()["loadcollection1"]["arguments"] == { + "id": "S2", + "spatial_extent": { + "type": "Polygon", + "coordinates": [[[3, 51], [4, 51], [4, 52], [3, 52], [3, 51]]], + }, + "temporal_extent": None, } @pytest.mark.parametrize("path_factory", [str, pathlib.Path]) - def test_load_collection_connectionless_local_path_spatial_extent(self, path_factory, test_data): + def test_load_collection_spatial_extent_local_path(self, dummy_backend, path_factory, test_data): path = path_factory(test_data.get_path("geojson/polygon02.json")) - cube = DataCube.load_collection("T3", spatial_extent=path) - assert cube.flat_graph() == { + cube = DataCube.load_collection("S2", spatial_extent=path, connection=dummy_backend.connection) + cube.execute() + assert dummy_backend.get_sync_pg()["loadcollection1"]["arguments"] == { + "id": "S2", + "spatial_extent": {"type": "Polygon", "coordinates": [[[3, 50], [4, 50], [4, 51], [3, 50]]]}, + "temporal_extent": None, + } + + def test_load_collection_spatial_extent_url(self, dummy_backend): + cube = DataCube.load_collection( + "S2", spatial_extent="https://geo.test/geometry.json", connection=dummy_backend.connection + ) + cube.execute() + assert dummy_backend.get_sync_pg() == { + "loadurl1": { + "process_id": "load_url", + "arguments": {"format": "GeoJSON", "url": "https://geo.test/geometry.json"}, + }, "loadcollection1": { - "arguments": {"id": "T3", "spatial_extent": - {"type": "Polygon", "coordinates": [[[3, 50], [4, 50], [4, 51], [3, 50]]]}, - "temporal_extent": None}, "process_id": "load_collection", + "arguments": { + "id": "S2", + "spatial_extent": {"from_node": "loadurl1"}, + "temporal_extent": None, + }, "result": True, - } + }, + } + + def test_load_collection_spatial_extent_parameter(self, dummy_backend): + cube = DataCube.load_collection( + "S2", spatial_extent=Parameter.geojson("zpatial_extent"), connection=dummy_backend.connection + ) + cube.execute() + assert dummy_backend.get_sync_pg()["loadcollection1"]["arguments"] == { + "id": "S2", + "spatial_extent": {"from_parameter": "zpatial_extent"}, + "temporal_extent": None, } def test_load_collection_connectionless_save_result(self): @@ -206,6 +245,71 @@ def test_load_stac_connectionless_save_result(self): }, } + def test_load_stac_spatial_extent_bbox(self, dummy_backend): + spatial_extent = {"west": 1, "south": 2, "east": 3, "north": 4} + cube = DataCube.load_stac( + "https://stac.test/data", spatial_extent=spatial_extent, connection=dummy_backend.connection + ) + cube.execute() + assert dummy_backend.get_sync_pg()["loadstac1"]["arguments"] == { + "url": "https://stac.test/data", + "spatial_extent": {"west": 1, "south": 2, "east": 3, "north": 4}, + } + + def test_load_stac_spatial_extent_shapely(self, dummy_backend): + polygon = shapely.geometry.Polygon([(3, 51), (4, 51), (4, 52), (3, 52)]) + cube = DataCube.load_stac("https://stac.test/data", spatial_extent=polygon, connection=dummy_backend.connection) + cube.execute() + assert dummy_backend.get_sync_pg()["loadstac1"]["arguments"] == { + "url": "https://stac.test/data", + "spatial_extent": { + "type": "Polygon", + "coordinates": [[[3, 51], [4, 51], [4, 52], [3, 52], [3, 51]]], + }, + } + + @pytest.mark.parametrize("path_factory", [str, pathlib.Path]) + def test_load_stac_spatial_extent_local_path(self, dummy_backend, path_factory, test_data): + path = path_factory(test_data.get_path("geojson/polygon02.json")) + cube = DataCube.load_stac("https://stac.test/data", spatial_extent=path, connection=dummy_backend.connection) + cube.execute() + assert dummy_backend.get_sync_pg()["loadstac1"]["arguments"] == { + "url": "https://stac.test/data", + "spatial_extent": {"type": "Polygon", "coordinates": [[[3, 50], [4, 50], [4, 51], [3, 50]]]}, + } + + def test_load_stac_spatial_extent_url(self, dummy_backend): + cube = DataCube.load_stac( + "https://stac.test/data", + spatial_extent="https://geo.test/geometry.json", + connection=dummy_backend.connection, + ) + cube.execute() + assert dummy_backend.get_sync_pg() == { + "loadurl1": { + "process_id": "load_url", + "arguments": {"format": "GeoJSON", "url": "https://geo.test/geometry.json"}, + }, + "loadstac1": { + "process_id": "load_stac", + "arguments": { + "url": "https://stac.test/data", + "spatial_extent": {"from_node": "loadurl1"}, + }, + "result": True, + }, + } + + def test_load_stac_spatial_extent_parameter(self, dummy_backend): + spatial_extent = Parameter.geojson("zpatial_extent") + cube = DataCube.load_stac( + "https://stac.test/data", spatial_extent=spatial_extent, connection=dummy_backend.connection + ) + cube.execute() + assert dummy_backend.get_sync_pg()["loadstac1"]["arguments"] == { + "url": "https://stac.test/data", + "spatial_extent": {"from_parameter": "zpatial_extent"}, + } def test_filter_temporal_basic_positional_args(s2cube): im = s2cube.filter_temporal("2016-01-01", "2016-03-10") diff --git a/tests/rest/test_connection.py b/tests/rest/test_connection.py index 408d79bd0..16f5cb894 100644 --- a/tests/rest/test_connection.py +++ b/tests/rest/test_connection.py @@ -2460,7 +2460,7 @@ def test_load_collection_spatial_extent_bbox(self, dummy_backend): cube.execute() assert dummy_backend.get_sync_pg()["loadcollection1"]["arguments"] == { "id": "S2", - "spatial_extent": {"east": 3, "north": 4, "south": 2, "west": 1}, + "spatial_extent": {"west": 1, "south": 2, "east": 3, "north": 4}, "temporal_extent": None, } @@ -2527,7 +2527,7 @@ def test_load_collection_spatial_extent_shapely_wrong_type(self, geojson, dummy_ ], ) @pytest.mark.parametrize("path_factory", [str, Path]) - def test_load_collection_spatial_extent_path(self, geojson, dummy_backend, tmp_path, path_factory): + def test_load_collection_spatial_extent_local_path(self, geojson, dummy_backend, tmp_path, path_factory): path = tmp_path / "geometry.json" with path.open("w") as f: json.dump(geojson, f) @@ -2935,8 +2935,8 @@ def test_load_stac_spatial_extent_bbox(self, dummy_backend): cube = dummy_backend.connection.load_stac("https://stac.test/data", spatial_extent=spatial_extent) cube.execute() assert dummy_backend.get_sync_pg()["loadstac1"]["arguments"] == { - "spatial_extent": {"east": 3, "north": 4, "south": 2, "west": 1}, "url": "https://stac.test/data", + "spatial_extent": {"west": 1, "south": 2, "east": 3, "north": 4}, } @pytest.mark.parametrize( @@ -2953,8 +2953,8 @@ def test_load_stac_spatial_extent_geojson(self, dummy_backend, spatial_extent): cube = dummy_backend.connection.load_stac("https://stac.test/data", spatial_extent=spatial_extent) cube.execute() assert dummy_backend.get_sync_pg()["loadstac1"]["arguments"] == { - "spatial_extent": spatial_extent, "url": "https://stac.test/data", + "spatial_extent": spatial_extent, } @pytest.mark.parametrize( @@ -3008,7 +3008,7 @@ def test_load_stac_spatial_extent_shapely_wront_type(self, dummy_backend, geojso ], ) @pytest.mark.parametrize("path_factory", [str, Path]) - def test_load_stac_spatial_extent_path(self, geojson, dummy_backend, tmp_path, path_factory): + def test_load_stac_spatial_extent_local_path(self, geojson, dummy_backend, tmp_path, path_factory): path = tmp_path / "geometry.json" with path.open("w") as f: json.dump(geojson, f)