From 80b6f6e9bdd2ba19cd7b5f4aebc0a4d528d21e0d Mon Sep 17 00:00:00 2001 From: Victor Verhaert <33786515+VictorVerhaert@users.noreply.github.com> Date: Fri, 23 Feb 2024 13:47:56 +0100 Subject: [PATCH 01/11] created CubeMetadata class issue #464 moved general methods from CollectionMetadata to CubeMetadata only collection parsing specific methods are left in CollectionMetadata This only has a refactoring effect, no functional changes for now --- openeo/metadata.py | 285 ++++++++++++++++++++++++--------------------- 1 file changed, 152 insertions(+), 133 deletions(-) diff --git a/openeo/metadata.py b/openeo/metadata.py index 84367de61..70d563dff 100644 --- a/openeo/metadata.py +++ b/openeo/metadata.py @@ -2,7 +2,7 @@ import logging import warnings -from typing import Any, Callable, List, NamedTuple, Optional, Tuple, Union +from typing import Any, Callable, List, NamedTuple, Optional, Tuple, Union, Self from openeo.internal.jupyter import render_component from openeo.util import deep_get @@ -199,23 +199,12 @@ def rename_labels(self, target, source) -> Dimension: return BandDimension(name=self.name, bands=new_bands) -class CollectionMetadata: +class CubeMetadata: """ - Wrapper for Image Collection metadata. - - Simplifies getting values from deeply nested mappings, - allows additional parsing and normalizing compatibility issues. - - Metadata is expected to follow format defined by - https://openeo.org/documentation/1.0/developers/api/reference.html#operation/describe-collection - (with partial support for older versions) - + Interface for metadata of a data cube. + allows interaction with the cube dimensions and their labels (if available). """ - # TODO: "CollectionMetadata" is also used as "cube metadata" where the link to original collection - # might be lost (if any). Better separation between rich EO raster collection metadata and - # essential cube metadata? E.g.: also thing of vector cubes. - def __init__(self, metadata: dict, dimensions: List[Dimension] = None): # Original collection metadata (actual cube metadata might be altered through processes) self._orig_metadata = metadata @@ -228,121 +217,37 @@ def __init__(self, metadata: dict, dimensions: List[Dimension] = None): self._temporal_dimension = None for dim in self._dimensions: # TODO: here we blindly pick last bands or temporal dimension if multiple. Let user choose? + # TODO: add spacial dimension handling? if dim.type == "bands": + # TODO: add check and/or cast to BandDimension self._band_dimension = dim if dim.type == "temporal": + # TODO: add check and/or cast to TemporalDimension self._temporal_dimension = dim def __eq__(self, o: Any) -> bool: - return isinstance(o, CollectionMetadata) and self._dimensions == o._dimensions + return isinstance(o, type(self)) and self._dimensions == o._dimensions def _clone_and_update( self, metadata: dict = None, dimensions: List[Dimension] = None, **kwargs - ) -> CollectionMetadata: + ) -> CubeMetadata: # python >= 3.11: -> Self to be more correct for subclasses """Create a new instance (of same class) with copied/updated fields.""" + # TODO: do we want to keep the type the same or force it to be CubeMetadata? + # this method is e.g. used by reduce_dimension, which should return a CubeMetadata + # If adjusted, name should be changed to e.g. _create_updated + # Alternative is to use an optional argument to specify the class to use cls = type(self) if dimensions == None: dimensions = self._dimensions return cls(metadata=metadata or self._orig_metadata, dimensions=dimensions, **kwargs) @classmethod - def _parse_dimensions(cls, spec: dict, complain: Callable[[str], None] = warnings.warn) -> List[Dimension]: - """ - Extract data cube dimension metadata from STAC-like description of a collection. - - Dimension metadata comes from different places in spec: - - 'cube:dimensions' has dimension names (e.g. 'x', 'y', 't'), dimension extent info - and band names for band dimensions - - 'eo:bands' has more detailed band information like "common" name and wavelength info - - This helper tries to normalize/combine these sources. - - :param spec: STAC like collection metadata dict - :param complain: handler for warnings - :return list: list of `Dimension` objects - - """ - - # Dimension info is in `cube:dimensions` (or 0.4-style `properties/cube:dimensions`) - cube_dimensions = ( - deep_get(spec, "cube:dimensions", default=None) - or deep_get(spec, "properties", "cube:dimensions", default=None) - or {} - ) - if not cube_dimensions: - complain("No cube:dimensions metadata") - dimensions = [] - for name, info in cube_dimensions.items(): - dim_type = info.get("type") - if dim_type == "spatial": - dimensions.append( - SpatialDimension( - name=name, - extent=info.get("extent"), - crs=info.get("reference_system", SpatialDimension.DEFAULT_CRS), - step=info.get("step", None), - ) - ) - elif dim_type == "temporal": - dimensions.append(TemporalDimension(name=name, extent=info.get("extent"))) - elif dim_type == "bands": - bands = [Band(name=b) for b in info.get("values", [])] - if not bands: - complain("No band names in dimension {d!r}".format(d=name)) - dimensions.append(BandDimension(name=name, bands=bands)) - else: - complain("Unknown dimension type {t!r}".format(t=dim_type)) - dimensions.append(Dimension(name=name, type=dim_type)) - - # Detailed band information: `summaries/[eo|raster]:bands` (and 0.4 style `properties/eo:bands`) - eo_bands = ( - deep_get(spec, "summaries", "eo:bands", default=None) - or deep_get(spec, "summaries", "raster:bands", default=None) - or deep_get(spec, "properties", "eo:bands", default=None) - ) - if eo_bands: - # center_wavelength is in micrometer according to spec - bands_detailed = [ - Band( - name=b["name"], - common_name=b.get("common_name"), - wavelength_um=b.get("center_wavelength"), - aliases=b.get("aliases"), - gsd=b.get("openeo:gsd"), - ) - for b in eo_bands - ] - # Update band dimension with more detailed info - band_dimensions = [d for d in dimensions if d.type == "bands"] - if len(band_dimensions) == 1: - dim = band_dimensions[0] - # Update band values from 'cube:dimensions' with more detailed 'eo:bands' info - eo_band_names = [b.name for b in bands_detailed] - cube_dimension_band_names = [b.name for b in dim.bands] - if eo_band_names == cube_dimension_band_names: - dim.bands = bands_detailed - else: - complain("Band name mismatch: {a} != {b}".format(a=cube_dimension_band_names, b=eo_band_names)) - elif len(band_dimensions) == 0: - if len(dimensions) == 0: - complain("Assuming name 'bands' for anonymous band dimension.") - dimensions.append(BandDimension(name="bands", bands=bands_detailed)) - else: - complain("No 'bands' dimension in 'cube:dimensions' while having 'eo:bands' or 'raster:bands'") - else: - complain("Multiple dimensions of type 'bands'") - - return dimensions + def _parse_dimensions(**kwargs): + pass def get(self, *args, default=None): return deep_get(self._orig_metadata, *args, default=default) - @property - def extent(self) -> dict: - # TODO: is this currently used and relevant? - # TODO: check against extent metadata in dimensions - return self._orig_metadata.get("extent") - def dimension_names(self) -> List[str]: return list(d.name for d in self._dimensions) @@ -394,29 +299,27 @@ def get_band_index(self, band: Union[int, str]) -> int: # TODO: eliminate this shortcut for smaller API surface return self.band_dimension.band_index(band) - def filter_bands(self, band_names: List[Union[int, str]]) -> CollectionMetadata: + def filter_bands(self, band_names: List[Union[int, str]]) -> CubeMetadata: """ - Create new `CollectionMetadata` with filtered band dimension + Create new `CubeMetadata` with filtered band dimension :param band_names: list of band names/indices to keep :return: """ assert self.band_dimension - return self._clone_and_update(dimensions=[ - d.filter_bands(band_names) if isinstance(d, BandDimension) else d - for d in self._dimensions - ]) + return self._clone_and_update( + dimensions=[d.filter_bands(band_names) if isinstance(d, BandDimension) else d for d in self._dimensions] + ) - def append_band(self, band: Band) -> CollectionMetadata: + def append_band(self, band: Band) -> CubeMetadata: """ - Create new `CollectionMetadata` with given band added to band dimension. + Create new `CubeMetadata` with given band added to band dimension. """ assert self.band_dimension - return self._clone_and_update(dimensions=[ - d.append_band(band) if isinstance(d, BandDimension) else d - for d in self._dimensions - ]) + return self._clone_and_update( + dimensions=[d.append_band(band) if isinstance(d, BandDimension) else d for d in self._dimensions] + ) - def rename_labels(self, dimension: str, target: list, source: list = None) -> CollectionMetadata: + def rename_labels(self, dimension: str, target: list, source: list = None) -> CubeMetadata: """ Renames the labels of the specified dimension from source to target. @@ -433,7 +336,7 @@ def rename_labels(self, dimension: str, target: list, source: list = None) -> Co return self._clone_and_update(dimensions=new_dimensions) - def rename_dimension(self, source: str, target: str) -> CollectionMetadata: + def rename_dimension(self, source: str, target: str) -> CubeMetadata: """ Rename source dimension into target, preserving other properties """ @@ -444,23 +347,23 @@ def rename_dimension(self, source: str, target: str) -> CollectionMetadata: return self._clone_and_update(dimensions=new_dimensions) - def reduce_dimension(self, dimension_name: str) -> CollectionMetadata: - """Create new metadata object by collapsing/reducing a dimension.""" + def reduce_dimension(self, dimension_name: str) -> CubeMetadata: + """Create new CubeMetadata object by collapsing/reducing a dimension.""" # TODO: option to keep reduced dimension (with a single value)? # TODO: rename argument to `name` for more internal consistency # TODO: merge with drop_dimension (which does the same). self.assert_valid_dimension(dimension_name) loc = self.dimension_names().index(dimension_name) - dimensions = self._dimensions[:loc] + self._dimensions[loc + 1:] + dimensions = self._dimensions[:loc] + self._dimensions[loc + 1 :] return self._clone_and_update(dimensions=dimensions) - def reduce_spatial(self) -> CollectionMetadata: - """Create new metadata object by reducing the spatial dimensions.""" + def reduce_spatial(self) -> CubeMetadata: + """Create new CubeMetadata object by reducing the spatial dimensions.""" dimensions = [d for d in self._dimensions if not isinstance(d, SpatialDimension)] return self._clone_and_update(dimensions=dimensions) - def add_dimension(self, name: str, label: Union[str, float], type: str = None) -> CollectionMetadata: - """Create new metadata object with added dimension""" + def add_dimension(self, name: str, label: Union[str, float], type: str = None) -> CubeMetadata: + """Create new CubeMetadata object with added dimension""" if any(d.name == name for d in self._dimensions): raise DimensionAlreadyExistsException(f"Dimension with name {name!r} already exists") if type == "bands": @@ -473,13 +376,129 @@ def add_dimension(self, name: str, label: Union[str, float], type: str = None) - dim = Dimension(type=type or "other", name=name) return self._clone_and_update(dimensions=self._dimensions + [dim]) - def drop_dimension(self, name: str = None) -> CollectionMetadata: - """Drop dimension with given name""" + def drop_dimension(self, name: str = None) -> CubeMetadata: + """Create new CubeMetadata object without dropped dimension with given name""" dimension_names = self.dimension_names() if name not in dimension_names: raise ValueError("No dimension named {n!r} (valid names: {ns!r})".format(n=name, ns=dimension_names)) return self._clone_and_update(dimensions=[d for d in self._dimensions if not d.name == name]) + def __str__(self) -> str: + bands = self.band_names if self.has_band_dimension() else "no bands dimension" + return f"CubeMetadata({bands} - {self.dimension_names()})" + + +class CollectionMetadata(CubeMetadata): + """ + Wrapper for Image Collection metadata. + + Simplifies getting values from deeply nested mappings, + allows additional parsing and normalizing compatibility issues. + + Metadata is expected to follow format defined by + https://openeo.org/documentation/1.0/developers/api/reference.html#operation/describe-collection + (with partial support for older versions) + + """ + + def __init__(self, metadata: dict, dimensions: List[Dimension] = None): + super().__init__(metadata=metadata, dimensions=dimensions) + + @classmethod + def _parse_dimensions(cls, spec: dict, complain: Callable[[str], None] = warnings.warn) -> List[Dimension]: + """ + Extract data cube dimension metadata from STAC-like description of a collection. + + Dimension metadata comes from different places in spec: + - 'cube:dimensions' has dimension names (e.g. 'x', 'y', 't'), dimension extent info + and band names for band dimensions + - 'eo:bands' has more detailed band information like "common" name and wavelength info + + This helper tries to normalize/combine these sources. + + :param spec: STAC like collection metadata dict + :param complain: handler for warnings + :return list: list of `Dimension` objects + + """ + + # Dimension info is in `cube:dimensions` (or 0.4-style `properties/cube:dimensions`) + cube_dimensions = ( + deep_get(spec, "cube:dimensions", default=None) + or deep_get(spec, "properties", "cube:dimensions", default=None) + or {} + ) + if not cube_dimensions: + complain("No cube:dimensions metadata") + dimensions = [] + for name, info in cube_dimensions.items(): + dim_type = info.get("type") + if dim_type == "spatial": + dimensions.append( + SpatialDimension( + name=name, + extent=info.get("extent"), + crs=info.get("reference_system", SpatialDimension.DEFAULT_CRS), + step=info.get("step", None), + ) + ) + elif dim_type == "temporal": + dimensions.append(TemporalDimension(name=name, extent=info.get("extent"))) + elif dim_type == "bands": + bands = [Band(name=b) for b in info.get("values", [])] + if not bands: + complain("No band names in dimension {d!r}".format(d=name)) + dimensions.append(BandDimension(name=name, bands=bands)) + else: + complain("Unknown dimension type {t!r}".format(t=dim_type)) + dimensions.append(Dimension(name=name, type=dim_type)) + + # Detailed band information: `summaries/[eo|raster]:bands` (and 0.4 style `properties/eo:bands`) + eo_bands = ( + deep_get(spec, "summaries", "eo:bands", default=None) + or deep_get(spec, "summaries", "raster:bands", default=None) + or deep_get(spec, "properties", "eo:bands", default=None) + ) + if eo_bands: + # center_wavelength is in micrometer according to spec + bands_detailed = [ + Band( + name=b["name"], + common_name=b.get("common_name"), + wavelength_um=b.get("center_wavelength"), + aliases=b.get("aliases"), + gsd=b.get("openeo:gsd"), + ) + for b in eo_bands + ] + # Update band dimension with more detailed info + band_dimensions = [d for d in dimensions if d.type == "bands"] + if len(band_dimensions) == 1: + dim = band_dimensions[0] + # Update band values from 'cube:dimensions' with more detailed 'eo:bands' info + eo_band_names = [b.name for b in bands_detailed] + cube_dimension_band_names = [b.name for b in dim.bands] + if eo_band_names == cube_dimension_band_names: + dim.bands = bands_detailed + else: + complain("Band name mismatch: {a} != {b}".format(a=cube_dimension_band_names, b=eo_band_names)) + elif len(band_dimensions) == 0: + if len(dimensions) == 0: + complain("Assuming name 'bands' for anonymous band dimension.") + dimensions.append(BandDimension(name="bands", bands=bands_detailed)) + else: + complain("No 'bands' dimension in 'cube:dimensions' while having 'eo:bands' or 'raster:bands'") + else: + complain("Multiple dimensions of type 'bands'") + + return dimensions + + @property + def extent(self) -> dict: + # TODO: is this currently used and relevant? + # TODO: check against extent metadata in dimensions + return self._orig_metadata.get("extent") + def _repr_html_(self): return render_component("collection", data=self._orig_metadata) From 498c4c7676f46d4ed6e858178cc08b1189a7699f Mon Sep 17 00:00:00 2001 From: Victor Verhaert <33786515+VictorVerhaert@users.noreply.github.com> Date: Fri, 23 Feb 2024 13:53:36 +0100 Subject: [PATCH 02/11] removed Self from import statements in metadata.py --- openeo/metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openeo/metadata.py b/openeo/metadata.py index 70d563dff..4ea1b8460 100644 --- a/openeo/metadata.py +++ b/openeo/metadata.py @@ -2,7 +2,7 @@ import logging import warnings -from typing import Any, Callable, List, NamedTuple, Optional, Tuple, Union, Self +from typing import Any, Callable, List, NamedTuple, Optional, Tuple, Union from openeo.internal.jupyter import render_component from openeo.util import deep_get From a420f80500f412366e72ea8221908f3e2e2b3e8a Mon Sep 17 00:00:00 2001 From: Victor Verhaert <33786515+VictorVerhaert@users.noreply.github.com> Date: Mon, 26 Feb 2024 18:16:34 +0100 Subject: [PATCH 03/11] removed _orig_metadata from CubeMetadata _clone_and_update had to be overridden by CollectionMetadata to keep the _orig_metadata attribute --- openeo/metadata.py | 65 ++++++++++++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 25 deletions(-) diff --git a/openeo/metadata.py b/openeo/metadata.py index 4ea1b8460..7e0fa730e 100644 --- a/openeo/metadata.py +++ b/openeo/metadata.py @@ -202,19 +202,23 @@ def rename_labels(self, target, source) -> Dimension: class CubeMetadata: """ Interface for metadata of a data cube. - allows interaction with the cube dimensions and their labels (if available). + + Allows interaction with the cube dimensions and their labels (if available). """ - def __init__(self, metadata: dict, dimensions: List[Dimension] = None): + def __init__(self, dimensions: List[Dimension] = None): # Original collection metadata (actual cube metadata might be altered through processes) - self._orig_metadata = metadata - - if dimensions == None: - self._dimensions = self._parse_dimensions(self._orig_metadata) - else: - self._dimensions = dimensions + self._dimensions = None self._band_dimension = None self._temporal_dimension = None + + if dimensions is not None: + self.set_dimensions(dimensions=dimensions) + + def set_dimensions(self, dimensions: List[Dimension]): + if dimensions == None: + raise ValueError("Dimensions can not be None.") + self._dimensions = dimensions for dim in self._dimensions: # TODO: here we blindly pick last bands or temporal dimension if multiple. Let user choose? # TODO: add spacial dimension handling? @@ -228,25 +232,12 @@ def __init__(self, metadata: dict, dimensions: List[Dimension] = None): def __eq__(self, o: Any) -> bool: return isinstance(o, type(self)) and self._dimensions == o._dimensions - def _clone_and_update( - self, metadata: dict = None, dimensions: List[Dimension] = None, **kwargs - ) -> CubeMetadata: # python >= 3.11: -> Self to be more correct for subclasses + def _clone_and_update(self, dimensions: List[Dimension] = None, **kwargs) -> CubeMetadata: """Create a new instance (of same class) with copied/updated fields.""" - # TODO: do we want to keep the type the same or force it to be CubeMetadata? - # this method is e.g. used by reduce_dimension, which should return a CubeMetadata - # If adjusted, name should be changed to e.g. _create_updated - # Alternative is to use an optional argument to specify the class to use cls = type(self) if dimensions == None: dimensions = self._dimensions - return cls(metadata=metadata or self._orig_metadata, dimensions=dimensions, **kwargs) - - @classmethod - def _parse_dimensions(**kwargs): - pass - - def get(self, *args, default=None): - return deep_get(self._orig_metadata, *args, default=default) + return cls(dimensions=dimensions, **kwargs) def dimension_names(self) -> List[str]: return list(d.name for d in self._dimensions) @@ -390,7 +381,7 @@ def __str__(self) -> str: class CollectionMetadata(CubeMetadata): """ - Wrapper for Image Collection metadata. + Wrapper for EO Data Collection metadata. Simplifies getting values from deeply nested mappings, allows additional parsing and normalizing compatibility issues. @@ -402,7 +393,13 @@ class CollectionMetadata(CubeMetadata): """ def __init__(self, metadata: dict, dimensions: List[Dimension] = None): - super().__init__(metadata=metadata, dimensions=dimensions) + super().__init__(dimensions=dimensions) + + self._orig_metadata = metadata + + if dimensions == None: + dimensions = self._parse_dimensions(self._orig_metadata) + self.set_dimensions(dimensions=dimensions) @classmethod def _parse_dimensions(cls, spec: dict, complain: Callable[[str], None] = warnings.warn) -> List[Dimension]: @@ -493,6 +490,24 @@ def _parse_dimensions(cls, spec: dict, complain: Callable[[str], None] = warning return dimensions + def _clone_and_update( + self, metadata: dict = None, dimensions: List[Dimension] = None, **kwargs + ) -> CollectionMetadata: + """ + Create a new instance (of same class) with copied/updated fields. + + This overrides the method in `CubeMetadata` to keep the original metadata. + """ + cls = type(self) + if metadata == None: + metadata = self._orig_metadata + if dimensions == None: + dimensions = self._dimensions + return cls(metadata=metadata, dimensions=dimensions, **kwargs) + + def get(self, *args, default=None): + return deep_get(self._orig_metadata, *args, default=default) + @property def extent(self) -> dict: # TODO: is this currently used and relevant? From 64b89fe9c942e0ea477c8234ee58d4499798067f Mon Sep 17 00:00:00 2001 From: Victor Verhaert <33786515+VictorVerhaert@users.noreply.github.com> Date: Tue, 27 Feb 2024 12:23:03 +0100 Subject: [PATCH 04/11] processed feedback == None -> is None set_dimensions -> _set_dimensions --- openeo/metadata.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/openeo/metadata.py b/openeo/metadata.py index 7e0fa730e..d2f52bfd9 100644 --- a/openeo/metadata.py +++ b/openeo/metadata.py @@ -213,10 +213,10 @@ def __init__(self, dimensions: List[Dimension] = None): self._temporal_dimension = None if dimensions is not None: - self.set_dimensions(dimensions=dimensions) + self._set_dimensions(dimensions=dimensions) - def set_dimensions(self, dimensions: List[Dimension]): - if dimensions == None: + def _set_dimensions(self, dimensions: List[Dimension]): + if dimensions is None: raise ValueError("Dimensions can not be None.") self._dimensions = dimensions for dim in self._dimensions: @@ -235,7 +235,7 @@ def __eq__(self, o: Any) -> bool: def _clone_and_update(self, dimensions: List[Dimension] = None, **kwargs) -> CubeMetadata: """Create a new instance (of same class) with copied/updated fields.""" cls = type(self) - if dimensions == None: + if dimensions is None: dimensions = self._dimensions return cls(dimensions=dimensions, **kwargs) @@ -397,9 +397,9 @@ def __init__(self, metadata: dict, dimensions: List[Dimension] = None): self._orig_metadata = metadata - if dimensions == None: + if dimensions is None: dimensions = self._parse_dimensions(self._orig_metadata) - self.set_dimensions(dimensions=dimensions) + self._set_dimensions(dimensions=dimensions) @classmethod def _parse_dimensions(cls, spec: dict, complain: Callable[[str], None] = warnings.warn) -> List[Dimension]: @@ -499,9 +499,9 @@ def _clone_and_update( This overrides the method in `CubeMetadata` to keep the original metadata. """ cls = type(self) - if metadata == None: + if metadata is None: metadata = self._orig_metadata - if dimensions == None: + if dimensions is None: dimensions = self._dimensions return cls(metadata=metadata, dimensions=dimensions, **kwargs) From 4a2d92dac4b12bff03056b5d51719954b58edb72 Mon Sep 17 00:00:00 2001 From: Victor Verhaert <33786515+VictorVerhaert@users.noreply.github.com> Date: Tue, 27 Feb 2024 13:51:16 +0100 Subject: [PATCH 05/11] moved _set_dimensions back to __init__ this was less complex and more readable than the previous implementation --- openeo/metadata.py | 31 ++++++++++++------------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/openeo/metadata.py b/openeo/metadata.py index d2f52bfd9..13fc6932e 100644 --- a/openeo/metadata.py +++ b/openeo/metadata.py @@ -213,21 +213,16 @@ def __init__(self, dimensions: List[Dimension] = None): self._temporal_dimension = None if dimensions is not None: - self._set_dimensions(dimensions=dimensions) - - def _set_dimensions(self, dimensions: List[Dimension]): - if dimensions is None: - raise ValueError("Dimensions can not be None.") - self._dimensions = dimensions - for dim in self._dimensions: - # TODO: here we blindly pick last bands or temporal dimension if multiple. Let user choose? - # TODO: add spacial dimension handling? - if dim.type == "bands": - # TODO: add check and/or cast to BandDimension - self._band_dimension = dim - if dim.type == "temporal": - # TODO: add check and/or cast to TemporalDimension - self._temporal_dimension = dim + self._dimensions = dimensions + for dim in self._dimensions: + # TODO: here we blindly pick last bands or temporal dimension if multiple. Let user choose? + # TODO: add spacial dimension handling? + if dim.type == "bands": + # TODO: add check and/or cast to BandDimension + self._band_dimension = dim + if dim.type == "temporal": + # TODO: add check and/or cast to TemporalDimension + self._temporal_dimension = dim def __eq__(self, o: Any) -> bool: return isinstance(o, type(self)) and self._dimensions == o._dimensions @@ -393,13 +388,11 @@ class CollectionMetadata(CubeMetadata): """ def __init__(self, metadata: dict, dimensions: List[Dimension] = None): - super().__init__(dimensions=dimensions) - self._orig_metadata = metadata - if dimensions is None: dimensions = self._parse_dimensions(self._orig_metadata) - self._set_dimensions(dimensions=dimensions) + + super().__init__(dimensions=dimensions) @classmethod def _parse_dimensions(cls, spec: dict, complain: Callable[[str], None] = warnings.warn) -> List[Dimension]: From a6f3b4ae773d407b3647b5a61f7e637c9324e356 Mon Sep 17 00:00:00 2001 From: Victor Verhaert <33786515+VictorVerhaert@users.noreply.github.com> Date: Tue, 27 Feb 2024 16:53:19 +0100 Subject: [PATCH 06/11] added type checks to __init__ of CubeMetadata BandsDimension and TemporalDimension needed to override rename and rename labels to ensure that the type checks were correct --- openeo/metadata.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/openeo/metadata.py b/openeo/metadata.py index 13fc6932e..7a1ac4223 100644 --- a/openeo/metadata.py +++ b/openeo/metadata.py @@ -84,6 +84,10 @@ def __init__(self, name: str, extent: Union[Tuple[str, str], List[str]]): def rename(self, name) -> Dimension: return TemporalDimension(name=name, extent=self.extent) + def rename_labels(self, target, source) -> Dimension: + # TODO should we check if the extend has changed with the new labels? + return TemporalDimension(name=self.name, extent=self.extent) + class Band(NamedTuple): """ @@ -198,6 +202,8 @@ def rename_labels(self, target, source) -> Dimension: new_bands = [Band(name=n) for n in target] return BandDimension(name=self.name, bands=new_bands) + def rename(self, name) -> Dimension: + return BandDimension(name=name, bands=self.bands) class CubeMetadata: """ @@ -218,11 +224,15 @@ def __init__(self, dimensions: List[Dimension] = None): # TODO: here we blindly pick last bands or temporal dimension if multiple. Let user choose? # TODO: add spacial dimension handling? if dim.type == "bands": - # TODO: add check and/or cast to BandDimension - self._band_dimension = dim + if isinstance(dim, BandDimension): + self._band_dimension = dim + else: + raise MetadataException("Invalid band dimension {d!r}".format(d=dim)) if dim.type == "temporal": - # TODO: add check and/or cast to TemporalDimension - self._temporal_dimension = dim + if isinstance(dim, TemporalDimension): + self._temporal_dimension = dim + else: + raise MetadataException("Invalid temporal dimension {d!r}".format(d=dim)) def __eq__(self, o: Any) -> bool: return isinstance(o, type(self)) and self._dimensions == o._dimensions From 995e700222d972adb5bfbc34f68c164df1b11d15 Mon Sep 17 00:00:00 2001 From: Victor Verhaert <33786515+VictorVerhaert@users.noreply.github.com> Date: Wed, 28 Feb 2024 11:20:45 +0100 Subject: [PATCH 07/11] issue #464 added tests for CubeMetadata --- tests/test_metadata.py | 783 ++++++++++++++++++++++++++--------------- 1 file changed, 491 insertions(+), 292 deletions(-) diff --git a/tests/test_metadata.py b/tests/test_metadata.py index 5a482673f..28ac81dc7 100644 --- a/tests/test_metadata.py +++ b/tests/test_metadata.py @@ -8,6 +8,7 @@ Band, BandDimension, CollectionMetadata, + CubeMetadata, Dimension, DimensionAlreadyExistsException, MetadataException, @@ -17,12 +18,7 @@ def test_metadata_get(): - metadata = CollectionMetadata({ - "foo": "bar", - "very": { - "deeply": {"nested": {"path": {"to": "somewhere"}}} - } - }) + metadata = CollectionMetadata({"foo": "bar", "very": {"deeply": {"nested": {"path": {"to": "somewhere"}}}}}) assert metadata.get("foo") == "bar" assert metadata.get("very", "deeply", "nested", "path", "to") == "somewhere" assert metadata.get("invalid", "key") is None @@ -30,9 +26,7 @@ def test_metadata_get(): def test_metadata_extent(): - metadata = CollectionMetadata({ - "extent": {"spatial": {"xmin": 4, "xmax": 10}} - }) + metadata = CollectionMetadata({"extent": {"spatial": {"xmin": 4, "xmax": 10}}}) assert metadata.extent == {"spatial": {"xmin": 4, "xmax": 10}} @@ -42,22 +36,28 @@ def test_band_minimal(): def test_band_dimension(): - bdim = BandDimension(name="spectral", bands=[ - Band("B02", "blue", 0.490), - Band("B03", "green", 0.560), - Band("B04", "red", 0.665), - ]) + bdim = BandDimension( + name="spectral", + bands=[ + Band("B02", "blue", 0.490), + Band("B03", "green", 0.560), + Band("B04", "red", 0.665), + ], + ) assert bdim.band_names == ["B02", "B03", "B04"] assert bdim.common_names == ["blue", "green", "red"] def test_band_dimension_band_index(): - bdim = BandDimension(name="spectral", bands=[ - Band("B02", "blue", 0.490), - Band("B03", "green", 0.560), - Band("B04", "red", 0.665), - Band("B08", "nir", 0.842), - ]) + bdim = BandDimension( + name="spectral", + bands=[ + Band("B02", "blue", 0.490), + Band("B03", "green", 0.560), + Band("B04", "red", 0.665), + Band("B08", "nir", 0.842), + ], + ) assert bdim.band_index(0) == 0 assert bdim.band_index(2) == 2 with pytest.raises(ValueError, match="Invalid band name/index"): @@ -78,10 +78,13 @@ def test_band_dimension_band_index(): def test_band_dimension_band_name(): - bdim = BandDimension(name="spectral", bands=[ - Band("B02", "blue", 0.490), - Band("B03", "green", 0.560), - ]) + bdim = BandDimension( + name="spectral", + bands=[ + Band("B02", "blue", 0.490), + Band("B03", "green", 0.560), + ], + ) assert bdim.band_name("B02") == "B02" assert bdim.band_name("B03") == "B03" with pytest.raises(ValueError, match="Invalid band name/index"): @@ -111,20 +114,30 @@ def test_band_dimension_rename_labels(): b03 = Band("B03", "green", 0.560) b04 = Band("B04", "red", 0.665) bdim = BandDimension(name="bs", bands=[b02, b03, b04]) + metadata = CollectionMetadata({}, dimensions=[bdim]) - newdim = metadata.rename_labels("bs", target=['1', '2', '3']).band_dimension + newdim = metadata.rename_labels("bs", target=["1", "2", "3"]).band_dimension + assert metadata.band_dimension.band_names == ["B02", "B03", "B04"] + assert newdim.band_names == ["1", "2", "3"] - assert metadata.band_dimension.band_names == ['B02', 'B03', 'B04'] - assert newdim.band_names == ['1', '2', '3'] + metadata = CubeMetadata(dimensions=[bdim]) + newdim = metadata.rename_labels("bs", target=["1", "2", "3"]).band_dimension + assert metadata.band_dimension.band_names == ["B02", "B03", "B04"] + assert newdim.band_names == ["1", "2", "3"] def test_band_dimension_set_labels(): - bdim = BandDimension(name="bs", bands=[Band('some_name', None, None)]) + bdim = BandDimension(name="bs", bands=[Band("some_name", None, None)]) + metadata = CollectionMetadata({}, dimensions=[bdim]) - newdim = metadata.rename_labels("bs", target=['1', '2', '3']).band_dimension + newdim = metadata.rename_labels("bs", target=["1", "2", "3"]).band_dimension + assert metadata.band_dimension.band_names == ["some_name"] + assert newdim.band_names == ["1", "2", "3"] - assert metadata.band_dimension.band_names == ['some_name'] - assert newdim.band_names == ['1', '2', '3'] + metadata = CubeMetadata(dimensions=[bdim]) + newdim = metadata.rename_labels("bs", target=["1", "2", "3"]).band_dimension + assert metadata.band_dimension.band_names == ["some_name"] + assert newdim.band_names == ["1", "2", "3"] def test_band_dimension_rename_labels_with_source(): @@ -132,20 +145,30 @@ def test_band_dimension_rename_labels_with_source(): b03 = Band("B03", "green", 0.560) b04 = Band("B04", "red", 0.665) bdim = BandDimension(name="bs", bands=[b02, b03, b04]) + metadata = CollectionMetadata({}, dimensions=[bdim]) - newdim = metadata.rename_labels("bs", target=['2'], source=['B03']).band_dimension + newdim = metadata.rename_labels("bs", target=["2"], source=["B03"]).band_dimension + assert metadata.band_dimension.band_names == ["B02", "B03", "B04"] + assert newdim.band_names == ["B02", "2", "B04"] - assert metadata.band_dimension.band_names == ['B02', 'B03', 'B04'] - assert newdim.band_names == ['B02', '2', 'B04'] + metadata = CubeMetadata(dimensions=[bdim]) + newdim = metadata.rename_labels("bs", target=["2"], source=["B03"]).band_dimension + assert metadata.band_dimension.band_names == ["B02", "B03", "B04"] + assert newdim.band_names == ["B02", "2", "B04"] def test_band_dimension_rename_labels_with_source_mismatch(): b02 = Band("B02", "blue", 0.490) b03 = Band("B03", "green", 0.560) bdim = BandDimension(name="bs", bands=[b02, b03]) + metadata = CollectionMetadata({}, dimensions=[bdim]) with pytest.raises(ValueError, match="should have same number of labels, but got"): - _ = metadata.rename_labels("bs", target=['2', "3"], source=['B03']) + _ = metadata.rename_labels("bs", target=["2", "3"], source=["B03"]) + + metadata = CubeMetadata(dimensions=[bdim]) + with pytest.raises(ValueError, match="should have same number of labels, but got"): + _ = metadata.rename_labels("bs", target=["2", "3"], source=["B03"]) def assert_same_dimensions(dims1: List[Dimension], dims2: List[Dimension]): @@ -158,219 +181,309 @@ def test_get_dimensions_cube_dimensions_empty(): def test_get_dimensions_cube_dimensions_spatial_xyt(): - dims = CollectionMetadata._parse_dimensions({ - "cube:dimensions": { - "xx": {"type": "spatial", "extent": [-10, 10]}, - "yy": {"type": "spatial", "extent": [-56, 83], "reference_system": 123}, - "tt": {"type": "temporal", "extent": ["2020-02-20", None]}, + dims = CollectionMetadata._parse_dimensions( + { + "cube:dimensions": { + "xx": {"type": "spatial", "extent": [-10, 10]}, + "yy": {"type": "spatial", "extent": [-56, 83], "reference_system": 123}, + "tt": {"type": "temporal", "extent": ["2020-02-20", None]}, + } } - }) - assert_same_dimensions(dims, [ - SpatialDimension(name="xx", extent=[-10, 10]), - SpatialDimension(name="yy", extent=[-56, 83], crs=123), - TemporalDimension(name="tt", extent=["2020-02-20", None]), - ]) + ) + assert_same_dimensions( + dims, + [ + SpatialDimension(name="xx", extent=[-10, 10]), + SpatialDimension(name="yy", extent=[-56, 83], crs=123), + TemporalDimension(name="tt", extent=["2020-02-20", None]), + ], + ) def test_get_dimensions_cube_dimensions_spatial_xyt_bands(): - dims = CollectionMetadata._parse_dimensions({ - "cube:dimensions": { - "x": {"type": "spatial", "extent": [-10, 10]}, - "y": {"type": "spatial", "extent": [-56, 83], "reference_system": 123}, - "t": {"type": "temporal", "extent": ["2020-02-20", None]}, - "spectral": {"type": "bands", "values": ["red", "green", "blue"]}, + dims = CollectionMetadata._parse_dimensions( + { + "cube:dimensions": { + "x": {"type": "spatial", "extent": [-10, 10]}, + "y": {"type": "spatial", "extent": [-56, 83], "reference_system": 123}, + "t": {"type": "temporal", "extent": ["2020-02-20", None]}, + "spectral": {"type": "bands", "values": ["red", "green", "blue"]}, + } } - }) - assert_same_dimensions(dims, [ - SpatialDimension(name="x", extent=[-10, 10]), - SpatialDimension(name="y", extent=[-56, 83], crs=123), - TemporalDimension(name="t", extent=["2020-02-20", None]), - BandDimension(name="spectral", bands=[ - Band("red", None, None), - Band("green", None, None), - Band("blue", None, None), - ]) - ]) + ) + assert_same_dimensions( + dims, + [ + SpatialDimension(name="x", extent=[-10, 10]), + SpatialDimension(name="y", extent=[-56, 83], crs=123), + TemporalDimension(name="t", extent=["2020-02-20", None]), + BandDimension( + name="spectral", + bands=[ + Band("red", None, None), + Band("green", None, None), + Band("blue", None, None), + ], + ), + ], + ) def test_get_dimensions_cube_dimensions_non_standard_type(): logs = [] - dims = CollectionMetadata._parse_dimensions({ - "cube:dimensions": { - "bar": {"type": "foo"}, + dims = CollectionMetadata._parse_dimensions( + { + "cube:dimensions": { + "bar": {"type": "foo"}, + }, }, - }, complain=logs.append) - assert_same_dimensions(dims, [ - Dimension(type="foo", name="bar") - ]) + complain=logs.append, + ) + assert_same_dimensions(dims, [Dimension(type="foo", name="bar")]) assert logs == ["Unknown dimension type 'foo'"] def test_get_dimensions_cube_dimensions_no_band_names(): logs = [] - dims = CollectionMetadata._parse_dimensions({ - "cube:dimensions": { - "spectral": {"type": "bands"}, + dims = CollectionMetadata._parse_dimensions( + { + "cube:dimensions": { + "spectral": {"type": "bands"}, + }, }, - }, complain=logs.append) - assert_same_dimensions(dims, [ - BandDimension(name="spectral", bands=[]) - ]) + complain=logs.append, + ) + assert_same_dimensions(dims, [BandDimension(name="spectral", bands=[])]) assert logs == ["No band names in dimension 'spectral'"] def test_get_dimensions_cube_dimensions_eo_bands(): - dims = CollectionMetadata._parse_dimensions({ - "cube:dimensions": { - "x": {"type": "spatial", "extent": [-10, 10]}, - "y": {"type": "spatial", "extent": [-56, 83], "reference_system": 123}, - "t": {"type": "temporal", "extent": ["2020-02-20", None]}, - "spectral": {"type": "bands", "values": ["r", "g", "b"]}, - }, - "summaries": { - "eo:bands": [ - {"name": "r", "common_name": "red", "center_wavelength": 5}, - {"name": "g", "center_wavelength": 8}, - {"name": "b", "common_name": "blue"}, - ] + dims = CollectionMetadata._parse_dimensions( + { + "cube:dimensions": { + "x": {"type": "spatial", "extent": [-10, 10]}, + "y": {"type": "spatial", "extent": [-56, 83], "reference_system": 123}, + "t": {"type": "temporal", "extent": ["2020-02-20", None]}, + "spectral": {"type": "bands", "values": ["r", "g", "b"]}, + }, + "summaries": { + "eo:bands": [ + {"name": "r", "common_name": "red", "center_wavelength": 5}, + {"name": "g", "center_wavelength": 8}, + {"name": "b", "common_name": "blue"}, + ] + }, } - }) - assert_same_dimensions(dims, [ - SpatialDimension(name="x", extent=[-10, 10]), - SpatialDimension(name="y", extent=[-56, 83], crs=123), - TemporalDimension(name="t", extent=["2020-02-20", None]), - BandDimension(name="spectral", bands=[ - Band("r", "red", 5), - Band("g", None, 8), - Band("b", "blue", None), - ]) - ]) + ) + assert_same_dimensions( + dims, + [ + SpatialDimension(name="x", extent=[-10, 10]), + SpatialDimension(name="y", extent=[-56, 83], crs=123), + TemporalDimension(name="t", extent=["2020-02-20", None]), + BandDimension( + name="spectral", + bands=[ + Band("r", "red", 5), + Band("g", None, 8), + Band("b", "blue", None), + ], + ), + ], + ) def test_get_dimensions_cube_dimensions_eo_bands_mismatch(): logs = [] - dims = CollectionMetadata._parse_dimensions({ - "cube:dimensions": { - "x": {"type": "spatial", "extent": [-10, 10]}, - "spectral": {"type": "bands", "values": ["r", "g", "b"]}, + dims = CollectionMetadata._parse_dimensions( + { + "cube:dimensions": { + "x": {"type": "spatial", "extent": [-10, 10]}, + "spectral": {"type": "bands", "values": ["r", "g", "b"]}, + }, + "summaries": { + "eo:bands": [ + {"name": "y", "common_name": "yellow", "center_wavelength": 5}, + {"name": "c", "center_wavelength": 8}, + {"name": "m", "common_name": "magenta"}, + ] + }, }, - "summaries": { - "eo:bands": [ - {"name": "y", "common_name": "yellow", "center_wavelength": 5}, - {"name": "c", "center_wavelength": 8}, - {"name": "m", "common_name": "magenta"}, - ] - } - }, complain=logs.append) - assert_same_dimensions(dims, [ - SpatialDimension(name="x", extent=[-10, 10]), - BandDimension(name="spectral", bands=[ - Band("r", None, None), - Band("g", None, None), - Band("b", None, None), - ]) - ]) + complain=logs.append, + ) + assert_same_dimensions( + dims, + [ + SpatialDimension(name="x", extent=[-10, 10]), + BandDimension( + name="spectral", + bands=[ + Band("r", None, None), + Band("g", None, None), + Band("b", None, None), + ], + ), + ], + ) assert logs == ["Band name mismatch: ['r', 'g', 'b'] != ['y', 'c', 'm']"] def test_get_dimensions_eo_bands_only(): logs = [] - dims = CollectionMetadata._parse_dimensions({ - "summaries": { - "eo:bands": [ - {"name": "y", "common_name": "yellow", "center_wavelength": 5}, - {"name": "c", "center_wavelength": 8}, - {"name": "m", "common_name": "magenta"}, - ] - } - }, complain=logs.append) - assert_same_dimensions(dims, [ - BandDimension(name="bands", bands=[ - Band("y", "yellow", 5), - Band("c", None, 8), - Band("m", "magenta", None), - ]) - ]) - assert logs == [ - 'No cube:dimensions metadata', - "Assuming name 'bands' for anonymous band dimension." - ] + dims = CollectionMetadata._parse_dimensions( + { + "summaries": { + "eo:bands": [ + {"name": "y", "common_name": "yellow", "center_wavelength": 5}, + {"name": "c", "center_wavelength": 8}, + {"name": "m", "common_name": "magenta"}, + ] + } + }, + complain=logs.append, + ) + assert_same_dimensions( + dims, + [ + BandDimension( + name="bands", + bands=[ + Band("y", "yellow", 5), + Band("c", None, 8), + Band("m", "magenta", None), + ], + ) + ], + ) + assert logs == ["No cube:dimensions metadata", "Assuming name 'bands' for anonymous band dimension."] def test_get_dimensions_no_band_dimension_with_eo_bands(): logs = [] - dims = CollectionMetadata._parse_dimensions({ - "cube:dimensions": { - "x": {"type": "spatial", "extent": [-10, 10]}, - }, - "summaries": { - "eo:bands": [ - {"name": "y", "common_name": "yellow", "center_wavelength": 5}, - {"name": "c", "center_wavelength": 8}, - {"name": "m", "common_name": "magenta"}, - ] + dims = CollectionMetadata._parse_dimensions( + { + "cube:dimensions": { + "x": {"type": "spatial", "extent": [-10, 10]}, + }, + "summaries": { + "eo:bands": [ + {"name": "y", "common_name": "yellow", "center_wavelength": 5}, + {"name": "c", "center_wavelength": 8}, + {"name": "m", "common_name": "magenta"}, + ] + }, }, - }, complain=logs.append) - assert_same_dimensions(dims, [ - SpatialDimension(name="x", extent=[-10, 10]), - ]) + complain=logs.append, + ) + assert_same_dimensions( + dims, + [ + SpatialDimension(name="x", extent=[-10, 10]), + ], + ) assert logs == ["No 'bands' dimension in 'cube:dimensions' while having 'eo:bands' or 'raster:bands'"] def test_get_dimensions_multiple_band_dimensions_with_eo_bands(): logs = [] - dims = CollectionMetadata._parse_dimensions({ - "cube:dimensions": { - "x": {"type": "spatial", "extent": [-10, 10]}, - "spectral": {"type": "bands", "values": ["alpha", "beta"]}, - "bands": {"type": "bands", "values": ["r", "g", "b"]}, - }, - "summaries": { - "eo:bands": [ - {"name": "zu", "common_name": "foo"}, - ] + dims = CollectionMetadata._parse_dimensions( + { + "cube:dimensions": { + "x": {"type": "spatial", "extent": [-10, 10]}, + "spectral": {"type": "bands", "values": ["alpha", "beta"]}, + "bands": {"type": "bands", "values": ["r", "g", "b"]}, + }, + "summaries": { + "eo:bands": [ + {"name": "zu", "common_name": "foo"}, + ] + }, }, - }, complain=logs.append) - assert_same_dimensions(dims, [ - SpatialDimension(name="x", extent=[-10, 10]), - BandDimension(name="spectral", bands=[Band("alpha", None, None), Band("beta", None, None), ]), - BandDimension(name="bands", bands=[Band("r", None, None), Band("g", None, None), Band("b", None, None), ]), - ]) + complain=logs.append, + ) + assert_same_dimensions( + dims, + [ + SpatialDimension(name="x", extent=[-10, 10]), + BandDimension( + name="spectral", + bands=[ + Band("alpha", None, None), + Band("beta", None, None), + ], + ), + BandDimension( + name="bands", + bands=[ + Band("r", None, None), + Band("g", None, None), + Band("b", None, None), + ], + ), + ], + ) assert logs == ["Multiple dimensions of type 'bands'"] -@pytest.mark.parametrize("spec", [ - # API 0.4 style - { - "properties": { - "cube:dimensions": { - "x": {"type": "spatial", "axis": "x"}, - "b": {"type": "bands", "values": ["foo", "bar"]} - }}}, - # API 1.0 style - { - "cube:dimensions": { - "x": {"type": "spatial", "axis": "x"}, - "b": {"type": "bands", "values": ["foo", "bar"]} - }}, -]) -def test_metadata_bands_dimension_cube_dimensions(spec): +@pytest.mark.parametrize( + "spec", + [ + # API 0.4 style + { + "properties": { + "cube:dimensions": { + "x": {"type": "spatial", "axis": "x"}, + "b": {"type": "bands", "values": ["foo", "bar"]}, + } + } + }, + # API 1.0 style + {"cube:dimensions": {"x": {"type": "spatial", "axis": "x"}, "b": {"type": "bands", "values": ["foo", "bar"]}}}, + ], +) +def test_collectionmetadata_bands_dimension_cube_dimensions(spec): metadata = CollectionMetadata(spec) assert metadata.band_dimension.name == "b" - assert metadata.bands == [ - Band("foo", None, None), - Band("bar", None, None) - ] + assert metadata.bands == [Band("foo", None, None), Band("bar", None, None)] assert metadata.band_names == ["foo", "bar"] assert metadata.band_common_names == [None, None] -def test_metadata_bands_dimension_no_band_dimensions(): - metadata = CollectionMetadata({ - "cube:dimensions": { - "x": {"type": "spatial", "axis": "x"}, +def test_cubemetdata_bands_dimension(): + metadata = CubeMetadata( + dimensions=[ + SpatialDimension(name="x", extent=None), + BandDimension(name="b", bands=[Band("foo"), Band("bar")]), + ] + ) + assert metadata.band_dimension.name == "b" + assert metadata.bands == [Band("foo", None, None), Band("bar", None, None)] + assert metadata.band_names == ["foo", "bar"] + assert metadata.band_common_names == [None, None] + + +def test_collectionmetadata_bands_dimension_no_band_dimensions(): + metadata = CollectionMetadata( + { + "cube:dimensions": { + "x": {"type": "spatial", "axis": "x"}, + } } - }) + ) + with pytest.raises(MetadataException, match="No band dimension"): + metadata.band_dimension + with pytest.raises(MetadataException, match="No band dimension"): + metadata.bands + with pytest.raises(MetadataException, match="No band dimension"): + metadata.band_common_names + with pytest.raises(MetadataException, match="No band dimension"): + metadata.get_band_index("red") + with pytest.raises(MetadataException, match="No band dimension"): + metadata.filter_bands(["red"]) + + +def test_cubemetadata_bands_dimension_no_band_dimensions(): + metadata = CubeMetadata(dimensions=[SpatialDimension(name="x", extent=None)]) with pytest.raises(MetadataException, match="No band dimension"): metadata.band_dimension with pytest.raises(MetadataException, match="No band dimension"): @@ -383,103 +496,106 @@ def test_metadata_bands_dimension_no_band_dimensions(): metadata.filter_bands(["red"]) -@pytest.mark.parametrize("spec", [ - # API 0.4 style - { - "properties": { - "eo:bands": [ - {"name": "foo", "common_name": "F00", "center_wavelength": 0.543}, - {"name": "bar"} - ] - }}, - # API 1.0 style - { - "summaries": { - "eo:bands": [ - {"name": "foo", "common_name": "F00", "center_wavelength": 0.543}, - {"name": "bar"} - ] - }}, -]) +@pytest.mark.parametrize( + "spec", + [ + # API 0.4 style + { + "properties": { + "eo:bands": [{"name": "foo", "common_name": "F00", "center_wavelength": 0.543}, {"name": "bar"}] + } + }, + # API 1.0 style + { + "summaries": { + "eo:bands": [{"name": "foo", "common_name": "F00", "center_wavelength": 0.543}, {"name": "bar"}] + } + }, + ], +) def test_metadata_bands_dimension_eo_bands(spec): metadata = CollectionMetadata(spec) assert metadata.band_dimension.name == "bands" - assert metadata.bands == [ - Band("foo", "F00", 0.543), - Band("bar", None, None) - ] + assert metadata.bands == [Band("foo", "F00", 0.543), Band("bar", None, None)] assert metadata.band_names == ["foo", "bar"] assert metadata.band_common_names == ["F00", None] -@pytest.mark.parametrize("spec", [ - # API 0.4 style - { - "properties": { +@pytest.mark.parametrize( + "spec", + [ + # API 0.4 style + { + "properties": { + "cube:dimensions": { + "x": {"type": "spatial", "axis": "x"}, + "b": {"type": "bands", "values": ["foo", "bar"]}, + }, + "eo:bands": [{"name": "foo", "common_name": "F00", "center_wavelength": 0.543}, {"name": "bar"}], + } + }, + # API 1.0 style + { "cube:dimensions": { "x": {"type": "spatial", "axis": "x"}, - "b": {"type": "bands", "values": ["foo", "bar"]} + "b": {"type": "bands", "values": ["foo", "bar"]}, + }, + "summaries": { + "eo:bands": [{"name": "foo", "common_name": "F00", "center_wavelength": 0.543}, {"name": "bar"}] }, - "eo:bands": [ - {"name": "foo", "common_name": "F00", "center_wavelength": 0.543}, - {"name": "bar"} - ] - } - }, - # API 1.0 style - { - "cube:dimensions": { - "x": {"type": "spatial", "axis": "x"}, - "b": {"type": "bands", "values": ["foo", "bar"]} }, - "summaries": { - "eo:bands": [ - {"name": "foo", "common_name": "F00", "center_wavelength": 0.543}, - {"name": "bar"} - ] - } - }, -]) + ], +) def test_metadata_bands_dimension(spec): metadata = CollectionMetadata(spec) assert metadata.band_dimension.name == "b" - assert metadata.bands == [ - Band("foo", "F00", 0.543), - Band("bar", None, None) - ] + assert metadata.bands == [Band("foo", "F00", 0.543), Band("bar", None, None)] assert metadata.band_names == ["foo", "bar"] assert metadata.band_common_names == ["F00", None] -def test_metadata_reduce_dimension(): - metadata = CollectionMetadata({ - "cube:dimensions": { - "x": {"type": "spatial"}, - "b": {"type": "bands", "values": ["red", "green"]} - } - }) +def test_collectionmetadata_reduce_dimension(): + metadata = CollectionMetadata( + {"cube:dimensions": {"x": {"type": "spatial"}, "b": {"type": "bands", "values": ["red", "green"]}}} + ) reduced = metadata.reduce_dimension("b") assert set(metadata.dimension_names()) == {"x", "b"} assert set(reduced.dimension_names()) == {"x"} -def test_metadata_reduce_dimension_invalid_name(): - metadata = CollectionMetadata({ - "cube:dimensions": { - "x": {"type": "spatial"}, - "b": {"type": "bands", "values": ["red", "green"]} - } - }) +def test_cubemetadata_reduce_dimension(): + metadata = CubeMetadata( + dimensions=[ + SpatialDimension(name="x", extent=None), + BandDimension(name="b", bands=[Band("red"), Band("green")]), + ] + ) + reduced = metadata.reduce_dimension("b") + assert set(metadata.dimension_names()) == {"x", "b"} + assert set(reduced.dimension_names()) == {"x"} + + +def test_collectionmetadata_reduce_dimension_invalid_name(): + metadata = CollectionMetadata( + {"cube:dimensions": {"x": {"type": "spatial"}, "b": {"type": "bands", "values": ["red", "green"]}}} + ) with pytest.raises(ValueError): metadata.reduce_dimension("y") -def test_metadata_add_band_dimension(): - metadata = CollectionMetadata({ - "cube:dimensions": { - "t": {"type": "temporal"} - } - }) +def test_cubemetadata_reduce_dimension_invalid_name(): + metadata = CubeMetadata( + dimensions=[ + SpatialDimension(name="x", extent=None), + BandDimension(name="b", bands=[Band("red"), Band("green")]), + ] + ) + with pytest.raises(ValueError): + metadata.reduce_dimension("y") + + +def test_collectionmetadata_add_band_dimension(): + metadata = CollectionMetadata({"cube:dimensions": {"t": {"type": "temporal"}}}) new = metadata.add_dimension("layer", "red", "bands") assert metadata.dimension_names() == ["t"] @@ -491,23 +607,35 @@ def test_metadata_add_band_dimension(): assert new.band_names == ["red"] -def test_metadata_add_band_dimension_duplicate(): - metadata = CollectionMetadata({ - "cube:dimensions": { - "t": {"type": "temporal"} - } - }) +def test_cubemetadata_add_band_dimension(): + metadata = CubeMetadata(dimensions=[TemporalDimension(name="t", extent=None)]) + new = metadata.add_dimension("layer", "red", "bands") + + assert metadata.dimension_names() == ["t"] + assert not metadata.has_band_dimension() + + assert new.has_band_dimension() + assert new.dimension_names() == ["t", "layer"] + assert new.band_dimension.name == "layer" + assert new.band_names == ["red"] + + +def test_collectionmetadata_add_band_dimension_duplicate(): + metadata = CollectionMetadata({"cube:dimensions": {"t": {"type": "temporal"}}}) metadata = metadata.add_dimension("layer", "red", "bands") with pytest.raises(DimensionAlreadyExistsException, match="Dimension with name 'layer' already exists"): _ = metadata.add_dimension("layer", "red", "bands") -def test_metadata_add_temporal_dimension(): - metadata = CollectionMetadata({ - "cube:dimensions": { - "x": {"type": "spatial"} - } - }) +def test_cubemetadata_add_band_dimension_dublicate(): + metadata = CubeMetadata(dimensions=[TemporalDimension(name="t", extent=None)]) + metadata = metadata.add_dimension("layer", "red", "bands") + with pytest.raises(DimensionAlreadyExistsException, match="Dimension with name 'layer' already exists"): + _ = metadata.add_dimension("layer", "red", "bands") + + +def test_collectionmetadata_add_temporal_dimension(): + metadata = CollectionMetadata({"cube:dimensions": {"x": {"type": "spatial"}}}) new = metadata.add_dimension("date", "2020-05-15", "temporal") assert metadata.dimension_names() == ["x"] @@ -519,24 +647,42 @@ def test_metadata_add_temporal_dimension(): assert new.temporal_dimension.extent == ["2020-05-15", "2020-05-15"] -def test_metadata_add_temporal_dimension_duplicate(): - metadata = CollectionMetadata({ - "cube:dimensions": { - "x": {"type": "spatial"} - } - }) +def test_cubemetadata_add_temporal_dimension(): + metadata = CubeMetadata(dimensions=[SpatialDimension(name="x", extent=None)]) + new = metadata.add_dimension("date", "2020-05-15", "temporal") + + assert metadata.dimension_names() == ["x"] + assert not metadata.has_temporal_dimension() + + assert new.has_temporal_dimension() + assert new.dimension_names() == ["x", "date"] + assert new.temporal_dimension.name == "date" + assert new.temporal_dimension.extent == ["2020-05-15", "2020-05-15"] + + +def test_collectionmetadata_add_temporal_dimension_duplicate(): + metadata = CollectionMetadata({"cube:dimensions": {"x": {"type": "spatial"}}}) metadata = metadata.add_dimension("date", "2020-05-15", "temporal") with pytest.raises(DimensionAlreadyExistsException, match="Dimension with name 'date' already exists"): _ = metadata.add_dimension("date", "2020-05-15", "temporal") -def test_metadata_drop_dimension(): - metadata = CollectionMetadata({ - "cube:dimensions": { - "t": {"type": "temporal"}, - "b": {"type": "bands", "values": ["red", "green"]}, +def test_cubemetadata_add_temporal_dimension_duplicate(): + metadata = CubeMetadata(dimensions=[SpatialDimension(name="x", extent=None)]) + metadata = metadata.add_dimension("date", "2020-05-15", "temporal") + with pytest.raises(DimensionAlreadyExistsException, match="Dimension with name 'date' already exists"): + _ = metadata.add_dimension("date", "2020-05-15", "temporal") + + +def test_collectionmetadata_drop_dimension(): + metadata = CollectionMetadata( + { + "cube:dimensions": { + "t": {"type": "temporal"}, + "b": {"type": "bands", "values": ["red", "green"]}, + } } - }) + ) new = metadata.drop_dimension("t") assert metadata.dimension_names() == ["t", "b"] @@ -552,14 +698,36 @@ def test_metadata_drop_dimension(): metadata.drop_dimension("x") -def test_metadata_subclass(): +def test_cubemetadata_drop_dimension(): + metadata = CubeMetadata( + dimensions=[ + TemporalDimension(name="t", extent=None), + BandDimension(name="b", bands=[Band("red"), Band("green")]), + ] + ) + + new = metadata.drop_dimension("t") + assert metadata.dimension_names() == ["t", "b"] + assert new.dimension_names() == ["b"] + assert new.band_dimension.band_names == ["red", "green"] + + new = metadata.drop_dimension("b") + assert metadata.dimension_names() == ["t", "b"] + assert new.dimension_names() == ["t"] + assert new.temporal_dimension.name == "t" + + with pytest.raises(ValueError): + metadata.drop_dimension("x") + + +def test_collectionmetadata_subclass(): class MyCollectionMetadata(CollectionMetadata): def __init__(self, metadata: dict, dimensions: List[Dimension] = None, bbox=None): super().__init__(metadata=metadata, dimensions=dimensions) self.bbox = bbox def _clone_and_update( - self, metadata: dict = None, dimensions: List[Dimension] = None, bbox=None, **kwargs + self, metadata: dict = None, dimensions: List[Dimension] = None, bbox=None, **kwargs ) -> MyCollectionMetadata: return super()._clone_and_update(metadata=metadata, dimensions=dimensions, bbox=bbox or self.bbox, **kwargs) @@ -583,3 +751,34 @@ def filter_bbox(self, bbox): assert isinstance(new, MyCollectionMetadata) assert orig.bbox is None assert new.bbox == (1, 2, 3, 4) + + +def test_cubemetadata_subclass(): + class MyCubeMetadata(CubeMetadata): + def __init__(self, dimensions: List[Dimension], bbox=None): + super().__init__(dimensions=dimensions) + self.bbox = bbox + + def _clone_and_update(self, dimensions: List[Dimension] = None, bbox=None, **kwargs) -> MyCubeMetadata: + return super()._clone_and_update(dimensions=dimensions, bbox=bbox or self.bbox, **kwargs) + + def filter_bbox(self, bbox): + return self._clone_and_update(bbox=bbox) + + orig = MyCubeMetadata([SpatialDimension(name="x", extent=None)]) + assert orig.bbox is None + + new = orig.add_dimension(name="layer", label="red", type="bands") + assert isinstance(new, MyCubeMetadata) + assert orig.bbox is None + assert new.bbox is None + + new = new.filter_bbox((1, 2, 3, 4)) + assert isinstance(new, MyCubeMetadata) + assert orig.bbox is None + assert new.bbox == (1, 2, 3, 4) + + new = new.add_dimension(name="time", label="2020", type="time") + assert isinstance(new, MyCubeMetadata) + assert orig.bbox is None + assert new.bbox == (1, 2, 3, 4) From 88e4360d5b4f4aade508b0f71d6b457df73e6635 Mon Sep 17 00:00:00 2001 From: Victor Verhaert <33786515+VictorVerhaert@users.noreply.github.com> Date: Thu, 29 Feb 2024 13:31:29 +0100 Subject: [PATCH 08/11] issue #464 added chanhelog entry --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a037b8f3..3598199f2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Changed default `chunk_size` of various `download` functions from None to 10MB. This improves the handling of large downloads and reduces memory usage. ([#528](https://github.com/Open-EO/openeo-python-client/issues/528)) - `Connection.execute()` and `DataCube.execute()` now have a `auto_decode` argument. If set to True (default) the response will be decoded as a JSON and throw an exception if this fails, if set to False the raw `requests.Response` object will be returned. ([#499](https://github.com/Open-EO/openeo-python-client/issues/499)) +- Created superclass `CubeMetadata` for `CollectionMetadata` containing methods to handle basic metadata like dimensions and their labels, without requiring STAC information. ([#464](https://github.com/Open-EO/openeo-python-client/issues/464)) ### Removed ### Fixed From b01197f723252a3b116dc640507476a735c808dc Mon Sep 17 00:00:00 2001 From: Victor Verhaert <33786515+VictorVerhaert@users.noreply.github.com> Date: Thu, 29 Feb 2024 13:34:09 +0100 Subject: [PATCH 09/11] Apply suggestions from code review Co-authored-by: Stefaan Lippens --- openeo/metadata.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openeo/metadata.py b/openeo/metadata.py index 7a1ac4223..dba6b392a 100644 --- a/openeo/metadata.py +++ b/openeo/metadata.py @@ -85,7 +85,7 @@ def rename(self, name) -> Dimension: return TemporalDimension(name=name, extent=self.extent) def rename_labels(self, target, source) -> Dimension: - # TODO should we check if the extend has changed with the new labels? + # TODO should we check if the extent has changed with the new labels? return TemporalDimension(name=self.name, extent=self.extent) @@ -212,7 +212,7 @@ class CubeMetadata: Allows interaction with the cube dimensions and their labels (if available). """ - def __init__(self, dimensions: List[Dimension] = None): + def __init__(self, dimensions: Optional[List[Dimension]] = None): # Original collection metadata (actual cube metadata might be altered through processes) self._dimensions = None self._band_dimension = None @@ -237,7 +237,7 @@ def __init__(self, dimensions: List[Dimension] = None): def __eq__(self, o: Any) -> bool: return isinstance(o, type(self)) and self._dimensions == o._dimensions - def _clone_and_update(self, dimensions: List[Dimension] = None, **kwargs) -> CubeMetadata: + def _clone_and_update(self, dimensions: Optional[List[Dimension]] = None, **kwargs) -> CubeMetadata: """Create a new instance (of same class) with copied/updated fields.""" cls = type(self) if dimensions is None: From a48552ef7fb500adb627830c295c87ffca09d4a0 Mon Sep 17 00:00:00 2001 From: Victor Verhaert <33786515+VictorVerhaert@users.noreply.github.com> Date: Thu, 29 Feb 2024 13:40:05 +0100 Subject: [PATCH 10/11] small changes to metadata.py --- openeo/metadata.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/openeo/metadata.py b/openeo/metadata.py index dba6b392a..31d97513f 100644 --- a/openeo/metadata.py +++ b/openeo/metadata.py @@ -214,12 +214,11 @@ class CubeMetadata: def __init__(self, dimensions: Optional[List[Dimension]] = None): # Original collection metadata (actual cube metadata might be altered through processes) - self._dimensions = None + self._dimensions = dimensions self._band_dimension = None self._temporal_dimension = None if dimensions is not None: - self._dimensions = dimensions for dim in self._dimensions: # TODO: here we blindly pick last bands or temporal dimension if multiple. Let user choose? # TODO: add spacial dimension handling? From 3f8fb048702205dc06a07490ce28efe3b1dc2361 Mon Sep 17 00:00:00 2001 From: Stefaan Lippens Date: Thu, 29 Feb 2024 16:08:11 +0100 Subject: [PATCH 11/11] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3598199f2..e3ff0b84c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed - Changed default `chunk_size` of various `download` functions from None to 10MB. This improves the handling of large downloads and reduces memory usage. ([#528](https://github.com/Open-EO/openeo-python-client/issues/528)) - `Connection.execute()` and `DataCube.execute()` now have a `auto_decode` argument. If set to True (default) the response will be decoded as a JSON and throw an exception if this fails, if set to False the raw `requests.Response` object will be returned. ([#499](https://github.com/Open-EO/openeo-python-client/issues/499)) -- Created superclass `CubeMetadata` for `CollectionMetadata` containing methods to handle basic metadata like dimensions and their labels, without requiring STAC information. ([#464](https://github.com/Open-EO/openeo-python-client/issues/464)) +- Introduced superclass `CubeMetadata` for `CollectionMetadata` for essential metadata handling (just dimensions for now) without collection-specific STAC metadata parsing. ([#464](https://github.com/Open-EO/openeo-python-client/issues/464)) ### Removed ### Fixed