Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Issue527 load stac metadata #548

Merged
merged 13 commits into from
Mar 25, 2024
Merged
Show file tree
Hide file tree
Changes from 8 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 59 additions & 0 deletions openeo/metadata.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from __future__ import annotations

import logging
import pystac
VincentVerelst marked this conversation as resolved.
Show resolved Hide resolved
import warnings
from typing import Any, Callable, List, NamedTuple, Optional, Tuple, Union

Expand Down Expand Up @@ -522,3 +523,61 @@ def _repr_html_(self):
def __str__(self) -> str:
bands = self.band_names if self.has_band_dimension() else "no bands dimension"
return f"CollectionMetadata({self.extent} - {bands} - {self.dimension_names()})"


def metadata_from_stac(url: str) -> CubeMetadata:
"""
Reads the band metadata a static STAC catalog or a STAC API Collection and returns it as a :py:class:`CubeMetadata`

:param url: The URL to a static STAC catalog (STAC Item, STAC Collection, or STAC Catalog) or a specific STAC API Collection
:return: A :py:class:`CubeMetadata` containing the DataCube band metadata from the url.
"""

def get_band_names(asst: pystac.Asset) -> List[Band]:
VincentVerelst marked this conversation as resolved.
Show resolved Hide resolved
return [Band(eo_band["name"]) for eo_band in asst.extra_fields["eo:bands"]]
VincentVerelst marked this conversation as resolved.
Show resolved Hide resolved

def is_band_asset(asset: pystac.Asset) -> bool:
return "eo:bands" in asset.extra_fields

stac_object = pystac.read_file(href=url)

band_names = []
VincentVerelst marked this conversation as resolved.
Show resolved Hide resolved
collection = None

if isinstance(stac_object, pystac.Item):
item = stac_object
if "eo:bands" in item.properties:
eo_bands_location = item.properties
elif item.get_collection() is not None:
collection = item.get_collection()
eo_bands_location = item.get_collection().summaries.lists
else:
eo_bands_location = {}
band_names = [Band(b["name"]) for b in eo_bands_location.get("eo:bands", [])]

elif isinstance(stac_object, pystac.Collection):
collection = stac_object
band_names = [Band(b["name"]) for b in collection.summaries.lists.get("eo:bands", [])]

# Summaries is not a required field in a STAC collection, so also check the assets
for itm in collection.get_items():
band_assets = {
asset_id: asset
for asset_id, asset in dict(sorted(itm.get_assets().items())).items()
VincentVerelst marked this conversation as resolved.
Show resolved Hide resolved
if is_band_asset(asset)
}

for asset in band_assets.values():
asset_band_names = get_band_names(asset)
for asset_band_name in asset_band_names:
if asset_band_name not in band_names:
band_names.append(asset_band_name)

else:
assert isinstance(stac_object, pystac.Catalog)
catalog = stac_object
band_names = [Band(b["name"]) for b in catalog.extra_fields.get("summaries", {}).get("eo:bands", [])]

band_dimension = BandDimension(name="bands", bands=band_names)
metadata = CubeMetadata(dimensions=[band_dimension])
return metadata
13 changes: 12 additions & 1 deletion openeo/rest/connection.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,14 @@
from openeo.internal.jupyter import VisualDict, VisualList
from openeo.internal.processes.builder import ProcessBuilderBase
from openeo.internal.warnings import deprecated, legacy_alias
from openeo.metadata import Band, BandDimension, CollectionMetadata, SpatialDimension, TemporalDimension
from openeo.metadata import (
Band,
BandDimension,
CollectionMetadata,
SpatialDimension,
TemporalDimension,
metadata_from_stac,
)
from openeo.rest import (
CapabilitiesException,
OpenEoApiError,
Expand Down Expand Up @@ -1361,6 +1368,10 @@ def load_stac(
prop: build_child_callback(pred, parent_parameters=["value"]) for prop, pred in properties.items()
}
cube = self.datacube_from_process(process_id="load_stac", **arguments)
try:
cube.metadata = metadata_from_stac(url)
except Exception:
_log.warning("Python client could not read band metadata from URL.")
VincentVerelst marked this conversation as resolved.
Show resolved Hide resolved
return cube

def load_ml_model(self, id: Union[str, BatchJob]) -> MlModel:
Expand Down
1 change: 1 addition & 0 deletions setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
"flake8>=5.0.0",
"time_machine",
"pyproj>=3.2.0", # Pyproj is an optional, best-effort runtime dependency
"pystac"
VincentVerelst marked this conversation as resolved.
Show resolved Hide resolved
]

docs_require = [
Expand Down
49 changes: 49 additions & 0 deletions tests/test_metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

from typing import List

import json
import pytest

from openeo.metadata import (
Expand All @@ -14,6 +15,7 @@
MetadataException,
SpatialDimension,
TemporalDimension,
metadata_from_stac,
)


Expand Down Expand Up @@ -782,3 +784,50 @@ def filter_bbox(self, bbox):
assert isinstance(new, MyCubeMetadata)
assert orig.bbox is None
assert new.bbox == (1, 2, 3, 4)


collection_json = {
VincentVerelst marked this conversation as resolved.
Show resolved Hide resolved
"type": "Collection",
"id": "test-collection",
"stac_version": "1.0.0",
"description": "Test collection",
"links": [],
"title": "Test Collection",
"extent": {
"spatial": {"bbox": [[-180.0, -90.0, 180.0, 90.0]]},
"temporal": {"interval": [["2020-01-01T00:00:00Z", "2020-01-10T00:00:00Z"]]},
},
"license": "proprietary",
"summaries": {"eo:bands": [{"name": "B01"}, {"name": "B02"}]},
}

catalog_json = {
"type": "Catalog",
"id": "test-catalog",
"stac_version": "1.0.0",
"description": "Test Catalog",
"links": [],
}

item_json = {
"type": "Feature",
"stac_version": "1.0.0",
"id": "test-item",
"properties": {"datetime": "2020-05-22T00:00:00Z", "eo:bands": [{"name": "SCL"}, {"name": "B08"}]},
"geometry": {"coordinates": [[[0, 0], [0, 1], [1, 1], [1, 0], [0, 0]]], "type": "Polygon"},
"links": [],
"assets": {},
"bbox": [0, 1, 0, 1],
"stac_extensions": [],
}


@pytest.mark.parametrize(
"test_stac, expected", [(collection_json, ["B01", "B02"]), (catalog_json, []), (item_json, ["SCL", "B08"])]
)
def test_metadata_from_stac(tmp_path, test_stac, expected):

path = tmp_path / "stac.json"
path.write_text(json.dumps(test_stac))
metadata = metadata_from_stac(path)
assert metadata.band_names == expected
Loading