Merge pull request #24 from ecmwf/develop

Develop

example_config.json

@@ -1 +1,106 @@
-{"datacube": {"type": "gribjump", "config": "config.yaml", "uri": "http://localhost:8001"}, "options": {"axis_config": [{"axis_name": "date", "transformations": [{"name": "merge", "other_axis": "time", "linkers": ["T", "00"]}]}, {"axis_name": "values", "transformations": [{"name": "mapper", "type": "octahedral", "resolution": 1280, "axes": ["latitude", "longitude"], "local": null}]}, {"axis_name": "latitude", "transformations": [{"name": "reverse", "is_reverse": true}]}, {"axis_name": "longitude", "transformations": [{"name": "cyclic", "range": [0.0, 360.0]}]}, {"axis_name": "step", "transformations": [{"name": "type_change", "type": "int"}]}, {"axis_name": "number", "transformations": [{"name": "type_change", "type": "int"}]}], "compressed_axes_config": ["longitude", "latitude", "levtype", "step", "date", "domain", "expver", "param", "class", "stream", "type"], "pre_path": {"class": "od", "expver": "0001", "levtype": "sfc", "stream": "oper", "type": "fc"}, "alternative_axes": []}, "coverageconfig": {"param_db": "ecmwf"}}
+{
+    "datacube": {
+        "type": "gribjump",
+        "config": "config.yaml",
+        "uri": "http://localhost:8001"
+    },
+    "options": {
+        "axis_config": [
+            {
+                "axis_name": "date",
+                "transformations": [
+                    {
+                        "name": "merge",
+                        "other_axis": "time",
+                        "linkers": [
+                            "T",
+                            "00"
+                        ]
+                    }
+                ]
+            },
+            {
+                "axis_name": "values",
+                "transformations": [
+                    {
+                        "name": "mapper",
+                        "type": "octahedral",
+                        "resolution": 1280,
+                        "axes": [
+                            "latitude",
+                            "longitude"
+                        ],
+                        "local": null
+                    }
+                ]
+            },
+            {
+                "axis_name": "latitude",
+                "transformations": [
+                    {
+                        "name": "reverse",
+                        "is_reverse": true
+                    }
+                ]
+            },
+            {
+                "axis_name": "longitude",
+                "transformations": [
+                    {
+                        "name": "cyclic",
+                        "range": [
+                            0.0,
+                            360.0
+                        ]
+                    }
+                ]
+            },
+            {
+                "axis_name": "step",
+                "transformations": [
+                    {
+                        "name": "type_change",
+                        "type": "int"
+                    }
+                ]
+            },
+            {
+                "axis_name": "number",
+                "transformations": [
+                    {
+                        "name": "type_change",
+                        "type": "int"
+                    }
+                ]
+            }
+        ],
+        "compressed_axes_config": [
+            "longitude",
+            "latitude",
+            "levtype",
+            "step",
+            "date",
+            "domain",
+            "expver",
+            "param",
+            "class",
+            "stream",
+            "type"
+        ],
+        "pre_path": {
+            "class": "od",
+            "expver": "0001",
+            "levtype": "sfc",
+            "stream": "oper",
+            "type": "fc"
+        },
+        "alternative_axes": []
+    },
+    "coverageconfig": {
+        "param_db": "ecmwf"
+    },
+    "polygonrules": {
+        "max_points": 1000,
+        "max_polygons": 1000.0
+    }
+}

polytope_mars/api.py

@@ -15,10 +15,10 @@
 from .features.boundingbox import BoundingBox
 from .features.frame import Frame
 from .features.path import Path
+from .features.polygon import Polygons
 from .features.shpfile import Shapefile
 from .features.timeseries import TimeSeries
 from .features.verticalprofile import VerticalProfile
-from .features.wkt import Wkt
 
 features = {
     "timeseries": TimeSeries,
@@ -27,7 +27,7 @@
     "frame": Frame,
     "path": Path,
     "shapefile": Shapefile,
-    "polygon": Wkt,
+    "polygon": Polygons,
 }
 
 
@@ -56,6 +56,8 @@ def __init__(self, config=None, log_context=None):
 
     def extract(self, request):
         # request expected in JSON or dict
+        start = time.time()
+        logging.debug(f"{self.id}: Gribjump/setup time start: {start}")  # noqa: E501
         if not isinstance(request, dict):
             try:
                 request = json.loads(request)
@@ -81,7 +83,9 @@ def extract(self, request):
         else:
             timeseries_type = None
 
-        feature = self._feature_factory(feature_type, feature_config)
+        feature = self._feature_factory(
+            feature_type, feature_config, self.conf
+        )  # noqa: E501
 
         feature.validate(request)
 
@@ -104,14 +108,20 @@ def extract(self, request):
             options=self.conf.options.model_dump(),
         )
 
+        end = time.time()
+        delta = end - start
+        logging.debug(f"{self.id}: Gribjump/setup time start: {end}")  # noqa: E501
+        logging.debug(f"{self.id}: Gribjump/setup time start: {delta}")  # noqa: E501
+
         logging.debug(
             f"{self.id}: The request we give polytope from polytope-mars is: {preq}"  # noqa: E501
         )
         start = time.time()
         logging.debug(f"{self.id}: Polytope time start: {start}")  # noqa: E501
 
         if self.log_context:
-            result = self.api.retrieve(preq, self.log_context)
+            logging.debug(f"Send log_context to polytope: {self.log_context}")
+            result = self.api.retrieve(preq, context=self.log_context)
         else:
             result = self.api.retrieve(preq)
 
@@ -120,7 +130,7 @@ def extract(self, request):
         logging.debug(f"{self.id}: Polytope time end: {end}")  # noqa: E501
         logging.debug(f"{self.id}: Polytope time taken: {delta}")  # noqa: E501
         start = time.time()
-        logging.debug(f"{self.id}: Polytope time start: {start}")  # noqa: E501
+        logging.debug(f"{self.id}: Covjson time start: {start}")  # noqa: E501
         encoder = Covjsonkit(self.conf.coverageconfig.model_dump()).encode(
             "CoverageCollection", feature_type
         )  # noqa: E501
@@ -150,10 +160,6 @@ def _create_base_shapes(self, request: dict) -> List[shapes.Shape]:
             raise NotImplementedError(
                 "Currently only one time is supported"
             )  # noqa: E501
-        # if str(time).split("/") != 1:
-        #   time = str(time).split("/")
-        # else:
-        #   time = [time]
 
         # TODO: not restricting certain keywords:
         #   * AREA, GRID
@@ -204,9 +210,9 @@ def _create_base_shapes(self, request: dict) -> List[shapes.Shape]:
 
         return base_shapes
 
-    def _feature_factory(self, feature_name, feature_config):
+    def _feature_factory(self, feature_name, feature_config, config=None):
         feature_class = features.get(feature_name)
         if feature_class:
-            return feature_class(feature_config)
+            return feature_class(feature_config, config)
         else:
             raise NotImplementedError(f"Feature '{feature_name}' not found")

polytope_mars/config.py

@@ -13,8 +13,18 @@ class CovjsonKitConfig(ConfigModel):
     param_db: str = "ecmwf"
 
 
+class PolygonRulesConfig(ConfigModel):
+    # Max points is the max number of points in all polygons requested allowed
+    max_points: int = 1000
+    # Max area is the max area of all polygons requested that is allowed.
+    # Area is in abstract units as a projection would otherwise be required
+    # to calculate the area
+    max_area: float = 1000.0
+
+
 class PolytopeMarsConfig(ConfigModel):
 
     datacube: DatacubeConfig = DatacubeConfig()
     options: Config = Config()
     coverageconfig: CovjsonKitConfig = CovjsonKitConfig()
+    polygonrules: PolygonRulesConfig = PolygonRulesConfig()

polytope_mars/features/boundingbox.py

@@ -4,11 +4,13 @@
 
 
 class BoundingBox(Feature):
-    def __init__(self, config):
-        assert config.pop("type") == "boundingbox"
-        self.points = config.pop("points", [])
+    def __init__(self, feature_config, client_config):
+        assert feature_config.pop("type") == "boundingbox"
+        self.points = feature_config.pop("points", [])
 
-        assert len(config) == 0, f"Unexpected keys in config: {config.keys()}"
+        assert (
+            len(feature_config) == 0
+        ), f"Unexpected keys in config: {feature_config.keys()}"
 
     def get_shapes(self):
         # Time-series is a squashed box from start_step to start_end for each point  # noqa: E501

polytope_mars/features/frame.py

@@ -4,13 +4,15 @@
 
 
 class Frame(Feature):
-    def __init__(self, config):
-        assert config.pop("type") == "frame"
-        self.points = config.pop("points", [])
-        self.outer_box = config.pop("outer_box", [])
-        self.inner_box = config.pop("inner_box", [])
-
-        assert len(config) == 0, f"Unexpected keys in config: {config.keys()}"
+    def __init__(self, feature_config, client_config):
+        assert feature_config.pop("type") == "frame"
+        self.points = feature_config.pop("points", [])
+        self.outer_box = feature_config.pop("outer_box", [])
+        self.inner_box = feature_config.pop("inner_box", [])
+
+        assert (
+            len(feature_config) == 0
+        ), f"Unexpected keys in config: {feature_config.keys()}"
 
     def get_shapes(self):
         # frame is a four seperate boxes requested based on the inner and outer boxes  # noqa: E501

polytope_mars/features/path.py

@@ -4,12 +4,14 @@
 
 
 class Path(Feature):
-    def __init__(self, config):
-        assert config.pop("type") == "path"
-        self.points = config.pop("points", [])
-        self.padding = config.pop("padding")
-
-        assert len(config) == 0, f"Unexpected keys in config: {config.keys()}"
+    def __init__(self, feature_config, client_config):
+        assert feature_config.pop("type") == "path"
+        self.points = feature_config.pop("points", [])
+        self.padding = feature_config.pop("padding")
+
+        assert (
+            len(feature_config) == 0
+        ), f"Unexpected keys in config: {feature_config.keys()}"
 
     def get_shapes(self):
         # Time-series is a squashed box from start_step to start_end for each point  # noqa: E501

polytope_mars/features/polygon.py

@@ -0,0 +1,67 @@
+from polytope import shapes
+from shapely.geometry import Polygon
+
+from ..feature import Feature
+
+
+def get_area(points):
+    x = []
+    y = []
+    for point in points:
+        x.append(point[0])
+        y.append(point[1])
+    pgon = Polygon(zip(x, y))
+    return pgon.area
+
+
+class Polygons(Feature):
+    def __init__(self, feature_config, client_config):
+        assert feature_config.pop("type") == "polygon"
+        self.shape = feature_config.pop("shape")
+        if type(self.shape[0][0]) is not list:
+            if len(self.shape) > client_config.polygonrules.max_points:
+                raise ValueError(
+                    f"Number of points {len(self.shape)} exceeds the maximum of {client_config.polygonrules.max_points}"  # noqa: E501
+                )
+            if get_area(self.shape) > client_config.polygonrules.max_area:
+                raise ValueError(
+                    f"Area of polygon {get_area(self.shape)} exceeds the maximum of {client_config.polygonrules.max_area}"  # noqa: E501
+                )
+            self.shape = [self.shape]
+        else:
+            len_polygons = 0
+            area_polygons = 0
+            for polygon in self.shape:
+                len_polygons += len(polygon)
+                area_polygons += get_area(polygon)
+            if len_polygons > client_config.polygonrules.max_points:
+                raise ValueError(
+                    f"Number of points {len_polygons} exceeds the maximum of {client_config.polygonrules.max_points}"  # noqa: E501
+                )
+            if area_polygons > client_config.polygonrules.max_area:
+                raise ValueError(
+                    f"Area of polygon {area_polygons} exceeds the maximum of {client_config.polygonrules.max_area}"  # noqa: E501
+                )
+
+        assert (
+            len(feature_config) == 0
+        ), f"Unexpected keys in config: {feature_config.keys()}"
+
+    def get_shapes(self):
+        # coordinates = get_coords(self.df)
+        polygons = []
+        for polygon in self.shape:
+            points = []
+            for point in polygon:
+                points.append([point[0], point[1]])
+            polygons.append(shapes.Polygon(["latitude", "longitude"], points))
+        return [shapes.Union(["latitude", "longitude"], *polygons)]
+
+    def incompatible_keys(self):
+        return []
+
+    def coverage_type(self):
+        return "MultiPoint"
+
+    def name(self):
+        return "Polygon"

polytope_mars/features/shpfile.py

@@ -22,12 +22,14 @@ def get_coords(geom):
 
 
 class Shapefile(Feature):
-    def __init__(self, config):
-        assert config.pop("type") == "shapefile"
-        self.file = config.pop("file")
+    def __init__(self, feature_config, client_config):
+        assert feature_config.pop("type") == "shapefile"
+        self.file = feature_config.pop("file")
         self.df = gpd.read_file(self.file)
 
-        assert len(config) == 0, f"Unexpected keys in config: {config.keys()}"
+        assert (
+            len(feature_config) == 0
+        ), f"Unexpected keys in config: {feature_config.keys()}"
 
     def get_shapes(self):
         self.df = self.df.head(1)

polytope_mars/features/timeseries.py

@@ -4,15 +4,17 @@
 
 
 class TimeSeries(Feature):
-    def __init__(self, config):
-        assert config.pop("type") == "timeseries"
+    def __init__(self, feature_config, client_config):
+        assert feature_config.pop("type") == "timeseries"
         # self.start_step = config.pop("start", None)
         # self.end_step = config.pop("end", None)
-        self.axis = config.pop("axis", [])
+        self.axis = feature_config.pop("axis", [])
 
-        self.points = config.pop("points", [])
+        self.points = feature_config.pop("points", [])
 
-        assert len(config) == 0, f"Unexpected keys in config: {config.keys()}"
+        assert (
+            len(feature_config) == 0
+        ), f"Unexpected keys in config: {feature_config.keys()}"
 
     def get_shapes(self):
         # Time-series is a squashed box from start_step to start_end for each point  # noqa: E501