diff --git a/README.md b/README.md index a463477..62ad76d 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ pip install -e . ## Example -**Create time series request**: Create a request for a time series request using the time series feature, set options for use by polytope feature extraction. "grib" indicates the type of data in this case. NB: Assumes data is in a local FDB. +**Create time series request**: Create a request for a time series request using the time series feature, set options for use by polytope feature extraction. "gribjump" indicates the type of data in this case. NB: Assumes data is in a local FDB or or environment variables have been set up to point to a gribjump server. ```python from polytope_mars.api import PolytopeMars @@ -89,6 +89,40 @@ If the user provides no arguments to PolytopeMars then a config is loaded from t The user can also pass in a config as a python dictionary to PolytopeMars for a custom config at runtime. +```python +from polytope_mars.api import PolytopeMars +from conflator import Conflator +from polytope_mars.config import PolytopeMarsConfig + +conf = Conflator(app_name="polytope_mars", model=PolytopeMarsConfig).load() +cf = conf.model_dump() +cf["options"] = options + +request = { + "class": "od", + "stream" : "enfo", + "type" : "pf", + "date" : "20231205", + "time" : "00:00:00", + "levtype" : "sfc", + "expver" : "0001", + "domain" : "g", + "param" : "228/49/164/165/166/167", + "number" : "1/to/5", + "step" : "0/1" + "feature" : { + "type" : "timeseries", + "points" : [[51.5073219, 2.1]], + "axis" : "step", + }, +} + +result = PolytopeMars(cf).extract(request) + +``` + +A context dictionary can also be passed to PolytopeMars for logging. + Result will be a coverageJSON file with the requested data if it is available, further manipulation of the coverage can be made using [covjsonkit](https://github.com/ecmwf/covjsonkit). ### Config diff --git a/polytope_mars/api.py b/polytope_mars/api.py index bd6c8e7..be693bf 100644 --- a/polytope_mars/api.py +++ b/polytope_mars/api.py @@ -1,5 +1,6 @@ import json import logging +import time from typing import List import pandas as pd @@ -31,17 +32,25 @@ class PolytopeMars: - def __init__(self, config=None): + def __init__(self, config=None, log_context=None): # Initialise polytope-mars configuration + self.log_context = log_context + self.id = log_context["id"] if log_context else "-1" # If no config check default locations if config is None: self.conf = Conflator( app_name="polytope_mars", model=PolytopeMarsConfig ).load() + logging.debug( + f"{self.id}: Config loaded from file: {self.conf}" + ) # noqa: E501 # else initialise with provided config else: self.conf = PolytopeMarsConfig.model_validate(config) + logging.debug( + f"{self.id}: Config loaded from dictionary: {self.conf}" + ) # noqa: E501 self.coverage = {} @@ -94,10 +103,24 @@ def extract(self, request): engine=slicer, options=self.conf.options.model_dump(), ) + logging.debug( - "The request we give polytope from polytope-mars are: %s", preq - ) # noqa: E501 - result = self.api.retrieve(preq) + f"{self.id}: The request we give polytope from polytope-mars is: {preq}" # noqa: E501 + ) + start = time.time() + logging.debug(f"{self.id}: Polytope time start: {start}") # noqa: E501 + + if self.log_context: + result = self.api.retrieve(preq, self.log_context) + else: + result = self.api.retrieve(preq) + + end = time.time() + delta = end - start + logging.debug(f"{self.id}: Polytope time end: {end}") # noqa: E501 + logging.debug(f"{self.id}: Polytope time taken: {delta}") # noqa: E501 + start = time.time() + logging.debug(f"{self.id}: Polytope time start: {start}") # noqa: E501 encoder = Covjsonkit(self.conf.coverageconfig.model_dump()).encode( "CoverageCollection", feature_type ) # noqa: E501 @@ -107,6 +130,11 @@ def extract(self, request): else: self.coverage = encoder.from_polytope(result) + end = time.time() + delta = end - start + logging.debug(f"{self.id}: Covjsonkit time end: {end}") # noqa: E501 + logging.debug(f"{self.id}: Covjsonkit time taken: {delta}") # noqa: E501 + return self.coverage def _create_base_shapes(self, request: dict) -> List[shapes.Shape]: @@ -118,6 +146,10 @@ def _create_base_shapes(self, request: dict) -> List[shapes.Shape]: # * enforcing strings are actually strings (e.g. type=fc) time = request.pop("time").replace(":", "") + if len(time.split("/")) != 1: + raise NotImplementedError( + "Currently only one time is supported" + ) # noqa: E501 # if str(time).split("/") != 1: # time = str(time).split("/") # else: @@ -142,12 +174,19 @@ def _create_base_shapes(self, request: dict) -> List[shapes.Shape]: # Range a/to/b, "by" not supported -> Span elif len(split) == 3 and split[1] == "to": + # if date then only get time of dates in span not + # all in times within date if k == "date": - split[0] = pd.Timestamp(split[0] + "T" + time) - split[2] = pd.Timestamp(split[2] + "T" + time) - base_shapes.append( - shapes.Span(k, lower=split[0], upper=split[2]) - ) # noqa: E501 + start = pd.Timestamp(split[0] + "T" + time) + end = pd.Timestamp(split[2] + "T" + time) + dates = [] + for s in pd.date_range(start, end): + dates.append(s) + base_shapes.append(shapes.Select(k, dates)) + else: + base_shapes.append( + shapes.Span(k, lower=split[0], upper=split[2]) + ) # noqa: E501 elif "by" in split: raise ValueError( diff --git a/tests/performance/test_bounding_box.py b/tests/performance/test_bounding_box.py new file mode 100644 index 0000000..dd6291a --- /dev/null +++ b/tests/performance/test_bounding_box.py @@ -0,0 +1,114 @@ +import pytest +from conflator import Conflator +from covjsonkit.api import Covjsonkit + +from polytope_mars.api import PolytopeMars +from polytope_mars.config import PolytopeMarsConfig + +# os.environ['DYLD_LIBRARY_PATH'] = + + +class TestBoundingBox: + def setup_method(self): + self.options = { + "axis_config": [ + { + "axis_name": "date", + "transformations": [ + { + "name": "merge", + "other_axis": "time", + "linkers": ["T", "00"], + } # noqa: E501 + ], + }, + { + "axis_name": "values", + "transformations": [ + { + "name": "mapper", + "type": "octahedral", + "resolution": 1280, + "axes": ["latitude", "longitude"], + } + ], + }, + { + "axis_name": "latitude", + "transformations": [ + {"name": "reverse", "is_reverse": True} + ], # noqa: E501 + }, + { + "axis_name": "longitude", + "transformations": [{"name": "cyclic", "range": [0, 360]}], + }, + { + "axis_name": "step", + "transformations": [ + {"name": "type_change", "type": "int"} + ], # noqa: E501 + }, + { + "axis_name": "number", + "transformations": [ + {"name": "type_change", "type": "int"} + ], # noqa: E501 + }, + ], + "compressed_axes_config": [ + "longitude", + "latitude", + "levtype", + "step", + "date", + "domain", + "expver", + "param", + "class", + "stream", + "type", + ], + "pre_path": { + "class": "od", + "expver": "0079", + "levtype": "sfc", + "stream": "oper", + "type": "fc", + }, + } + + self.request = { + "class": "od", + "stream": "oper", + "type": "fc", + "date": "20240915", + "time": "1200", + "levtype": "sfc", + "expver": "0079", + "domain": "g", + "param": "167/169", + "step": "0", + "feature": { + "type": "boundingbox", + "points": [[38, -9.5], [39, -8.5]], + }, + } + + conf = Conflator( + app_name="polytope_mars", model=PolytopeMarsConfig + ).load() # noqa: E501 + self.cf = conf.model_dump() + self.cf["options"] = self.options + + @pytest.mark.skip(reason="Gribjump not set up for ci actions yet") + def test_basic_boundingbox(self): + polytope_mars = PolytopeMars(self.cf) + result = polytope_mars.extract(self.request) + assert result is not None + + decoder = Covjsonkit().decode(result) + ds = decoder.to_xarray() + assert ds is not None + assert len(ds["datetimes"]) == 1 diff --git a/tests/performance/test_timeseries.py b/tests/performance/test_timeseries.py new file mode 100644 index 0000000..f15328f --- /dev/null +++ b/tests/performance/test_timeseries.py @@ -0,0 +1,115 @@ +import pytest +from conflator import Conflator +from covjsonkit.api import Covjsonkit + +from polytope_mars.api import PolytopeMars +from polytope_mars.config import PolytopeMarsConfig + +# os.environ['DYLD_LIBRARY_PATH'] = + + +class TestTimeSeries: + def setup_method(self): + self.options = { + "axis_config": [ + { + "axis_name": "date", + "transformations": [ + { + "name": "merge", + "other_axis": "time", + "linkers": ["T", "00"], + } # noqa: E501 + ], + }, + { + "axis_name": "values", + "transformations": [ + { + "name": "mapper", + "type": "octahedral", + "resolution": 1280, + "axes": ["latitude", "longitude"], + } + ], + }, + { + "axis_name": "latitude", + "transformations": [ + {"name": "reverse", "is_reverse": True} + ], # noqa: E501 + }, + { + "axis_name": "longitude", + "transformations": [{"name": "cyclic", "range": [0, 360]}], + }, + { + "axis_name": "step", + "transformations": [ + {"name": "type_change", "type": "int"} + ], # noqa: E501 + }, + { + "axis_name": "number", + "transformations": [ + {"name": "type_change", "type": "int"} + ], # noqa: E501 + }, + ], + "compressed_axes_config": [ + "longitude", + "latitude", + "levtype", + "step", + "date", + "domain", + "expver", + "param", + "class", + "stream", + "type", + ], + "pre_path": { + "class": "od", + "expver": "0079", + "levtype": "sfc", + "stream": "oper", + "type": "fc", + }, + } + + self.request = { + "class": "od", + "stream": "oper", + "type": "fc", + "date": "20240915", + "time": "0000", + "levtype": "sfc", + "expver": "0079", + "domain": "g", + "param": "167/168/169", + "step": "0/to/10", + "feature": { + "type": "timeseries", + "points": [[38, -9.5]], + "axis": "step", + }, + } + + conf = Conflator( + app_name="polytope_mars", model=PolytopeMarsConfig + ).load() # noqa: E501 + self.cf = conf.model_dump() + self.cf["options"] = self.options + + @pytest.mark.skip(reason="Gribjump not set up for ci actions yet") + def test_basic_timeseries(self): + polytope_mars = PolytopeMars(self.cf) + result = polytope_mars.extract(self.request) + assert result is not None + + decoder = Covjsonkit().decode(result) + ds = decoder.to_xarray() + assert ds is not None + assert len(ds["datetime"]) == 1 diff --git a/tests/performance/test_vertical_profile.py b/tests/performance/test_vertical_profile.py new file mode 100644 index 0000000..63e93ee --- /dev/null +++ b/tests/performance/test_vertical_profile.py @@ -0,0 +1,123 @@ +import pytest +from conflator import Conflator +from covjsonkit.api import Covjsonkit + +from polytope_mars.api import PolytopeMars +from polytope_mars.config import PolytopeMarsConfig + +# os.environ['DYLD_LIBRARY_PATH'] = + + +class TestVerticalProfile: + def setup_method(self): + self.options = { + "axis_config": [ + { + "axis_name": "date", + "transformations": [ + { + "name": "merge", + "other_axis": "time", + "linkers": ["T", "00"], + } # noqa: E501 + ], + }, + { + "axis_name": "values", + "transformations": [ + { + "name": "mapper", + "type": "octahedral", + "resolution": 1280, + "axes": ["latitude", "longitude"], + } + ], + }, + { + "axis_name": "latitude", + "transformations": [ + {"name": "reverse", "is_reverse": True} + ], # noqa: E501 + }, + { + "axis_name": "longitude", + "transformations": [{"name": "cyclic", "range": [0, 360]}], + }, + { + "axis_name": "step", + "transformations": [ + {"name": "type_change", "type": "int"} + ], # noqa: E501 + }, + { + "axis_name": "number", + "transformations": [ + {"name": "type_change", "type": "int"} + ], # noqa: E501 + }, + { + "axis_name": "levelist", + "transformations": [ + {"name": "type_change", "type": "int"} + ], # noqa: E501 + }, + ], + "compressed_axes_config": [ + "longitude", + "latitude", + "levtype", + "levelist", + "step", + "date", + "domain", + "expver", + "param", + "class", + "stream", + "type", + "number", + ], + "pre_path": { + "class": "od", + "expver": "0079", + "levtype": "pl", + "stream": "enfo", + "type": "pf", + }, + } + + self.request = { + "class": "od", + "stream": "enfo", + "type": "pf", + "date": "20240915", + "time": "0000", + "levtype": "pl", + "expver": "0079", + "domain": "g", + "param": "203/133", + "number": "1", + "step": "0", + "levelist": "1/to/1000", + "feature": { + "type": "verticalprofile", + "points": [[38.9, -9.1]], + }, + } + + conf = Conflator( + app_name="polytope_mars", model=PolytopeMarsConfig + ).load() # noqa: E501 + self.cf = conf.model_dump() + self.cf["options"] = self.options + + @pytest.mark.skip(reason="Gribjump not set up for ci actions yet") + def test_basic_verticalprofile(self): + polytope_mars = PolytopeMars(self.cf) + result = polytope_mars.extract(self.request) + assert result is not None + + decoder = Covjsonkit().decode(result) + ds = decoder.to_xarray() + assert ds is not None