From f54bbfc86fe47c4f0ebec01c89b9060a10a4160a Mon Sep 17 00:00:00 2001 From: awarde96 Date: Tue, 17 Sep 2024 09:58:28 +0000 Subject: [PATCH 1/5] Update api with context, fix span of dates to include a single time --- polytope_mars/api.py | 45 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 7 deletions(-) diff --git a/polytope_mars/api.py b/polytope_mars/api.py index bd6c8e7..5fe8e62 100644 --- a/polytope_mars/api.py +++ b/polytope_mars/api.py @@ -1,5 +1,6 @@ import json import logging +import time from typing import List import pandas as pd @@ -31,7 +32,7 @@ class PolytopeMars: - def __init__(self, config=None): + def __init__(self, config=None, log_context=None): # Initialise polytope-mars configuration # If no config check default locations @@ -39,11 +40,14 @@ def __init__(self, config=None): self.conf = Conflator( app_name="polytope_mars", model=PolytopeMarsConfig ).load() + logging.debug("Config loaded from file: %s", self.conf) # noqa: E501 # else initialise with provided config else: self.conf = PolytopeMarsConfig.model_validate(config) + logging.debug("Config loaded from dictionary: %s", self.conf) # noqa: E501 self.coverage = {} + self.log_context = log_context def extract(self, request): # request expected in JSON or dict @@ -94,10 +98,24 @@ def extract(self, request): engine=slicer, options=self.conf.options.model_dump(), ) + logging.debug( "The request we give polytope from polytope-mars are: %s", preq ) # noqa: E501 - result = self.api.retrieve(preq) + start = time.time() + logging.debug("Polytope time start: %s", start) # noqa: E501 + + if self.log_context: + result = self.api.retrieve(preq, self.log_context) + else: + result = self.api.retrieve(preq) + + end = time.time() + delta = end - start + logging.debug("Polytope time end: %s", end) # noqa: E501 + logging.debug("Polytope time taken: %s", delta) # noqa: E501 + start = time.time() + logging.debug("Polytope time start: %s", start) # noqa: E501 encoder = Covjsonkit(self.conf.coverageconfig.model_dump()).encode( "CoverageCollection", feature_type ) # noqa: E501 @@ -107,6 +125,11 @@ def extract(self, request): else: self.coverage = encoder.from_polytope(result) + end = time.time() + delta = end - start + logging.debug("Covjsonkit time end: %s", end) # noqa: E501 + logging.debug("Covjsonkit time taken: %s", delta) # noqa: E501 + return self.coverage def _create_base_shapes(self, request: dict) -> List[shapes.Shape]: @@ -118,6 +141,8 @@ def _create_base_shapes(self, request: dict) -> List[shapes.Shape]: # * enforcing strings are actually strings (e.g. type=fc) time = request.pop("time").replace(":", "") + if len(time.split('/')) != 1: + raise NotImplementedError(f"Currently only one time is supported") # noqa: E501 # if str(time).split("/") != 1: # time = str(time).split("/") # else: @@ -142,12 +167,18 @@ def _create_base_shapes(self, request: dict) -> List[shapes.Shape]: # Range a/to/b, "by" not supported -> Span elif len(split) == 3 and split[1] == "to": + # if date then only get time of dates in span not all in times within date if k == "date": - split[0] = pd.Timestamp(split[0] + "T" + time) - split[2] = pd.Timestamp(split[2] + "T" + time) - base_shapes.append( - shapes.Span(k, lower=split[0], upper=split[2]) - ) # noqa: E501 + start = pd.Timestamp(split[0] + "T" + time) + end = pd.Timestamp(split[2] + "T" + time) + dates = [] + for s in pd.date_range(start, end): + dates.append(s) + base_shapes.append(shapes.Select(k, dates)) + else: + base_shapes.append( + shapes.Span(k, lower=split[0], upper=split[2]) + ) # noqa: E501 elif "by" in split: raise ValueError( From 417b77cf87648aa4b7d8f4671357a4c52957dc8f Mon Sep 17 00:00:00 2001 From: awarde96 Date: Tue, 17 Sep 2024 10:10:33 +0000 Subject: [PATCH 2/5] Format files correctly --- polytope_mars/api.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/polytope_mars/api.py b/polytope_mars/api.py index 5fe8e62..60512da 100644 --- a/polytope_mars/api.py +++ b/polytope_mars/api.py @@ -141,8 +141,10 @@ def _create_base_shapes(self, request: dict) -> List[shapes.Shape]: # * enforcing strings are actually strings (e.g. type=fc) time = request.pop("time").replace(":", "") - if len(time.split('/')) != 1: - raise NotImplementedError(f"Currently only one time is supported") # noqa: E501 + if len(time.split("/")) != 1: + raise NotImplementedError( + "Currently only one time is supported" + ) # noqa: E501 # if str(time).split("/") != 1: # time = str(time).split("/") # else: @@ -167,7 +169,8 @@ def _create_base_shapes(self, request: dict) -> List[shapes.Shape]: # Range a/to/b, "by" not supported -> Span elif len(split) == 3 and split[1] == "to": - # if date then only get time of dates in span not all in times within date + # if date then only get time of dates in span not + # all in times within date if k == "date": start = pd.Timestamp(split[0] + "T" + time) end = pd.Timestamp(split[2] + "T" + time) From b6a625a0e1cef4adf56387021c4c354f9e8cbe7e Mon Sep 17 00:00:00 2001 From: awarde96 Date: Tue, 17 Sep 2024 10:41:51 +0000 Subject: [PATCH 3/5] Add id to logging --- polytope_mars/api.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/polytope_mars/api.py b/polytope_mars/api.py index 60512da..be693bf 100644 --- a/polytope_mars/api.py +++ b/polytope_mars/api.py @@ -34,20 +34,25 @@ class PolytopeMars: def __init__(self, config=None, log_context=None): # Initialise polytope-mars configuration + self.log_context = log_context + self.id = log_context["id"] if log_context else "-1" # If no config check default locations if config is None: self.conf = Conflator( app_name="polytope_mars", model=PolytopeMarsConfig ).load() - logging.debug("Config loaded from file: %s", self.conf) # noqa: E501 + logging.debug( + f"{self.id}: Config loaded from file: {self.conf}" + ) # noqa: E501 # else initialise with provided config else: self.conf = PolytopeMarsConfig.model_validate(config) - logging.debug("Config loaded from dictionary: %s", self.conf) # noqa: E501 + logging.debug( + f"{self.id}: Config loaded from dictionary: {self.conf}" + ) # noqa: E501 self.coverage = {} - self.log_context = log_context def extract(self, request): # request expected in JSON or dict @@ -100,10 +105,10 @@ def extract(self, request): ) logging.debug( - "The request we give polytope from polytope-mars are: %s", preq - ) # noqa: E501 + f"{self.id}: The request we give polytope from polytope-mars is: {preq}" # noqa: E501 + ) start = time.time() - logging.debug("Polytope time start: %s", start) # noqa: E501 + logging.debug(f"{self.id}: Polytope time start: {start}") # noqa: E501 if self.log_context: result = self.api.retrieve(preq, self.log_context) @@ -112,10 +117,10 @@ def extract(self, request): end = time.time() delta = end - start - logging.debug("Polytope time end: %s", end) # noqa: E501 - logging.debug("Polytope time taken: %s", delta) # noqa: E501 + logging.debug(f"{self.id}: Polytope time end: {end}") # noqa: E501 + logging.debug(f"{self.id}: Polytope time taken: {delta}") # noqa: E501 start = time.time() - logging.debug("Polytope time start: %s", start) # noqa: E501 + logging.debug(f"{self.id}: Polytope time start: {start}") # noqa: E501 encoder = Covjsonkit(self.conf.coverageconfig.model_dump()).encode( "CoverageCollection", feature_type ) # noqa: E501 @@ -127,8 +132,8 @@ def extract(self, request): end = time.time() delta = end - start - logging.debug("Covjsonkit time end: %s", end) # noqa: E501 - logging.debug("Covjsonkit time taken: %s", delta) # noqa: E501 + logging.debug(f"{self.id}: Covjsonkit time end: {end}") # noqa: E501 + logging.debug(f"{self.id}: Covjsonkit time taken: {delta}") # noqa: E501 return self.coverage From 19ed5532094286b04d3d767a4df2decfcfa209e3 Mon Sep 17 00:00:00 2001 From: awarde96 Date: Tue, 17 Sep 2024 13:52:40 +0000 Subject: [PATCH 4/5] Add new tests using gribjump server --- tests/performance/test_bounding_box.py | 114 +++++++++++++++++++ tests/performance/test_timeseries.py | 115 +++++++++++++++++++ tests/performance/test_vertical_profile.py | 123 +++++++++++++++++++++ 3 files changed, 352 insertions(+) create mode 100644 tests/performance/test_bounding_box.py create mode 100644 tests/performance/test_timeseries.py create mode 100644 tests/performance/test_vertical_profile.py diff --git a/tests/performance/test_bounding_box.py b/tests/performance/test_bounding_box.py new file mode 100644 index 0000000..dd6291a --- /dev/null +++ b/tests/performance/test_bounding_box.py @@ -0,0 +1,114 @@ +import pytest +from conflator import Conflator +from covjsonkit.api import Covjsonkit + +from polytope_mars.api import PolytopeMars +from polytope_mars.config import PolytopeMarsConfig + +# os.environ['DYLD_LIBRARY_PATH'] = + + +class TestBoundingBox: + def setup_method(self): + self.options = { + "axis_config": [ + { + "axis_name": "date", + "transformations": [ + { + "name": "merge", + "other_axis": "time", + "linkers": ["T", "00"], + } # noqa: E501 + ], + }, + { + "axis_name": "values", + "transformations": [ + { + "name": "mapper", + "type": "octahedral", + "resolution": 1280, + "axes": ["latitude", "longitude"], + } + ], + }, + { + "axis_name": "latitude", + "transformations": [ + {"name": "reverse", "is_reverse": True} + ], # noqa: E501 + }, + { + "axis_name": "longitude", + "transformations": [{"name": "cyclic", "range": [0, 360]}], + }, + { + "axis_name": "step", + "transformations": [ + {"name": "type_change", "type": "int"} + ], # noqa: E501 + }, + { + "axis_name": "number", + "transformations": [ + {"name": "type_change", "type": "int"} + ], # noqa: E501 + }, + ], + "compressed_axes_config": [ + "longitude", + "latitude", + "levtype", + "step", + "date", + "domain", + "expver", + "param", + "class", + "stream", + "type", + ], + "pre_path": { + "class": "od", + "expver": "0079", + "levtype": "sfc", + "stream": "oper", + "type": "fc", + }, + } + + self.request = { + "class": "od", + "stream": "oper", + "type": "fc", + "date": "20240915", + "time": "1200", + "levtype": "sfc", + "expver": "0079", + "domain": "g", + "param": "167/169", + "step": "0", + "feature": { + "type": "boundingbox", + "points": [[38, -9.5], [39, -8.5]], + }, + } + + conf = Conflator( + app_name="polytope_mars", model=PolytopeMarsConfig + ).load() # noqa: E501 + self.cf = conf.model_dump() + self.cf["options"] = self.options + + @pytest.mark.skip(reason="Gribjump not set up for ci actions yet") + def test_basic_boundingbox(self): + polytope_mars = PolytopeMars(self.cf) + result = polytope_mars.extract(self.request) + assert result is not None + + decoder = Covjsonkit().decode(result) + ds = decoder.to_xarray() + assert ds is not None + assert len(ds["datetimes"]) == 1 diff --git a/tests/performance/test_timeseries.py b/tests/performance/test_timeseries.py new file mode 100644 index 0000000..f15328f --- /dev/null +++ b/tests/performance/test_timeseries.py @@ -0,0 +1,115 @@ +import pytest +from conflator import Conflator +from covjsonkit.api import Covjsonkit + +from polytope_mars.api import PolytopeMars +from polytope_mars.config import PolytopeMarsConfig + +# os.environ['DYLD_LIBRARY_PATH'] = + + +class TestTimeSeries: + def setup_method(self): + self.options = { + "axis_config": [ + { + "axis_name": "date", + "transformations": [ + { + "name": "merge", + "other_axis": "time", + "linkers": ["T", "00"], + } # noqa: E501 + ], + }, + { + "axis_name": "values", + "transformations": [ + { + "name": "mapper", + "type": "octahedral", + "resolution": 1280, + "axes": ["latitude", "longitude"], + } + ], + }, + { + "axis_name": "latitude", + "transformations": [ + {"name": "reverse", "is_reverse": True} + ], # noqa: E501 + }, + { + "axis_name": "longitude", + "transformations": [{"name": "cyclic", "range": [0, 360]}], + }, + { + "axis_name": "step", + "transformations": [ + {"name": "type_change", "type": "int"} + ], # noqa: E501 + }, + { + "axis_name": "number", + "transformations": [ + {"name": "type_change", "type": "int"} + ], # noqa: E501 + }, + ], + "compressed_axes_config": [ + "longitude", + "latitude", + "levtype", + "step", + "date", + "domain", + "expver", + "param", + "class", + "stream", + "type", + ], + "pre_path": { + "class": "od", + "expver": "0079", + "levtype": "sfc", + "stream": "oper", + "type": "fc", + }, + } + + self.request = { + "class": "od", + "stream": "oper", + "type": "fc", + "date": "20240915", + "time": "0000", + "levtype": "sfc", + "expver": "0079", + "domain": "g", + "param": "167/168/169", + "step": "0/to/10", + "feature": { + "type": "timeseries", + "points": [[38, -9.5]], + "axis": "step", + }, + } + + conf = Conflator( + app_name="polytope_mars", model=PolytopeMarsConfig + ).load() # noqa: E501 + self.cf = conf.model_dump() + self.cf["options"] = self.options + + @pytest.mark.skip(reason="Gribjump not set up for ci actions yet") + def test_basic_timeseries(self): + polytope_mars = PolytopeMars(self.cf) + result = polytope_mars.extract(self.request) + assert result is not None + + decoder = Covjsonkit().decode(result) + ds = decoder.to_xarray() + assert ds is not None + assert len(ds["datetime"]) == 1 diff --git a/tests/performance/test_vertical_profile.py b/tests/performance/test_vertical_profile.py new file mode 100644 index 0000000..63e93ee --- /dev/null +++ b/tests/performance/test_vertical_profile.py @@ -0,0 +1,123 @@ +import pytest +from conflator import Conflator +from covjsonkit.api import Covjsonkit + +from polytope_mars.api import PolytopeMars +from polytope_mars.config import PolytopeMarsConfig + +# os.environ['DYLD_LIBRARY_PATH'] = + + +class TestVerticalProfile: + def setup_method(self): + self.options = { + "axis_config": [ + { + "axis_name": "date", + "transformations": [ + { + "name": "merge", + "other_axis": "time", + "linkers": ["T", "00"], + } # noqa: E501 + ], + }, + { + "axis_name": "values", + "transformations": [ + { + "name": "mapper", + "type": "octahedral", + "resolution": 1280, + "axes": ["latitude", "longitude"], + } + ], + }, + { + "axis_name": "latitude", + "transformations": [ + {"name": "reverse", "is_reverse": True} + ], # noqa: E501 + }, + { + "axis_name": "longitude", + "transformations": [{"name": "cyclic", "range": [0, 360]}], + }, + { + "axis_name": "step", + "transformations": [ + {"name": "type_change", "type": "int"} + ], # noqa: E501 + }, + { + "axis_name": "number", + "transformations": [ + {"name": "type_change", "type": "int"} + ], # noqa: E501 + }, + { + "axis_name": "levelist", + "transformations": [ + {"name": "type_change", "type": "int"} + ], # noqa: E501 + }, + ], + "compressed_axes_config": [ + "longitude", + "latitude", + "levtype", + "levelist", + "step", + "date", + "domain", + "expver", + "param", + "class", + "stream", + "type", + "number", + ], + "pre_path": { + "class": "od", + "expver": "0079", + "levtype": "pl", + "stream": "enfo", + "type": "pf", + }, + } + + self.request = { + "class": "od", + "stream": "enfo", + "type": "pf", + "date": "20240915", + "time": "0000", + "levtype": "pl", + "expver": "0079", + "domain": "g", + "param": "203/133", + "number": "1", + "step": "0", + "levelist": "1/to/1000", + "feature": { + "type": "verticalprofile", + "points": [[38.9, -9.1]], + }, + } + + conf = Conflator( + app_name="polytope_mars", model=PolytopeMarsConfig + ).load() # noqa: E501 + self.cf = conf.model_dump() + self.cf["options"] = self.options + + @pytest.mark.skip(reason="Gribjump not set up for ci actions yet") + def test_basic_verticalprofile(self): + polytope_mars = PolytopeMars(self.cf) + result = polytope_mars.extract(self.request) + assert result is not None + + decoder = Covjsonkit().decode(result) + ds = decoder.to_xarray() + assert ds is not None From 6188093c24a3f2898aeca4f7f9fd2c373199b501 Mon Sep 17 00:00:00 2001 From: awarde96 Date: Tue, 17 Sep 2024 15:56:14 +0000 Subject: [PATCH 5/5] Update readme --- README.md | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index a463477..62ad76d 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ pip install -e . ## Example -**Create time series request**: Create a request for a time series request using the time series feature, set options for use by polytope feature extraction. "grib" indicates the type of data in this case. NB: Assumes data is in a local FDB. +**Create time series request**: Create a request for a time series request using the time series feature, set options for use by polytope feature extraction. "gribjump" indicates the type of data in this case. NB: Assumes data is in a local FDB or or environment variables have been set up to point to a gribjump server. ```python from polytope_mars.api import PolytopeMars @@ -89,6 +89,40 @@ If the user provides no arguments to PolytopeMars then a config is loaded from t The user can also pass in a config as a python dictionary to PolytopeMars for a custom config at runtime. +```python +from polytope_mars.api import PolytopeMars +from conflator import Conflator +from polytope_mars.config import PolytopeMarsConfig + +conf = Conflator(app_name="polytope_mars", model=PolytopeMarsConfig).load() +cf = conf.model_dump() +cf["options"] = options + +request = { + "class": "od", + "stream" : "enfo", + "type" : "pf", + "date" : "20231205", + "time" : "00:00:00", + "levtype" : "sfc", + "expver" : "0001", + "domain" : "g", + "param" : "228/49/164/165/166/167", + "number" : "1/to/5", + "step" : "0/1" + "feature" : { + "type" : "timeseries", + "points" : [[51.5073219, 2.1]], + "axis" : "step", + }, +} + +result = PolytopeMars(cf).extract(request) + +``` + +A context dictionary can also be passed to PolytopeMars for logging. + Result will be a coverageJSON file with the requested data if it is available, further manipulation of the coverage can be made using [covjsonkit](https://github.com/ecmwf/covjsonkit). ### Config