Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Feature/add logging and tests #17

Merged
merged 5 commits into from
Sep 18, 2024
Merged

Feature/add logging and tests #17

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
f54bbfc
Update api with context, fix span of dates to include a single time
awarde96 Sep 17, 2024
417b77c
Format files correctly
awarde96 Sep 17, 2024
b6a625a
Add id to logging
awarde96 Sep 17, 2024
19ed553
Add new tests using gribjump server
awarde96 Sep 17, 2024
6188093
Update readme
awarde96 Sep 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 35 additions & 1 deletion README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ pip install -e .

## Example

**Create time series request**: Create a request for a time series request using the time series feature, set options for use by polytope feature extraction. "grib" indicates the type of data in this case. NB: Assumes data is in a local FDB.
**Create time series request**: Create a request for a time series request using the time series feature, set options for use by polytope feature extraction. "gribjump" indicates the type of data in this case. NB: Assumes data is in a local FDB or or environment variables have been set up to point to a gribjump server.

```python
from polytope_mars.api import PolytopeMars
Expand Down Expand Up @@ -89,6 +89,40 @@ If the user provides no arguments to PolytopeMars then a config is loaded from t

The user can also pass in a config as a python dictionary to PolytopeMars for a custom config at runtime.

```python
from polytope_mars.api import PolytopeMars
from conflator import Conflator
from polytope_mars.config import PolytopeMarsConfig

conf = Conflator(app_name="polytope_mars", model=PolytopeMarsConfig).load()
cf = conf.model_dump()
cf["options"] = options

request = {
"class": "od",
"stream" : "enfo",
"type" : "pf",
"date" : "20231205",
"time" : "00:00:00",
"levtype" : "sfc",
"expver" : "0001",
"domain" : "g",
"param" : "228/49/164/165/166/167",
"number" : "1/to/5",
"step" : "0/1"
"feature" : {
"type" : "timeseries",
"points" : [[51.5073219, 2.1]],
"axis" : "step",
},
}

result = PolytopeMars(cf).extract(request)

```

A context dictionary can also be passed to PolytopeMars for logging.

Result will be a coverageJSON file with the requested data if it is available, further manipulation of the coverage can be made using [covjsonkit](https://github.com/ecmwf/covjsonkit).

### Config
Expand Down
57 changes: 48 additions & 9 deletions polytope_mars/api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import json
import logging
import time
from typing import List

import pandas as pd
Expand Down Expand Up @@ -31,17 +32,25 @@


class PolytopeMars:
def __init__(self, config=None):
def __init__(self, config=None, log_context=None):
# Initialise polytope-mars configuration
self.log_context = log_context
self.id = log_context["id"] if log_context else "-1"

# If no config check default locations
if config is None:
self.conf = Conflator(
app_name="polytope_mars", model=PolytopeMarsConfig
).load()
logging.debug(
f"{self.id}: Config loaded from file: {self.conf}"
) # noqa: E501
# else initialise with provided config
else:
self.conf = PolytopeMarsConfig.model_validate(config)
logging.debug(
f"{self.id}: Config loaded from dictionary: {self.conf}"
) # noqa: E501

self.coverage = {}

Expand Down Expand Up @@ -94,10 +103,24 @@ def extract(self, request):
engine=slicer,
options=self.conf.options.model_dump(),
)

logging.debug(
"The request we give polytope from polytope-mars are: %s", preq
) # noqa: E501
result = self.api.retrieve(preq)
f"{self.id}: The request we give polytope from polytope-mars is: {preq}" # noqa: E501
)
start = time.time()
logging.debug(f"{self.id}: Polytope time start: {start}") # noqa: E501

if self.log_context:
result = self.api.retrieve(preq, self.log_context)
else:
result = self.api.retrieve(preq)

end = time.time()
delta = end - start
logging.debug(f"{self.id}: Polytope time end: {end}") # noqa: E501
logging.debug(f"{self.id}: Polytope time taken: {delta}") # noqa: E501
start = time.time()
logging.debug(f"{self.id}: Polytope time start: {start}") # noqa: E501
encoder = Covjsonkit(self.conf.coverageconfig.model_dump()).encode(
"CoverageCollection", feature_type
) # noqa: E501
Expand All @@ -107,6 +130,11 @@ def extract(self, request):
else:
self.coverage = encoder.from_polytope(result)

end = time.time()
delta = end - start
logging.debug(f"{self.id}: Covjsonkit time end: {end}") # noqa: E501
logging.debug(f"{self.id}: Covjsonkit time taken: {delta}") # noqa: E501

return self.coverage

def _create_base_shapes(self, request: dict) -> List[shapes.Shape]:
Expand All @@ -118,6 +146,10 @@ def _create_base_shapes(self, request: dict) -> List[shapes.Shape]:
# * enforcing strings are actually strings (e.g. type=fc)

time = request.pop("time").replace(":", "")
if len(time.split("/")) != 1:
raise NotImplementedError(
"Currently only one time is supported"
) # noqa: E501
# if str(time).split("/") != 1:
# time = str(time).split("/")
# else:
Expand All @@ -142,12 +174,19 @@ def _create_base_shapes(self, request: dict) -> List[shapes.Shape]:

# Range a/to/b, "by" not supported -> Span
elif len(split) == 3 and split[1] == "to":
# if date then only get time of dates in span not
# all in times within date
if k == "date":
split[0] = pd.Timestamp(split[0] + "T" + time)
split[2] = pd.Timestamp(split[2] + "T" + time)
base_shapes.append(
shapes.Span(k, lower=split[0], upper=split[2])
) # noqa: E501
start = pd.Timestamp(split[0] + "T" + time)
end = pd.Timestamp(split[2] + "T" + time)
dates = []
for s in pd.date_range(start, end):
dates.append(s)
base_shapes.append(shapes.Select(k, dates))
else:
base_shapes.append(
shapes.Span(k, lower=split[0], upper=split[2])
) # noqa: E501

elif "by" in split:
raise ValueError(
Expand Down
114 changes: 114 additions & 0 deletions tests/performance/test_bounding_box.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,114 @@
import pytest
from conflator import Conflator
from covjsonkit.api import Covjsonkit

from polytope_mars.api import PolytopeMars
from polytope_mars.config import PolytopeMarsConfig

# os.environ['DYLD_LIBRARY_PATH'] = <path gribjump library
# os.environ['GRIBJUMP_CONFIG_FILE']= <path to gribjump config file>


class TestBoundingBox:
def setup_method(self):
self.options = {
"axis_config": [
{
"axis_name": "date",
"transformations": [
{
"name": "merge",
"other_axis": "time",
"linkers": ["T", "00"],
} # noqa: E501
],
},
{
"axis_name": "values",
"transformations": [
{
"name": "mapper",
"type": "octahedral",
"resolution": 1280,
"axes": ["latitude", "longitude"],
}
],
},
{
"axis_name": "latitude",
"transformations": [
{"name": "reverse", "is_reverse": True}
], # noqa: E501
},
{
"axis_name": "longitude",
"transformations": [{"name": "cyclic", "range": [0, 360]}],
},
{
"axis_name": "step",
"transformations": [
{"name": "type_change", "type": "int"}
], # noqa: E501
},
{
"axis_name": "number",
"transformations": [
{"name": "type_change", "type": "int"}
], # noqa: E501
},
],
"compressed_axes_config": [
"longitude",
"latitude",
"levtype",
"step",
"date",
"domain",
"expver",
"param",
"class",
"stream",
"type",
],
"pre_path": {
"class": "od",
"expver": "0079",
"levtype": "sfc",
"stream": "oper",
"type": "fc",
},
}

self.request = {
"class": "od",
"stream": "oper",
"type": "fc",
"date": "20240915",
"time": "1200",
"levtype": "sfc",
"expver": "0079",
"domain": "g",
"param": "167/169",
"step": "0",
"feature": {
"type": "boundingbox",
"points": [[38, -9.5], [39, -8.5]],
},
}

conf = Conflator(
app_name="polytope_mars", model=PolytopeMarsConfig
).load() # noqa: E501
self.cf = conf.model_dump()
self.cf["options"] = self.options

@pytest.mark.skip(reason="Gribjump not set up for ci actions yet")
def test_basic_boundingbox(self):
polytope_mars = PolytopeMars(self.cf)
result = polytope_mars.extract(self.request)
assert result is not None

decoder = Covjsonkit().decode(result)
ds = decoder.to_xarray()
assert ds is not None
assert len(ds["datetimes"]) == 1
115 changes: 115 additions & 0 deletions tests/performance/test_timeseries.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
import pytest
from conflator import Conflator
from covjsonkit.api import Covjsonkit

from polytope_mars.api import PolytopeMars
from polytope_mars.config import PolytopeMarsConfig

# os.environ['DYLD_LIBRARY_PATH'] = <path gribjump library
# os.environ['GRIBJUMP_CONFIG_FILE']= <path to gribjump config file>


class TestTimeSeries:
def setup_method(self):
self.options = {
"axis_config": [
{
"axis_name": "date",
"transformations": [
{
"name": "merge",
"other_axis": "time",
"linkers": ["T", "00"],
} # noqa: E501
],
},
{
"axis_name": "values",
"transformations": [
{
"name": "mapper",
"type": "octahedral",
"resolution": 1280,
"axes": ["latitude", "longitude"],
}
],
},
{
"axis_name": "latitude",
"transformations": [
{"name": "reverse", "is_reverse": True}
], # noqa: E501
},
{
"axis_name": "longitude",
"transformations": [{"name": "cyclic", "range": [0, 360]}],
},
{
"axis_name": "step",
"transformations": [
{"name": "type_change", "type": "int"}
], # noqa: E501
},
{
"axis_name": "number",
"transformations": [
{"name": "type_change", "type": "int"}
], # noqa: E501
},
],
"compressed_axes_config": [
"longitude",
"latitude",
"levtype",
"step",
"date",
"domain",
"expver",
"param",
"class",
"stream",
"type",
],
"pre_path": {
"class": "od",
"expver": "0079",
"levtype": "sfc",
"stream": "oper",
"type": "fc",
},
}

self.request = {
"class": "od",
"stream": "oper",
"type": "fc",
"date": "20240915",
"time": "0000",
"levtype": "sfc",
"expver": "0079",
"domain": "g",
"param": "167/168/169",
"step": "0/to/10",
"feature": {
"type": "timeseries",
"points": [[38, -9.5]],
"axis": "step",
},
}

conf = Conflator(
app_name="polytope_mars", model=PolytopeMarsConfig
).load() # noqa: E501
self.cf = conf.model_dump()
self.cf["options"] = self.options

@pytest.mark.skip(reason="Gribjump not set up for ci actions yet")
def test_basic_timeseries(self):
polytope_mars = PolytopeMars(self.cf)
result = polytope_mars.extract(self.request)
assert result is not None

decoder = Covjsonkit().decode(result)
ds = decoder.to_xarray()
assert ds is not None
assert len(ds["datetime"]) == 1
Loading
Loading