Skip to content

Commit

Permalink
Merge pull request #858 from dcs4cop/alicja-norman-viewer289-fix_time…
Browse files Browse the repository at this point in the history 
…stamp_rounding

fixes time rounding issue
  • Loading branch information
@forman
forman authored Jun 6, 2023
2 parents b10f32d + 73b9d55 commit a1b0042
Show file tree
Hide file tree
Showing 5 changed files with 205 additions and 72 deletions.
5 changes: 5 additions & 0 deletions CHANGES.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,11 @@
* Bundled [xcube-viewer 1.1.0-dev.1](https://github.com/dcs4cop/xcube-viewer/releases/tag/v1.1.0-dev.1).

* Fixed various issues with the auto-generated Python API documentation.
* Fixed rounding of timestamp issue observed in xcube viewer https://github.com/dcs4cop/xcube-viewer/issues/289.
xcube server now rounds the time dimension labels for a dataset as follows (rounding frequency is always 1 second):
- First timesstamp: floor(time[0])
- Last timesstamp: ceil(time[-1])
- In-between timestamps: round(time[1: -1])

* Fixed a problem where time series requests may have missed outer values
of a requested time range. Introduced query parameter `tolerance` for
Expand Down
3 changes: 2 additions & 1 deletion docs/source/overview.rst
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@ Overview

*xcube* is an open-source Python package and toolkit that has been developed to provide Earth observation (EO) data in an
analysis-ready form to users. xcube achieves this by carefully converting EO data sources into self-contained *data cubes*
that can be published in the cloud.
that can be published in the cloud. The Python package is expanded and maintained by
[Brockmann Consult GmbH](https://www.brockmann-consult.de)

Data Cube
=========
Expand Down
182 changes: 140 additions & 42 deletions test/core/test_timecoord.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import numpy as np
import pandas as pd
import pytest

from test.sampledata import create_highroc_dataset
from xcube.core.new import new_cube
Expand All @@ -13,6 +14,8 @@
from xcube.core.timecoord import get_time_range_from_data
from xcube.core.timecoord import timestamp_to_iso_string
from xcube.core.timecoord import to_time_in_days_since_1970


# from xcube.core.timecoord import find_datetime_format
# from xcube.core.timecoord import get_timestamp_from_string
# from xcube.core.timecoord import get_timestamps_from_string
Expand All @@ -22,15 +25,17 @@ class AddTimeCoordsTest(unittest.TestCase):

def test_add_time_coords_point(self):
dataset = create_highroc_dataset()
dataset_with_time = add_time_coords(dataset, (365 * 47 + 20, 365 * 47 + 20))
dataset_with_time = add_time_coords(dataset,
(365 * 47 + 20, 365 * 47 + 20))
self.assertIsNot(dataset_with_time, dataset)
self.assertIn('time', dataset_with_time)
self.assertEqual(dataset_with_time.time.shape, (1,))
self.assertNotIn('time_bnds', dataset_with_time)

def test_add_time_coords_range(self):
dataset = create_highroc_dataset()
dataset_with_time = add_time_coords(dataset, (365 * 47 + 20, 365 * 47 + 21))
dataset_with_time = add_time_coords(dataset,
(365 * 47 + 20, 365 * 47 + 21))
self.assertIsNot(dataset_with_time, dataset)
self.assertIn('time', dataset_with_time)
self.assertEqual(dataset_with_time.time.shape, (1,))
Expand All @@ -47,7 +52,8 @@ def test_to_time_in_days_since_1970(self):
self.assertEqual(17690.5,
to_time_in_days_since_1970('2018-06-08T12:00'))
self.assertEqual(18173.42625622898,
to_time_in_days_since_1970('04-OCT-2019 10:13:48.538184'))
to_time_in_days_since_1970(
'04-OCT-2019 10:13:48.538184'))

def test_from_time_in_days_since_1970(self):
self.assertEqual('2017-06-07T12:00:00.000000000',
Expand All @@ -64,7 +70,8 @@ def test_from_time_in_days_since_1970(self):
to_time_in_days_since_1970('2018-06-08T12:00'))))
self.assertEqual('2019-10-04T10:13:48.538000000',
str(from_time_in_days_since_1970(
to_time_in_days_since_1970('04-OCT-2019 10:13:48.538184'))))
to_time_in_days_since_1970(
'04-OCT-2019 10:13:48.538184'))))


class GetTimeRangeTest(unittest.TestCase):
Expand All @@ -73,16 +80,20 @@ def test_get_time_range_from_data(self):
cube = new_cube(drop_bounds=True)
time_range = get_time_range_from_data(cube)
self.assertIsNotNone(time_range)
self.assertEqual('2010-01-01T00:00:00', pd.Timestamp(time_range[0]).isoformat())
self.assertEqual('2010-01-06T00:00:00', pd.Timestamp(time_range[1]).isoformat())
self.assertEqual('2010-01-01T00:00:00',
pd.Timestamp(time_range[0]).isoformat())
self.assertEqual('2010-01-06T00:00:00',
pd.Timestamp(time_range[1]).isoformat())

def test_get_time_range_from_data_with_irregular_data(self):
cube = new_cube(drop_bounds=True,
time_freq='M')
time_range = get_time_range_from_data(cube)
self.assertIsNotNone(time_range)
self.assertEqual('2010-01-31T00:00:00', pd.Timestamp(time_range[0]).isoformat())
self.assertEqual('2010-06-30T00:00:00', pd.Timestamp(time_range[1]).isoformat())
self.assertEqual('2010-01-31T00:00:00',
pd.Timestamp(time_range[0]).isoformat())
self.assertEqual('2010-06-30T00:00:00',
pd.Timestamp(time_range[1]).isoformat())

def test_get_time_range_from_data_with_irregular_data_and_no_metadata(self):
cube = new_cube(drop_bounds=True,
Expand All @@ -91,17 +102,21 @@ def test_get_time_range_from_data_with_irregular_data_and_no_metadata(self):
cube.attrs.pop('time_coverage_end')
time_range = get_time_range_from_data(cube)
self.assertIsNotNone(time_range)
self.assertEqual('2010-02-14T00:00:00', pd.Timestamp(time_range[0]).isoformat())
self.assertEqual('2010-06-14T00:00:00', pd.Timestamp(time_range[1]).isoformat())
self.assertEqual('2010-02-14T00:00:00',
pd.Timestamp(time_range[0]).isoformat())
self.assertEqual('2010-06-14T00:00:00',
pd.Timestamp(time_range[1]).isoformat())

def test_get_time_range_from_data_cftime(self):
cube = new_cube(drop_bounds=True,
use_cftime=True,
time_dtype=None)
time_range = get_time_range_from_data(cube)
self.assertIsNotNone(time_range)
self.assertEqual('2010-01-01T00:00:00', pd.Timestamp(time_range[0]).isoformat())
self.assertEqual('2010-01-06T00:00:00', pd.Timestamp(time_range[1]).isoformat())
self.assertEqual('2010-01-01T00:00:00',
pd.Timestamp(time_range[0]).isoformat())
self.assertEqual('2010-01-06T00:00:00',
pd.Timestamp(time_range[1]).isoformat())

def test_get_time_range_from_data_with_irregular_cftime_data(self):
cube = new_cube(drop_bounds=True,
Expand All @@ -110,10 +125,13 @@ def test_get_time_range_from_data_with_irregular_cftime_data(self):
time_dtype=None)
time_range = get_time_range_from_data(cube)
self.assertIsNotNone(time_range)
self.assertEqual('2010-01-31T00:00:00', pd.Timestamp(time_range[0]).isoformat())
self.assertEqual('2010-06-30T00:00:00', pd.Timestamp(time_range[1]).isoformat())
self.assertEqual('2010-01-31T00:00:00',
pd.Timestamp(time_range[0]).isoformat())
self.assertEqual('2010-06-30T00:00:00',
pd.Timestamp(time_range[1]).isoformat())

def test_get_time_range_from_data_with_irregular_cftime_data_and_no_metadata(self):
def test_get_time_range_from_data_with_irregular_cftime_data_and_no_metadata(
self):
cube = new_cube(drop_bounds=True,
time_freq='M',
use_cftime=True,
Expand All @@ -122,94 +140,174 @@ def test_get_time_range_from_data_with_irregular_cftime_data_and_no_metadata(sel
cube.attrs.pop('time_coverage_end')
time_range = get_time_range_from_data(cube)
self.assertIsNotNone(time_range)
self.assertEqual('2010-02-14T00:00:00', pd.Timestamp(time_range[0]).isoformat())
self.assertEqual('2010-06-14T00:00:00', pd.Timestamp(time_range[1]).isoformat())
self.assertEqual('2010-02-14T00:00:00',
pd.Timestamp(time_range[0]).isoformat())
self.assertEqual('2010-06-14T00:00:00',
pd.Timestamp(time_range[1]).isoformat())

def test_get_time_range_from_data_time_named_t(self):
cube = new_cube(drop_bounds=True, time_name='t')
time_range = get_time_range_from_data(cube)
self.assertIsNotNone(time_range)
self.assertEqual('2010-01-01T00:00:00', pd.Timestamp(time_range[0]).isoformat())
self.assertEqual('2010-01-06T00:00:00', pd.Timestamp(time_range[1]).isoformat())
self.assertEqual('2010-01-01T00:00:00',
pd.Timestamp(time_range[0]).isoformat())
self.assertEqual('2010-01-06T00:00:00',
pd.Timestamp(time_range[1]).isoformat())

def test_get_time_range_from_data_additional_t_variable(self):
import xarray as xr
start_time_data = pd.date_range(start='2010-01-03T12:00:00',
periods=5,
freq='5D').values.astype(dtype='datetime64[s]')
freq='5D').values.astype(
dtype='datetime64[s]')
start_time = xr.DataArray(start_time_data, dims='time')
end_time_data = pd.date_range(start='2010-01-07T12:00:00',
periods=5,
freq='5D').values.astype(dtype='datetime64[s]')
freq='5D').values.astype(
dtype='datetime64[s]')
end_time = xr.DataArray(end_time_data, dims='time')
cube = new_cube(drop_bounds=True,
time_start='2010-01-05T12:00:00',
time_freq='5D',
variables=dict(start_time=start_time, end_time=end_time))
variables=dict(start_time=start_time,
end_time=end_time))
time_range = get_time_range_from_data(cube)
self.assertIsNotNone(time_range)
self.assertEqual('2010-01-03T12:00:00', pd.Timestamp(time_range[0]).isoformat())
self.assertEqual('2010-01-27T12:00:00', pd.Timestamp(time_range[1]).isoformat())
self.assertEqual('2010-01-03T12:00:00',
pd.Timestamp(time_range[0]).isoformat())
self.assertEqual('2010-01-27T12:00:00',
pd.Timestamp(time_range[1]).isoformat())

def test_get_time_range_from_data_start_and_end_time_arrays(self):
cube = new_cube(drop_bounds=True,
use_cftime=True,
time_dtype=None)
time_range = get_time_range_from_data(cube)
self.assertIsNotNone(time_range)
self.assertEqual('2010-01-01T00:00:00', pd.Timestamp(time_range[0]).isoformat())
self.assertEqual('2010-01-06T00:00:00', pd.Timestamp(time_range[1]).isoformat())
self.assertEqual('2010-01-01T00:00:00',
pd.Timestamp(time_range[0]).isoformat())
self.assertEqual('2010-01-06T00:00:00',
pd.Timestamp(time_range[1]).isoformat())

def test_get_time_range_from_data_bounds(self):
cube = new_cube()
time_range = get_time_range_from_data(cube)
self.assertIsNotNone(time_range)
self.assertEqual('2010-01-01T00:00:00', pd.Timestamp(time_range[0]).isoformat())
self.assertEqual('2010-01-06T00:00:00', pd.Timestamp(time_range[1]).isoformat())
self.assertEqual('2010-01-01T00:00:00',
pd.Timestamp(time_range[0]).isoformat())
self.assertEqual('2010-01-06T00:00:00',
pd.Timestamp(time_range[1]).isoformat())

def test_get_time_range_from_attrs(self):
cube = new_cube()
time_range = get_time_range_from_attrs(cube)
self.assertIsNotNone(time_range)
self.assertEqual('2010-01-01T00:00:00', pd.Timestamp(time_range[0]).isoformat())
self.assertEqual('2010-01-06T00:00:00', pd.Timestamp(time_range[1]).isoformat())
self.assertEqual('2010-01-01T00:00:00',
pd.Timestamp(time_range[0]).isoformat())
self.assertEqual('2010-01-06T00:00:00',
pd.Timestamp(time_range[1]).isoformat())

def test_get_start_time_from_attrs(self):
cube = new_cube()
start_time = get_start_time_from_attrs(cube)
self.assertEqual('2010-01-01T00:00:00', pd.Timestamp(start_time).isoformat())
self.assertEqual('2010-01-01T00:00:00',
pd.Timestamp(start_time).isoformat())

def test_get_end_time_from_attrs(self):
cube = new_cube()
end_time = get_end_time_from_attrs(cube)
self.assertEqual('2010-01-06T00:00:00', pd.Timestamp(end_time).isoformat())
self.assertEqual('2010-01-06T00:00:00',
pd.Timestamp(end_time).isoformat())


class TimestampToIsoStringTest(unittest.TestCase):
def test_it_with_default_res(self):
self.assertEqual("2018-09-05T00:00:00Z",
timestamp_to_iso_string(np.datetime64("2018-09-05")))
self.assertEqual("2018-09-05T10:35:42Z",
timestamp_to_iso_string(np.datetime64("2018-09-05 10:35:42")))
timestamp_to_iso_string(
np.datetime64("2018-09-05 10:35:42")))
self.assertEqual("2018-09-05T10:35:42Z",
timestamp_to_iso_string(
np.datetime64("2018-09-05 10:35:42.164")))
self.assertEqual("2019-10-04T10:13:49Z",
timestamp_to_iso_string(
pd.to_datetime("04-OCT-2019 10:13:48.538184")))

def test_it_with_ceil_round_fn(self):
self.assertEqual("2018-09-05T00:00:00Z",
timestamp_to_iso_string(np.datetime64("2018-09-05"),
round_fn="ceil"))
self.assertEqual("2018-09-05T10:35:42Z",
timestamp_to_iso_string(np.datetime64("2018-09-05 10:35:42.164")))
timestamp_to_iso_string(
np.datetime64("2018-09-05 10:35:42"),
round_fn="ceil"))
self.assertEqual("2018-09-05T10:35:43Z",
timestamp_to_iso_string(
np.datetime64("2018-09-05 10:35:42.164"),
round_fn="ceil"))
self.assertEqual("2019-10-04T10:13:49Z",
timestamp_to_iso_string(pd.to_datetime("04-OCT-2019 10:13:48.538184")))
timestamp_to_iso_string(
pd.to_datetime("04-OCT-2019 10:13:48.538184"),
round_fn="ceil"))

def test_it_with_floor_round_fn(self):
self.assertEqual("2018-09-05T00:00:00Z",
timestamp_to_iso_string(np.datetime64("2018-09-05"),
round_fn="floor"))
self.assertEqual("2018-09-05T10:35:42Z",
timestamp_to_iso_string(
np.datetime64("2018-09-05 10:35:42"),
round_fn="floor"))
self.assertEqual("2018-09-05T10:35:42Z",
timestamp_to_iso_string(
np.datetime64("2018-09-05 10:35:42.164"),
round_fn="floor"))
self.assertEqual("2019-10-04T10:13:48Z",
timestamp_to_iso_string(
pd.to_datetime("04-OCT-2019 10:13:48.538184"),
round_fn="floor"))

def test_it_with_array_round_fn(self):
var = [np.datetime64("2018-09-05 10:35:42.564"),
np.datetime64("2018-09-06 10:35:42.564"),
np.datetime64("2018-09-07 10:35:42.564"),
pd.to_datetime("04-OCT-2019 10:13:48.038184")
]
expected_values = ["2018-09-05T10:35:42Z",
"2018-09-06T10:35:43Z",
"2018-09-07T10:35:43Z",
"2019-10-04T10:13:49Z"]
values = [timestamp_to_iso_string(var[0], round_fn="floor")] +\
list(map(timestamp_to_iso_string, var[1:-1])) +\
[timestamp_to_iso_string(var[-1], round_fn="ceil")]
self.assertEqual(expected_values, values)


# noinspection PyMethodMayBeStatic
def test_it_with_invalid_round_fn(self):
with pytest.raises(ValueError,
match=r"round_fn must be one of"
r" \('ceil', 'floor', 'round'\)"):
timestamp_to_iso_string(np.datetime64("2018-09-05 10:35:42.164"),
round_fn="foo")

def test_it_with_h_res(self):
self.assertEqual("2018-09-05T00:00:00Z",
timestamp_to_iso_string(np.datetime64("2018-09-05"),
freq="H"))
self.assertEqual("2018-09-05T11:00:00Z",
timestamp_to_iso_string(np.datetime64("2018-09-05 10:35:42"),
freq="H"))
timestamp_to_iso_string(
np.datetime64("2018-09-05 10:35:42"),
freq="H"))
self.assertEqual("2018-09-05T11:00:00Z",
timestamp_to_iso_string(np.datetime64("2018-09-05 10:35:42.164"),
freq="H"))
timestamp_to_iso_string(
np.datetime64("2018-09-05 10:35:42.164"),
freq="H"))
self.assertEqual("2019-10-04T10:00:00Z",
timestamp_to_iso_string(pd.to_datetime("04-OCT-2019 10:13:48.538184"),
freq="H"))

timestamp_to_iso_string(
pd.to_datetime("04-OCT-2019 10:13:48.538184"),
freq="H"))

# class TimeStampsTest(unittest.TestCase):
#
Expand Down
Loading

0 comments on commit a1b0042

Please sign in to comment.