Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

fixes time rounding issue #858

Merged
merged 5 commits into from
Jun 6, 2023
Merged

fixes time rounding issue #858

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
caef789
fixes time rounding issue
AliceBalfanz Jun 5, 2023
d60e433
added changes entry
AliceBalfanz Jun 5, 2023
1269bdc
Update xcube/webapi/datasets/controllers.py
AliceBalfanz Jun 5, 2023
55cfc61
Update CHANGES.md
AliceBalfanz Jun 5, 2023
73b9d55
test for a time array
AliceBalfanz Jun 5, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGES.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
* Updated [xcube Dataset Specification](docs/source/cubespec.md).
* Bundled [xcube-viewer 1.1.0-dev.1](https://github.com/dcs4cop/xcube-viewer/releases/tag/v1.1.0-dev.1).
* Fixed various issues with the auto-generated Python API documentation.
* Fixed rounding of timestamp issue observed in xcube viewer https://github.com/dcs4cop/xcube-viewer/issues/289
AliceBalfanz marked this conversation as resolved.
Show resolved Hide resolved

## Changes in 1.0.5

Expand Down
3 changes: 2 additions & 1 deletion docs/source/overview.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,8 @@ Overview

*xcube* is an open-source Python package and toolkit that has been developed to provide Earth observation (EO) data in an
analysis-ready form to users. xcube achieves this by carefully converting EO data sources into self-contained *data cubes*
that can be published in the cloud.
that can be published in the cloud. The Python package is expanded and maintained by
[Brockmann Consult GmbH](https://www.brockmann-consult.de)

Data Cube
=========
Expand Down
166 changes: 124 additions & 42 deletions test/core/test_timecoord.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import numpy as np
import pandas as pd
import pytest

from test.sampledata import create_highroc_dataset
from xcube.core.new import new_cube
Expand All @@ -13,6 +14,8 @@
from xcube.core.timecoord import get_time_range_from_data
from xcube.core.timecoord import timestamp_to_iso_string
from xcube.core.timecoord import to_time_in_days_since_1970


# from xcube.core.timecoord import find_datetime_format
# from xcube.core.timecoord import get_timestamp_from_string
# from xcube.core.timecoord import get_timestamps_from_string
Expand All @@ -22,15 +25,17 @@ class AddTimeCoordsTest(unittest.TestCase):

def test_add_time_coords_point(self):
dataset = create_highroc_dataset()
dataset_with_time = add_time_coords(dataset, (365 * 47 + 20, 365 * 47 + 20))
dataset_with_time = add_time_coords(dataset,
(365 * 47 + 20, 365 * 47 + 20))
self.assertIsNot(dataset_with_time, dataset)
self.assertIn('time', dataset_with_time)
self.assertEqual(dataset_with_time.time.shape, (1,))
self.assertNotIn('time_bnds', dataset_with_time)

def test_add_time_coords_range(self):
dataset = create_highroc_dataset()
dataset_with_time = add_time_coords(dataset, (365 * 47 + 20, 365 * 47 + 21))
dataset_with_time = add_time_coords(dataset,
(365 * 47 + 20, 365 * 47 + 21))
self.assertIsNot(dataset_with_time, dataset)
self.assertIn('time', dataset_with_time)
self.assertEqual(dataset_with_time.time.shape, (1,))
Expand All @@ -47,7 +52,8 @@ def test_to_time_in_days_since_1970(self):
self.assertEqual(17690.5,
to_time_in_days_since_1970('2018-06-08T12:00'))
self.assertEqual(18173.42625622898,
to_time_in_days_since_1970('04-OCT-2019 10:13:48.538184'))
to_time_in_days_since_1970(
'04-OCT-2019 10:13:48.538184'))

def test_from_time_in_days_since_1970(self):
self.assertEqual('2017-06-07T12:00:00.000000000',
Expand All @@ -64,7 +70,8 @@ def test_from_time_in_days_since_1970(self):
to_time_in_days_since_1970('2018-06-08T12:00'))))
self.assertEqual('2019-10-04T10:13:48.538000000',
str(from_time_in_days_since_1970(
to_time_in_days_since_1970('04-OCT-2019 10:13:48.538184'))))
to_time_in_days_since_1970(
'04-OCT-2019 10:13:48.538184'))))


class GetTimeRangeTest(unittest.TestCase):
Expand All @@ -73,16 +80,20 @@ def test_get_time_range_from_data(self):
cube = new_cube(drop_bounds=True)
time_range = get_time_range_from_data(cube)
self.assertIsNotNone(time_range)
self.assertEqual('2010-01-01T00:00:00', pd.Timestamp(time_range[0]).isoformat())
self.assertEqual('2010-01-06T00:00:00', pd.Timestamp(time_range[1]).isoformat())
self.assertEqual('2010-01-01T00:00:00',
pd.Timestamp(time_range[0]).isoformat())
self.assertEqual('2010-01-06T00:00:00',
pd.Timestamp(time_range[1]).isoformat())

def test_get_time_range_from_data_with_irregular_data(self):
cube = new_cube(drop_bounds=True,
time_freq='M')
time_range = get_time_range_from_data(cube)
self.assertIsNotNone(time_range)
self.assertEqual('2010-01-31T00:00:00', pd.Timestamp(time_range[0]).isoformat())
self.assertEqual('2010-06-30T00:00:00', pd.Timestamp(time_range[1]).isoformat())
self.assertEqual('2010-01-31T00:00:00',
pd.Timestamp(time_range[0]).isoformat())
self.assertEqual('2010-06-30T00:00:00',
pd.Timestamp(time_range[1]).isoformat())

def test_get_time_range_from_data_with_irregular_data_and_no_metadata(self):
cube = new_cube(drop_bounds=True,
Expand All @@ -91,17 +102,21 @@ def test_get_time_range_from_data_with_irregular_data_and_no_metadata(self):
cube.attrs.pop('time_coverage_end')
time_range = get_time_range_from_data(cube)
self.assertIsNotNone(time_range)
self.assertEqual('2010-02-14T00:00:00', pd.Timestamp(time_range[0]).isoformat())
self.assertEqual('2010-06-14T00:00:00', pd.Timestamp(time_range[1]).isoformat())
self.assertEqual('2010-02-14T00:00:00',
pd.Timestamp(time_range[0]).isoformat())
self.assertEqual('2010-06-14T00:00:00',
pd.Timestamp(time_range[1]).isoformat())

def test_get_time_range_from_data_cftime(self):
cube = new_cube(drop_bounds=True,
use_cftime=True,
time_dtype=None)
time_range = get_time_range_from_data(cube)
self.assertIsNotNone(time_range)
self.assertEqual('2010-01-01T00:00:00', pd.Timestamp(time_range[0]).isoformat())
self.assertEqual('2010-01-06T00:00:00', pd.Timestamp(time_range[1]).isoformat())
self.assertEqual('2010-01-01T00:00:00',
pd.Timestamp(time_range[0]).isoformat())
self.assertEqual('2010-01-06T00:00:00',
pd.Timestamp(time_range[1]).isoformat())

def test_get_time_range_from_data_with_irregular_cftime_data(self):
cube = new_cube(drop_bounds=True,
Expand All @@ -110,10 +125,13 @@ def test_get_time_range_from_data_with_irregular_cftime_data(self):
time_dtype=None)
time_range = get_time_range_from_data(cube)
self.assertIsNotNone(time_range)
self.assertEqual('2010-01-31T00:00:00', pd.Timestamp(time_range[0]).isoformat())
self.assertEqual('2010-06-30T00:00:00', pd.Timestamp(time_range[1]).isoformat())
self.assertEqual('2010-01-31T00:00:00',
pd.Timestamp(time_range[0]).isoformat())
self.assertEqual('2010-06-30T00:00:00',
pd.Timestamp(time_range[1]).isoformat())

def test_get_time_range_from_data_with_irregular_cftime_data_and_no_metadata(self):
def test_get_time_range_from_data_with_irregular_cftime_data_and_no_metadata(
self):
cube = new_cube(drop_bounds=True,
time_freq='M',
use_cftime=True,
Expand All @@ -122,94 +140,158 @@ def test_get_time_range_from_data_with_irregular_cftime_data_and_no_metadata(sel
cube.attrs.pop('time_coverage_end')
time_range = get_time_range_from_data(cube)
self.assertIsNotNone(time_range)
self.assertEqual('2010-02-14T00:00:00', pd.Timestamp(time_range[0]).isoformat())
self.assertEqual('2010-06-14T00:00:00', pd.Timestamp(time_range[1]).isoformat())
self.assertEqual('2010-02-14T00:00:00',
pd.Timestamp(time_range[0]).isoformat())
self.assertEqual('2010-06-14T00:00:00',
pd.Timestamp(time_range[1]).isoformat())

def test_get_time_range_from_data_time_named_t(self):
cube = new_cube(drop_bounds=True, time_name='t')
time_range = get_time_range_from_data(cube)
self.assertIsNotNone(time_range)
self.assertEqual('2010-01-01T00:00:00', pd.Timestamp(time_range[0]).isoformat())
self.assertEqual('2010-01-06T00:00:00', pd.Timestamp(time_range[1]).isoformat())
self.assertEqual('2010-01-01T00:00:00',
pd.Timestamp(time_range[0]).isoformat())
self.assertEqual('2010-01-06T00:00:00',
pd.Timestamp(time_range[1]).isoformat())

def test_get_time_range_from_data_additional_t_variable(self):
import xarray as xr
start_time_data = pd.date_range(start='2010-01-03T12:00:00',
periods=5,
freq='5D').values.astype(dtype='datetime64[s]')
freq='5D').values.astype(
dtype='datetime64[s]')
start_time = xr.DataArray(start_time_data, dims='time')
end_time_data = pd.date_range(start='2010-01-07T12:00:00',
periods=5,
freq='5D').values.astype(dtype='datetime64[s]')
freq='5D').values.astype(
dtype='datetime64[s]')
end_time = xr.DataArray(end_time_data, dims='time')
cube = new_cube(drop_bounds=True,
time_start='2010-01-05T12:00:00',
time_freq='5D',
variables=dict(start_time=start_time, end_time=end_time))
variables=dict(start_time=start_time,
end_time=end_time))
time_range = get_time_range_from_data(cube)
self.assertIsNotNone(time_range)
self.assertEqual('2010-01-03T12:00:00', pd.Timestamp(time_range[0]).isoformat())
self.assertEqual('2010-01-27T12:00:00', pd.Timestamp(time_range[1]).isoformat())
self.assertEqual('2010-01-03T12:00:00',
pd.Timestamp(time_range[0]).isoformat())
self.assertEqual('2010-01-27T12:00:00',
pd.Timestamp(time_range[1]).isoformat())

def test_get_time_range_from_data_start_and_end_time_arrays(self):
cube = new_cube(drop_bounds=True,
use_cftime=True,
time_dtype=None)
time_range = get_time_range_from_data(cube)
self.assertIsNotNone(time_range)
self.assertEqual('2010-01-01T00:00:00', pd.Timestamp(time_range[0]).isoformat())
self.assertEqual('2010-01-06T00:00:00', pd.Timestamp(time_range[1]).isoformat())
self.assertEqual('2010-01-01T00:00:00',
pd.Timestamp(time_range[0]).isoformat())
self.assertEqual('2010-01-06T00:00:00',
pd.Timestamp(time_range[1]).isoformat())

def test_get_time_range_from_data_bounds(self):
cube = new_cube()
time_range = get_time_range_from_data(cube)
self.assertIsNotNone(time_range)
self.assertEqual('2010-01-01T00:00:00', pd.Timestamp(time_range[0]).isoformat())
self.assertEqual('2010-01-06T00:00:00', pd.Timestamp(time_range[1]).isoformat())
self.assertEqual('2010-01-01T00:00:00',
pd.Timestamp(time_range[0]).isoformat())
self.assertEqual('2010-01-06T00:00:00',
pd.Timestamp(time_range[1]).isoformat())

def test_get_time_range_from_attrs(self):
cube = new_cube()
time_range = get_time_range_from_attrs(cube)
self.assertIsNotNone(time_range)
self.assertEqual('2010-01-01T00:00:00', pd.Timestamp(time_range[0]).isoformat())
self.assertEqual('2010-01-06T00:00:00', pd.Timestamp(time_range[1]).isoformat())
self.assertEqual('2010-01-01T00:00:00',
pd.Timestamp(time_range[0]).isoformat())
self.assertEqual('2010-01-06T00:00:00',
pd.Timestamp(time_range[1]).isoformat())

def test_get_start_time_from_attrs(self):
cube = new_cube()
start_time = get_start_time_from_attrs(cube)
self.assertEqual('2010-01-01T00:00:00', pd.Timestamp(start_time).isoformat())
self.assertEqual('2010-01-01T00:00:00',
pd.Timestamp(start_time).isoformat())

def test_get_end_time_from_attrs(self):
cube = new_cube()
end_time = get_end_time_from_attrs(cube)
self.assertEqual('2010-01-06T00:00:00', pd.Timestamp(end_time).isoformat())
self.assertEqual('2010-01-06T00:00:00',
pd.Timestamp(end_time).isoformat())


class TimestampToIsoStringTest(unittest.TestCase):
def test_it_with_default_res(self):
self.assertEqual("2018-09-05T00:00:00Z",
timestamp_to_iso_string(np.datetime64("2018-09-05")))
self.assertEqual("2018-09-05T10:35:42Z",
timestamp_to_iso_string(np.datetime64("2018-09-05 10:35:42")))
timestamp_to_iso_string(
np.datetime64("2018-09-05 10:35:42")))
self.assertEqual("2018-09-05T10:35:42Z",
timestamp_to_iso_string(np.datetime64("2018-09-05 10:35:42.164")))
timestamp_to_iso_string(
np.datetime64("2018-09-05 10:35:42.164")))
self.assertEqual("2019-10-04T10:13:49Z",
timestamp_to_iso_string(pd.to_datetime("04-OCT-2019 10:13:48.538184")))
timestamp_to_iso_string(
pd.to_datetime("04-OCT-2019 10:13:48.538184")))

def test_it_with_ceil_round_fn(self):
self.assertEqual("2018-09-05T00:00:00Z",
timestamp_to_iso_string(np.datetime64("2018-09-05"),
round_fn="ceil"))
self.assertEqual("2018-09-05T10:35:42Z",
timestamp_to_iso_string(
np.datetime64("2018-09-05 10:35:42"),
round_fn="ceil"))
self.assertEqual("2018-09-05T10:35:43Z",
timestamp_to_iso_string(
np.datetime64("2018-09-05 10:35:42.164"),
round_fn="ceil"))
self.assertEqual("2019-10-04T10:13:49Z",
timestamp_to_iso_string(
pd.to_datetime("04-OCT-2019 10:13:48.538184"),
round_fn="ceil"))

def test_it_with_floor_round_fn(self):
self.assertEqual("2018-09-05T00:00:00Z",
timestamp_to_iso_string(np.datetime64("2018-09-05"),
round_fn="floor"))
self.assertEqual("2018-09-05T10:35:42Z",
timestamp_to_iso_string(
np.datetime64("2018-09-05 10:35:42"),
round_fn="floor"))
self.assertEqual("2018-09-05T10:35:42Z",
timestamp_to_iso_string(
np.datetime64("2018-09-05 10:35:42.164"),
round_fn="floor"))
self.assertEqual("2019-10-04T10:13:48Z",
timestamp_to_iso_string(
pd.to_datetime("04-OCT-2019 10:13:48.538184"),
round_fn="floor"))

# noinspection PyMethodMayBeStatic
def test_it_with_invalid_round_fn(self):
with pytest.raises(ValueError,
match=r"round_fn must be one of"
r" \('ceil', 'floor', 'round'\)"):
timestamp_to_iso_string(np.datetime64("2018-09-05 10:35:42.164"),
round_fn="foo")

def test_it_with_h_res(self):
self.assertEqual("2018-09-05T00:00:00Z",
timestamp_to_iso_string(np.datetime64("2018-09-05"),
freq="H"))
self.assertEqual("2018-09-05T11:00:00Z",
timestamp_to_iso_string(np.datetime64("2018-09-05 10:35:42"),
freq="H"))
timestamp_to_iso_string(
np.datetime64("2018-09-05 10:35:42"),
freq="H"))
self.assertEqual("2018-09-05T11:00:00Z",
timestamp_to_iso_string(np.datetime64("2018-09-05 10:35:42.164"),
freq="H"))
timestamp_to_iso_string(
np.datetime64("2018-09-05 10:35:42.164"),
freq="H"))
self.assertEqual("2019-10-04T10:00:00Z",
timestamp_to_iso_string(pd.to_datetime("04-OCT-2019 10:13:48.538184"),
freq="H"))

timestamp_to_iso_string(
pd.to_datetime("04-OCT-2019 10:13:48.538184"),
freq="H"))

# class TimeStampsTest(unittest.TestCase):
#
Expand Down
Loading