From aa0ae3d2aba942523e7dd36077fac5f07edfa37c Mon Sep 17 00:00:00 2001 From: dzelge Date: Fri, 7 May 2021 12:42:41 +0200 Subject: [PATCH 1/7] Ensured that gen2 prints error tracebacks --- xcube/cli/gen2.py | 1 + 1 file changed, 1 insertion(+) diff --git a/xcube/cli/gen2.py b/xcube/cli/gen2.py index ef5d0fd97..9707e8794 100644 --- a/xcube/cli/gen2.py +++ b/xcube/cli/gen2.py @@ -135,6 +135,7 @@ def dump_cube_info(cube_info: CubeInfo): print(**print_kwargs) print('Remote traceback:', **print_kwargs) print('=================', **print_kwargs) + print(e.remote_traceback, file=sys.stderr) raise click.ClickException(f'{e}') from e except DataStoreError as e: From 8cef98e15ed624cef0c8389d0e4e82daeb060242 Mon Sep 17 00:00:00 2001 From: dzelge Date: Fri, 7 May 2021 12:52:02 +0200 Subject: [PATCH 2/7] Updated CHANGELOG for fixing issue 448 --- CHANGES.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 316ed0efe..72d40991a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,7 @@ ## Changes in 0.8.2 (in development) - +* Fixed the issue that xcube gen2 would not print tracebacks to stderr when raising + CubeGeneratorErrors (#448). ## Changes in 0.8.1 From 94e103dc24fb32a2695b5bb13fb1f40baece7854 Mon Sep 17 00:00:00 2001 From: Helge Dzierzon <42441761+dzelge@users.noreply.github.com> Date: Mon, 10 May 2021 10:36:43 +0200 Subject: [PATCH 3/7] Update xcube/cli/gen2.py Co-authored-by: Norman Fomferra --- xcube/cli/gen2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xcube/cli/gen2.py b/xcube/cli/gen2.py index 9707e8794..ce4dc524c 100644 --- a/xcube/cli/gen2.py +++ b/xcube/cli/gen2.py @@ -135,7 +135,7 @@ def dump_cube_info(cube_info: CubeInfo): print(**print_kwargs) print('Remote traceback:', **print_kwargs) print('=================', **print_kwargs) - print(e.remote_traceback, file=sys.stderr) + print(e.remote_traceback, **print_kwargs) raise click.ClickException(f'{e}') from e except DataStoreError as e: From 8dfb7dd619fa4bbe096d6084a786d72c19a8c022 Mon Sep 17 00:00:00 2001 From: Tonio Fincke Date: Tue, 18 May 2021 14:29:10 +0200 Subject: [PATCH 4/7] improved support of cftime --- test/core/test_timecoord.py | 40 +++++++++++++++++++++++++++++++++++++ xcube/core/timecoord.py | 15 ++++++-------- 2 files changed, 46 insertions(+), 9 deletions(-) diff --git a/test/core/test_timecoord.py b/test/core/test_timecoord.py index 471169881..462fd6e83 100644 --- a/test/core/test_timecoord.py +++ b/test/core/test_timecoord.py @@ -73,6 +73,24 @@ def test_get_time_range_from_data(self): self.assertEqual('2010-01-01T00:00:00', pd.Timestamp(time_range[0]).isoformat()) self.assertEqual('2010-01-06T00:00:00', pd.Timestamp(time_range[1]).isoformat()) + def test_get_time_range_from_data_with_irregular_data(self): + cube = new_cube(drop_bounds=True, + time_freq='M') + time_range = get_time_range_from_data(cube) + self.assertIsNotNone(time_range) + self.assertEqual('2010-01-31T00:00:00', pd.Timestamp(time_range[0]).isoformat()) + self.assertEqual('2010-06-30T00:00:00', pd.Timestamp(time_range[1]).isoformat()) + + def test_get_time_range_from_data_with_irregular_data_and_no_metadata(self): + cube = new_cube(drop_bounds=True, + time_freq='M') + cube.attrs.pop('time_coverage_start') + cube.attrs.pop('time_coverage_end') + time_range = get_time_range_from_data(cube) + self.assertIsNotNone(time_range) + self.assertEqual('2010-02-14T00:00:00', pd.Timestamp(time_range[0]).isoformat()) + self.assertEqual('2010-06-14T00:00:00', pd.Timestamp(time_range[1]).isoformat()) + def test_get_time_range_from_data_cftime(self): cube = new_cube(drop_bounds=True, use_cftime=True, @@ -82,6 +100,28 @@ def test_get_time_range_from_data_cftime(self): self.assertEqual('2010-01-01T00:00:00', pd.Timestamp(time_range[0]).isoformat()) self.assertEqual('2010-01-06T00:00:00', pd.Timestamp(time_range[1]).isoformat()) + def test_get_time_range_from_data_with_irregular_cftime_data(self): + cube = new_cube(drop_bounds=True, + time_freq='M', + use_cftime=True, + time_dtype=None) + time_range = get_time_range_from_data(cube) + self.assertIsNotNone(time_range) + self.assertEqual('2010-01-31T00:00:00', pd.Timestamp(time_range[0]).isoformat()) + self.assertEqual('2010-06-30T00:00:00', pd.Timestamp(time_range[1]).isoformat()) + + def test_get_time_range_from_data_with_irregular_cftime_data_and_no_metadata(self): + cube = new_cube(drop_bounds=True, + time_freq='M', + use_cftime=True, + time_dtype=None) + cube.attrs.pop('time_coverage_start') + cube.attrs.pop('time_coverage_end') + time_range = get_time_range_from_data(cube) + self.assertIsNotNone(time_range) + self.assertEqual('2010-02-14T00:00:00', pd.Timestamp(time_range[0]).isoformat()) + self.assertEqual('2010-06-14T00:00:00', pd.Timestamp(time_range[1]).isoformat()) + def test_get_time_range_from_data_time_named_t(self): cube = new_cube(drop_bounds=True, time_name='t') time_range = get_time_range_from_data(cube) diff --git a/xcube/core/timecoord.py b/xcube/core/timecoord.py index 60ec531bc..15ca3bdd7 100644 --- a/xcube/core/timecoord.py +++ b/xcube/core/timecoord.py @@ -118,17 +118,14 @@ def get_time_range_from_data(dataset: xr.Dataset, maybe_consider_metadata: bool= time_diff = time.diff(dim=time.dims[0]).values time_res = time_diff[0] time_regular = all([time_res - diff == np.timedelta64(0) for diff in time_diff[1:]]) + is_cf_time = type(time.values[0]).__module__.startswith('cftime') + data_start = pd.to_datetime(time.values[0].isoformat()) if is_cf_time else time.values[0] + data_end = pd.to_datetime(time.values[-1].isoformat()) if is_cf_time else time.values[-1] if time_regular: - try: - return time.values[0] - time_res / 2, time.values[-1] + time_res / 2 - except TypeError: - # Time is probably given as cftime.DatetimeJulian or cftime.DatetimeGregorian - # To convert it to datetime, we must derive its isoformat first. - return (pd.to_datetime(time.values[0].isoformat()) - time_res / 2).to_datetime64(), \ - (pd.to_datetime(time.values[-1].isoformat()) + time_res / 2).to_datetime64() + return data_start - time_res / 2, data_end + time_res / 2 return _maybe_return_time_range_from_metadata(dataset, - time.values[0], - time.values[-1], + data_start, + data_end, maybe_consider_metadata) From 3072dfff6b3ad10f542c28680b21ddbb477f9eda Mon Sep 17 00:00:00 2001 From: Tonio Fincke Date: Thu, 20 May 2021 16:01:07 +0200 Subject: [PATCH 5/7] improved support of cftime --- xcube/core/timecoord.py | 47 +++++++++++++++++++++-------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/xcube/core/timecoord.py b/xcube/core/timecoord.py index 15ca3bdd7..eb4718b24 100644 --- a/xcube/core/timecoord.py +++ b/xcube/core/timecoord.py @@ -19,6 +19,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +import cftime import datetime from typing import Optional, Sequence, Tuple, Union @@ -56,15 +57,16 @@ def add_time_coords(dataset: xr.Dataset, time_range: Tuple[float, float]) -> xr. time_var.encoding['calendar'] = DATETIME_CALENDAR if t1 != t2: time_var.attrs['bounds'] = 'time_bnds' - dataset = dataset.assign_coords(time_bnds=(['time', 'bnds'], - from_time_in_days_since_1970([t1, t2]).reshape(1, 2))) + dataset = dataset.assign_coords( + time_bnds=(['time', 'bnds'], from_time_in_days_since_1970([t1, t2]).reshape(1, 2))) time_bnds_var = dataset.coords['time_bnds'] time_bnds_var.attrs['long_name'] = 'time' time_bnds_var.attrs['standard_name'] = 'time' # Avoiding xarray error: - # ValueError: failed to prevent overwriting existing key units in attrs on variable 'time'. - # This is probably an encoding field used by xarray to describe how a variable is serialized. - # To proceed, remove this key from the variable's attributes manually. + # ValueError: failed to prevent overwriting existing key units in attrs on variable + # 'time'. This is probably an encoding field used by xarray to describe how a variable + # is serialized. + # To proceed, remove this key from the variable's attributes manually. # time_bnds_var.attrs['units'] = DATETIME_UNITS # time_bnds_var.attrs['calendar'] = DATETIME_CALENDAR time_bnds_var.encoding['units'] = DATETIME_UNITS @@ -72,7 +74,7 @@ def add_time_coords(dataset: xr.Dataset, time_range: Tuple[float, float]) -> xr. return dataset -def get_time_range_from_data(dataset: xr.Dataset, maybe_consider_metadata: bool=True) \ +def get_time_range_from_data(dataset: xr.Dataset, maybe_consider_metadata: bool = True) \ -> Tuple[Optional[float], Optional[float]]: """ Determines a time range from a dataset by inspecting its time_bounds or time data arrays. @@ -80,7 +82,7 @@ def get_time_range_from_data(dataset: xr.Dataset, maybe_consider_metadata: bool= metadata may be considered. :param dataset: The dataset of which the time range shall be determined - "param maybe_consider_metadata": Whether metadata shall be considered. + :param maybe_consider_metadata: Whether metadata shall be considered. Only used when the dataset has no time bounds array and no time periodicity. The values will only be set when they do not contradict the values from the data arrays. :return: A tuple with two float values: The first one represents the start time, @@ -105,22 +107,19 @@ def get_time_range_from_data(dataset: xr.Dataset, maybe_consider_metadata: bool= time_bnds_name = time.attrs.get("bounds", "time_bnds") if time_bnds_name in dataset: return _get_time_range_from_time_bounds(dataset, time_bnds_name) - if time.size == 1: + is_cf_time = isinstance(time[0].values.item(), cftime.datetime) + data_start = pd.to_datetime(time[0].values.item().isoformat()) \ + if is_cf_time else time[0].values + data_end = pd.to_datetime(time[-1].values.item().isoformat()) \ + if is_cf_time else time[-1].values + if time.size < 3: return _maybe_return_time_range_from_metadata(dataset, - time.values[0], - time.values[0], - maybe_consider_metadata) - if time.size == 2: - return _maybe_return_time_range_from_metadata(dataset, - time.values[0], - time.values[1], + data_start, + data_end, maybe_consider_metadata) time_diff = time.diff(dim=time.dims[0]).values time_res = time_diff[0] time_regular = all([time_res - diff == np.timedelta64(0) for diff in time_diff[1:]]) - is_cf_time = type(time.values[0]).__module__.startswith('cftime') - data_start = pd.to_datetime(time.values[0].isoformat()) if is_cf_time else time.values[0] - data_end = pd.to_datetime(time.values[-1].isoformat()) if is_cf_time else time.values[-1] if time_regular: return data_start - time_res / 2, data_end + time_res / 2 return _maybe_return_time_range_from_metadata(dataset, @@ -186,15 +185,16 @@ def remove_time_part_from_isoformat(datetime_str: str) -> str: def to_time_in_days_since_1970(time_str: str, pattern=None) -> float: - datetime = pd.to_datetime(time_str, format=pattern, infer_datetime_format=False, utc=True) - timedelta = datetime - REF_DATETIME + date_time = pd.to_datetime(time_str, format=pattern, infer_datetime_format=False, utc=True) + timedelta = date_time - REF_DATETIME return timedelta.days + timedelta.seconds / SECONDS_PER_DAY + \ timedelta.microseconds / MICROSECONDS_PER_DAY def from_time_in_days_since_1970(time_value: Union[float, Sequence[float]]) -> np.ndarray: if isinstance(time_value, int) or isinstance(time_value, float): - return pd.to_datetime(time_value, utc=True, unit='d', origin='unix').round(freq='ms').to_datetime64() + return pd.to_datetime(time_value, utc=True, unit='d', origin='unix').round(freq='ms').\ + to_datetime64() else: return np.array(list(map(from_time_in_days_since_1970, time_value))) @@ -204,8 +204,9 @@ def timestamp_to_iso_string(time: Union[np.datetime64, datetime.datetime], freq= Convert a UTC timestamp given as nanos, millis, seconds, etc. since 1970-01-01 00:00:00 to an ISO-format string. - :param time: UTC timestamp given as time delta since since 1970-01-01 00:00:00 in the units given by - the numpy datetime64 type, so it can be as nanos, millis, seconds since 1970-01-01 00:00:00. + :param time: UTC timestamp given as time delta since since 1970-01-01 00:00:00 in the units + given by the numpy datetime64 type, so it can be as nanos, millis, + seconds since 1970-01-01 00:00:00. :param freq: time rounding resolution. See pandas.Timestamp.round(). :return: ISO-format string. """ From 9852a5311b7ec7e8ef14c4a74847d89ec7514601 Mon Sep 17 00:00:00 2001 From: Tonio Fincke Date: Wed, 26 May 2021 14:50:37 +0200 Subject: [PATCH 6/7] pep-8 Co-authored-by: Norman Fomferra --- xcube/core/timecoord.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/xcube/core/timecoord.py b/xcube/core/timecoord.py index eb4718b24..f2ff1f58c 100644 --- a/xcube/core/timecoord.py +++ b/xcube/core/timecoord.py @@ -58,7 +58,8 @@ def add_time_coords(dataset: xr.Dataset, time_range: Tuple[float, float]) -> xr. if t1 != t2: time_var.attrs['bounds'] = 'time_bnds' dataset = dataset.assign_coords( - time_bnds=(['time', 'bnds'], from_time_in_days_since_1970([t1, t2]).reshape(1, 2))) + time_bnds=(['time', 'bnds'], from_time_in_days_since_1970([t1, t2]).reshape(1, 2)) + ) time_bnds_var = dataset.coords['time_bnds'] time_bnds_var.attrs['long_name'] = 'time' time_bnds_var.attrs['standard_name'] = 'time' From 3082d772589dfbc5a4224a9c47edc12b8309af07 Mon Sep 17 00:00:00 2001 From: Tonio Fincke Date: Wed, 26 May 2021 14:50:56 +0200 Subject: [PATCH 7/7] pep-8 Co-authored-by: Norman Fomferra --- xcube/core/timecoord.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/xcube/core/timecoord.py b/xcube/core/timecoord.py index f2ff1f58c..b1cdfb3c6 100644 --- a/xcube/core/timecoord.py +++ b/xcube/core/timecoord.py @@ -194,8 +194,8 @@ def to_time_in_days_since_1970(time_str: str, pattern=None) -> float: def from_time_in_days_since_1970(time_value: Union[float, Sequence[float]]) -> np.ndarray: if isinstance(time_value, int) or isinstance(time_value, float): - return pd.to_datetime(time_value, utc=True, unit='d', origin='unix').round(freq='ms').\ - to_datetime64() + return pd.to_datetime(time_value, utc=True, unit='d', origin='unix').round(freq='ms') \ + .to_datetime64() else: return np.array(list(map(from_time_in_days_since_1970, time_value)))