From c33fdc934806e0f6dc3b6435be188d80953df62f Mon Sep 17 00:00:00 2001 From: Andreas Motl Date: Wed, 25 Sep 2024 01:51:36 +0200 Subject: [PATCH] MongoDB: Fix timestamp conversion --- CHANGES.md | 1 + src/commons_codec/transform/mongodb.py | 5 ++--- tests/transform/mongodb/data.py | 12 ++++++------ tests/transform/mongodb/test_mongodb_cdc.py | 4 ++-- tests/transform/mongodb/test_mongodb_convert.py | 14 +++++++------- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 762444c..d86e6cb 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -7,6 +7,7 @@ removes the element altogether if it's empty - Zyp/Moksha/jq: Improve error reporting at `MokshaTransformation.apply` - MongoDB: Improved `MongoDBCrateDBConverter.decode_extended_json` +- MongoDB: Fixed timestamp conversion ## 2024/09/22 v0.0.17 - MongoDB: Fixed edge case when decoding MongoDB Extended JSON elements diff --git a/src/commons_codec/transform/mongodb.py b/src/commons_codec/transform/mongodb.py index d2fa01c..2878547 100644 --- a/src/commons_codec/transform/mongodb.py +++ b/src/commons_codec/transform/mongodb.py @@ -2,7 +2,6 @@ # Distributed under the terms of the LGPLv3 license, see LICENSE. # ruff: noqa: S608 import base64 -import calendar import datetime as dt import logging import typing as t @@ -134,7 +133,7 @@ def decode_extended_json(self, value: t.Dict[str, t.Any]) -> t.Any: out = self.convert_epoch(out) if self.timestamp_use_milliseconds: out *= 1000 - return out + return int(out) elif self.timestamp_to_iso8601: return self.convert_iso8601(out) @@ -163,7 +162,7 @@ def convert_epoch(value: t.Any) -> float: datetime = dateparser.parse(value) else: raise ValueError(f"Unable to convert datetime value: {value}") - return calendar.timegm(datetime.utctimetuple()) + return datetime.timestamp() @staticmethod def convert_iso8601(value: t.Any) -> str: diff --git a/tests/transform/mongodb/data.py b/tests/transform/mongodb/data.py index 037ae2a..f42a039 100644 --- a/tests/transform/mongodb/data.py +++ b/tests/transform/mongodb/data.py @@ -115,7 +115,7 @@ "_id": "56027fcae4b09385a85f9344", "python": { "boolean": True, - "datetime": 1721140162000, + "datetime": 1721132962907, "dict_basic": {"foo": "bar"}, "dict_dollarkey": {"$a": "foo"}, # "dict_dottedkey": {'a.b': 'foo'}, # TODO: InvalidColumnNameException["." contains a dot] @@ -144,7 +144,7 @@ "foo": "bar", }, }, - "datetimems": 1721140162000, + "datetimems": 1721132962987, "binary_uuid": "73636373-6363-7363-6373-636373636373", "decimal128": "42.42", "dbref": { @@ -168,7 +168,7 @@ "x": 42, }, }, - "date_iso8601": 1443004362000, + "date_iso8601": 1442997162330, "date_numberlong": 1356351330000, "dbref": { "$id": "56027fcae4b09385a85f9344", @@ -183,13 +183,13 @@ "int32": -2147483648, "int64": "-9223372036854775808", # TODO: Representation as string is just fine? "list_date": [ - 1443090762000, + 1443083562330, 2147483647000, -2147483648000, ], "list_dict": [ - {"id": "bar", "value": 1443090762000}, - {"value": 1443090762000}, + {"id": "bar", "value": 1443083562330}, + {"value": 1443083562330}, ], "list_int": [ -2147483648, diff --git a/tests/transform/mongodb/test_mongodb_cdc.py b/tests/transform/mongodb/test_mongodb_cdc.py index 9980b8b..214bb72 100644 --- a/tests/transform/mongodb/test_mongodb_cdc.py +++ b/tests/transform/mongodb/test_mongodb_cdc.py @@ -126,7 +126,7 @@ def test_decode_cdc_insert(): "_id": "669683c2b0750b2c84893f3e", "id": "5F9E", "data": {"temperature": 42.42, "humidity": 84.84}, - "meta": {"timestamp": 1720739862000, "device": "foo"}, + "meta": {"timestamp": 1720732662000, "device": "foo"}, }, }, ) @@ -140,7 +140,7 @@ def test_decode_cdc_update(): "_id": "669683c2b0750b2c84893f3e", "id": "5F9E", "data": {"temperature": 42.5}, - "meta": {"timestamp": 1720739862000, "device": "foo"}, + "meta": {"timestamp": 1720732662000, "device": "foo"}, } }, ) diff --git a/tests/transform/mongodb/test_mongodb_convert.py b/tests/transform/mongodb/test_mongodb_convert.py index a3e5afa..6f5ad11 100644 --- a/tests/transform/mongodb/test_mongodb_convert.py +++ b/tests/transform/mongodb/test_mongodb_convert.py @@ -28,8 +28,8 @@ def test_epoch_ms_converter_iso8601(): """ Datetime values encoded as ISO8601 values will be parsed. """ - assert convert_epoch("2015-09-23T10:32:42.33Z") == 1443004362 - assert convert_epoch(b"2015-09-23T10:32:42.33Z") == 1443004362 + assert convert_epoch("2015-09-23T10:32:42.33Z") == 1443004362.33 + assert convert_epoch(b"2015-09-23T10:32:42.33Z") == 1443004362.33 def test_epoch_ms_converter_invalid(): @@ -109,12 +109,12 @@ class DateConversionCase: DateConversionCase( converter=MongoDBCrateDBConverter(timestamp_to_epoch=True, timestamp_use_milliseconds=True), data_in={"$date": "2015-09-23T10:32:42.123456Z"}, - data_out=1443004362000, + data_out=1442997162123, ), DateConversionCase( converter=MongoDBCrateDBConverter(timestamp_to_epoch=True, timestamp_use_milliseconds=True), data_in={"$date": {"$numberLong": "1655210544987"}}, - data_out=1655210544000, + data_out=1655210544987, ), DateConversionCase( converter=MongoDBCrateDBConverter(timestamp_to_iso8601=True), @@ -168,7 +168,7 @@ def test_convert_with_treatment_ignore_complex_lists(): "_id": "56027fcae4b09385a85f9344", "value": { "id": 42, - "date": 1443004362000, + "date": 1442997162330, }, } @@ -203,7 +203,7 @@ def test_convert_with_treatment_normalize_complex_lists(): "_id": "56027fcae4b09385a85f9344", "value": { "id": 42, - "date": 1443004362000, + "date": 1442997162330, "some_complex_list": [ {"id": "foo", "value": "something"}, # FIXME: `normalize_complex_lists` does not see it's a timestamp. @@ -244,7 +244,7 @@ def test_convert_with_treatment_all_options(): data_out = { "_id": "56027fcae4b09385a85f9344", "value": { - "date": 1443004362000, + "date": 1442997162330, "id": 42, }, "to_list": [42],