MongoDB: Fix timestamp conversion

CHANGES.md

@@ -7,6 +7,7 @@
   removes the element altogether if it's empty
 - Zyp/Moksha/jq: Improve error reporting at `MokshaTransformation.apply`
 - MongoDB: Improved `MongoDBCrateDBConverter.decode_extended_json`
+- MongoDB: Fixed timestamp conversion
 
 ## 2024/09/22 v0.0.17
 - MongoDB: Fixed edge case when decoding MongoDB Extended JSON elements

src/commons_codec/transform/mongodb.py

@@ -2,7 +2,6 @@
 # Distributed under the terms of the LGPLv3 license, see LICENSE.
 # ruff: noqa: S608
 import base64
-import calendar
 import datetime as dt
 import logging
 import typing as t
@@ -134,7 +133,7 @@ def decode_extended_json(self, value: t.Dict[str, t.Any]) -> t.Any:
                 out = self.convert_epoch(out)
                 if self.timestamp_use_milliseconds:
                     out *= 1000
-                return out
+                return int(out)
             elif self.timestamp_to_iso8601:
                 return self.convert_iso8601(out)
 
@@ -163,7 +162,7 @@ def convert_epoch(value: t.Any) -> float:
             datetime = dateparser.parse(value)
         else:
             raise ValueError(f"Unable to convert datetime value: {value}")
-        return calendar.timegm(datetime.utctimetuple())
+        return datetime.timestamp()
 
     @staticmethod
     def convert_iso8601(value: t.Any) -> str:

tests/transform/mongodb/data.py

@@ -115,7 +115,7 @@
     "_id": "56027fcae4b09385a85f9344",
     "python": {
         "boolean": True,
-        "datetime": 1721140162000,
+        "datetime": 1721132962907,
         "dict_basic": {"foo": "bar"},
         "dict_dollarkey": {"$a": "foo"},
         # "dict_dottedkey": {'a.b': 'foo'},  # TODO: InvalidColumnNameException["." contains a dot]
@@ -144,7 +144,7 @@
                 "foo": "bar",
             },
         },
-        "datetimems": 1721140162000,
+        "datetimems": 1721132962987,
         "binary_uuid": "73636373-6363-7363-6373-636373636373",
         "decimal128": "42.42",
         "dbref": {
@@ -168,7 +168,7 @@
                 "x": 42,
             },
         },
-        "date_iso8601": 1443004362000,
+        "date_iso8601": 1442997162330,
         "date_numberlong": 1356351330000,
         "dbref": {
             "$id": "56027fcae4b09385a85f9344",
@@ -183,13 +183,13 @@
         "int32": -2147483648,
         "int64": "-9223372036854775808",  # TODO: Representation as string is just fine?
         "list_date": [
-            1443090762000,
+            1443083562330,
             2147483647000,
             -2147483648000,
         ],
         "list_dict": [
-            {"id": "bar", "value": 1443090762000},
-            {"value": 1443090762000},
+            {"id": "bar", "value": 1443083562330},
+            {"value": 1443083562330},
         ],
         "list_int": [
             -2147483648,

tests/transform/mongodb/test_mongodb_cdc.py

@@ -126,7 +126,7 @@ def test_decode_cdc_insert():
                 "_id": "669683c2b0750b2c84893f3e",
                 "id": "5F9E",
                 "data": {"temperature": 42.42, "humidity": 84.84},
-                "meta": {"timestamp": 1720739862000, "device": "foo"},
+                "meta": {"timestamp": 1720732662000, "device": "foo"},
             },
         },
     )
@@ -140,7 +140,7 @@ def test_decode_cdc_update():
                 "_id": "669683c2b0750b2c84893f3e",
                 "id": "5F9E",
                 "data": {"temperature": 42.5},
-                "meta": {"timestamp": 1720739862000, "device": "foo"},
+                "meta": {"timestamp": 1720732662000, "device": "foo"},
             }
         },
     )

tests/transform/mongodb/test_mongodb_convert.py

@@ -28,8 +28,8 @@ def test_epoch_ms_converter_iso8601():
     """
     Datetime values encoded as ISO8601 values will be parsed.
     """
-    assert convert_epoch("2015-09-23T10:32:42.33Z") == 1443004362
-    assert convert_epoch(b"2015-09-23T10:32:42.33Z") == 1443004362
+    assert convert_epoch("2015-09-23T10:32:42.33Z") == 1443004362.33
+    assert convert_epoch(b"2015-09-23T10:32:42.33Z") == 1443004362.33
 
 
 def test_epoch_ms_converter_invalid():
@@ -109,12 +109,12 @@ class DateConversionCase:
     DateConversionCase(
         converter=MongoDBCrateDBConverter(timestamp_to_epoch=True, timestamp_use_milliseconds=True),
         data_in={"$date": "2015-09-23T10:32:42.123456Z"},
-        data_out=1443004362000,
+        data_out=1442997162123,
     ),
     DateConversionCase(
         converter=MongoDBCrateDBConverter(timestamp_to_epoch=True, timestamp_use_milliseconds=True),
         data_in={"$date": {"$numberLong": "1655210544987"}},
-        data_out=1655210544000,
+        data_out=1655210544987,
     ),
     DateConversionCase(
         converter=MongoDBCrateDBConverter(timestamp_to_iso8601=True),
@@ -168,7 +168,7 @@ def test_convert_with_treatment_ignore_complex_lists():
         "_id": "56027fcae4b09385a85f9344",
         "value": {
             "id": 42,
-            "date": 1443004362000,
+            "date": 1442997162330,
         },
     }
 
@@ -203,7 +203,7 @@ def test_convert_with_treatment_normalize_complex_lists():
         "_id": "56027fcae4b09385a85f9344",
         "value": {
             "id": 42,
-            "date": 1443004362000,
+            "date": 1442997162330,
             "some_complex_list": [
                 {"id": "foo", "value": "something"},
                 # FIXME: `normalize_complex_lists` does not see it's a timestamp.
@@ -244,7 +244,7 @@ def test_convert_with_treatment_all_options():
     data_out = {
         "_id": "56027fcae4b09385a85f9344",
         "value": {
-            "date": 1443004362000,
+            "date": 1442997162330,
             "id": 42,
         },
         "to_list": [42],