Added some f;ags for handling timestampls in parquet

conbench · Aug 30, 2022 · 389a548 · 389a548
1 parent 84ed050
commit 389a548
Showing 1 changed file with 4 additions and 1 deletion.
diff --git a/datalogistik/util.py b/datalogistik/util.py
@@ -488,7 +488,10 @@ def convert_dataset(
                 if parquet_compression is None:
                     parquet_compression = "snappy"  # Use snappy by default
                 write_options = dataset_write_format.make_write_options(
-                    compression=parquet_compression
+                    compression=parquet_compression,
+                    use_deprecated_int96_timestamps=False,
+                    coerce_timestamps="us",
+                    allow_truncated_timestamps=True,
                 )
             if new_format == "csv":
                 dataset_write_format = ds.CsvFileFormat()