Let {:datetime, _} be an alias for {:naive_datetime, _} for backw…

…ards compatibility (#924)

* fix typo

* allow for alias

* fix bug I think?

* document alias

* test that alias works with warning

* don't document the deprecated alias

Co-authored-by: José Valim <[email protected]>

* use microsecond

* ci test fix attempt

---------

Co-authored-by: José Valim <[email protected]>

lib/explorer/data_frame.ex

@@ -2863,10 +2863,10 @@ defmodule Explorer.DataFrame do
         LazySeries.new(:lazy, [date], :date)
 
       naive_datetime = %NaiveDateTime{} ->
-        LazySeries.new(:lazy, [naive_datetime], {:naive_datetime, :nanosecond})
+        LazySeries.new(:lazy, [naive_datetime], {:naive_datetime, :microsecond})
 
-      datetime = %DateTime{} ->
-        LazySeries.new(:lazy, [datetime], {:datetime, :nanosecond})
+      datetime = %DateTime{time_zone: time_zone} ->
+        LazySeries.new(:lazy, [datetime], {:datetime, :microsecond, time_zone})
 
       duration = %Explorer.Duration{precision: precision} ->
         LazySeries.new(:lazy, [duration], {:duration, precision})

lib/explorer/polars_backend/expression.ex

@@ -333,7 +333,8 @@ defmodule Explorer.PolarsBackend.Expression do
   def to_expr(number) when is_integer(number), do: Native.expr_integer(number)
   def to_expr(number) when is_float(number), do: Native.expr_float(number)
   def to_expr(%Date{} = date), do: Native.expr_date(date)
-  def to_expr(%NaiveDateTime{} = datetime), do: Native.expr_datetime(datetime)
+  def to_expr(%NaiveDateTime{} = naive_datetime), do: Native.expr_naive_datetime(naive_datetime)
+  # def to_expr(%DateTime{} = datetime), do: Native.expr_datetime(datetime)
   def to_expr(%Explorer.Duration{} = duration), do: Native.expr_duration(duration)
   def to_expr(%PolarsSeries{} = polars_series), do: Native.expr_series(polars_series)
 

lib/explorer/polars_backend/native.ex

@@ -193,7 +193,8 @@ defmodule Explorer.PolarsBackend.Native do
   def expr_atom(_atom), do: err()
   def expr_boolean(_bool), do: err()
   def expr_date(_date), do: err()
-  def expr_datetime(_datetime), do: err()
+  def expr_naive_datetime(_datetime), do: err()
+  # def expr_datetime(_datetime), do: err()
   def expr_duration(_duration), do: err()
   def expr_describe_filter_plan(_df, _expr), do: err()
   def expr_float(_number), do: err()

lib/explorer/series.ex

@@ -30,7 +30,7 @@ defmodule Explorer.Series do
   and compatibility with the Elixir ecosystem:
 
     * All numeric dtypes (signed integer, unsigned integer, and floats) can
-      be specified as an atom in the form of `:s32`, `:u8`, `:f32` and o son
+      be specified as an atom in the form of `:s32`, `:u8`, `:f32` and so on
     * The atom `:float` as an alias for `{:f, 64}` to mirror Elixir's floats
     * The atom `:integer` as an alias for `{:s, 64}` to mirror Elixir's integers
 
@@ -370,7 +370,7 @@ defmodule Explorer.Series do
   A list of `Date`, `Time`, `NaiveDateTime`, `DateTime`, and
   `Explorer.Duration` structs are also supported, and they will become series
   with the respective dtypes: `:date`, `:time`, `{:naive_datetime, :microsecond}`,
-  and `{:duration, precision}`.
+  `{:datetime, :microsecond, time_zone}` and `{:duration, precision}`.
   For example:
 
       iex> Explorer.Series.from_list([~D[0001-01-01], ~D[1970-01-01], ~D[1986-10-13]])

lib/explorer/shared.ex

@@ -2,6 +2,8 @@ defmodule Explorer.Shared do
   # A collection of **private** helpers shared in Explorer.
   @moduledoc false
 
+  require Logger
+
   @integer_types [
     {:s, 8},
     {:s, 16},
@@ -87,6 +89,16 @@ defmodule Explorer.Shared do
   def normalise_dtype(:u32), do: {:u, 32}
   def normalise_dtype(:u64), do: {:u, 64}
 
+  def normalise_dtype({:datetime, precision}) do
+    :ok =
+      Logger.warning("""
+      The `{:datetime, _}` dtype has been deprecated.
+      Please use `{:naive_datetime, _}` instead.
+      """)
+
+    {:naive_datetime, precision}
+  end
+
   def normalise_dtype(_dtype), do: nil
 
   @doc """

native/explorer/src/datatypes.rs

@@ -363,7 +363,19 @@ impl From<NaiveDateTime> for ExNaiveDateTime {
 
 impl Literal for ExNaiveDateTime {
     fn lit(self) -> Expr {
-        NaiveDateTime::from(self).lit()
+        let ndt = NaiveDateTime::from(self);
+        // We can't call `ndt.lit()` because Polars defaults to nanoseconds
+        // for all years between 1386 and 2554, but Elixir only represents
+        // microsecond precision natively. This code is copied from the
+        // microsecond branch of their `.lit()` implementation.
+        //
+        // See here for details:
+        // polars-time-0.38.3/src/date_range.rs::in_nanoseconds_window
+        Expr::Literal(LiteralValue::DateTime(
+            ndt.and_utc().timestamp_micros(),
+            TimeUnit::Microseconds,
+            None,
+        ))
     }
 }
 

native/explorer/src/expressions.rs

@@ -72,8 +72,8 @@ pub fn expr_date(date: ExDate) -> ExExpr {
 }
 
 #[rustler::nif]
-pub fn expr_datetime(datetime: ExNaiveDateTime) -> ExExpr {
-    ExExpr::new(datetime.lit())
+pub fn expr_naive_datetime(naive_datetime: ExNaiveDateTime) -> ExExpr {
+    ExExpr::new(naive_datetime.lit())
 }
 
 #[rustler::nif]

native/explorer/src/lib.rs

@@ -139,7 +139,8 @@ rustler::init!(
         expr_cast,
         expr_column,
         expr_date,
-        expr_datetime,
+        expr_naive_datetime,
+        // expr_datetime,
         expr_duration,
         expr_day_of_week,
         expr_day_of_year,

test/explorer/data_frame_test.exs

@@ -861,7 +861,7 @@ defmodule Explorer.DataFrameTest do
                "g" => :string,
                "h" => :boolean,
                "i" => :date,
-               "j" => {:naive_datetime, :nanosecond}
+               "j" => {:naive_datetime, :microsecond}
              }
     end
 

test/explorer/series_test.exs

@@ -5,6 +5,8 @@ defmodule Explorer.SeriesTest do
 
   alias Explorer.Series
 
+  import ExUnit.CaptureLog
+
   doctest Explorer.Series
 
   test "defines doc metadata" do
@@ -286,6 +288,29 @@ defmodule Explorer.SeriesTest do
                dates
     end
 
+    test "with naive datetimes aliased" do
+      naive_datetimes = [
+        ~N[2022-04-13 15:44:31.560227],
+        ~N[1022-01-04 21:18:31.224123],
+        ~N[1988-11-23 06:36:16.158432],
+        ~N[2353-03-07 00:39:35.702789]
+      ]
+
+      dtype_alias = {:datetime, :microsecond}
+      dtype_actual = {:naive_datetime, :microsecond}
+
+      {series, warning} =
+        with_log(fn -> Series.from_list(naive_datetimes, dtype: dtype_alias) end)
+
+      assert Series.to_list(series) == naive_datetimes
+      assert Series.dtype(series) == dtype_actual
+
+      assert warning =~ """
+             The `{:datetime, _}` dtype has been deprecated.
+             Please use `{:naive_datetime, _}` instead.
+             """
+    end
+
     test "with 32-bit integers" do
       for dtype <- [:s32, {:s, 32}] do
         s = Series.from_list([-1, 0, 1, 2, 3, nil], dtype: dtype)