diff --git a/CHANGELOG.md b/CHANGELOG.md index 836a735cb..f40caca47 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed + +- `Explorer.Series.pow/2` no longer casts to float when the exponent is a signed + integer. We are following the way Polars works now, which is to try to execute + the operation or raise an exception in case the exponent is negative. + +- `Explorer.Series.pivot_wider/4` no longer includes the `names_from` column + name in the new columns when `values_from` is a list of columns. This is more + consistent with its behaviour when `values_from` is a single column. + +- `Explorer.Series.substring/3` no longer cycles to the end of the string if the + negative offset surpasses the beginning of that string. In that case, an empty + string is returned. + +- The `Explorer.Series.ewm_*` functions no longer replace `nil` values with the + value at the previous index. They now propogate `nil` values through to the + result series. + +- Saving a dataframe as a Parquet file to S3 services no longer works when + streaming is enabled. This is temporary due to a bug in Polars. An exception + should be raised instead. + ## [v0.8.3] - 2024-06-10 ### Added diff --git a/lib/explorer/data_frame.ex b/lib/explorer/data_frame.ex index 3b85753d1..6b12ad853 100644 --- a/lib/explorer/data_frame.ex +++ b/lib/explorer/data_frame.ex @@ -2788,7 +2788,7 @@ defmodule Explorer.DataFrame do #Explorer.DataFrame< Polars[3 x 2] a string ["a", "b", "c"] - b f64 [1.0, 4.0, 9.0] + b s64 [1, 4, 9] > It's possible to "reuse" a variable for different computations: @@ -4755,8 +4755,7 @@ defmodule Explorer.DataFrame do Multiple columns are accepted for the `values_from` parameter, but the behaviour is slightly different for the naming of new columns in the resultant dataframe. The new columns are going - to be prefixed by the name of the original value column, followed by an underscore and the - original column name, followed by the name of the variable. + to be prefixed by the name of the original value column, followed by the name of the variable. iex> df = Explorer.DataFrame.new( iex> product_id: [1, 1, 1, 1, 2, 2, 2, 2], @@ -4768,14 +4767,14 @@ defmodule Explorer.DataFrame do #Explorer.DataFrame< Polars[2 x 9] product_id s64 [1, 2] - property_value_property_product_id s64 [1, 2] - property_value_property_width_cm s64 [42, 35] - property_value_property_height_cm s64 [40, 20] - property_value_property_length_cm s64 [64, 40] - another_value_property_product_id s64 [1, 2] - another_value_property_width_cm s64 [43, 36] - another_value_property_height_cm s64 [41, 21] - another_value_property_length_cm s64 [65, 42] + property_value_product_id s64 [1, 2] + property_value_width_cm s64 [42, 35] + property_value_height_cm s64 [40, 20] + property_value_length_cm s64 [64, 40] + another_value_product_id s64 [1, 2] + another_value_width_cm s64 [43, 36] + another_value_height_cm s64 [41, 21] + another_value_length_cm s64 [65, 42] > ## Grouped examples @@ -6073,7 +6072,7 @@ defmodule Explorer.DataFrame do Basic example: iex> df = Explorer.DataFrame.new(a: [1, 2, 3], b: ["x", "y", "y"]) - iex> Explorer.DataFrame.sql(df, "select a, b from df group by b order by b") + iex> Explorer.DataFrame.sql(df, "select ARRAY_AGG(a), b from df group by b order by b") #Explorer.DataFrame< Polars[2 x 2] a list[s64] [[1], [2, 3]] diff --git a/lib/explorer/polars_backend/data_frame.ex b/lib/explorer/polars_backend/data_frame.ex index 54d64c73f..da27b437e 100644 --- a/lib/explorer/polars_backend/data_frame.ex +++ b/lib/explorer/polars_backend/data_frame.ex @@ -32,7 +32,7 @@ defmodule Explorer.PolarsBackend.DataFrame do with {:ok, df_result} <- adbc_result, {:ok, df} <- df_result, - do: {:ok, Shared.create_dataframe(df)} + do: Shared.create_dataframe(df) end @impl true @@ -122,7 +122,7 @@ defmodule Explorer.PolarsBackend.DataFrame do ) case df do - {:ok, df} -> {:ok, Shared.create_dataframe(df)} + {:ok, df} -> Shared.create_dataframe(df) {:error, error} -> {:error, RuntimeError.exception(error)} end end @@ -195,6 +195,11 @@ defmodule Explorer.PolarsBackend.DataFrame do {columns, with_projection} = column_names_or_projection(columns) + dtypes_list = + if not Enum.empty?(dtypes) do + Map.to_list(dtypes) + end + df = Native.df_load_csv( contents, @@ -207,7 +212,7 @@ defmodule Explorer.PolarsBackend.DataFrame do delimiter, true, columns, - Map.to_list(dtypes), + dtypes_list, encoding, nil_values, parse_dates, @@ -215,7 +220,7 @@ defmodule Explorer.PolarsBackend.DataFrame do ) case df do - {:ok, df} -> {:ok, Shared.create_dataframe(df)} + {:ok, df} -> Shared.create_dataframe(df) {:error, error} -> {:error, RuntimeError.exception(error)} end end @@ -242,7 +247,7 @@ defmodule Explorer.PolarsBackend.DataFrame do @impl true def from_ndjson(%Local.Entry{} = entry, infer_schema_length, batch_size) do case Native.df_from_ndjson(entry.path, infer_schema_length, batch_size) do - {:ok, df} -> {:ok, Shared.create_dataframe(df)} + {:ok, df} -> Shared.create_dataframe(df) {:error, error} -> {:error, RuntimeError.exception(error)} end end @@ -274,7 +279,7 @@ defmodule Explorer.PolarsBackend.DataFrame do @impl true def load_ndjson(contents, infer_schema_length, batch_size) when is_binary(contents) do case Native.df_load_ndjson(contents, infer_schema_length, batch_size) do - {:ok, df} -> {:ok, Shared.create_dataframe(df)} + {:ok, df} -> Shared.create_dataframe(df) {:error, error} -> {:error, RuntimeError.exception(error)} end end @@ -284,7 +289,7 @@ defmodule Explorer.PolarsBackend.DataFrame do # We first read using a lazy dataframe, then we collect. with {:ok, ldf} <- Native.lf_from_parquet_cloud(entry, max_rows, columns), {:ok, df} <- Native.lf_compute(ldf) do - {:ok, Shared.create_dataframe(df)} + Shared.create_dataframe(df) end end @@ -316,7 +321,7 @@ defmodule Explorer.PolarsBackend.DataFrame do ) case df do - {:ok, df} -> {:ok, Shared.create_dataframe(df)} + {:ok, df} -> Shared.create_dataframe(df) {:error, error} -> {:error, RuntimeError.exception(error)} end end @@ -370,7 +375,7 @@ defmodule Explorer.PolarsBackend.DataFrame do @impl true def load_parquet(contents) when is_binary(contents) do case Native.df_load_parquet(contents) do - {:ok, df} -> {:ok, Shared.create_dataframe(df)} + {:ok, df} -> Shared.create_dataframe(df) {:error, error} -> {:error, RuntimeError.exception(error)} end end @@ -394,7 +399,7 @@ defmodule Explorer.PolarsBackend.DataFrame do {columns, projection} = column_names_or_projection(columns) case Native.df_from_ipc(entry.path, columns, projection) do - {:ok, df} -> {:ok, Shared.create_dataframe(df)} + {:ok, df} -> Shared.create_dataframe(df) {:error, error} -> {:error, RuntimeError.exception(error)} end end @@ -428,7 +433,7 @@ defmodule Explorer.PolarsBackend.DataFrame do {columns, projection} = column_names_or_projection(columns) case Native.df_load_ipc(contents, columns, projection) do - {:ok, df} -> {:ok, Shared.create_dataframe(df)} + {:ok, df} -> Shared.create_dataframe(df) {:error, error} -> {:error, RuntimeError.exception(error)} end end @@ -452,7 +457,7 @@ defmodule Explorer.PolarsBackend.DataFrame do {columns, projection} = column_names_or_projection(columns) case Native.df_from_ipc_stream(entry.path, columns, projection) do - {:ok, df} -> {:ok, Shared.create_dataframe(df)} + {:ok, df} -> Shared.create_dataframe(df) {:error, error} -> {:error, RuntimeError.exception(error)} end end @@ -486,7 +491,7 @@ defmodule Explorer.PolarsBackend.DataFrame do {columns, projection} = column_names_or_projection(columns) case Native.df_load_ipc_stream(contents, columns, projection) do - {:ok, df} -> {:ok, Shared.create_dataframe(df)} + {:ok, df} -> Shared.create_dataframe(df) {:error, error} -> {:error, RuntimeError.exception(error)} end end @@ -556,7 +561,7 @@ defmodule Explorer.PolarsBackend.DataFrame do list = Enum.map(list, & &1.data) Shared.apply(:df_from_series, [list]) - |> Shared.create_dataframe() + |> Shared.create_dataframe!() end defp to_column_name!(column_name) when is_binary(column_name), do: column_name @@ -669,7 +674,7 @@ defmodule Explorer.PolarsBackend.DataFrame do @impl true def nil_count(%DataFrame{} = df) do Shared.apply(:df_nil_count, [df.data]) - |> Shared.create_dataframe() + |> Shared.create_dataframe!() end @impl true @@ -707,6 +712,7 @@ defmodule Explorer.PolarsBackend.DataFrame do expressions, directions, maintain_order?, + multithreaded?, nulls_last?, df.groups ]) @@ -811,7 +817,7 @@ defmodule Explorer.PolarsBackend.DataFrame do values_from, names_prefix_optional ]) - |> Shared.create_dataframe() + |> Shared.create_dataframe!() end @impl true diff --git a/lib/explorer/polars_backend/lazy_frame.ex b/lib/explorer/polars_backend/lazy_frame.ex index 43111ee84..147d9fb67 100644 --- a/lib/explorer/polars_backend/lazy_frame.ex +++ b/lib/explorer/polars_backend/lazy_frame.ex @@ -170,7 +170,7 @@ defmodule Explorer.PolarsBackend.LazyFrame do ) case result do - {:ok, polars_ldf} -> {:ok, Shared.create_dataframe(polars_ldf)} + {:ok, polars_ldf} -> Shared.create_dataframe(polars_ldf) {:error, error} -> {:error, RuntimeError.exception(error)} end end @@ -204,7 +204,7 @@ defmodule Explorer.PolarsBackend.LazyFrame do @impl true def from_parquet(%S3.Entry{} = entry, max_rows, columns, _rechunk) do case Native.lf_from_parquet_cloud(entry, max_rows, columns) do - {:ok, polars_ldf} -> {:ok, Shared.create_dataframe(polars_ldf)} + {:ok, polars_ldf} -> Shared.create_dataframe(polars_ldf) {:error, error} -> {:error, RuntimeError.exception(error)} end end @@ -212,7 +212,7 @@ defmodule Explorer.PolarsBackend.LazyFrame do @impl true def from_parquet(%Local.Entry{} = entry, max_rows, columns, _rechunk) do case Native.lf_from_parquet(entry.path, max_rows, columns) do - {:ok, polars_ldf} -> {:ok, Shared.create_dataframe(polars_ldf)} + {:ok, polars_ldf} -> Shared.create_dataframe(polars_ldf) {:error, error} -> {:error, RuntimeError.exception(error)} end end @@ -226,7 +226,7 @@ defmodule Explorer.PolarsBackend.LazyFrame do @impl true def from_ndjson(%Local.Entry{} = entry, infer_schema_length, batch_size) do case Native.lf_from_ndjson(entry.path, infer_schema_length, batch_size) do - {:ok, polars_ldf} -> {:ok, Shared.create_dataframe(polars_ldf)} + {:ok, polars_ldf} -> Shared.create_dataframe(polars_ldf) {:error, error} -> {:error, RuntimeError.exception(error)} end end @@ -240,7 +240,7 @@ defmodule Explorer.PolarsBackend.LazyFrame do @impl true def from_ipc(%Local.Entry{} = entry, columns) when is_nil(columns) do case Native.lf_from_ipc(entry.path) do - {:ok, polars_ldf} -> {:ok, Shared.create_dataframe(polars_ldf)} + {:ok, polars_ldf} -> Shared.create_dataframe(polars_ldf) {:error, error} -> {:error, RuntimeError.exception(error)} end end @@ -364,15 +364,8 @@ defmodule Explorer.PolarsBackend.LazyFrame do end @impl true - def to_parquet(%DF{} = ldf, %S3.Entry{} = entry, {compression, level}, _streaming = true) do - case Native.lf_to_parquet_cloud( - ldf.data, - entry, - Shared.parquet_compression(compression, level) - ) do - {:ok, _} -> :ok - {:error, error} -> {:error, RuntimeError.exception(error)} - end + def to_parquet(%DF{} = _ldf, %S3.Entry{} = _entry, {_compression, _level}, _streaming = true) do + raise "streaming of a lazy frame to the cloud using parquet is currently unavailable. Please try again disabling the `:streaming` option." end @impl true diff --git a/lib/explorer/polars_backend/native.ex b/lib/explorer/polars_backend/native.ex index 44e317457..037df1bd4 100644 --- a/lib/explorer/polars_backend/native.ex +++ b/lib/explorer/polars_backend/native.ex @@ -67,6 +67,7 @@ defmodule Explorer.PolarsBackend.Native do _expressions, _directions, _maintain_order?, + _multithreaded?, _nulls_last?, _groups ), diff --git a/lib/explorer/polars_backend/series.ex b/lib/explorer/polars_backend/series.ex index 989c799b4..5d5af3869 100644 --- a/lib/explorer/polars_backend/series.ex +++ b/lib/explorer/polars_backend/series.ex @@ -518,7 +518,7 @@ defmodule Explorer.PolarsBackend.Series do def frequencies(%Series{} = series) do Shared.apply(:s_frequencies, [series.data]) - |> Shared.create_dataframe() + |> Shared.create_dataframe!() |> DataFrame.rename(["values", "counts"]) end @@ -534,7 +534,7 @@ defmodule Explorer.PolarsBackend.Series do category_label ) do {:ok, polars_df} -> - Shared.create_dataframe(polars_df) + Shared.create_dataframe!(polars_df) {:error, "Polars Error: lengths don't match: " <> _rest} -> raise ArgumentError, "lengths don't match: labels count must equal bins count" @@ -553,7 +553,7 @@ defmodule Explorer.PolarsBackend.Series do break_point_label, category_label ]) - |> Shared.create_dataframe() + |> Shared.create_dataframe!() end # Window @@ -596,6 +596,13 @@ defmodule Explorer.PolarsBackend.Series do end defp window_function(operation, series, window_size, weights, min_periods, center) do + series = + if List.wrap(weights) == [] do + series + else + cast(series, {:f, 64}) + end + Shared.apply_series(series, operation, [window_size, weights, min_periods, center]) end diff --git a/lib/explorer/polars_backend/shared.ex b/lib/explorer/polars_backend/shared.ex index c5e6dd234..215057b27 100644 --- a/lib/explorer/polars_backend/shared.ex +++ b/lib/explorer/polars_backend/shared.ex @@ -49,11 +49,11 @@ defmodule Explorer.PolarsBackend.Shared do check_df = if match?(%PolarsLazyFrame{}, new_df) do case Native.lf_compute(new_df) do - {:ok, new_df} -> create_dataframe(new_df) + {:ok, new_df} -> create_dataframe!(new_df) {:error, error} -> raise runtime_error(error) end else - create_dataframe(new_df) + create_dataframe!(new_df) end if Enum.sort(out_df.names) != Enum.sort(check_df.names) or @@ -95,27 +95,34 @@ defmodule Explorer.PolarsBackend.Shared do end def create_dataframe(polars_df) do - Explorer.Backend.DataFrame.new(polars_df, df_names(polars_df), df_dtypes(polars_df)) + with {:ok, names} <- df_names(polars_df), {:ok, dtypes} <- df_dtypes(polars_df) do + {:ok, Explorer.Backend.DataFrame.new(polars_df, names, dtypes)} + else + {:error, error} -> {:error, runtime_error(error)} + end + end + + def create_dataframe!(polars_df) do + case create_dataframe(polars_df) do + {:ok, df} -> df + {:error, error} -> raise error + end end defp df_names(%PolarsDataFrame{} = polars_df) do - {:ok, names} = Native.df_names(polars_df) - names + Native.df_names(polars_df) end defp df_names(%PolarsLazyFrame{} = polars_df) do - {:ok, names} = Native.lf_names(polars_df) - names + Native.lf_names(polars_df) end defp df_dtypes(%PolarsDataFrame{} = polars_df) do - {:ok, dtypes} = Native.df_dtypes(polars_df) - dtypes + Native.df_dtypes(polars_df) end defp df_dtypes(%PolarsLazyFrame{} = polars_df) do - {:ok, dtypes} = Native.lf_dtypes(polars_df) - dtypes + Native.lf_dtypes(polars_df) end def from_list(list, dtype), do: from_list(list, dtype, "") diff --git a/lib/explorer/series.ex b/lib/explorer/series.ex index d4490b16b..a4128342f 100644 --- a/lib/explorer/series.ex +++ b/lib/explorer/series.ex @@ -3595,9 +3595,13 @@ defmodule Explorer.Series do Raises a numeric series to the power of the exponent. At least one of the arguments must be a series. If both - sizes are series, the series must have the same size or + sides are series, the series must have the same size or at last one of them must have size of 1. + Note that this operation can fail if the exponent is a + signed integer series or scalar containing negative values, + and the base is also of an integer type. + ## Supported dtypes * floats: #{Shared.inspect_dtypes(@float_dtypes, backsticks: true)} @@ -3616,7 +3620,7 @@ defmodule Explorer.Series do iex> Explorer.Series.pow(s, 3) #Explorer.Series< Polars[3] - f64 [8.0, 64.0, 216.0] + s64 [8, 64, 216] > iex> s = [2, 4, 6] |> Explorer.Series.from_list() @@ -3657,7 +3661,7 @@ defmodule Explorer.Series do defp cast_to_pow({:f, l}, {:f, r}), do: {:f, max(l, r)} defp cast_to_pow({:f, l}, {n, _}) when K.in(n, [:u, :s]), do: {:f, l} defp cast_to_pow({n, _}, {:f, r}) when K.in(n, [:u, :s]), do: {:f, r} - defp cast_to_pow({n, _}, {:s, _}) when K.in(n, [:u, :s]), do: {:f, 64} + defp cast_to_pow({n, _}, {:s, _}) when K.in(n, [:u, :s]), do: {:s, 64} defp cast_to_pow(_, _), do: nil @doc """ diff --git a/native/explorer/Cargo.lock b/native/explorer/Cargo.lock index 5619f9cae..75d312e2c 100644 --- a/native/explorer/Cargo.lock +++ b/native/explorer/Cargo.lock @@ -4,9 +4,9 @@ version = 3 [[package]] name = "addr2line" -version = "0.21.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678" dependencies = [ "gimli", ] @@ -77,9 +77,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.82" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f538837af36e6f6a9be0faa67f9a314f8119e4e4b5867c6ab40ed60360142519" +checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" [[package]] name = "argminmax" @@ -96,6 +96,18 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf7d0a018de4f6aa429b9d33d69edf69072b1c5b1cb8d3e4a5f7ef898fc3eb76" +[[package]] +name = "arrayref" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d151e35f61089500b617991b791fc8bfd237ae50cd5950803758a179b41e67a" + +[[package]] +name = "arrayvec" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" + [[package]] name = "async-stream" version = "0.3.5" @@ -115,18 +127,18 @@ checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.72", ] [[package]] name = "async-trait" -version = "0.1.80" +version = "0.1.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" +checksum = "6e0c28dcc82d7c8ead5cb13beb15405b57b8546e93215673ff8ca0349a028107" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.72", ] [[package]] @@ -144,17 +156,23 @@ version = "0.15.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ae037714f313c1353189ead58ef9eec30a8e8dc101b2622d461418fd59e28a9" +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1fdabc7756949593fe60f30ec81974b613357de856987752631dea1e3394c80" +checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" [[package]] name = "backtrace" -version = "0.3.71" +version = "0.3.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b05800d2e817c8b3b4b54abd461726265fa9789ae34330622f2db9ee696f9d" +checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a" dependencies = [ "addr2line", "cc", @@ -171,6 +189,12 @@ version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "bitflags" version = "1.3.2" @@ -179,9 +203,25 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.5.0" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +dependencies = [ + "serde", +] + +[[package]] +name = "blake3" +version = "1.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" +checksum = "e9ec96fe9a81b5e365f9db71fe00edc4fe4ca2cc7dcb7861f0603012a7caa210" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", +] [[package]] name = "block-buffer" @@ -194,9 +234,9 @@ dependencies = [ [[package]] name = "brotli" -version = "3.5.0" +version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d640d25bc63c50fb1f0b545ffd80207d2e10a4c965530809b40ba3386825c391" +checksum = "19483b140a7ac7174d34b5a581b406c64f84da5409d3e09cf4fff604f9270e67" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -205,9 +245,9 @@ dependencies = [ [[package]] name = "brotli-decompressor" -version = "2.5.1" +version = "4.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e2e4afe60d7dd600fdd3de8d0f08c2b7ec039712e3b6137ff98b7004e82de4f" +checksum = "9a45bd2e4095a8b518033b128020dd4a55aab1c0a381ba4404a472630f4bc362" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -221,35 +261,35 @@ checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "bytemuck" -version = "1.15.0" +version = "1.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d6d68c57235a3a081186990eca2867354726650f42f7516ca50c28d6281fd15" +checksum = "b236fc92302c97ed75b38da1f4917b5cdda4984745740f153a5d3059e48d725e" dependencies = [ "bytemuck_derive", ] [[package]] name = "bytemuck_derive" -version = "1.6.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4da9a32f3fed317401fa3c862968128267c3106685286e15d5aaa3d7389c2f60" +checksum = "1ee891b04274a59bd38b412188e24b849617b2e45a0fd8d057deb63e7403761b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.72", ] [[package]] name = "bytes" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" +checksum = "a12916984aab3fa6e39d655a33e09c0071eb36d6ab3aea5c2d78551f1df6d952" [[package]] name = "cc" -version = "1.0.94" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17f6e324229dc011159fcc089755d1e2e216a90d43a7dea6853ca740b84f35e7" +checksum = "2aba8f4e9906c7ce3c73463f62a7f0c65183ada1a2d47e397cc8810827f9694f" dependencies = [ "jobserver", "libc", @@ -273,7 +313,7 @@ dependencies = [ "num-traits", "serde", "wasm-bindgen", - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] @@ -329,6 +369,12 @@ dependencies = [ "cc", ] +[[package]] +name = "constant_time_eq" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7144d30dcf0fafbce74250a3963025d8d52177934239851c917d29f1df280c2" + [[package]] name = "core-foundation" version = "0.9.4" @@ -347,18 +393,18 @@ checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" [[package]] name = "crc32fast" -version = "1.4.0" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" dependencies = [ "cfg-if", ] [[package]] name = "crossbeam-channel" -version = "0.5.12" +version = "0.5.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab3db02a9c5b5121e1e42fbdb1aeb65f5e02624cc58c43f2884c6ccac0b82f95" +checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" dependencies = [ "crossbeam-utils", ] @@ -393,9 +439,9 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.19" +version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" [[package]] name = "crypto-common" @@ -431,9 +477,9 @@ checksum = "0d6ef0072f8a535281e4876be788938b528e9a1d43900b82c2569af7da799125" [[package]] name = "either" -version = "1.11.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a47c1c47d2f5964e29c61246e81db715514cd532db6b5116a25ea3c03d6780a2" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" [[package]] name = "encoding_rs" @@ -453,7 +499,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.72", ] [[package]] @@ -462,6 +508,16 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +[[package]] +name = "errno" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "ethnum" version = "1.5.0" @@ -503,9 +559,9 @@ checksum = "95765f67b4b18863968b4a1bd5bb576f732b29a4a28c7cd84c09fa3e2875f33c" [[package]] name = "flate2" -version = "1.0.28" +version = "1.0.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46303f565772937ffe1d394a4fac6f411c6013172fadde9dcdb1e147a086940e" +checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae" dependencies = [ "crc32fast", "libz-ng-sys", @@ -542,6 +598,16 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs4" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7e180ac76c23b45e767bd7ae9579bc0bb458618c4bc71835926e098e61d15f8" +dependencies = [ + "rustix", + "windows-sys 0.52.0", +] + [[package]] name = "futures" version = "0.3.30" @@ -598,7 +664,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.72", ] [[package]] @@ -643,9 +709,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.14" +version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94b22e06ecb0110981051723910cbf0b5f5e09a2062dd7663334ee79a9d1286c" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", "js-sys", @@ -656,9 +722,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.28.1" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" +checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" [[package]] name = "glob" @@ -677,7 +743,26 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http", + "http 0.2.12", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "h2" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa82e28a107a8cc405f0839610bdc9b15f1e25ec7d696aa5cf173edbcb1486ab" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http 1.1.0", "indexmap", "slab", "tokio", @@ -697,13 +782,14 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.14.3" +version = "0.14.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ "ahash", "allocator-api2", "rayon", + "serde", ] [[package]] @@ -750,6 +836,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http-body" version = "0.4.6" @@ -757,15 +854,38 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", - "http", + "http 0.2.12", + "pin-project-lite", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http 1.1.0", +] + +[[package]] +name = "http-body-util" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" +dependencies = [ + "bytes", + "futures-util", + "http 1.1.0", + "http-body 1.0.1", "pin-project-lite", ] [[package]] name = "httparse" -version = "1.8.0" +version = "1.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" +checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9" [[package]] name = "httpdate" @@ -781,17 +901,17 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hyper" -version = "0.14.28" +version = "0.14.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf96e135eb83a2a8ddf766e426a841d8ddd7449d5f00d34ea02b41d2f19eef80" +checksum = "a152ddd61dfaec7273fe8419ab357f33aee0d914c5f4efbf0d96fa749eea5ec9" dependencies = [ "bytes", "futures-channel", "futures-core", "futures-util", - "h2", - "http", - "http-body", + "h2 0.3.26", + "http 0.2.12", + "http-body 0.4.6", "httparse", "httpdate", "itoa", @@ -803,18 +923,62 @@ dependencies = [ "want", ] +[[package]] +name = "hyper" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50dfd22e0e76d0f662d429a5f80fcaf3855009297eab6a0a9f8543834744ba05" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "h2 0.4.5", + "http 1.1.0", + "http-body 1.0.1", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + [[package]] name = "hyper-rustls" -version = "0.24.2" +version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" +checksum = "5ee4be2c948921a1a5320b629c4193916ed787a7f7f293fd3f7f5a6c9de74155" dependencies = [ "futures-util", - "http", - "hyper", + "http 1.1.0", + "hyper 1.4.1", + "hyper-util", "rustls", + "rustls-native-certs", + "rustls-pki-types", "tokio", "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ab92f4f49ee4fb4f997c784b7a2e0fa70050211e0b6a287f898c3c9785ca956" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http 1.1.0", + "http-body 1.0.1", + "hyper 1.4.1", + "pin-project-lite", + "socket2", + "tokio", + "tower", + "tower-service", + "tracing", ] [[package]] @@ -858,6 +1022,7 @@ checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", "hashbrown", + "serde", ] [[package]] @@ -874,9 +1039,9 @@ checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3" [[package]] name = "itertools" -version = "0.12.1" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" dependencies = [ "either", ] @@ -895,9 +1060,9 @@ checksum = "9028f49264629065d057f340a86acb84867925865f73bbf8d47b4d149a7e88b8" [[package]] name = "jobserver" -version = "0.1.30" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "685a7d121ee3f65ae4fddd72b25a04bb36b6af81bc0828f7d5434c0fe60fa3a2" +checksum = "48d1dbcbbeb6a7fec7e059840aa538bd62aaccf972c7346c4d9d2059312853d0" dependencies = [ "libc", ] @@ -988,9 +1153,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.153" +version = "0.2.155" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" +checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" [[package]] name = "libm" @@ -1000,9 +1165,9 @@ checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" [[package]] name = "libmimalloc-sys" -version = "0.1.35" +version = "0.1.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3979b5c37ece694f1f5e51e7ecc871fdb0f517ed04ee45f88d15d6d553cb9664" +checksum = "23aa6811d3bd4deb8a84dde645f943476d13b248d818edcf8ce0b2f37f036b44" dependencies = [ "cc", "libc", @@ -1018,11 +1183,17 @@ dependencies = [ "libc", ] +[[package]] +name = "linux-raw-sys" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" + [[package]] name = "lock_api" -version = "0.4.11" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" dependencies = [ "autocfg", "scopeguard", @@ -1030,15 +1201,15 @@ dependencies = [ [[package]] name = "log" -version = "0.4.21" +version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" [[package]] name = "lz4" -version = "1.24.0" +version = "1.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e9e2dd86df36ce760a60f6ff6ad526f7ba1f14ba0356f8254fb6905e6494df1" +checksum = "958b4caa893816eea05507c20cfe47574a43d9a697138a7872990bba8a0ece68" dependencies = [ "libc", "lz4-sys", @@ -1046,9 +1217,9 @@ dependencies = [ [[package]] name = "lz4-sys" -version = "1.9.4" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57d27b317e207b10f69f5e75494119e391a96f48861ae870d1da6edac98ca900" +checksum = "109de74d5d2353660401699a4174a4ff23fcc649caf553df71933c7fb45ad868" dependencies = [ "cc", "libc", @@ -1066,9 +1237,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.2" +version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "memmap2" @@ -1081,9 +1252,9 @@ dependencies = [ [[package]] name = "mimalloc" -version = "0.1.39" +version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa01922b5ea280a911e323e4d2fd24b7fe5cc4042e0d2cda3c40775cdc4bdc9c" +checksum = "68914350ae34959d83f732418d51e2427a794055d0b9529f48259ac07af65633" dependencies = [ "libmimalloc-sys", ] @@ -1096,22 +1267,23 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "miniz_oxide" -version = "0.7.2" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" +checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" dependencies = [ "adler", ] [[package]] name = "mio" -version = "0.8.11" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" +checksum = "4569e456d394deccd22ce1c1913e6ea0e54519f577285001215d33557431afe4" dependencies = [ + "hermit-abi", "libc", "wasi", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -1156,53 +1328,43 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.18" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", "libm", ] -[[package]] -name = "num_cpus" -version = "1.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" -dependencies = [ - "hermit-abi", - "libc", -] - [[package]] name = "object" -version = "0.32.2" +version = "0.36.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6a622008b6e321afc04970976f62ee297fdbaa6f95318ca343e3eebb9648441" +checksum = "3f203fa8daa7bb185f760ae12bd8e097f63d17041dcdcaf675ac54cdf863170e" dependencies = [ "memchr", ] [[package]] name = "object_store" -version = "0.9.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8718f8b65fdf67a45108d1548347d4af7d71fb81ce727bbf9e3b2535e079db3" +checksum = "e6da452820c715ce78221e8202ccc599b4a52f3e1eb3eedb487b680c81a8e3f3" dependencies = [ "async-trait", - "base64", + "base64 0.22.1", "bytes", "chrono", "futures", "humantime", - "hyper", + "hyper 1.4.1", "itertools", "md-5", "parking_lot", "percent-encoding", "quick-xml", "rand", - "reqwest", + "reqwest 0.12.5", "ring", "serde", "serde_json", @@ -1227,9 +1389,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "parking_lot" -version = "0.12.1" +version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" dependencies = [ "lock_api", "parking_lot_core", @@ -1237,15 +1399,15 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.9" +version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c42a9226546d68acdd9c0a280d17ce19bfe27a46bf68784e4066115788d008e" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" dependencies = [ "cfg-if", "libc", "redox_syscall", "smallvec", - "windows-targets 0.48.5", + "windows-targets 0.52.6", ] [[package]] @@ -1260,9 +1422,9 @@ dependencies = [ [[package]] name = "parse-zoneinfo" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c705f256449c60da65e11ff6626e0c16a0a0b96aaa348de61376b249bc340f41" +checksum = "1f2a05b18d44e2957b88f96ba460715e295bc1d7510468a2f3d3b44535d26c24" dependencies = [ "regex", ] @@ -1311,6 +1473,26 @@ dependencies = [ "siphasher", ] +[[package]] +name = "pin-project" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.72", +] + [[package]] name = "pin-project-lite" version = "0.2.14" @@ -1340,9 +1522,9 @@ dependencies = [ [[package]] name = "polars" -version = "0.38.3" +version = "0.41.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f01006048a264047d6cba081fed8e11adbd69c15956f9e53185a9ac4a541853c" +checksum = "8e3351ea4570e54cd556e6755b78fe7a2c85368d820c0307cca73c96e796a7ba" dependencies = [ "getrandom", "polars-arrow", @@ -1360,9 +1542,9 @@ dependencies = [ [[package]] name = "polars-arrow" -version = "0.38.3" +version = "0.41.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25197f40d71f82b2f79bb394f03e555d3cc1ce4db1dd052c28318721c71e96ad" +checksum = "ba65fc4bcabbd64fca01fd30e759f8b2043f0963c57619e331d4b534576c0b47" dependencies = [ "ahash", "atoi", @@ -1387,6 +1569,7 @@ dependencies = [ "polars-error", "polars-utils", "ryu", + "serde", "simdutf8", "streaming-iterator", "strength_reduce", @@ -1406,9 +1589,9 @@ dependencies = [ [[package]] name = "polars-compute" -version = "0.38.3" +version = "0.41.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c354515f73cdbbad03c2bf723fcd68e6825943b3ec503055abc8a8cb08ce46bb" +checksum = "9f099516af30ac9ae4b4480f4ad02aa017d624f2f37b7a16ad4e9ba52f7e5269" dependencies = [ "bytemuck", "either", @@ -1422,12 +1605,12 @@ dependencies = [ [[package]] name = "polars-core" -version = "0.38.3" +version = "0.41.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f20d3c227186f74aa3c228c64ef72f5a15617322fed30b4323eaf53b25f8e7b" +checksum = "b2439484be228b8c302328e2f953e64cfd93930636e5c7ceed90339ece7fef6c" dependencies = [ "ahash", - "bitflags 2.5.0", + "bitflags 2.6.0", "bytemuck", "chrono", "chrono-tz 0.8.6", @@ -1445,6 +1628,7 @@ dependencies = [ "rand_distr", "rayon", "regex", + "serde", "smartstring", "thiserror", "version_check", @@ -1453,9 +1637,9 @@ dependencies = [ [[package]] name = "polars-error" -version = "0.38.3" +version = "0.41.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d66dd0ce51f8bd620eb8bd376502fe68a2b1a446d5433ecd2e75270b0755ce76" +checksum = "0c9b06dfbe79cabe50a7f0a90396864b5ee2c0e0f8d6a9353b2343c29c56e937" dependencies = [ "object_store", "polars-arrow-format", @@ -1464,20 +1648,42 @@ dependencies = [ "thiserror", ] +[[package]] +name = "polars-expr" +version = "0.41.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c630385a56a867c410a20f30772d088f90ec3d004864562b84250b35268f97" +dependencies = [ + "ahash", + "bitflags 2.6.0", + "once_cell", + "polars-arrow", + "polars-core", + "polars-io", + "polars-ops", + "polars-plan", + "polars-time", + "polars-utils", + "rayon", + "smartstring", +] + [[package]] name = "polars-io" -version = "0.38.3" +version = "0.41.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b40bef2edcdc58394792c4d779465144283a09ff1836324e7b72df7978a6e992" +checksum = "9d7363cd14e4696a28b334a56bd11013ff49cc96064818ab3f91a126e453462d" dependencies = [ "ahash", "async-trait", "atoi_simd", + "blake3", "bytes", "chrono", "chrono-tz 0.8.6", "fast-float", "flate2", + "fs4", "futures", "home", "itoa", @@ -1496,8 +1702,9 @@ dependencies = [ "polars-utils", "rayon", "regex", - "reqwest", + "reqwest 0.11.27", "ryu", + "serde", "serde_json", "simd-json", "simdutf8", @@ -1510,12 +1717,13 @@ dependencies = [ [[package]] name = "polars-json" -version = "0.38.3" +version = "0.41.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef86aca08f10ddc939fe95aabb44e1d2582dcb08b55d4dadb93353ce42adc248" +checksum = "543d7d3853f2c52dbfedee9ebf0d58c4ff3b92aadee5309150b2d14df49d6253" dependencies = [ "ahash", "chrono", + "chrono-tz 0.8.6", "fallible-streaming-iterator", "hashbrown", "indexmap", @@ -1531,19 +1739,22 @@ dependencies = [ [[package]] name = "polars-lazy" -version = "0.38.3" +version = "0.41.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c27df26a19d3092298d31d47614ad84dc330c106e38aa8cd53727cd91c07cf56" +checksum = "03877e74e42b5340ae52ded705f6d5d14563d90554c9177b01b91ed2412a56ed" dependencies = [ "ahash", - "bitflags 2.5.0", + "bitflags 2.6.0", "futures", "glob", + "memchr", "once_cell", "polars-arrow", "polars-core", + "polars-expr", "polars-io", "polars-json", + "polars-mem-engine", "polars-ops", "polars-pipe", "polars-plan", @@ -1555,15 +1766,36 @@ dependencies = [ "version_check", ] +[[package]] +name = "polars-mem-engine" +version = "0.41.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dea9e17771af750c94bf959885e4b3f5b14149576c62ef3ec1c9ef5827b2a30f" +dependencies = [ + "futures", + "polars-arrow", + "polars-core", + "polars-error", + "polars-expr", + "polars-io", + "polars-json", + "polars-ops", + "polars-plan", + "polars-time", + "polars-utils", + "rayon", + "tokio", +] + [[package]] name = "polars-ops" -version = "0.38.3" +version = "0.41.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f8a51c3bdc9e7c34196ff6f5c3cb17da134e5aafb1756aaf24b76c7118e63dc" +checksum = "6066552eb577d43b307027fb38096910b643ffb2c89a21628c7e41caf57848d0" dependencies = [ "ahash", "argminmax", - "base64", + "base64 0.22.1", "bytemuck", "chrono", "chrono-tz 0.8.6", @@ -1592,13 +1824,13 @@ dependencies = [ [[package]] name = "polars-parquet" -version = "0.38.3" +version = "0.41.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8824ee00fbbe83d69553f2711014c50361238d210ed81a7a297695b7db97d42" +checksum = "2b35b2592a2e7ef7ce9942dc2120dc4576142626c0e661668e4c6b805042e461" dependencies = [ "ahash", "async-stream", - "base64", + "base64 0.22.1", "brotli", "ethnum", "flate2", @@ -1607,9 +1839,10 @@ dependencies = [ "num-traits", "parquet-format-safe", "polars-arrow", + "polars-compute", "polars-error", "polars-utils", - "seq-macro", + "serde", "simdutf8", "snap", "streaming-decompression", @@ -1618,9 +1851,9 @@ dependencies = [ [[package]] name = "polars-pipe" -version = "0.38.3" +version = "0.41.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c5e2c1f14e81d60cfa9afe4e611a9bad9631a2cb7cd19b7c0094d0dc32f0231" +checksum = "021bce7768c330687d735340395a77453aa18dd70d57c184cbb302311e87c1b9" dependencies = [ "crossbeam-channel", "crossbeam-queue", @@ -1631,6 +1864,7 @@ dependencies = [ "polars-arrow", "polars-compute", "polars-core", + "polars-expr", "polars-io", "polars-ops", "polars-plan", @@ -1645,14 +1879,16 @@ dependencies = [ [[package]] name = "polars-plan" -version = "0.38.3" +version = "0.41.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff48362bd1b078bbbec7e7ba9ec01fea58fee2887db22a8e3deaf78f322fa3c4" +checksum = "220d0d7c02d1c4375802b2813dbedcd1a184df39c43b74689e729ede8d5c2921" dependencies = [ "ahash", "bytemuck", "chrono-tz 0.8.6", + "either", "futures", + "hashbrown", "once_cell", "percent-encoding", "polars-arrow", @@ -1664,6 +1900,7 @@ dependencies = [ "polars-time", "polars-utils", "rayon", + "recursive", "regex", "smartstring", "strum_macros", @@ -1672,9 +1909,9 @@ dependencies = [ [[package]] name = "polars-row" -version = "0.38.3" +version = "0.41.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63029da56ff6a720b190490bbc7b6263f9b72d1134311b1f381fc8d306d37770" +checksum = "c1d70d87a2882a64a43b431aea1329cb9a2c4100547c95c417cc426bb82408b3" dependencies = [ "bytemuck", "polars-arrow", @@ -1684,16 +1921,19 @@ dependencies = [ [[package]] name = "polars-sql" -version = "0.38.3" +version = "0.41.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3652c362959f608d1297196b973d1e3acb508a9562b886ac39bf7606b841052b" +checksum = "a6fc1c9b778862f09f4a347f768dfdd3d0ba9957499d306d83c7103e0fa8dc5b" dependencies = [ "hex", + "once_cell", "polars-arrow", "polars-core", "polars-error", "polars-lazy", + "polars-ops", "polars-plan", + "polars-time", "rand", "serde", "serde_json", @@ -1702,11 +1942,12 @@ dependencies = [ [[package]] name = "polars-time" -version = "0.38.3" +version = "0.41.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86eb74ea6ddfe675aa5c3f33c00dadbe2b85f0e8e3887b85db1fd5a3397267fd" +checksum = "179f98313a15c0bfdbc8cc0f1d3076d08d567485b9952d46439f94fbc3085df5" dependencies = [ "atoi", + "bytemuck", "chrono", "chrono-tz 0.8.6", "now", @@ -1722,9 +1963,9 @@ dependencies = [ [[package]] name = "polars-utils" -version = "0.38.3" +version = "0.41.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "694656a7d2b0cd8f07660dbc8d0fb7a81066ff57a452264907531d805c1e58c4" +checksum = "53e6dd89fcccb1ec1a62f752c9a9f2d482a85e9255153f46efecc617b4996d50" dependencies = [ "ahash", "bytemuck", @@ -1736,6 +1977,7 @@ dependencies = [ "raw-cpuid", "rayon", "smartstring", + "stacker", "sysinfo", "version_check", ] @@ -1748,23 +1990,78 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] name = "proc-macro2" -version = "1.0.81" +version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d1597b0c024618f09a9c3b8655b7e430397a36d23fdafec26d6965e9eec3eba" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" dependencies = [ "unicode-ident", ] +[[package]] +name = "psm" +version = "0.1.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5787f7cda34e3033a72192c018bc5883100330f362ef279a8cbccfce8bb4e874" +dependencies = [ + "cc", +] + [[package]] name = "quick-xml" -version = "0.31.0" +version = "0.36.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" +checksum = "96a05e2e8efddfa51a84ca47cec303fac86c8541b686d37cac5efc0e094417bc" dependencies = [ "memchr", "serde", ] +[[package]] +name = "quinn" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4ceeeeabace7857413798eb1ffa1e9c905a9946a57d81fb69b4b71c4d8eb3ad" +dependencies = [ + "bytes", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "thiserror", + "tokio", + "tracing", +] + +[[package]] +name = "quinn-proto" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddf517c03a109db8100448a4be38d498df8a210a99fe0e1b9eaf39e78c640efe" +dependencies = [ + "bytes", + "rand", + "ring", + "rustc-hash", + "rustls", + "slab", + "thiserror", + "tinyvec", + "tracing", +] + +[[package]] +name = "quinn-udp" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bffec3605b73c6f1754535084a85229fa8a30f86014e6c81aeec4abb68b0285" +dependencies = [ + "libc", + "once_cell", + "socket2", + "windows-sys 0.52.0", +] + [[package]] name = "quote" version = "1.0.36" @@ -1825,11 +2122,11 @@ dependencies = [ [[package]] name = "raw-cpuid" -version = "11.0.1" +version = "11.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d86a7c4638d42c44551f4791a20e687dbb4c3de1f33c43dd71e355cd429def1" +checksum = "cb9ee317cfe3fbd54b36a511efc1edd42e216903c9cd575e686dd68a2ba90d8d" dependencies = [ - "bitflags 2.5.0", + "bitflags 2.6.0", ] [[package]] @@ -1852,40 +2149,60 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "recursive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0786a43debb760f491b1bc0269fe5e84155353c67482b9e60d0cfb596054b43e" +dependencies = [ + "recursive-proc-macro-impl", + "stacker", +] + +[[package]] +name = "recursive-proc-macro-impl" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" +dependencies = [ + "quote", + "syn 2.0.72", +] + [[package]] name = "redox_syscall" -version = "0.4.1" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa" +checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.6.0", ] [[package]] name = "ref-cast" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4846d4c50d1721b1a3bef8af76924eef20d5e723647333798c1b519b3a9473f" +checksum = "ccf0a6f84d5f1d581da8b41b47ec8600871962f2a528115b542b362d4b744931" dependencies = [ "ref-cast-impl", ] [[package]] name = "ref-cast-impl" -version = "1.0.22" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fddb4f8d99b0a2ebafc65a87a69a7b9875e4b1ae1f00db265d300ef7f28bccc" +checksum = "bcc303e793d3734489387d205e9b186fac9c6cfacedd98cbb2e8a5943595f3e6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.72", ] [[package]] name = "regex" -version = "1.10.4" +version = "1.10.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" +checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" dependencies = [ "aho-corasick", "memchr", @@ -1895,9 +2212,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" +checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" dependencies = [ "aho-corasick", "memchr", @@ -1906,9 +2223,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" +checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" [[package]] name = "reqwest" @@ -1916,16 +2233,53 @@ version = "0.11.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" dependencies = [ - "base64", + "base64 0.21.7", "bytes", "encoding_rs", "futures-core", "futures-util", - "h2", - "http", - "http-body", - "hyper", + "h2 0.3.26", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.30", + "ipnet", + "js-sys", + "log", + "mime", + "once_cell", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper 0.1.2", + "system-configuration", + "tokio", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg 0.50.0", +] + +[[package]] +name = "reqwest" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7d6d2a27d57148378eb5e111173f4276ad26340ecc5c49a4a2152167a2d6a37" +dependencies = [ + "base64 0.22.1", + "bytes", + "futures-core", + "futures-util", + "h2 0.4.5", + "http 1.1.0", + "http-body 1.0.1", + "http-body-util", + "hyper 1.4.1", "hyper-rustls", + "hyper-util", "ipnet", "js-sys", "log", @@ -1933,14 +2287,15 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", + "quinn", "rustls", "rustls-native-certs", "rustls-pemfile", + "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", - "sync_wrapper", - "system-configuration", + "sync_wrapper 1.0.1", "tokio", "tokio-rustls", "tokio-util", @@ -1950,7 +2305,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "winreg", + "winreg 0.52.0", ] [[package]] @@ -1970,9 +2325,28 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.23" +version = "0.1.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" + +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustix" +version = "0.38.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +dependencies = [ + "bitflags 2.6.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] [[package]] name = "rustler" @@ -1995,7 +2369,7 @@ dependencies = [ "inventory", "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.72", ] [[package]] @@ -2010,58 +2384,69 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.11" +version = "0.23.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fecbfb7b1444f477b345853b1fce097a2c6fb637b2bfb87e6bc5db0f043fae4" +checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044" dependencies = [ - "log", + "once_cell", "ring", + "rustls-pki-types", "rustls-webpki", - "sct", + "subtle", + "zeroize", ] [[package]] name = "rustls-native-certs" -version = "0.6.3" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +checksum = "a88d6d420651b496bdd98684116959239430022a115c1240e6c3993be0b15fba" dependencies = [ "openssl-probe", "rustls-pemfile", + "rustls-pki-types", "schannel", "security-framework", ] [[package]] name = "rustls-pemfile" -version = "1.0.4" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" dependencies = [ - "base64", + "base64 0.22.1", + "rustls-pki-types", ] +[[package]] +name = "rustls-pki-types" +version = "1.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" + [[package]] name = "rustls-webpki" -version = "0.101.7" +version = "0.102.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +checksum = "8e6b52d4fda176fd835fdc55a835d4a89b8499cad995885a21149d5ad62f852e" dependencies = [ "ring", + "rustls-pki-types", "untrusted", ] [[package]] name = "rustversion" -version = "1.0.15" +version = "1.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80af6f9131f277a45a3fba6ce8e2258037bb0477a67e610d3c1fe046ab31de47" +checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" [[package]] name = "ryu" -version = "1.0.17" +version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "same-file" @@ -2087,23 +2472,13 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "sct" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "security-framework" -version = "2.10.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "770452e37cad93e0a50d5abc3990d2bc351c36d0328f86cefec2f2fb206eaef6" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.6.0", "core-foundation", "core-foundation-sys", "libc", @@ -2112,45 +2487,39 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.10.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41f3cc463c0ef97e11c3461a9d3787412d30e8e7eb907c79180c4a57bf7c04ef" +checksum = "75da29fe9b9b08fe9d6b22b5b4bcbc75d8db3aa31e639aa56bb62e9d46bfceaf" dependencies = [ "core-foundation-sys", "libc", ] -[[package]] -name = "seq-macro" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" - [[package]] name = "serde" -version = "1.0.198" +version = "1.0.204" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9846a40c979031340571da2545a4e5b7c4163bdae79b301d5f86d03979451fcc" +checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.198" +version = "1.0.204" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e88edab869b01783ba905e7d0153f9fc1a6505a96e4ad3018011eedb838566d9" +checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.72", ] [[package]] name = "serde_json" -version = "1.0.116" +version = "1.0.120" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e17db7126d17feb94eb3fad46bf1a96b034e8aacbc2e775fe81505f8b0b2813" +checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5" dependencies = [ "indexmap", "itoa", @@ -2222,6 +2591,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fb72c633efbaa2dd666986505016c32c3044395ceaf881518399d2f4127ee29" dependencies = [ "autocfg", + "serde", "static_assertions", "version_check", ] @@ -2256,9 +2626,9 @@ checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" [[package]] name = "socket2" -version = "0.5.6" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05ffd9c0a93b7543e062e759284fcf5f5e3b098501104bfbdde4d404db792871" +checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" dependencies = [ "libc", "windows-sys 0.52.0", @@ -2272,13 +2642,26 @@ checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" [[package]] name = "sqlparser" -version = "0.39.0" +version = "0.47.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "743b4dc2cbde11890ccb254a8fc9d537fa41b36da00de2a1c5e9848c9bc42bd7" +checksum = "295e9930cd7a97e58ca2a070541a3ca502b17f5d1fa7157376d0fabd85324f25" dependencies = [ "log", ] +[[package]] +name = "stacker" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c886bd4480155fd3ef527d45e9ac8dd7118a898a46530b7b94c3e21866259fce" +dependencies = [ + "cc", + "cfg-if", + "libc", + "psm", + "winapi", +] + [[package]] name = "static_assertions" version = "1.1.0" @@ -2308,17 +2691,23 @@ checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" [[package]] name = "strum_macros" -version = "0.25.3" +version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23dc1fa9ac9c169a78ba62f0b841814b7abae11bdd047b9c58f893439e309ea0" +checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" dependencies = [ - "heck 0.4.1", + "heck 0.5.0", "proc-macro2", "quote", "rustversion", - "syn 2.0.60", + "syn 2.0.72", ] +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "1.0.109" @@ -2332,9 +2721,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.60" +version = "2.0.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "909518bc7b1c9b779f1bbf07f2929d35af9f0f37e47c6e9ef7f9dddc1e1821f3" +checksum = "dc4b9b9bf2add8093d3f2c0204471e951b2285580335de42f9d2534f3ae7a8af" dependencies = [ "proc-macro2", "quote", @@ -2347,11 +2736,17 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +[[package]] +name = "sync_wrapper" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" + [[package]] name = "sysinfo" -version = "0.30.11" +version = "0.30.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87341a165d73787554941cd5ef55ad728011566fe714e987d1b976c15dbc3a83" +checksum = "0a5b4ddaee55fb2bea2bf0e5000747e5f5c0de765e5a5ff87f4cd106439f4bb3" dependencies = [ "cfg-if", "core-foundation-sys", @@ -2390,29 +2785,29 @@ checksum = "c1bbb9f3c5c463a01705937a24fdabc5047929ac764b2d5b9cf681c1f5041ed5" [[package]] name = "thiserror" -version = "1.0.58" +version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03468839009160513471e86a034bb2c5c0e4baae3b43f79ffc55c4a5427b3297" +checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.58" +version = "1.0.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7" +checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.72", ] [[package]] name = "tinyvec" -version = "1.6.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" +checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938" dependencies = [ "tinyvec_macros", ] @@ -2425,56 +2820,76 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.37.0" +version = "1.39.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787" +checksum = "d040ac2b29ab03b09d4129c2f5bbd012a3ac2f79d38ff506a4bf8dd34b0eac8a" dependencies = [ "backtrace", "bytes", "libc", "mio", - "num_cpus", "pin-project-lite", "socket2", "tokio-macros", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "tokio-macros" -version = "2.2.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" +checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.72", ] [[package]] name = "tokio-rustls" -version = "0.24.1" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ "rustls", + "rustls-pki-types", "tokio", ] [[package]] name = "tokio-util" -version = "0.7.10" +version = "0.7.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5419f34732d9eb6ee4c3578b7989078579b7f039cbbb9ca2c4da015749371e15" +checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" dependencies = [ "bytes", "futures-core", "futures-sink", "pin-project-lite", "tokio", - "tracing", ] +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "pin-project", + "pin-project-lite", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + [[package]] name = "tower-service" version = "0.3.2" @@ -2500,7 +2915,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.72", ] [[package]] @@ -2577,9 +2992,9 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.0" +version = "2.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" +checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" dependencies = [ "form_urlencoded", "idna", @@ -2588,9 +3003,9 @@ dependencies = [ [[package]] name = "uuid" -version = "1.8.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a183cf7feeba97b4dd1c0d46788634f6221d87fa961b305bed08c851829efcc0" +checksum = "81dfa00651efa65069b0b6b651f4aaa31ba9e3c3ce0137aaad053604ee7e0314" dependencies = [ "getrandom", ] @@ -2665,7 +3080,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.72", "wasm-bindgen-shared", ] @@ -2699,7 +3114,7 @@ checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.72", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -2751,11 +3166,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.6" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f29e6f9198ba0d26b4c9f07dbe6f9ed633e1f3d5b8b414090084349e46a52596" +checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b" dependencies = [ - "winapi", + "windows-sys 0.52.0", ] [[package]] @@ -2771,7 +3186,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" dependencies = [ "windows-core", - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] @@ -2780,7 +3195,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] @@ -2798,7 +3213,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.5", + "windows-targets 0.52.6", ] [[package]] @@ -2818,18 +3233,18 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ - "windows_aarch64_gnullvm 0.52.5", - "windows_aarch64_msvc 0.52.5", - "windows_i686_gnu 0.52.5", + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", "windows_i686_gnullvm", - "windows_i686_msvc 0.52.5", - "windows_x86_64_gnu 0.52.5", - "windows_x86_64_gnullvm 0.52.5", - "windows_x86_64_msvc 0.52.5", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", ] [[package]] @@ -2840,9 +3255,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_msvc" @@ -2852,9 +3267,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_i686_gnu" @@ -2864,15 +3279,15 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" [[package]] name = "windows_i686_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_msvc" @@ -2882,9 +3297,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_x86_64_gnu" @@ -2894,9 +3309,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnullvm" @@ -2906,9 +3321,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_msvc" @@ -2918,9 +3333,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "windows_x86_64_msvc" -version = "0.52.5" +version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winreg" @@ -2932,55 +3347,71 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "winreg" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + [[package]] name = "xxhash-rust" -version = "0.8.10" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "927da81e25be1e1a2901d59b81b37dd2efd1fc9c9345a55007f09bf5a2d3ee03" +checksum = "63658493314859b4dfdf3fb8c1defd61587839def09582db50b8a4e93afca6bb" [[package]] name = "zerocopy" -version = "0.7.32" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.7.32" +version = "0.7.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.60", + "syn 2.0.72", ] +[[package]] +name = "zeroize" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" + [[package]] name = "zstd" -version = "0.13.1" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d789b1514203a1120ad2429eae43a7bd32b90976a7bb8a05f7ec02fa88cc23a" +checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9" dependencies = [ "zstd-safe", ] [[package]] name = "zstd-safe" -version = "7.1.0" +version = "7.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd99b45c6bc03a018c8b8a86025678c87e55526064e38f9df301989dce7ec0a" +checksum = "fa556e971e7b568dc775c136fc9de8c779b1c2fc3a63defaafadffdbd3181afa" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.10+zstd.1.5.6" +version = "2.0.12+zstd.1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c253a4914af5bafc8fa8c86ee400827e83cf6ec01195ec1f1ed8441bf00d65aa" +checksum = "0a4e40c320c3cb459d9a9ff6de98cff88f4751ee9275d140e2be94a2b74e4c13" dependencies = [ "cc", "pkg-config", diff --git a/native/explorer/Cargo.toml b/native/explorer/Cargo.toml index 65d49a13a..8fd22dd81 100644 --- a/native/explorer/Cargo.toml +++ b/native/explorer/Cargo.toml @@ -30,7 +30,7 @@ tokio-util = { version = "0.7", default-features = false, features = [ "io", "io-util", ], optional = true } -object_store = { version = "0.9", default-features = false, optional = true } +object_store = { version = "0.10", default-features = false, optional = true } # MiMalloc won´t compile on Windows with the GCC compiler. # On Linux with Musl it won´t load correctly. @@ -38,7 +38,7 @@ object_store = { version = "0.9", default-features = false, optional = true } mimalloc = { version = "*", default-features = false } [dependencies.polars] -version = "0.38" +version = "0.41.3" default-features = false features = [ "abs", @@ -55,7 +55,6 @@ features = [ "ewma", "extract_groups", "extract_jsonpath", - "group_by_list", "ipc_streaming", "ipc", "is_in", @@ -87,7 +86,7 @@ features = [ ] [dependencies.polars-ops] -version = "0.38" +version = "0.41.3" features = ["abs", "ewma", "cum_agg", "cov"] [features] diff --git a/native/explorer/rust-toolchain.toml b/native/explorer/rust-toolchain.toml index 526ab5a6f..bce747f05 100644 --- a/native/explorer/rust-toolchain.toml +++ b/native/explorer/rust-toolchain.toml @@ -1,4 +1,4 @@ [toolchain] -channel = "nightly-2024-06-23" +channel = "nightly-2024-06-24" components = ["rustfmt", "clippy"] profile = "minimal" diff --git a/native/explorer/src/cloud_writer.rs b/native/explorer/src/cloud_writer.rs index e32b979fa..fecced8c8 100644 --- a/native/explorer/src/cloud_writer.rs +++ b/native/explorer/src/cloud_writer.rs @@ -1,127 +1,112 @@ -use tokio::io::{AsyncWrite, AsyncWriteExt}; - use crate::ExplorerError; use object_store::path::Path; -use object_store::MultipartId; -use object_store::ObjectStore; +use object_store::{ObjectMeta, ObjectStore}; +use std::sync::Arc; + +use object_store::buffered::BufWriter as OSBufWriter; +use tokio::io::AsyncWriteExt; -/// CloudWriter wraps the asynchronous interface of [ObjectStore::put_multipart](https://docs.rs/object_store/latest/object_store/trait.ObjectStore.html#tymethod.put_multipart) +#[derive(Debug, PartialEq)] +enum CloudWriterStatus { + Running, + Stopped, + Aborted, +} +/// CloudWriter wraps the asynchronous interface of [ObjectStore's BufWriter](https://docs.rs/object_store/latest/object_store/buffered/struct.BufWriter.html) /// in a synchronous interface which implements `std::io::Write`. /// /// This allows it to be used in sync code which would otherwise write to a simple File or byte stream, /// such as with `polars::prelude::CsvWriter`. pub struct CloudWriter { - // Hold a reference to the store. The store itself is thread-safe. - object_store: Box, - // The path in the object_store which we want to write to - path: Path, - // ID of a partially-done upload, used to abort the upload on error - multipart_id: MultipartId, // The Tokio runtime which the writer uses internally. runtime: tokio::runtime::Runtime, // Internal writer, constructed at creation - writer: Box, + writer: OSBufWriter, + // The copy of the object_store + object_store: Arc, + // Keep the path for the file, so we can use to read head. + path: Path, + // Private status of the current writer + status: CloudWriterStatus, } impl CloudWriter { /// Construct a new CloudWriter /// /// Creates a new (current-thread) Tokio runtime - /// which bridges the sync writing process with the async ObjectStore multipart uploading. - pub fn new(object_store: Box, path: Path) -> Result { + /// which bridges the sync writing process with the async ObjectStore uploading. + pub fn new(object_store: Arc, path: Path) -> Result { let runtime = tokio::runtime::Builder::new_current_thread() .enable_time() .enable_io() .build()?; + let writer = OSBufWriter::new(object_store.clone(), path.clone()); - let (multipart_id, writer) = - runtime.block_on(async { Self::build_writer(&object_store, &path).await })?; Ok(CloudWriter { + writer, + runtime, object_store, path, - multipart_id, - runtime, - writer, + status: CloudWriterStatus::Running, }) } - async fn build_writer( - object_store: &dyn ObjectStore, - path: &Path, - ) -> Result<(MultipartId, Box), ExplorerError> { - let (multipart_id, async_s3_writer) = (object_store.put_multipart(path).await) - .map_err(|_| ExplorerError::Other(format!("Could not put multipart to path {path}")))?; - Ok((multipart_id, async_s3_writer)) - } - - fn abort(&self) { - let _ = self.runtime.block_on(async { - self.object_store - .abort_multipart(&self.path, &self.multipart_id) - .await - }); + /// Make a head request to check if the upload has finished. + pub fn finish(&mut self) -> Result { + if self.status != CloudWriterStatus::Stopped { + self.status = CloudWriterStatus::Stopped; + let _ = self.runtime.block_on(self.writer.shutdown()); + self.runtime + .block_on(self.object_store.head(&self.path)) + .map_err(|err| { + ExplorerError::Other(format!( + "cannot read information from file, which means the upload failed. {err}" + )) + }) + } else { + Err(ExplorerError::Other( + "cannot finish cloud writer due to an error, or it was already finished.".into(), + )) + } } } impl std::io::Write for CloudWriter { fn write(&mut self, buf: &[u8]) -> std::io::Result { - let res = self.runtime.block_on(self.writer.write(buf)); - if res.is_err() { - self.abort(); - } - res + // SAFETY: + // We extend the lifetime for the duration of this function. This is safe as well block the + // async runtime here + // This was copied from Polars' own CloudWriter. + let buf = unsafe { std::mem::transmute::<&[u8], &'static [u8]>(buf) }; + + self.runtime.block_on(async { + // TODO: use writer.put to avoid copying data + let res = self.writer.write(buf).await; + if res.is_err() { + let _ = self.writer.abort().await; + self.status = CloudWriterStatus::Aborted; + } + res + }) } fn flush(&mut self) -> std::io::Result<()> { - let res = self.runtime.block_on(self.writer.flush()); - if res.is_err() { - self.abort(); - } - res + self.runtime.block_on(async { + let res = self.writer.flush().await; + if res.is_err() { + let _ = self.writer.abort().await; + self.status = CloudWriterStatus::Aborted; + } + res + }) } } impl Drop for CloudWriter { fn drop(&mut self) { - let _ = self.runtime.block_on(self.writer.shutdown()); - } -} - -#[cfg(test)] -mod tests { - use object_store::ObjectStore; - - use super::*; - - use polars::df; - use polars::prelude::DataFrame; - use polars::prelude::NamedFrom; - - fn example_dataframe() -> DataFrame { - df!( - "foo" => &[1, 2, 3], - "bar" => &[None, Some("bak"), Some("baz")], - ) - .unwrap() - } - - #[test] - fn csv_to_local_objectstore_cloudwriter() { - use polars::prelude::{CsvWriter, SerWriter}; - - let mut df = example_dataframe(); - - let object_store: Box = Box::new( - object_store::local::LocalFileSystem::new_with_prefix("/tmp/") - .expect("Could not initialize connection"), - ); - let object_store: Box = object_store; - - let path: object_store::path::Path = "cloud_writer_example.csv".into(); - - let mut cloud_writer = CloudWriter::new(object_store, path).unwrap(); - CsvWriter::new(&mut cloud_writer) - .finish(&mut df) - .expect("Could not write dataframe as CSV to remote location"); + if self.status != CloudWriterStatus::Stopped { + self.status = CloudWriterStatus::Stopped; + let _ = self.runtime.block_on(self.writer.shutdown()); + } } } diff --git a/native/explorer/src/dataframe.rs b/native/explorer/src/dataframe.rs index d7e7713c1..7ca7c9f73 100644 --- a/native/explorer/src/dataframe.rs +++ b/native/explorer/src/dataframe.rs @@ -260,31 +260,17 @@ pub fn df_sort_by( nulls_last: bool, groups: Vec, ) -> Result { + let sort_options = SortMultipleOptions::new() + .with_maintain_order(maintain_order) + .with_multithreaded(multithreaded) + .with_nulls_last(nulls_last) + .with_order_descending_multi(reverse); + let new_df = if groups.is_empty() { - // Note: we cannot use either df.sort or df.sort_with_options. - // df.sort does not allow a nulls_last option. - // df.sort_with_options only allows a single column. - let by_columns = df.select_series(by_columns)?; - df.sort_impl( - by_columns, - reverse, - nulls_last, - maintain_order, - None, - multithreaded, - )? + df.sort(by_columns, sort_options)? } else { - df.group_by_stable(groups)?.apply(|df| { - let by_columns = df.select_series(&by_columns)?; - df.sort_impl( - by_columns, - reverse.clone(), - nulls_last, - maintain_order, - None, - multithreaded, - ) - })? + df.group_by_stable(groups)? + .apply(|df| df.sort(by_columns.clone(), sort_options.clone()))? }; Ok(ExDataFrame::new(new_df)) @@ -296,20 +282,25 @@ pub fn df_sort_with( expressions: Vec, directions: Vec, maintain_order: bool, + multithreaded: bool, nulls_last: bool, groups: Vec, ) -> Result { let df = data.clone_inner(); let exprs = ex_expr_to_exprs(expressions); + let sort_options = SortMultipleOptions::new() + .with_maintain_order(maintain_order) + .with_multithreaded(multithreaded) + .with_nulls_last(nulls_last) + .with_order_descending_multi(directions); + let new_df = if groups.is_empty() { - df.lazy() - .sort_by_exprs(exprs, directions, nulls_last, maintain_order) - .collect()? + df.lazy().sort_by_exprs(exprs, sort_options).collect()? } else { df.group_by_stable(groups)?.apply(|df| { df.lazy() - .sort_by_exprs(&exprs, &directions, nulls_last, maintain_order) + .sort_by_exprs(&exprs, sort_options.clone()) .collect() })? }; @@ -415,8 +406,8 @@ pub fn df_pivot_wider( let mut new_df = pivot_stable( &df, - &temp_id_names, [pivot_column], + Some(temp_id_names), Some(values_column), false, Some(PivotAgg::First), diff --git a/native/explorer/src/dataframe/io.rs b/native/explorer/src/dataframe/io.rs index eda3921d1..676b36c68 100644 --- a/native/explorer/src/dataframe/io.rs +++ b/native/explorer/src/dataframe/io.rs @@ -18,6 +18,9 @@ use std::io::{BufReader, BufWriter, Cursor}; use crate::datatypes::{ExParquetCompression, ExS3Entry, ExSeriesDtype}; use crate::{ExDataFrame, ExplorerError}; +#[cfg(feature = "cloud")] +use crate::cloud_writer::CloudWriter; + // ============ CSV ============ // #[rustler::nif(schedule = "DirtyIo")] @@ -44,24 +47,29 @@ pub fn df_from_csv( _ => CsvEncoding::Utf8, }; - let reader = CsvReader::from_path(filename)? - .infer_schema(infer_schema_length) - .has_header(has_header) - .truncate_ragged_lines(true) - .with_try_parse_dates(parse_dates) + let dataframe = CsvReadOptions::default() + .with_infer_schema_length(infer_schema_length) + .with_has_header(has_header) .with_n_rows(stop_after_n_rows) - .with_separator(delimiter_as_byte) .with_skip_rows(skip_rows) .with_skip_rows_after_header(skip_rows_after_header) - .with_projection(projection) + .with_projection(projection.map(Arc::new)) .with_rechunk(do_rechunk) - .with_encoding(encoding) - .with_columns(column_names) - .with_dtypes(Some(schema_from_dtypes_pairs(dtypes)?)) - .with_null_values(Some(NullValues::AllColumns(null_vals))) - .with_end_of_line_char(eol_delimiter.unwrap_or(b'\n')); - - Ok(ExDataFrame::new(reader.finish()?)) + .with_columns(column_names.map(Arc::from)) + .with_schema_overwrite(Some(schema_from_dtypes_pairs(dtypes)?)) + .with_parse_options( + CsvParseOptions::default() + .with_encoding(encoding) + .with_truncate_ragged_lines(true) + .with_try_parse_dates(parse_dates) + .with_separator(delimiter_as_byte) + .with_eol_char(eol_delimiter.unwrap_or(b'\n')) + .with_null_values(Some(NullValues::AllColumns(null_vals))), + ) + .try_into_reader_with_file_path(Some(filename.into()))? + .finish(); + + Ok(ExDataFrame::new(dataframe?)) } pub fn schema_from_dtypes_pairs( @@ -105,6 +113,9 @@ pub fn df_to_csv_cloud( .include_header(include_headers) .with_separator(delimiter) .finish(&mut data.clone())?; + + let _ = cloud_writer.finish()?; + Ok(()) } @@ -141,7 +152,7 @@ pub fn df_load_csv( delimiter_as_byte: u8, do_rechunk: bool, column_names: Option>, - dtypes: Vec<(&str, ExSeriesDtype)>, + dtypes: Option>, encoding: &str, null_vals: Vec, parse_dates: bool, @@ -154,23 +165,32 @@ pub fn df_load_csv( let cursor = Cursor::new(binary.as_slice()); - let reader = CsvReader::new(cursor) - .infer_schema(infer_schema_length) - .has_header(has_header) - .with_try_parse_dates(parse_dates) + let read_options = match dtypes { + Some(val) => CsvReadOptions::default().with_schema(Some(schema_from_dtypes_pairs(val)?)), + None => CsvReadOptions::default(), + }; + + let dataframe = read_options + .with_has_header(has_header) + .with_infer_schema_length(infer_schema_length) .with_n_rows(stop_after_n_rows) - .with_separator(delimiter_as_byte) + .with_columns(column_names.map(Arc::from)) .with_skip_rows(skip_rows) .with_skip_rows_after_header(skip_rows_after_header) - .with_projection(projection) + .with_projection(projection.map(Arc::new)) .with_rechunk(do_rechunk) - .with_encoding(encoding) - .with_columns(column_names) - .with_dtypes(Some(schema_from_dtypes_pairs(dtypes)?)) - .with_null_values(Some(NullValues::AllColumns(null_vals))) - .with_end_of_line_char(eol_delimiter.unwrap_or(b'\n')); + .with_parse_options( + CsvParseOptions::default() + .with_separator(delimiter_as_byte) + .with_encoding(encoding) + .with_null_values(Some(NullValues::AllColumns(null_vals))) + .with_try_parse_dates(parse_dates) + .with_eol_char(eol_delimiter.unwrap_or(b'\n')), + ) + .into_reader_with_file_handle(cursor) + .finish(); - Ok(ExDataFrame::new(reader.finish()?)) + Ok(ExDataFrame::new(dataframe?)) } // ============ Parquet ============ // @@ -226,6 +246,9 @@ pub fn df_to_parquet_cloud( ParquetWriter::new(&mut cloud_writer) .with_compression(compression) .finish(&mut data.clone())?; + + let _ = cloud_writer.finish()?; + Ok(()) } @@ -235,9 +258,7 @@ fn object_store_to_explorer_error(error: impl std::fmt::Debug) -> ExplorerError } #[cfg(feature = "aws")] -fn build_aws_s3_cloud_writer( - ex_entry: ExS3Entry, -) -> Result { +fn build_aws_s3_cloud_writer(ex_entry: ExS3Entry) -> Result { let config = ex_entry.config; let mut aws_builder = object_store::aws::AmazonS3Builder::new() .with_region(&config.region) @@ -264,9 +285,8 @@ fn build_aws_s3_cloud_writer( .build() .map_err(object_store_to_explorer_error)?; - let object_store: Box = Box::new(aws_s3); - - crate::cloud_writer::CloudWriter::new(object_store, ex_entry.key.into()) + let object_store: Arc = Arc::new(aws_s3); + CloudWriter::new(object_store, ex_entry.key.into()) } #[rustler::nif(schedule = "DirtyCpu")] @@ -350,6 +370,9 @@ pub fn df_to_ipc_cloud( IpcWriter::new(&mut cloud_writer) .with_compression(compression) .finish(&mut data.clone())?; + + let _ = cloud_writer.finish()?; + Ok(()) } @@ -452,6 +475,9 @@ pub fn df_to_ipc_stream_cloud( IpcStreamWriter::new(&mut cloud_writer) .with_compression(compression) .finish(&mut data.clone())?; + + let _ = cloud_writer.finish()?; + Ok(()) } @@ -509,7 +535,7 @@ pub fn df_from_ndjson( let reader = JsonReader::new(buf_reader) .with_json_format(JsonFormat::JsonLines) .with_batch_size(batch_size) - .infer_schema_len(infer_schema_length); + .infer_schema_len(infer_schema_length.and_then(NonZeroUsize::new)); Ok(ExDataFrame::new(reader.finish()?)) } @@ -534,6 +560,9 @@ pub fn df_to_ndjson_cloud(data: ExDataFrame, ex_entry: ExS3Entry) -> Result<(), JsonWriter::new(&mut cloud_writer) .with_json_format(JsonFormat::JsonLines) .finish(&mut data.clone())?; + + let _ = cloud_writer.finish()?; + Ok(()) } @@ -566,7 +595,7 @@ pub fn df_load_ndjson( let reader = JsonReader::new(cursor) .with_json_format(JsonFormat::JsonLines) .with_batch_size(batch_size) - .infer_schema_len(infer_schema_length); + .infer_schema_len(infer_schema_length.and_then(NonZeroUsize::new)); Ok(ExDataFrame::new(reader.finish()?)) } diff --git a/native/explorer/src/expressions.rs b/native/explorer/src/expressions.rs index 5f7e53872..a30b29e5f 100644 --- a/native/explorer/src/expressions.rs +++ b/native/explorer/src/expressions.rs @@ -4,16 +4,11 @@ // or an expression and returns an expression that is // wrapped in an Elixir struct. -use polars::error::PolarsError; - -use polars::prelude::{GetOutput, IntoSeries, Utf8JsonPathImpl}; -use polars::series::Series; - use crate::datatypes::{ ExCorrelationMethod, ExDate, ExDateTime, ExDuration, ExNaiveDateTime, ExRankMethod, ExSeriesDtype, ExValidValue, }; -use crate::series::{cast_str_to_f64, ewm_opts, rolling_opts}; +use crate::series::{cast_str_to_f64, ewm_opts, rolling_opts_fixed_window}; use crate::{ExDataFrame, ExExpr, ExSeries}; use polars::lazy::dsl; use polars::prelude::{ @@ -38,13 +33,13 @@ pub fn expr_nil() -> ExExpr { #[rustler::nif] pub fn expr_integer(number: i64) -> ExExpr { - let expr = number.lit(); + let expr = Expr::Literal(LiteralValue::Int64(number)); ExExpr::new(expr) } #[rustler::nif] pub fn expr_float(number: f64) -> ExExpr { - let expr = number.lit(); + let expr = Expr::Literal(LiteralValue::Float64(number)); ExExpr::new(expr) } @@ -657,8 +652,11 @@ macro_rules! init_window_expr_fun { min_periods: Option, center: bool, ) -> ExExpr { - let expr = data.clone_inner(); - let opts = rolling_opts(window_size, weights, min_periods, center); + let expr = match weights.as_deref() { + Some([]) | None => data.clone_inner(), + _ => data.clone_inner().cast(DataType::Float64), + }; + let opts = rolling_opts_fixed_window(window_size, weights, min_periods, center); ExExpr::new(expr.$fun(opts)) } }; @@ -679,7 +677,7 @@ pub fn expr_window_standard_deviation( center: bool, ) -> ExExpr { let expr = data.clone_inner(); - let opts = rolling_opts(window_size, weights, min_periods, center); + let opts = rolling_opts_fixed_window(window_size, weights, min_periods, center); ExExpr::new(expr.rolling_std(opts).cast(DataType::Float64)) } @@ -784,7 +782,7 @@ pub fn expr_sort( nulls_last, }; - ExExpr::new(expr.sort_with(opts)) + ExExpr::new(expr.sort(opts)) } #[rustler::nif] @@ -831,7 +829,10 @@ pub fn expr_unary_not(expr: ExExpr) -> ExExpr { pub fn expr_describe_filter_plan(data: ExDataFrame, expr: ExExpr) -> String { let df = data.clone(); let expressions = expr.clone_inner(); - df.lazy().filter(expressions).describe_plan() + df.lazy() + .filter(expressions) + .describe_plan() + .expect("error") } #[rustler::nif] @@ -1119,18 +1120,8 @@ pub fn expr_json_decode(expr: ExExpr, ex_dtype: ExSeriesDtype) -> ExExpr { } #[rustler::nif] -pub fn expr_json_path_match(expr: ExExpr, json_path: &str) -> ExExpr { - let p = json_path.to_owned(); - let function = move |s: Series| { - let ca = s.str()?; - match ca.json_path_match(&p) { - Ok(ca) => Ok(Some(ca.into_series())), - Err(e) => Err(PolarsError::ComputeError(format!("{e:?}").into())), - } - }; - let expr = expr - .clone_inner() - .map(function, GetOutput::from_type(DataType::String)); +pub fn expr_json_path_match(expr: ExExpr, json_path: String) -> ExExpr { + let expr = expr.clone_inner().str().json_path_match(json_path.lit()); ExExpr::new(expr) } diff --git a/native/explorer/src/lazyframe.rs b/native/explorer/src/lazyframe.rs index 4bc8fb34b..ec7e72b47 100644 --- a/native/explorer/src/lazyframe.rs +++ b/native/explorer/src/lazyframe.rs @@ -24,7 +24,7 @@ pub fn lf_describe_plan(data: ExLazyFrame, optimized: bool) -> Result lf.describe_optimized_plan()?, - false => lf.describe_plan(), + false => lf.describe_plan().expect("error"), }; Ok(plan) } @@ -63,7 +63,7 @@ pub fn lf_tail( #[rustler::nif] pub fn lf_names(data: ExLazyFrame) -> Result, ExplorerError> { - let lf = data.clone_inner(); + let mut lf = data.clone_inner(); let names = lf .schema()? .iter_names() @@ -146,9 +146,12 @@ pub fn lf_sort_with( nulls_last: bool, ) -> Result { let exprs = ex_expr_to_exprs(expressions); - let ldf = data - .clone_inner() - .sort_by_exprs(exprs, directions, nulls_last, maintain_order); + let sort_options = SortMultipleOptions::new() + .with_nulls_last(nulls_last) + .with_maintain_order(maintain_order) + .with_order_descending_multi(directions); + + let ldf = data.clone_inner().sort_by_exprs(exprs, sort_options); Ok(ExLazyFrame::new(ldf)) } @@ -160,12 +163,13 @@ pub fn lf_grouped_sort_with( groups: Vec, directions: Vec, ) -> Result { + let sort_options = SortMultipleOptions::new().with_order_descending_multi(directions); // For grouped lazy frames, we need to use the `#sort_by` method that is // less powerful, but can be used with `over`. // See: https://docs.pola.rs/user-guide/expressions/window/#operations-per-group let ldf = data .clone_inner() - .with_columns([col("*").sort_by(expressions, directions).over(groups)]); + .with_columns([col("*").sort_by(expressions, sort_options).over(groups)]); Ok(ExLazyFrame::new(ldf)) } @@ -258,14 +262,14 @@ pub fn lf_pivot_longer( values_to: String, ) -> Result { let ldf = data.clone_inner(); - let melt_opts = MeltArgs { - id_vars: to_smart_strings(id_vars), - value_vars: to_smart_strings(value_vars), + let unpivot_opts = UnpivotArgs { + index: to_smart_strings(id_vars), + on: to_smart_strings(value_vars), variable_name: Some(names_to.into()), value_name: Some(values_to.into()), streamable: true, }; - let new_df = ldf.melt(melt_opts); + let new_df = ldf.unpivot(unpivot_opts); Ok(ExLazyFrame::new(new_df)) } @@ -281,7 +285,7 @@ pub fn lf_join( let how = match how { "left" => JoinType::Left, "inner" => JoinType::Inner, - "outer" => JoinType::Outer { coalesce: false }, + "outer" => JoinType::Full, "cross" => JoinType::Cross, _ => { return Err(ExplorerError::Other(format!( @@ -293,14 +297,24 @@ pub fn lf_join( let ldf = data.clone_inner(); let ldf1 = other.clone_inner(); - let new_ldf = ldf - .join_builder() - .with(ldf1) - .how(how) - .left_on(ex_expr_to_exprs(left_on)) - .right_on(ex_expr_to_exprs(right_on)) - .suffix(suffix) - .finish(); + let new_ldf = match how { + // Cross-joins no longer accept keys. + // https://github.com/pola-rs/polars/pull/17305 + JoinType::Cross => ldf + .join_builder() + .with(ldf1) + .how(JoinType::Cross) + .suffix(suffix) + .finish(), + _ => ldf + .join_builder() + .with(ldf1) + .how(how) + .left_on(ex_expr_to_exprs(left_on)) + .right_on(ex_expr_to_exprs(right_on)) + .suffix(suffix) + .finish(), + }; Ok(ExLazyFrame::new(new_ldf)) } @@ -322,7 +336,7 @@ pub fn lf_concat_columns(ldfs: Vec) -> Result = ldf .schema() .expect("should be able to get schema") diff --git a/native/explorer/src/lazyframe/io.rs b/native/explorer/src/lazyframe/io.rs index a075df5d3..1048cf802 100644 --- a/native/explorer/src/lazyframe/io.rs +++ b/native/explorer/src/lazyframe/io.rs @@ -81,14 +81,14 @@ pub fn lf_to_parquet( if streaming { let options = ParquetWriteOptions { compression, - statistics: false, + statistics: StatisticsOptions::empty(), row_group_size: None, data_pagesize_limit: None, maintain_order: false, }; lf.with_comm_subplan_elim(false) - .sink_parquet(filename.into(), options)?; + .sink_parquet(filename, options)?; Ok(()) } else { let mut df = lf.collect()?; @@ -117,7 +117,7 @@ pub fn lf_to_parquet_cloud( let options = ParquetWriteOptions { compression, - statistics: false, + statistics: StatisticsOptions::empty(), row_group_size: None, data_pagesize_limit: None, maintain_order: false, @@ -172,7 +172,7 @@ pub fn lf_to_ipc( maintain_order: false, }; lf.with_comm_subplan_elim(false) - .sink_ipc(filename.into(), options)?; + .sink_ipc(filename, options)?; Ok(()) } else { let mut df = lf.collect()?; @@ -209,7 +209,7 @@ pub fn lf_from_csv( let df = LazyCsvReader::new(filename) .with_infer_schema_length(infer_schema_length) - .has_header(has_header) + .with_has_header(has_header) .with_try_parse_dates(parse_dates) .with_n_rows(stop_after_n_rows) .with_separator(delimiter_as_byte) @@ -217,9 +217,9 @@ pub fn lf_from_csv( .with_skip_rows_after_header(skip_rows_after_header) .with_rechunk(do_rechunk) .with_encoding(encoding) - .with_dtype_overwrite(Some(schema_from_dtypes_pairs(dtypes)?.as_ref())) + .with_dtype_overwrite(Some(schema_from_dtypes_pairs(dtypes)?)) .with_null_values(Some(NullValues::AllColumns(null_vals))) - .with_end_of_line_char(eol_delimiter.unwrap_or(b'\n')) + .with_eol_char(eol_delimiter.unwrap_or(b'\n')) .finish()?; Ok(ExLazyFrame::new(df)) @@ -248,7 +248,7 @@ pub fn lf_to_csv( }; lf.with_comm_subplan_elim(false) - .sink_csv(filename.into(), options)?; + .sink_csv(filename, options)?; Ok(()) } else { let df = lf.collect()?; @@ -274,7 +274,7 @@ pub fn lf_from_ndjson( "\"batch_size\" expected to be non zero.".to_string(), ))?; let lf = LazyJsonLineReader::new(filename) - .with_infer_schema_length(infer_schema_length) + .with_infer_schema_length(infer_schema_length.and_then(NonZeroUsize::new)) .with_batch_size(Some(batch_size)) .finish()?; diff --git a/native/explorer/src/series.rs b/native/explorer/src/series.rs index 34fecf344..d081b979a 100644 --- a/native/explorer/src/series.rs +++ b/native/explorer/src/series.rs @@ -9,7 +9,7 @@ use crate::{ use encoding::encode_naive_datetime; // use encoding::encode_datetime; -use polars::prelude::*; +use polars::prelude::*; //{lazy::dsl::Expr, }; use polars_ops::chunked_array::cov::{cov, pearson_corr}; use polars_ops::prelude::peaks::*; use rustler::{Binary, Encoder, Env, Term}; @@ -70,29 +70,32 @@ pub fn s_mask(series: ExSeries, filter: ExSeries) -> Result Result { let s = data.clone_inner(); let s1 = other.clone_inner(); - Ok(ExSeries::new(s + s1)) + let result = s + s1; + Ok(ExSeries::new(result?)) } #[rustler::nif(schedule = "DirtyCpu")] pub fn s_subtract(lhs: ExSeries, rhs: ExSeries) -> Result { let left = lhs.clone_inner(); let right = rhs.clone_inner(); - - Ok(ExSeries::new(left - right)) + let result = left - right; + Ok(ExSeries::new(result?)) } #[rustler::nif(schedule = "DirtyCpu")] pub fn s_multiply(data: ExSeries, other: ExSeries) -> Result { let s = data.clone_inner(); let s1 = other.clone_inner(); - Ok(ExSeries::new(s * s1)) + let result = s * s1; + Ok(ExSeries::new(result?)) } #[rustler::nif(schedule = "DirtyCpu")] pub fn s_divide(data: ExSeries, other: ExSeries) -> Result { let s = data.clone_inner().cast(&DataType::Float64)?; let s1 = other.clone_inner().cast(&DataType::Float64)?; - Ok(ExSeries::new(s / s1)) + let result = s / s1; + Ok(ExSeries::new(result?)) } #[rustler::nif(schedule = "DirtyCpu")] @@ -106,9 +109,9 @@ pub fn s_remainder(data: ExSeries, other: ExSeries) -> Result Result #[rustler::nif(schedule = "DirtyCpu")] pub fn s_frequencies(series: ExSeries) -> Result { - let df = series.value_counts(true, true)?; + let df = series.value_counts(true, true, "counts".to_string(), false)?; Ok(ExDataFrame::new(df)) } @@ -559,8 +562,8 @@ pub fn s_window_sum( min_periods: Option, center: bool, ) -> Result { - let opts = rolling_opts(window_size, weights, min_periods, center); - let s1 = series.rolling_sum(opts.try_into()?)?; + let opts = rolling_opts_fixed_window(window_size, weights, min_periods, center); + let s1 = series.rolling_sum(opts)?; Ok(ExSeries::new(s1)) } @@ -572,8 +575,8 @@ pub fn s_window_mean( min_periods: Option, center: bool, ) -> Result { - let opts = rolling_opts(window_size, weights, min_periods, center); - let s1 = series.rolling_mean(opts.try_into()?)?; + let opts = rolling_opts_fixed_window(window_size, weights, min_periods, center); + let s1 = series.rolling_mean(opts)?; Ok(ExSeries::new(s1)) } @@ -585,7 +588,7 @@ pub fn s_window_median( min_periods: Option, center: bool, ) -> Result { - let opts = rolling_opts(window_size, weights, min_periods, center); + let opts = rolling_opts_fixed_window(window_size, weights, min_periods, center); let s1 = series .clone_inner() .into_frame() @@ -605,8 +608,8 @@ pub fn s_window_max( min_periods: Option, center: bool, ) -> Result { - let opts = rolling_opts(window_size, weights, min_periods, center); - let s1 = series.rolling_max(opts.try_into()?)?; + let opts = rolling_opts_fixed_window(window_size, weights, min_periods, center); + let s1 = series.rolling_max(opts)?; Ok(ExSeries::new(s1)) } @@ -618,8 +621,8 @@ pub fn s_window_min( min_periods: Option, center: bool, ) -> Result { - let opts = rolling_opts(window_size, weights, min_periods, center); - let s1 = series.rolling_min(opts.try_into()?)?; + let opts = rolling_opts_fixed_window(window_size, weights, min_periods, center); + let s1 = series.rolling_min(opts)?; Ok(ExSeries::new(s1)) } @@ -631,27 +634,27 @@ pub fn s_window_standard_deviation( min_periods: Option, center: bool, ) -> Result { - let opts = rolling_opts(window_size, weights, min_periods, center); - let s1 = series.rolling_std(opts.try_into()?)?; + let opts = rolling_opts_fixed_window(window_size, weights, min_periods, center); + let s1 = series.rolling_std(opts)?; Ok(ExSeries::new(s1)) } // Used for rolling functions - also see "expressions" module -pub fn rolling_opts( +pub fn rolling_opts_fixed_window( window_size: usize, weights: Option>, min_periods: Option, center: bool, -) -> RollingOptions { - let min_periods = if let Some(mp) = min_periods { +) -> RollingOptionsFixedWindow { + let min_periods: usize = if let Some(mp) = min_periods { mp } else { window_size }; - let window_size_duration = Duration::new(window_size as i64); + // let window_size_duration = Duration::new(window_size as i64); - RollingOptions { - window_size: window_size_duration, + RollingOptionsFixedWindow { + window_size, weights, min_periods, center, @@ -838,7 +841,16 @@ pub fn s_mode(s: ExSeries) -> Result { #[rustler::nif(schedule = "DirtyCpu")] pub fn s_product(s: ExSeries) -> Result { if s.dtype().is_numeric() { - Ok(ExSeries::new(s.product()?)) + let series = s + .clone_inner() + .into_frame() + .lazy() + .select([col(s.name()).product()]) + .collect()? + .column(s.name())? + .clone(); + + Ok(ExSeries::new(series)) } else { panic!("product/1 not implemented for {:?}", &s.dtype()) } @@ -999,7 +1011,8 @@ pub fn s_quantile<'a>( .encode(env)), }, _ => encoding::term_from_value( - s.quantile_as_series(quantile, strategy)? + s.quantile_reduce(quantile, strategy)? + .into_series("quantile") .cast(dtype)? .get(0)?, env, @@ -1612,26 +1625,17 @@ pub fn s_json_decode(s: ExSeries, ex_dtype: ExSeriesDtype) -> Result Result { - let p = json_path.to_owned(); - let function = move |s: Series| { - let ca = s.str()?; - match ca.json_path_match(&p) { - Ok(ca) => Ok(Some(ca.into_series())), - Err(e) => Err(PolarsError::ComputeError(format!("{e:?}").into())), - } - }; - let s2 = s +pub fn s_json_path_match(s: ExSeries, json_path: String) -> Result { + let var_series = s .clone_inner() .into_frame() .lazy() - .select([col(s.name()) - .map(function, GetOutput::from_type(DataType::String)) - .alias(s.name())]) + .select([col(s.name()).str().json_path_match(json_path.lit())]) .collect()? .column(s.name())? .clone(); - Ok(ExSeries::new(s2)) + + Ok(ExSeries::new(var_series)) } #[rustler::nif] diff --git a/test/explorer/data_frame/ipc_test.exs b/test/explorer/data_frame/ipc_test.exs index e0e45d14a..d4dce332c 100644 --- a/test/explorer/data_frame/ipc_test.exs +++ b/test/explorer/data_frame/ipc_test.exs @@ -188,8 +188,11 @@ defmodule Explorer.DataFrame.IPCTest do assert {:error, error} = DF.to_ipc(df, path, config: s3_config) - assert error == - RuntimeError.exception("Generic Error: Could not put multipart to path " <> key) + assert %RuntimeError{message: message} = error + + assert message =~ "cannot read information from file, which means the upload failed." + assert message =~ "Object at location test-writes/wine" + assert message =~ "Client error with status 404 Not Found: No Body" end end diff --git a/test/explorer/data_frame/lazy_test.exs b/test/explorer/data_frame/lazy_test.exs index da04861cb..aa162b70f 100644 --- a/test/explorer/data_frame/lazy_test.exs +++ b/test/explorer/data_frame/lazy_test.exs @@ -261,7 +261,7 @@ defmodule Explorer.DataFrame.LazyTest do ) assert RuntimeError.message(error) =~ - "Polars Error: Object at location oranges.parquet not found:" + "Polars Error: expected at least 1 path: 'parquet scan' failed: 'select' input failed to resolve" end end @@ -411,12 +411,19 @@ defmodule Explorer.DataFrame.LazyTest do path = "s3://test-bucket/test-lazy-writes/wine-#{System.monotonic_time()}.parquet" ldf = DF.head(ldf, 15) - assert :ok = DF.to_parquet(ldf, path, streaming: true, config: config) + # assert :ok = DF.to_parquet(ldf, path, streaming: true, config: config) - df = DF.compute(ldf) - df1 = DF.from_parquet!(path, config: config) + # df = DF.compute(ldf) + # df1 = DF.from_parquet!(path, config: config) + + # assert DF.to_rows(df) |> Enum.sort() == DF.to_rows(df1) |> Enum.sort() - assert DF.to_rows(df) |> Enum.sort() == DF.to_rows(df1) |> Enum.sort() + message = + "streaming of a lazy frame to the cloud using parquet is currently unavailable. Please try again disabling the `:streaming` option." + + assert_raise RuntimeError, message, fn -> + DF.to_parquet(ldf, path, streaming: true, config: config) + end end @tag :cloud_integration diff --git a/test/explorer/data_frame_test.exs b/test/explorer/data_frame_test.exs index 3bd85fd6c..777174c57 100644 --- a/test/explorer/data_frame_test.exs +++ b/test/explorer/data_frame_test.exs @@ -299,7 +299,7 @@ defmodule Explorer.DataFrameTest do df = DF.new(a: [1, 2, 3, 4, 5, 6, 5], b: [9, 8, 7, 6, 5, 4, 3]) message = - "expecting the function to return a boolean LazySeries, but instead it returned a LazySeries of type {:f, 64}" + "expecting the function to return a boolean LazySeries, but instead it returned a LazySeries of type {:s, 64}" assert_raise ArgumentError, message, fn -> DF.filter_with(df, fn ldf -> @@ -811,7 +811,7 @@ defmodule Explorer.DataFrameTest do df = DF.new([%{a: ~s({"n": 1})}, %{a: ~s({"m": 1})}]) assert_raise RuntimeError, - "Polars Error: ComputeError(ErrString(\"error compiling JSONpath expression path error: \\nEof\\n\"))", + "Polars Error: error compiling JSON path expression path error: \nEof\n", fn -> DF.mutate(df, n: json_path_match(a, "$.")) end @@ -948,7 +948,7 @@ defmodule Explorer.DataFrameTest do calc2: [-1, 0, 2], calc3: [2, 4, 8], calc4: [0.5, 1.0, 2.0], - calc5: [1.0, 4.0, 16.0], + calc5: [1, 4, 16], calc6: [0, 1, 2], calc7: [1, 0, 0], calc8: [:nan, :nan, :nan], @@ -964,7 +964,7 @@ defmodule Explorer.DataFrameTest do "calc2" => {:s, 64}, "calc3" => {:s, 64}, "calc4" => {:f, 64}, - "calc5" => {:f, 64}, + "calc5" => {:s, 64}, "calc6" => {:s, 64}, "calc7" => {:s, 64}, "calc8" => {:f, 64}, @@ -996,7 +996,7 @@ defmodule Explorer.DataFrameTest do calc2: [1, 0, -2], calc3: [2, 4, 8], calc4: [2.0, 1.0, 0.5], - calc5: [2.0, 4.0, 16.0], + calc5: [2, 4, 16], calc5_1: [2.0, 4.0, 16.0], calc6: [2, 1, 0], calc7: [0, 0, 2] @@ -1008,7 +1008,7 @@ defmodule Explorer.DataFrameTest do "calc2" => {:s, 64}, "calc3" => {:s, 64}, "calc4" => {:f, 64}, - "calc5" => {:f, 64}, + "calc5" => {:s, 64}, "calc5_1" => {:f, 64}, "calc6" => {:s, 64}, "calc7" => {:s, 64} @@ -1036,7 +1036,7 @@ defmodule Explorer.DataFrameTest do calc2: [-1, 1, 2], calc3: [2, 2, 8], calc4: [0.5, 2.0, 2.0], - calc5: [1.0, 2.0, 16.0], + calc5: [1, 2, 16], calc6: [0, 2, 2], calc7: [1, 0, 0] } @@ -1047,7 +1047,7 @@ defmodule Explorer.DataFrameTest do "calc2" => {:s, 64}, "calc3" => {:s, 64}, "calc4" => {:f, 64}, - "calc5" => {:f, 64}, + "calc5" => {:s, 64}, "calc6" => {:s, 64}, "calc7" => {:s, 64} } @@ -1074,7 +1074,7 @@ defmodule Explorer.DataFrameTest do calc2: [-1, 1, 2], calc3: [2, 2, 8], calc4: [0.5, 2.0, 2.0], - calc5: [1.0, 2.0, 16.0], + calc5: [1, 2, 16], calc6: [0, 2, 2], calc7: [1, 0, 0] } @@ -1085,7 +1085,7 @@ defmodule Explorer.DataFrameTest do "calc2" => {:s, 64}, "calc3" => {:s, 64}, "calc4" => {:f, 64}, - "calc5" => {:f, 64}, + "calc5" => {:s, 64}, "calc6" => {:s, 64}, "calc7" => {:s, 64} } @@ -1114,7 +1114,7 @@ defmodule Explorer.DataFrameTest do calc2: [19, 38, 57], calc3: [3, 4, 3], calc4: [2.0, :infinity, 7.5], - calc5: [1.0, 4.0, 3.0], + calc5: [1, 4, 3], calc6: [2, nil, 7], calc7: [0, nil, 4] } @@ -1128,7 +1128,7 @@ defmodule Explorer.DataFrameTest do "calc2" => {:s, 64}, "calc3" => {:s, 64}, "calc4" => {:f, 64}, - "calc5" => {:f, 64}, + "calc5" => {:s, 64}, "calc6" => {:s, 64}, "calc7" => {:s, 64} } @@ -1714,8 +1714,7 @@ defmodule Explorer.DataFrameTest do f: substring(a, 1), g: substring(b, 2, 5), h: substring(c, -3), - i: substring(d, 6, 10), - j: substring(e, -15, 2) + i: substring(d, 6, 10) ) assert DF.to_columns(df1, atom_keys: true) == %{ @@ -1727,8 +1726,7 @@ defmodule Explorer.DataFrameTest do f: ["hello", "world", "foo", "bar"], g: ["nus", "rth", "rs", "piter"], h: ["foo", "bar", "baz", "uox"], - i: ["", "", "", ""], - j: ["_f", "_b", "_b", "_q"] + i: ["", "", "", ""] } end @@ -3362,10 +3360,10 @@ defmodule Explorer.DataFrameTest do assert DF.to_columns(df2, atom_keys: true) == %{ id: [1], - value_variable_a: [1], - value_variable_b: [2], - another_value_variable_a: [6], - another_value_variable_b: [9] + value_a: [1], + value_b: [2], + another_value_a: [6], + another_value_b: [9] } end @@ -3434,6 +3432,7 @@ defmodule Explorer.DataFrameTest do assert df.names == ["variable", "value"] assert df.dtypes == %{"variable" => :string, "value" => {:s, 64}} + assert DF.shape(df) == {3282, 2} end diff --git a/test/explorer/polars_backend/expression_test.exs b/test/explorer/polars_backend/expression_test.exs index 1fafea9f8..354c31e58 100644 --- a/test/explorer/polars_backend/expression_test.exs +++ b/test/explorer/polars_backend/expression_test.exs @@ -19,7 +19,7 @@ defmodule Explorer.PolarsBackend.ExpressionTest do assert Expression.describe_filter_plan(df, expr) == String.trim(""" FILTER [(col("col_a")) == (5)] FROM - DF ["col_a", "col_b"]; PROJECT */2 COLUMNS; SELECTION: "None" + DF ["col_a", "col_b"]; PROJECT */2 COLUMNS; SELECTION: None """) end @@ -93,8 +93,8 @@ defmodule Explorer.PolarsBackend.ExpressionTest do assert Expression.describe_filter_plan(df, expr) == String.trim(""" - FILTER [(col("col_a")) == (col("col_b"))] FROM - DF ["col_a", "col_b"]; PROJECT */2 COLUMNS; SELECTION: "None" + FILTER [(col("col_a").cast(Float64)) == (col("col_b"))] FROM + DF ["col_a", "col_b"]; PROJECT */2 COLUMNS; SELECTION: None """) end end diff --git a/test/explorer/series_test.exs b/test/explorer/series_test.exs index 84a7bf446..ae1ff898b 100644 --- a/test/explorer/series_test.exs +++ b/test/explorer/series_test.exs @@ -2283,14 +2283,14 @@ defmodule Explorer.SeriesTest do end end - test "pow(uint, sint) == float64" do + test "pow(uint, sint) == sint" do for u_base <- [8, 16, 32, 64], s_power <- [8, 16, 32, 64] do base = Series.from_list([1, 2, 3], dtype: {:u, u_base}) power = Series.from_list([3, 2, 1], dtype: {:s, s_power}) result = Series.pow(base, power) - assert result.dtype == {:f, 64} + assert result.dtype == {:s, 64} assert Series.to_list(result) == [1, 4, 3] end end @@ -2308,15 +2308,15 @@ defmodule Explorer.SeriesTest do end end - test "pow(sint, sint) == float64" do + test "pow(sint, sint) == sint" do for s_base <- [8, 16, 32, 64], s_power <- [8, 16, 32, 64] do base = Series.from_list([1, 2, 3], dtype: {:s, s_base}) power = Series.from_list([3, 2, 1], dtype: {:s, s_power}) result = Series.pow(base, power) - assert result.dtype == {:f, 64} - assert Series.to_list(result) === [1.0, 4.0, 3.0] + assert result.dtype == {:s, 64} + assert Series.to_list(result) === [1, 4, 3] end end @@ -2360,10 +2360,13 @@ defmodule Explorer.SeriesTest do s1 = Series.from_list([1, 2, 3]) s2 = Series.from_list([1, -2, 3]) - result = Series.pow(s1, s2) + message = + "Polars Error: invalid operation: invalid operation: conversion from `i64` to `u32` failed in column 'exponent' for 1 out of 3 values: [-2]\n\n" <> + "Hint: if you were trying to raise an integer to a negative integer power, please cast your base or exponent to float first." - assert result.dtype == {:f, 64} - assert Series.to_list(result) === [1.0, 0.25, 27.0] + assert_raise RuntimeError, message, fn -> + Series.pow(s1, s2) + end end test "pow of an integer series with a float series" do @@ -2392,7 +2395,7 @@ defmodule Explorer.SeriesTest do result = Series.pow(s1, s2) - assert result.dtype == {:f, 64} + assert result.dtype == {:s, 64} assert Series.to_list(result) == [1, nil, 3] end @@ -2402,7 +2405,7 @@ defmodule Explorer.SeriesTest do result = Series.pow(s1, s2) - assert result.dtype == {:f, 64} + assert result.dtype == {:s, 64} assert Series.to_list(result) == [1, nil, 3] end @@ -2412,7 +2415,7 @@ defmodule Explorer.SeriesTest do result = Series.pow(s1, s2) - assert result.dtype == {:f, 64} + assert result.dtype == {:s, 64} assert Series.to_list(result) == [1, nil, 3] end @@ -2421,17 +2424,20 @@ defmodule Explorer.SeriesTest do result = Series.pow(s1, 2) - assert result.dtype == {:f, 64} + assert result.dtype == {:s, 64} assert Series.to_list(result) == [1, 4, 9] end test "pow of an integer series with a negative integer scalar value on the right-hand side" do s1 = Series.from_list([1, 2, 3]) - result = Series.pow(s1, -2) + message = + "Polars Error: invalid operation: invalid operation: conversion from `i64` to `u32` failed in column 'literal' for 1 out of 1 values: [-2]\n\n" <> + "Hint: if you were trying to raise an integer to a negative integer power, please cast your base or exponent to float first." - assert result.dtype == {:f, 64} - assert Series.to_list(result) === [1.0, 1 / 4, 1 / 9] + assert_raise RuntimeError, message, fn -> + Series.pow(s1, -2) + end end test "pow of an integer series with a float scalar value on the right-hand side" do @@ -2484,17 +2490,20 @@ defmodule Explorer.SeriesTest do result = Series.pow(2, s1) - assert result.dtype == {:f, 64} - assert Series.to_list(result) === [2.0, 4.0, 8.0] + assert result.dtype == {:s, 64} + assert Series.to_list(result) === [2, 4, 8] end test "pow of an integer series that contains negative integer with an integer scalar value on the left-hand side" do s1 = Series.from_list([1, -2, 3]) - result = Series.pow(2, s1) + message = + "Polars Error: invalid operation: invalid operation: conversion from `i64` to `u32` failed in column 'exponent' for 1 out of 3 values: [-2]\n\n" <> + "Hint: if you were trying to raise an integer to a negative integer power, please cast your base or exponent to float first." - assert result.dtype == {:f, 64} - assert Series.to_list(result) === [2.0, 0.25, 8.0] + assert_raise RuntimeError, message, fn -> + Series.pow(2, s1) + end end test "pow of an integer series with a negative integer scalar value on the left-hand side" do @@ -2502,7 +2511,7 @@ defmodule Explorer.SeriesTest do result = Series.pow(-2, s1) - assert result.dtype == {:f, 64} + assert result.dtype == {:s, 64} assert Series.to_list(result) == [-2, 4, -8] end @@ -4176,9 +4185,9 @@ defmodule Explorer.SeriesTest do assert Series.to_list(s2) == [ 1.0, - 1.0, - 1.6666666666666667, + nil, 1.6666666666666667, + nil, 2.4285714285714284, 3.2666666666666666, 4.161290322580645, @@ -4189,14 +4198,16 @@ defmodule Explorer.SeriesTest do end test "does not ignore nil if set ignore_nils option to false and calculates ewma" do + # The idea is to ignore or not the `nils` in the calculation. But `nils` are + # always propagated. s1 = Series.from_list([1, nil, 2, nil, 3, 4, 5, 6, 7, 8]) s2 = Series.ewm_mean(s1, ignore_nils: false) assert Series.to_list(s2) == [ 1.0, - 1.0, - 1.8, + nil, 1.8, + nil, 2.7142857142857144, 3.490566037735849, 4.316239316239316, @@ -4281,14 +4292,16 @@ defmodule Explorer.SeriesTest do end test "ignores nil by default and calculates ewm std" do + # The idea is to ignore or not the `nils` in the calculation. But `nils` are + # always propagated. s1 = Series.from_list([1, nil, 2, nil, 3, 4, 5, 6, 7, 8]) s2 = Series.ewm_standard_deviation(s1, ignore_nils: true) assert Series.to_list(s2) == [ 0.0, - 0.0, - 0.7071067811865476, + nil, 0.7071067811865476, + nil, 0.9636241116594314, 1.1771636613972951, 1.3452425132127066, @@ -4299,14 +4312,16 @@ defmodule Explorer.SeriesTest do end test "does not ignore nil if set ignore_nils option to false and calculates ewm std" do + # The idea is to ignore or not the `nils` in the calculation. But `nils` are + # always propagated. s1 = Series.from_list([1, nil, 2, nil, 3, 4, 5, 6, 7, 8]) s2 = Series.ewm_standard_deviation(s1, ignore_nils: false) assert Series.to_list(s2) == [ 0.0, - 0.0, - 0.7071067811865476, + nil, 0.7071067811865476, + nil, 0.8864052604279183, 0.9772545497599153, 1.1470897308102692, @@ -4409,14 +4424,16 @@ defmodule Explorer.SeriesTest do end test "ignores nil by default and calculates ewm var" do + # The idea is to ignore or not the `nils` in the calculation. But `nils` are + # always propagated. s1 = Series.from_list([1, nil, 2, nil, 3, 4, 5, 6, 7, 8]) s2 = Series.ewm_variance(s1, ignore_nils: true) assert Series.to_list(s2) == [ 0.0, - 0.0, - 0.5, + nil, 0.5, + nil, 0.9285714285714284, 1.385714285714286, 1.8096774193548393, @@ -4427,14 +4444,16 @@ defmodule Explorer.SeriesTest do end test "does not ignore nil if set ignore_nils option to false and calculates ewm var" do + # The idea is to ignore or not the `nils` in the calculation. But `nils` are + # always propagated. s1 = Series.from_list([1, nil, 2, nil, 3, 4, 5, 6, 7, 8]) s2 = Series.ewm_variance(s1, ignore_nils: false) assert Series.to_list(s2) == [ 0.0, - 0.0, - 0.5, + nil, 0.5, + nil, 0.7857142857142857, 0.9550264550264549, 1.315814850530376, @@ -5549,7 +5568,7 @@ defmodule Explorer.SeriesTest do series = Series.from_list(["earth", "mars", "neptune"]) assert Series.substring(series, -4, 4) |> Series.to_list() == ["arth", "mars", "tune"] - assert Series.substring(series, -20, 4) |> Series.to_list() == ["eart", "mars", "nept"] + assert Series.substring(series, -6, 2) |> Series.to_list() == ["e", "", "ep"] end end