Merge branch 'main' into main

.github/release-drafter.yml

@@ -1,8 +1,10 @@
 categories:
   - title: 🏆 Highlights
     labels: highlight
-  - title: ⚠️ Breaking changes
+  - title: 💥 Breaking changes
     labels: breaking
+  - title: ⚠️ Deprecations
+    labels: deprecation
   - title: 🚀 Performance improvements
     labels: performance
   - title: ✨ Enhancements
@@ -12,13 +14,8 @@ categories:
   - title: 🛠️ Other improvements
     labels:
       - build
-      - chore
-      - ci
       - documentation
-      - refactor
-      - revert
-      - style
-      - test
+      - internal
 
 exclude-labels:
   - skip-changelog
@@ -28,7 +25,7 @@ change-template: '- $TITLE (#$NUMBER)'
 change-title-escapes: '\<*_&'
 replacers:
   # Remove conventional commits from titles
-  - search: '/- (build|chore|ci|docs|feat|fix|perf|refactor|release|revert|style|test)(\(.*\))?(\!)?\: /g'
+  - search: '/- (build|chore|depr|docs|feat|fix|perf|release)(\(.*\))?(\!)?\: /g'
     replace: '- '
 
 version-resolver:
@@ -39,22 +36,25 @@ version-resolver:
 autolabeler:
   - label: rust
     title:
-      - '/^(build|chore|ci|docs|feat|fix|perf|refactor|release|revert|style|test)\(.*rust.*\)/'
+      - '/^(build|chore|depr|docs|feat|fix|perf|release)(\(.*rust.*\))?\!?\: /'
   - label: python
     title:
-      - '/^(build|chore|ci|docs|feat|fix|perf|refactor|release|revert|style|test)\(.*python.*\)/'
+      - '/^(build|chore|depr|docs|feat|fix|perf|release)(\(.*python.*\))?\!?\: /'
+  - label: cli
+    title:
+      - '/^(build|chore|depr|docs|feat|fix|perf|release)\(.*cli.*\)\!?\: /'  # CLI tag not in global scope
   - label: breaking
     title:
-      - '/^(build|chore|ci|docs|feat|fix|perf|refactor|release|revert|style|test)(\(.*\))?\!\: /'
+      - '/^(build|chore|depr|docs|feat|fix|perf|release)(\(.*\))?\!\: /'
   - label: build
     title:
       - '/^build/'
-  - label: chore
+  - label: internal
     title:
       - '/^chore/'
-  - label: ci
+  - label: deprecation
     title:
-      - '/^ci/'
+      - '/^depr/'
   - label: documentation
     title:
       - '/^docs/'
@@ -67,21 +67,9 @@ autolabeler:
   - label: performance
     title:
       - '/^perf/'
-  - label: refactor
-    title:
-      - '/^refactor/'
   - label: release
     title:
       - '/^release/'
-  - label: revert
-    title:
-      - '/^revert/'
-  - label: style
-    title:
-      - '/^style/'
-  - label: test
-    title:
-      - '/^test/'
 
 template: |
   $CHANGES

.github/workflows/docs-rust.yml

@@ -32,7 +32,8 @@ jobs:
       - name: Build Rust documentation
         env:
           RUSTDOCFLAGS: --cfg docsrs -D warnings
-        run: cargo doc --features=docs-selection --package polars
+        working-directory: polars
+        run: make doctest
 
       - name: Create redirect to Polars crate and set no-jekyll
         if: ${{ github.ref_name == 'main' }}

.github/workflows/lint-rust.yml

@@ -60,7 +60,7 @@ jobs:
           save-if: ${{ github.ref_name == 'main' }}
 
       - name: Run cargo clippy
-        run: cargo clippy -- -D warnings
+        run: cargo clippy --all-targets -- -D warnings
 
   rustfmt:
     if: github.ref_name != 'main'
@@ -90,9 +90,9 @@ jobs:
         env:
           MIRIFLAGS: -Zmiri-disable-isolation -Zmiri-ignore-leaks -Zmiri-disable-stacked-borrows
           POLARS_ALLOW_EXTENSION: '1'
-        run: |
-          cargo miri test \
-          --no-default-features \
-          --features object \
-          -p polars-core \
+        run: >
+          cargo miri test
+          --no-default-features
+          --features object
+          -p polars-core
           -p polars-arrow

Cargo.toml

@@ -3,29 +3,16 @@ resolver = "2"
 members = [
   "polars",
   "polars-cli",
-  "polars/polars-core",
-  "polars/polars-io",
-  "polars/polars-time",
-  "polars/polars-utils",
-  "polars/polars-ops",
-  "polars/polars-algo",
-  "polars/polars-lazy",
-  "polars/polars-lazy/polars-plan",
-  "polars/polars-lazy/polars-pipe",
-  "polars/polars-sql",
-  "polars/polars-error",
-  "polars/polars-row",
-  "polars/polars-json",
-  "examples/read_csv",
-  "examples/read_json",
-  "examples/read_parquet",
-  "examples/read_parquet_cloud",
-  "examples/string_filter",
-  "examples/python_rust_compiled_function",
+  "polars/polars-*",
+  "polars/polars-lazy/polars-*",
+  "examples/*",
+]
+exclude = [
+  "examples/datasets",
 ]
 
 [workspace.package]
-version = "0.30.0"
+version = "0.31.1"
 
 [workspace.dependencies]
 rayon = "1.6"
@@ -46,11 +33,11 @@ strum_macros = "0.25"
 [workspace.dependencies.arrow]
 package = "arrow2"
 # git = "https://github.com/jorgecarleitao/arrow2"
-git = "https://github.com/ritchie46/arrow2"
+# git = "https://github.com/ritchie46/arrow2"
 # rev = "2d2e7053f9a50810bfe9cecff25ab39089aef98e"
 # path = "../arrow2"
-branch = "polars_2023-06-26"
-version = "0.17"
+# branch = "polars_2023-06-26"
+version = "0.17.2"
 default-features = false
 features = [
   "compute_aggregate",

polars-cli/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "polars-cli"
-version = "0.2.0"
+version = "0.3.0"
 edition = "2021"
 license = "MIT"
 repository = "https://github.com/pola-rs/polars"
@@ -28,11 +28,11 @@ ciborium = "0.2.0"
 clap = { version = "4.2.2", features = ["derive", "cargo"] }
 nu-ansi-term = { version = "0.47.0", optional = true }
 once_cell.workspace = true
-polars = { version = "0.30.0", path = "../polars", features = ["lazy", "sql", "dtype-full", "serde-lazy"] }
+polars = { version = "0.31.1", path = "../polars", features = ["lazy", "sql", "dtype-full", "serde-lazy"] }
 reedline = { version = "0.21.0" }
 serde = { version = "1.0.160", features = ["derive"] }
 sqlparser = "0.34"
 tmp_env = "0.1.1"
 
 [target.'cfg(target_os = "linux")'.dependencies]
-jemallocator = { version = "0.5", features = ["disable_initial_exec_tls"] }
+jemallocator = { version = "0.5.0", features = ["disable_initial_exec_tls"] }

polars/Cargo.toml

@@ -312,13 +312,13 @@ bench = [
 ]
 
 [dependencies]
-polars-algo = { version = "0.30.0", path = "./polars-algo", optional = true }
-polars-core = { version = "0.30.0", path = "./polars-core", features = ["docs"], default-features = false }
-polars-io = { version = "0.30.0", path = "./polars-io", features = [], default-features = false, optional = true }
-polars-lazy = { version = "0.30.0", path = "./polars-lazy", features = [], default-features = false, optional = true }
-polars-ops = { version = "0.30.0", path = "./polars-ops" }
-polars-sql = { version = "0.30.0", path = "./polars-sql", default-features = false, optional = true }
-polars-time = { version = "0.30.0", path = "./polars-time", default-features = false, optional = true }
+polars-algo = { version = "0.31.1", path = "./polars-algo", optional = true }
+polars-core = { version = "0.31.1", path = "./polars-core", features = ["docs"], default-features = false }
+polars-io = { version = "0.31.1", path = "./polars-io", features = [], default-features = false, optional = true }
+polars-lazy = { version = "0.31.1", path = "./polars-lazy", features = [], default-features = false, optional = true }
+polars-ops = { version = "0.31.1", path = "./polars-ops" }
+polars-sql = { version = "0.31.1", path = "./polars-sql", default-features = false, optional = true }
+polars-time = { version = "0.31.1", path = "./polars-time", default-features = false, optional = true }
 
 # enable js feature for getrandom to work in wasm
 [target.'cfg(target_family = "wasm")'.dependencies.getrandom]

polars/Makefile

@@ -1,26 +1,46 @@
 .DEFAULT_GOAL := help
 
+SHELL=/bin/bash
 BASE ?= main
 
-.PHONY: fmt check check-features clippy clippy-default test test-doc integration-tests
-
-fmt:
+.PHONY: fmt
+fmt:  ## Run rustfmt and dprint
 	cargo fmt --all
 	dprint fmt
 
-generate_test_files:
-	cargo run -p polars-cli "select * from read_csv('../examples/datasets/foods1.csv')" -o parquet > ../examples/datasets/foods1.parquet
-	cargo run -p polars-cli "select * from read_csv('../examples/datasets/foods1.csv')" -o arrow > ../examples/datasets/foods1.ipc
-
-check:
+.PHONY: check
+check:  ## Run cargo check with all features
 	cargo check --all-targets --all-features
 
-clippy:
+.PHONY: clippy
+clippy:  ## Run clippy with all features
 	cargo clippy --all-targets --all-features
 
-clippy-default:
-	cargo clippy
+.PHONY: clippy-default
+clippy-default:  ## Run clippy with default features
+	cargo clippy --all-targets
+
+.PHONY: pre-commit
+pre-commit: fmt clippy clippy-default  ## Run autoformatting and linting
+
+.PHONY: check-features
+check-features:  ## Run cargo check for feature flag combinations (warning: slow)
+	cargo hack check --each-feature --no-dev-deps
+
+.PHONY: miri
+miri:  ## Run miri
+	# not tested on all features because miri does not support SIMD
+	# some tests are also filtered, because miri cannot deal with the rayon threadpool
+	# we ignore leaks because the thread pool of rayon is never killed.
+	MIRIFLAGS="-Zmiri-disable-isolation -Zmiri-ignore-leaks -Zmiri-disable-stacked-borrows" \
+	POLARS_ALLOW_EXTENSION=1 \
+	cargo miri test \
+	    --no-default-features \
+	    --features object \
+	    -p polars-core \
+	    -p polars-arrow
 
+.PHONY: test
 test:  ## Run tests
 	cargo test --all-features \
 		-p polars-lazy \
@@ -34,42 +54,34 @@ test:  ## Run tests
 		-- \
 		--test-threads=2
 
-integration-tests:
-	cargo test --all-features --test it -- --test-threads=2
-
-miri:
-	# not tested on all features because miri does not support SIMD
-	# some tests are also filtered, because miri cannot deal with the rayon threadpool
-	# we ignore leaks because the thread pool of rayon is never killed.
-	MIRIFLAGS="-Zmiri-disable-isolation -Zmiri-ignore-leaks -Zmiri-disable-stacked-borrows" \
-	POLARS_ALLOW_EXTENSION=1 \
-	cargo miri test \
-	    --no-default-features \
-	    --features object \
-	    -p polars-core \
-	    -p polars-arrow
+.PHONY: integration-tests
+integration-tests:  ## Run integration tests
+	cargo test --all-features --test it
 
-test-doc:
+.PHONY: test-doc
+test-doc:  ## Run doc examples
 	cargo test --doc \
 	    -p polars-lazy \
 	    -p polars-io \
 	    -p polars-core \
 	    -p polars-arrow \
 		-p polars-sql
 
-pre-commit: fmt clippy clippy-default  ## Run autoformatting and linting
-
-
-check-features:
-	cargo hack check --each-feature --no-dev-deps
+.PHONY: generate_test_files
+generate_test_files:  ## Generate some datasets
+	cargo run -p polars-cli "select * from read_csv('../examples/datasets/foods1.csv')" -o parquet > ../examples/datasets/foods1.parquet
+	cargo run -p polars-cli "select * from read_csv('../examples/datasets/foods1.csv')" -o arrow > ../examples/datasets/foods1.ipc
 
-bench-save:
+.PHONY: bench-save
+bench-save:  ## Run benchmark and save
 	cargo bench --features=random --bench $(BENCH) -- --save-baseline $(SAVE)
 
-bench-cmp:
+.PHONY: bench-cmp
+bench-cmp:  ## Run benchmark and compare
 	cargo bench --features=random --bench $(BENCH) -- --load-baseline $(FEAT) --baseline $(BASE)
 
-doctest:
+.PHONY: doctest
+doctest:  ## Check that documentation builds
 	cargo doc --all-features -p polars-arrow
 	cargo doc --all-features -p polars-utils
 	cargo doc --features=docs-selection -p polars-core
@@ -80,7 +92,8 @@ doctest:
 	cargo doc --features=docs-selection -p polars
 	cargo doc --all-features -p polars-sql
 
-publish:
+.PHONY: publish
+publish:  ## Publish Polars crates
 	cargo publish --allow-dirty -p polars-error
 	cargo publish --allow-dirty -p polars-utils
 	cargo publish --allow-dirty -p polars-row
@@ -100,5 +113,5 @@ publish:
 
 .PHONY: help
 help:  ## Display this help screen
-	@echo -e "\033[1mAvailable commands:\033[0m\n"
+	@echo -e "\033[1mAvailable commands:\033[0m"
 	@grep -E '^[a-z.A-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "  \033[36m%-18s\033[0m %s\n", $$1, $$2}' | sort

polars/polars-algo/Cargo.toml

@@ -9,9 +9,9 @@ description = "Algorithms built upon Polars primitives"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-polars-core = { version = "0.30.0", path = "../polars-core", features = ["dtype-categorical", "asof_join"], default-features = false }
-polars-lazy = { version = "0.30.0", path = "../polars-lazy", features = ["asof_join", "concat_str", "strings"] }
-polars-ops = { version = "0.30.0", path = "../polars-ops", features = ["dtype-categorical", "asof_join"], default-features = false }
+polars-core = { version = "0.31.1", path = "../polars-core", features = ["dtype-categorical", "asof_join"], default-features = false }
+polars-lazy = { version = "0.31.1", path = "../polars-lazy", features = ["asof_join", "concat_str", "strings"] }
+polars-ops = { version = "0.31.1", path = "../polars-ops", features = ["dtype-categorical", "asof_join"], default-features = false }
 
 [package.metadata.docs.rs]
 all-features = true

polars/polars-arrow/Cargo.toml

@@ -17,7 +17,7 @@ ethnum = { version = "1.3.2", optional = true }
 hashbrown.workspace = true
 multiversion.workspace = true
 num-traits.workspace = true
-polars-error = { version = "0.30.0", path = "../polars-error" }
+polars-error = { version = "0.31.1", path = "../polars-error" }
 serde = { version = "1", features = ["derive"], optional = true }
 thiserror.workspace = true
 

polars/polars-arrow/src/kernels/rolling/mod.rs

@@ -10,7 +10,7 @@ use std::sync::Arc;
 use arrow::array::PrimitiveArray;
 use arrow::bitmap::{Bitmap, MutableBitmap};
 use arrow::types::NativeType;
-use num_traits::{Bounded, Float, NumCast, One, ToPrimitive, Zero};
+use num_traits::{Bounded, Float, NumCast, One, Zero};
 use window::*;
 
 use crate::data_types::IsFloat;
@@ -142,3 +142,9 @@ where
 pub struct RollingVarParams {
     pub ddof: u8,
 }
+
+#[derive(Clone, Copy, Debug)]
+pub struct RollingQuantileParams {
+    pub prob: f64,
+    pub interpol: QuantileInterpolOptions,
+}