Merge branch 'main' into feature/sink_csv_cloud

# Conflicts:
#	crates/polars-pipe/src/executors/sinks/file_sink.rs

.github/CODEOWNERS

@@ -1,7 +1,8 @@
 *  @ritchie46
 
-/.github/ @ritchie46 @stinodego
+/.github/  @ritchie46 @stinodego
 /crates/  @ritchie46 @orlp
 /crates/polars-sql/  @ritchie46 @orlp @universalmind303
 /crates/polars-time/  @ritchie46 @orlp @MarcoGorelli
 /py-polars/  @ritchie46 @stinodego @alexander-beedie
+/docs/  @ritchie46 @c-peters @braaannigan

.github/ISSUE_TEMPLATE/documentation.yml

@@ -13,11 +13,11 @@ body:
       required: true
 
   - type: input
-    id: location
+    id: link
     attributes:
-      label: Location
+      label: Link
       description: >
         Provide a link to the existing documentation, if applicable.
-      placeholder: https://pola-rs.github.io/polars/docs/python/dev/reference/api/polars.read_csv.html
+      placeholder: ex. https://pola-rs.github.io/polars/docs/python/dev/...
     validations:
       required: false

.github/workflows/docs-global.yml

@@ -0,0 +1,85 @@
+name: Build documentation
+
+on:
+  pull_request:
+    paths:
+      - docs/**
+      - mkdocs.yml
+      - .github/workflows/docs-global.yml
+  push:
+    tags:
+      - py-**
+
+jobs:
+  markdown-link-check:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - uses: gaurav-nelson/github-action-markdown-link-check@v1
+      with:
+        folder-path: docs
+
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: psf/black@stable
+        with:
+          src: docs/src/python
+          version: "23.7.0"
+
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - name: Create virtual environment
+        run: |
+          python -m venv .venv
+          echo "$GITHUB_WORKSPACE/.venv/bin" >> $GITHUB_PATH
+
+      - name: Install dependencies
+        run: |
+          pip install -r py-polars/requirements-dev.txt
+          pip install -r docs/requirements.txt
+
+      - name: Set up Rust
+        run: rustup show
+
+      - name: Cache Rust
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: py-polars
+          save-if: ${{ github.ref_name == 'main' }}
+
+      - name: Install Polars
+        working-directory: py-polars
+        run: |
+          source activate
+          maturin develop
+
+      - name: Set up Graphviz
+        uses: ts-graphviz/setup-graphviz@v1
+
+      - name: Build documentation
+        run: mkdocs build
+
+      - name: Add .nojekyll
+        if: ${{ github.ref_type == 'tag' }}
+        working-directory: site
+        run: touch .nojekyll
+
+      - name: Deploy docs
+        if: ${{ github.ref_type == 'tag' }}
+        uses: JamesIves/github-pages-deploy-action@v4
+        with:
+          folder: site
+          clean-exclude: |
+            docs/
+            py-polars/
+          single-commit: true

.github/workflows/lint-rust.yml

@@ -38,7 +38,7 @@ jobs:
           save-if: ${{ github.ref_name == 'main' }}
 
       - name: Run cargo clippy with all features enabled
-        run: cargo clippy --workspace --all-targets --all-features -- -D warnings
+        run: cargo clippy -p polars --all-features -- -D warnings
 
   # Default feature set should compile on the stable toolchain
   clippy-stable:
@@ -58,7 +58,7 @@ jobs:
           save-if: ${{ github.ref_name == 'main' }}
 
       - name: Run cargo clippy
-        run: cargo clippy --workspace --all-targets -- -D warnings
+        run: cargo clippy -p polars -- -D warnings
 
   rustfmt:
     if: github.ref_name != 'main'
@@ -90,7 +90,6 @@ jobs:
           POLARS_ALLOW_EXTENSION: '1'
         run: >
           cargo miri test
-          --no-default-features
           --features object
           -p polars-core
           -p polars-arrow

.github/workflows/test-python.yml

@@ -41,6 +41,9 @@ jobs:
         with:
           python-version: ${{ matrix.python-version }}
 
+      - name: Set up Graphviz
+        uses: ts-graphviz/setup-graphviz@v1
+
       - name: Create virtual environment
         run: |
           python -m venv .venv
@@ -65,11 +68,13 @@ jobs:
 
       - name: Run tests and report coverage
         if: github.ref_name != 'main'
-        run: pytest --cov -n auto --dist loadgroup -m "not benchmark"
+        run: pytest --cov -n auto --dist loadgroup -m "not benchmark and not docs"
 
       - name: Run doctests
         if: github.ref_name != 'main'
-        run: python tests/docs/run_doctest.py
+        run: |
+          python tests/docs/run_doctest.py
+          pytest tests/docs/test_user_guide.py -m docs
 
       - name: Check import without optional dependencies
         if: github.ref_name != 'main'
@@ -80,6 +85,7 @@ jobs:
           "matplotlib"
           "backports.zoneinfo"
           "connectorx"
+          "pyiceberg"
           "deltalake"
           "xlsx2csv"
           )
@@ -125,7 +131,7 @@ jobs:
 
       - name: Run tests
         if: github.ref_name != 'main'
-        run: pytest -n auto --dist loadgroup -m "not benchmark"
+        run: pytest -n auto --dist loadgroup -m "not benchmark and not docs"
 
       - name: Check import without optional dependencies
         if: github.ref_name != 'main'

.gitignore

@@ -1,27 +1,37 @@
 *.iml
 *.so
 *.ipynb
-.DS_Store
 .ENV
-.coverage
 .env
-.hypothesis/
-.idea/
 .ipynb_checkpoints/
-.mypy_cache/
-.pytest_cache/
 .python-version
 .yarn/
-.vscode/
-__pycache__/
-AUTO_CHANGELOG.md
-Cargo.lock
 coverage.lcov
 coverage.xml
 data/
-node_modules/
 polars/vendor
-target/
-venv*/
-.venv*/
+
+# OS
+.DS_Store
+
+# IDE
+.idea/
+.vscode/
 .vim
+
+# Python
+.hypothesis/
+.mypy_cache/
+.pytest_cache/
+.venv/
+__pycache__/
+.coverage
+
+# Rust
+target/
+Cargo.lock
+
+# Project
+/docs/data/
+/docs/images/
+/docs/people.md

CONTRIBUTING.md

@@ -148,12 +148,12 @@ If you are stuck or unsure about your solution, feel free to open a draft pull r
 
 ## Contributing to documentation
 
-The most important components of Polars documentation are the [user guide](https://pola-rs.github.io/polars-book/user-guide/), the API references, and the database of questions on [StackOverflow](https://stackoverflow.com/).
+The most important components of Polars documentation are the [user guide](https://pola-rs.github.io/polars/user-guide/), the API references, and the database of questions on [StackOverflow](https://stackoverflow.com/).
 
 ### User guide
 
-The user guide is maintained in the [polars-book](https://github.com/pola-rs/polars-book) repository.
-For contributing to the user guide, please refer to the [contributing guide](https://github.com/pola-rs/polars-book/blob/master/CONTRIBUTING.md) in that repository.
+The user guide is maintained in the `docs` folder.
+Further contributing information will be added shortly.
 
 ### API reference
 

Cargo.toml

@@ -13,7 +13,7 @@ exclude = [
 ]
 
 [workspace.package]
-version = "0.32.0"
+version = "0.33.2"
 authors = ["Ritchie Vink <[email protected]>"]
 edition = "2021"
 homepage = "https://www.pola.rs/"
@@ -28,7 +28,7 @@ bytemuck = { version = "1", features = ["derive", "extern_crate_alloc"] }
 chrono = { version = "0.4", default-features = false, features = ["std"] }
 chrono-tz = "0.8.1"
 ciborium = "0.2"
-either = "1.8"
+either = "1.9"
 futures = "0.3.25"
 hashbrown = { version = "0.14", features = ["rayon", "ahash"] }
 indexmap = { version = "2", features = ["std"] }
@@ -50,14 +50,36 @@ strum_macros = "0.25"
 thiserror = "1"
 url = "2.3.1"
 version_check = "0.9.4"
+simdutf8 = "0.1.4"
+hex = "0.4.3"
+base64 = "0.21.2"
+fallible-streaming-iterator = "0.1.9"
+streaming-iterator = "0.1.9"
+itoa = "1.0.6"
+ryu = "1.0.13"
+lexical-core = "0.8.5"
 xxhash-rust = { version = "0.8.6", features = ["xxh3"] }
+polars-core = { version = "0.33.2", path = "crates/polars-core", default-features = false }
+polars-arrow = { version = "0.33.2", path = "crates/polars-arrow", default-features = false }
+polars-plan = { version = "0.33.2", path = "crates/polars-plan", default-features = false }
+polars-lazy = { version = "0.33.2", path = "crates/polars-lazy", default-features = false }
+polars-pipe = { version = "0.33.2", path = "crates/polars-pipe", default-features = false }
+polars-row = { version = "0.33.2", path = "crates/polars-row", default-features = false }
+polars-ffi = { version = "0.33.2", path = "crates/polars-ffi", default-features = false }
+polars-ops = { version = "0.33.2", path = "crates/polars-ops", default-features = false }
+polars-sql = { version = "0.33.2", path = "crates/polars-sql", default-features = false }
+polars-algo = { version = "0.33.2", path = "crates/polars-algo", default-features = false }
+polars-time = { version = "0.33.2", path = "crates/polars-time", default-features = false }
+polars-utils = { version = "0.33.2", path = "crates/polars-utils", default-features = false }
+polars-io = { version = "0.33.2", path = "crates/polars-io", default-features = false }
+polars-error = { version = "0.33.2", path = "crates/polars-error", default-features = false }
+polars-json = { version = "0.33.2", path = "crates/polars-json", default-features = false }
+polars = { version = "0.33.2", path = "crates/polars", default-features = false }
 
 [workspace.dependencies.arrow]
-package = "arrow2"
-git = "https://github.com/jorgecarleitao/arrow2"
-rev = "7c93e358fc400bf3c0c0219c22eefc6b38fc2d12"
-# branch = ""
-# version = "0.17.4"
+package = "nano-arrow"
+version = "0.1.0"
+path = "crates/nano-arrow"
 default-features = false
 features = [
   "compute_aggregate",

Makefile

@@ -20,6 +20,11 @@ requirements: .venv  ## Install/refresh Python project requirements
 	$(VENV_BIN)/pip install --upgrade -r py-polars/requirements-dev.txt
 	$(VENV_BIN)/pip install --upgrade -r py-polars/requirements-lint.txt
 	$(VENV_BIN)/pip install --upgrade -r py-polars/docs/requirements-docs.txt
+	$(VENV_BIN)/pip install --upgrade -r docs/requirements.txt
+
+.PHONY: build-python
+build-python: .venv  ## Compile and install Python Polars for development
+	@$(MAKE) -s -C py-polars build
 
 .PHONY: clean
 clean:  ## Clean up caches and build artifacts

README.md

@@ -40,7 +40,7 @@
   -
   <a href="https://stackoverflow.com/questions/tagged/r-polars">R</a>
   |
-  <a href="https://pola-rs.github.io/polars-book/">User Guide</a>
+  <a href="https://pola-rs.github.io/polars/">User Guide</a>
   |
   <a href="https://discord.gg/4UfP5cfBE7">Discord</a>
 </p>
@@ -58,7 +58,7 @@ Polars is a DataFrame interface on top of an OLAP Query Engine implemented in Ru
 - Hybrid Streaming (larger than RAM datasets)
 - Rust | Python | NodeJS | R | ...
 
-To learn more, read the [User Guide](https://pola-rs.github.io/polars-book/).
+To learn more, read the [User Guide](https://pola-rs.github.io/polars/).
 
 ## Python
 
@@ -208,6 +208,7 @@ You can also install the dependencies directly.
 | xlsx2csv   | Support for reading from Excel files                                         |
 | openpyxl   | Support for reading from Excel files with native types                       |
 | deltalake  | Support for reading from Delta Lake Tables                                   |
+| pyiceberg  | Support for reading from Apache Iceberg tables                               |
 | timezone   | Timezone support, only needed if are on Python<3.9 or you are on Windows     |
 
 Releases happen quite often (weekly / every few days) at the moment, so updating polars regularly to get the latest bugfixes / features might not be a bad idea.

_typos.toml

@@ -7,6 +7,7 @@ extend-ignore-identifiers-re = [
 ba = "ba"
 Fo = "Fo"
 nd = "nd"
+ND = "ND"
 opt_nd = "opt_nd"
 ser = "ser"
 strat = "strat"

crates/Makefile

@@ -10,22 +10,22 @@ fmt:  ## Run rustfmt and dprint
 
 .PHONY: check
 check:  ## Run cargo check with all features
-	cargo check --workspace --all-targets --all-features
+	cargo check --workspace --all-targets --exclude nano-arrow --all-features
 
 .PHONY: clippy
 clippy:  ## Run clippy with all features
-	cargo clippy --workspace --all-targets --all-features
+	cargo clippy -p polars --all-features
 
 .PHONY: clippy-default
 clippy-default:  ## Run clippy with default features
-	cargo clippy --workspace --all-targets
+	cargo clippy -p polars
 
 .PHONY: pre-commit
 pre-commit: fmt clippy clippy-default  ## Run autoformatting and linting
 
 .PHONY: check-features
 check-features:  ## Run cargo check for feature flag combinations (warning: slow)
-	cargo hack check --each-feature --no-dev-deps
+	cargo hack check -p polars --each-feature --no-dev-deps
 
 .PHONY: miri
 miri:  ## Run miri
@@ -35,7 +35,6 @@ miri:  ## Run miri
 	MIRIFLAGS="-Zmiri-disable-isolation -Zmiri-ignore-leaks -Zmiri-disable-stacked-borrows" \
 	POLARS_ALLOW_EXTENSION=1 \
 	cargo miri test \
-	    --no-default-features \
 	    --features object \
 	    -p polars-core \
 	    -p polars-arrow
@@ -109,6 +108,7 @@ publish:  ## Publish Polars crates
 	cargo publish --allow-dirty -p polars-arrow
 	cargo publish --allow-dirty -p polars-json
 	cargo publish --allow-dirty -p polars-core
+	cargo publish --allow-dirty -p polars-ffi
 	cargo publish --allow-dirty -p polars-ops
 	cargo publish --allow-dirty -p polars-time
 	cargo publish --allow-dirty -p polars-io