Merge branch 'pola-rs:main' into round-durations

.github/scripts/test_bytecode_parser.py

@@ -12,13 +12,14 @@
 
 Running it without `PYTHONPATH` set will result in the test failing.
 """
+
 import datetime as dt  # noqa: F401
 import subprocess
 from datetime import datetime  # noqa: F401
 from typing import Any, Callable
 
 import pytest
-from polars.utils.udfs import BytecodeParser
+from polars._utils.udfs import BytecodeParser
 from tests.unit.operations.map.test_inefficient_map_warning import (
     MY_DICT,
     NOOP_TEST_CASES,
@@ -44,7 +45,7 @@ def test_bytecode_parser_expression_in_ipython(
     col: str, func: Callable[[Any], Any], expected: str
 ) -> None:
     script = (
-        "from polars.utils.udfs import BytecodeParser; "
+        "from polars._utils.udfs import BytecodeParser; "
         "import datetime as dt; "
         "from datetime import datetime; "
         "import numpy as np; "
@@ -73,7 +74,7 @@ def test_bytecode_parser_expression_noop(func: str) -> None:
 )
 def test_bytecode_parser_expression_noop_in_ipython(func: str) -> None:
     script = (
-        "from polars.utils.udfs import BytecodeParser; "
+        "from polars._utils.udfs import BytecodeParser; "
         f"MY_DICT = {MY_DICT};"
         f'parser = BytecodeParser({func}, map_target="expr");'
         f'print(not parser.can_attempt_rewrite() or not parser.to_expression("x"));'

.github/workflows/benchmark.yml

@@ -87,9 +87,7 @@ jobs:
         env:
           RUSTFLAGS: -C embed-bitcode -D warnings
         working-directory: py-polars
-        run: |
-          source activate
-          maturin develop --release -- -C codegen-units=8 -C lto=thin -C target-cpu=native
+        run: maturin develop --release -- -C codegen-units=8 -C lto=thin -C target-cpu=native
 
       - name: Run H2O AI database benchmark - on strings
         working-directory: py-polars/tests/benchmark

.github/workflows/codecov.yml

@@ -0,0 +1,105 @@
+name: Code coverage
+
+on:
+  pull_request:
+    paths:
+      - '**.rs'
+      - '**.py'
+      - .github/workflows/codecov.yml
+  push:
+    branches:
+      - main
+    paths:
+      - '**.rs'
+      - '**.py'
+      - .github/workflows/codecov.yml
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+defaults:
+  run:
+    working-directory: py-polars
+    shell: bash
+
+jobs:
+  coverage:
+    name: Code Coverage
+    runs-on: macos-latest
+    env:
+      RUSTFLAGS: '-C instrument-coverage --cfg=coverage --cfg=coverage_nightly --cfg=trybuild_no_target'
+      RUST_BACKTRACE: 1
+      LLVM_PROFILE_FILE: '/Users/runner/work/polars/polars/target/polars-%p-%3m.profraw'
+      CARGO_LLVM_COV: 1
+      CARGO_LLVM_COV_SHOW_ENV: 1
+      CARGO_LLVM_COV_TARGET_DIR: '/Users/runner/work/polars/polars/target'
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Create virtual environment
+        run: |
+          python -m venv .venv
+          echo "$GITHUB_WORKSPACE/py-polars/.venv/bin" >> $GITHUB_PATH
+
+      - name: Install dependencies
+        run: pip install -r requirements-dev.txt
+
+      - name: Set up Rust
+        run: rustup component add llvm-tools-preview
+
+      - name: Install cargo-llvm-cov
+        uses: taiki-e/install-action@cargo-llvm-cov
+
+      - uses: Swatinem/rust-cache@v2
+        with:
+          save-if: ${{ github.ref_name == 'main' }}
+
+      - name: Prepare coverage
+        run: cargo llvm-cov clean --workspace
+
+      - name: Run tests
+        run: >
+          cargo test --all-features
+          -p polars-arrow
+          -p polars-compute
+          -p polars-core
+          -p polars-io
+          -p polars-lazy
+          -p polars-ops
+          -p polars-plan
+          -p polars-row
+          -p polars-sql
+          -p polars-time
+          -p polars-utils
+
+      - name: Run Rust integration tests
+        run: cargo test --all-features -p polars --test it
+
+      - name: Install Polars
+        run: maturin develop
+
+      - name: Run Python tests
+        run: pytest --cov -n auto --dist loadgroup -m "not benchmark and not docs" --cov-report xml:main.xml
+        continue-on-error: true
+
+      - name: Run Python tests - async reader
+        env:
+          POLARS_FORCE_ASYNC: 1
+        run: pytest --cov -m "not benchmark and not docs" tests/unit/io/ --cov-report xml:async.xml
+        continue-on-error: true
+
+      - name: Report coverage
+        run: cargo llvm-cov report --lcov --output-path coverage.lcov
+
+      - name: Upload coverage information
+        uses: codecov/codecov-action@v4
+        with:
+          files: py-polars/coverage.lcov,py-polars/main.xml,py-polars/async.xml
+          name: macos
+          token: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/docs-global.yml

@@ -82,9 +82,7 @@ jobs:
 
       - name: Install Polars
         working-directory: py-polars
-        run: |
-          source activate
-          maturin develop
+        run: maturin develop
 
       - name: Set up Graphviz
         uses: ts-graphviz/setup-graphviz@v2

.github/workflows/lint-global.yml

@@ -15,4 +15,4 @@ jobs:
       - name: Lint Markdown and TOML
         uses: dprint/[email protected]
       - name: Spell Check with Typos
-        uses: crate-ci/typos@v1.17.2
+        uses: crate-ci/typos@v1.18.2

.github/workflows/release-python.yml

@@ -129,13 +129,14 @@ jobs:
         env:
           IS_LTS_CPU: ${{ matrix.package == 'polars-lts-cpu' }}
           IS_MACOS: ${{ matrix.os == 'macos-latest' }}
+        # IMPORTANT: All features enabled here should also be included in py-polars/polars/_cpu_check.py
         run: |
           if [[ "$IS_LTS_CPU" = true ]]; then
             FEATURES=+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt
           elif [[ "$IS_MACOS" = true ]]; then
-            FEATURES=+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt,+avx,+fma
+            FEATURES=+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt,+avx,+fma,+pclmulqdq
           else
-            FEATURES=+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt,+avx,+avx2,+fma,+bmi1,+bmi2,+lzcnt
+            FEATURES=+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt,+avx,+avx2,+fma,+bmi1,+bmi2,+lzcnt,+pclmulqdq
           fi
           echo "features=$FEATURES" >> $GITHUB_OUTPUT
 
@@ -178,6 +179,13 @@ jobs:
             --out dist
           manylinux: ${{ matrix.architecture == 'aarch64' && '2_24' || 'auto' }}
 
+      - name: Test wheel
+        # Only test on x86-64 for now as this matches the runner architecture
+        if: matrix.architecture == 'x86-64'
+        run: |
+          pip install --force-reinstall --verbose dist/*.whl
+          python -c 'import polars'
+
       - name: Upload wheel
         uses: actions/upload-artifact@v4
         with:

.github/workflows/test-python.yml

@@ -73,9 +73,7 @@ jobs:
           save-if: ${{ github.ref_name == 'main' }}
 
       - name: Install Polars
-        run: |
-          source activate
-          maturin develop
+        run: maturin develop
 
       - name: Run doctests
         if: github.ref_name != 'main' && matrix.python-version == '3.12' && matrix.os == 'ubuntu-latest'
@@ -94,7 +92,9 @@ jobs:
 
       - name: Run tests async reader tests
         if: github.ref_name != 'main' && matrix.os != 'windows-latest'
-        run: POLARS_FORCE_ASYNC=1 pytest -m "not benchmark and not docs" tests/unit/io/
+        env:
+          POLARS_FORCE_ASYNC: 1
+        run: pytest -m "not benchmark and not docs" tests/unit/io/
 
       - name: Check import without optional dependencies
         if: github.ref_name != 'main' && matrix.python-version == '3.12' && matrix.os == 'ubuntu-latest'

.github/workflows/test-rust.yml

@@ -46,6 +46,7 @@ jobs:
         run: >
           cargo test --all-features --no-run
           -p polars-arrow
+          -p polars-compute
           -p polars-core
           -p polars-io
           -p polars-lazy
@@ -61,6 +62,7 @@ jobs:
         run: >
           cargo test --all-features
           -p polars-arrow
+          -p polars-compute
           -p polars-core
           -p polars-io
           -p polars-lazy