From 1c35b59502d3522c9071a2980ee98208dbb16d67 Mon Sep 17 00:00:00 2001 From: Henry Harbeck Date: Fri, 31 Jan 2025 22:19:04 +1000 Subject: [PATCH 1/6] grammar --- docs/source/user-guide/plugins/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/user-guide/plugins/index.md b/docs/source/user-guide/plugins/index.md index cbbfa03e3f75..3c1912ca7685 100644 --- a/docs/source/user-guide/plugins/index.md +++ b/docs/source/user-guide/plugins/index.md @@ -1,6 +1,6 @@ # Plugins -Polars allows you to extend it's functionality with either Expression plugins or IO plugins. +Polars allows you to extend its functionality with either Expression plugins or IO plugins. - [Expression plugins](./expr_plugins.md) - [IO plugins](./io_plugins.md) From 3de4d062e8e2a3ccbf5a33375f13e44ff1b54977 Mon Sep 17 00:00:00 2001 From: Henry Harbeck Date: Fri, 31 Jan 2025 22:19:26 +1000 Subject: [PATCH 2/6] more specific user guide link --- py-polars/polars/plugins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/py-polars/polars/plugins.py b/py-polars/polars/plugins.py index 762a3e56a029..a649686183b1 100644 --- a/py-polars/polars/plugins.py +++ b/py-polars/polars/plugins.py @@ -36,7 +36,7 @@ def register_plugin_function( """ Register a plugin function. - See the `user guide `_ + See the `user guide `_ for more information about plugins. Parameters From e0b143684f94d4096aac7a5d1c306d1dcb916489 Mon Sep 17 00:00:00 2001 From: Henry Harbeck Date: Fri, 31 Jan 2025 22:19:58 +1000 Subject: [PATCH 3/6] docstring improvements --- py-polars/polars/io/plugins.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/py-polars/polars/io/plugins.py b/py-polars/polars/io/plugins.py index 8c9e6581bf7e..0bb754fa5df4 100644 --- a/py-polars/polars/io/plugins.py +++ b/py-polars/polars/io/plugins.py @@ -26,6 +26,14 @@ def register_io_source( """ Register your IO plugin and initialize a LazyFrame. + See the `user guide `_ + for more information about plugins. + + .. warning:: + This functionality is considered **unstable**. It may be changed + at any point without it being considered a breaking change. + + Parameters ---------- callable @@ -36,17 +44,21 @@ def register_io_source( predicate Polars expression. The reader must filter their rows accordingly. - n_rows: + n_rows Materialize only n rows from the source. The reader can stop when `n_rows` are read. batch_size A hint of the ideal batch size the reader's generator must produce. + The function should return a DataFrame batch (an iterator over individual DataFrames). schema Schema that the reader will produce before projection pushdown. + Returns + ------- + LazyFrame """ def wrap( From 10d1447a7a1d570c49aa48431a21928f84786c3e Mon Sep 17 00:00:00 2001 From: Henry Harbeck Date: Fri, 31 Jan 2025 22:20:35 +1000 Subject: [PATCH 4/6] better user guide link --- docs/source/user-guide/plugins/io_plugins.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/user-guide/plugins/io_plugins.md b/docs/source/user-guide/plugins/io_plugins.md index dd1a975e75a6..0a59f2969aa9 100644 --- a/docs/source/user-guide/plugins/io_plugins.md +++ b/docs/source/user-guide/plugins/io_plugins.md @@ -1,6 +1,6 @@ # IO Plugins -Besides [expression plugins](./index.md), we also support IO plugins. These allow you to register +Besides [expression plugins](./expr_plugins.md), we also support IO plugins. These allow you to register different file formats as sources to the Polars engines. Because sources can move data zero copy via Arrow FFI and sources can produce large chunks of data before returning, we've decided to interface to IO plugins via Python for now, as we don't think the short time the GIL is needed should lead to From 0356c65a377c111c2d0bc884d56acd8dfb11c371 Mon Sep 17 00:00:00 2001 From: Henry Harbeck Date: Fri, 31 Jan 2025 22:21:13 +1000 Subject: [PATCH 5/6] updated API reference --- py-polars/docs/source/reference/plugins.rst | 48 +++++++++++++++++---- 1 file changed, 40 insertions(+), 8 deletions(-) diff --git a/py-polars/docs/source/reference/plugins.rst b/py-polars/docs/source/reference/plugins.rst index 7a121ee11dce..bab812ec5b56 100644 --- a/py-polars/docs/source/reference/plugins.rst +++ b/py-polars/docs/source/reference/plugins.rst @@ -3,13 +3,45 @@ Plugins ======= .. currentmodule:: polars -Plugins allow for extending Polars' functionality. See the -`user guide `_ for more information -and resources. +Polars allows you to extend its functionality with either Expression plugins or IO plugins. +See the `user guide `_ for more information and resources. -Available plugin utility functions are: +Expression plugins +------------------ -.. automodule:: polars.plugins - :members: - :autosummary: - :autosummary-no-titles: +Expression plugins are the preferred way to create user defined functions. They allow you to compile +a Rust function and register that as an expression into the Polars library. The Polars engine will +dynamically link your function at runtime and your expression will run almost as fast as native +expressions. Note that this works without any interference of Python and thus no GIL contention. + +See the `expression plugins section of the user guide `_ +for more information. + +.. autosummary:: + :toctree: api/ + + plugins.register_plugin_function + + +IO plugins +------------------ + +IO plugins allow you to register different file formats as sources to the Polars engines. + +See the `IO plugins section of the user guide `_ +for more information. + +.. note:: + + The ``io.plugins`` module is not imported by default in order to optimise import speed of + the primary ``polars`` module. Either import ``polars.io.plugins`` and *then* use that + namespace, or import ``register_io_source`` from the full module path, e.g.: + + .. code-block:: python + + from polars.io.plugins import register_io_source + +.. autosummary:: + :toctree: api/ + + io.plugins.register_io_source From f5dec3d964c52d5cfee89c3a64393408167f1fc3 Mon Sep 17 00:00:00 2001 From: Henry Harbeck Date: Fri, 31 Jan 2025 22:50:16 +1000 Subject: [PATCH 6/6] fmt --- docs/source/user-guide/plugins/io_plugins.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/source/user-guide/plugins/io_plugins.md b/docs/source/user-guide/plugins/io_plugins.md index 0a59f2969aa9..1e176a5214f1 100644 --- a/docs/source/user-guide/plugins/io_plugins.md +++ b/docs/source/user-guide/plugins/io_plugins.md @@ -1,10 +1,10 @@ # IO Plugins -Besides [expression plugins](./expr_plugins.md), we also support IO plugins. These allow you to register -different file formats as sources to the Polars engines. Because sources can move data zero copy via -Arrow FFI and sources can produce large chunks of data before returning, we've decided to interface -to IO plugins via Python for now, as we don't think the short time the GIL is needed should lead to -any contention. +Besides [expression plugins](./expr_plugins.md), we also support IO plugins. These allow you to +register different file formats as sources to the Polars engines. Because sources can move data zero +copy via Arrow FFI and sources can produce large chunks of data before returning, we've decided to +interface to IO plugins via Python for now, as we don't think the short time the GIL is needed +should lead to any contention. E.g. an IO source can read their dataframe's in rust and only at the rendez-vous move the data zero-copy having only a short time the GIL is needed.