Arrow array resolvers and builders.

Signed-off-by: Tao He <[email protected]>

.github/workflows/rust-ci.yml

@@ -45,10 +45,15 @@ jobs:
           override: true
           components: rustfmt, clippy
 
-      - name: Test
+      - name: Check
         run: |
           cd rust
           cargo fmt --all -- --check
-          # cargo clippy -- -D warnings
+          cargo clippy -- -D warnings
           cargo check
-          # cargo test --lib
+
+      - name: Unittest
+        if: false
+        run: |
+          cd rust
+          cargo test --lib

rust/Cargo.toml

@@ -1,4 +1,35 @@
+[workspace]
+members = [
+    "vineyard",
+    "vineyard-polars",
+    "vineyard-integration-testing",
+]
+resolver = "2"
+
+[workspace.package]
+version = "0.16.5"
+homepage = "https://v6d.io"
+repository = "https://github.com/v6d-io/v6d.git"
+authors = ["Vineyard <[email protected]>"]
+license = "Apache-2.0"
+keywords = ["vineyard"]
+include = [
+    "src/**/*.rs",
+    "Cargo.toml",
+]
 edition = "2021"
+readme = "README.md"
 
-[workspace]
-members = ["vineyard"]
+[workspace.dependencies]
+arrow-array = ">=40, <44"
+arrow-buffer = ">=40, <44"
+arrow-ipc = ">=40, <44"
+arrow-schema = ">=40, <44"
+arrow2 = { version = "0.17", features = ["arrow"] }
+inline-python = "0.12"
+polars-core = "0.32"
+spectral = "0.6"
+
+vineyard = { version = "0.16.5", path = "./vineyard" }
+vineyard-polars = { version = "0.16.5", path = "./vineyard-polars" }
+vineyard-integration-testing = { version = "0.16.5", path = "./vineyard-integration-testing" }

rust/README.md

@@ -1 +1,172 @@
 # Vineyard Rust SDK
+
+> [!NOTE]
+> Rust nightly is required. The vineyard Rust SDK is still under development.
+> The API may change in the future.
+
+[![crates.io](https://img.shields.io/crates/v/vineyard.svg)](https://crates.io/crates/vineyard)
+[![Downloads](https://img.shields.io/crates/d/vineyard)](https://crates.io/crates/vineyard)
+[![Docs.rs](https://img.shields.io/docsrs/vineyard/latest)](https://docs.rs/vineyard/latest/vineyard/)
+
+Connecting to Vineyard
+----------------------
+
+- Resolve the UNIX-domain socket from the environment variable `VINEYARD_IPC_SOCKET`:
+
+    ```rust
+    use vineyard::client::*;
+
+    let mut client = vineyard::default().unwrap();
+    ```
+
+- Or, using explicit parameter:
+
+    ```rust
+    use vineyard::client::*;
+
+    let mut client = vineyard::connect("/var/run/vineyard.sock").unwrap();
+    ```
+
+Interact with Vineyard
+----------------------
+
+- Creating blob:
+
+    ```rust
+    let mut blob_writer = client.create_blob(N)?;
+    ```
+
+- Get object:
+
+    ```rust
+    let mut meta_writer = client.get::<DataFrame>(object_id)?;
+    ```
+
+Inter-op with Python: `numpy.ndarray`
+-------------------------------------
+
+- Python:
+
+    ```python
+    import numpy as np
+    import vineyard
+
+    client = vineyard.connect()
+
+    np_array = np.random.rand(10, 20).astype(np.int32)
+    object_id = int(client.put(np_array))
+    ```
+
+- Rust:
+
+    ```rust
+    let mut client = IPCClient::default()?;
+    let tensor = client.get::<Int32Tensor>(object_id)?;
+    assert_that!(tensor.shape().to_vec()).is_equal_to(vec![10, 20]);
+    ```
+
+Inter-op with Python: `pandas.DataFrame`
+----------------------------------------
+
+- Python
+
+    ```python
+    import pandas as pd
+    import vineyard
+
+    client = vineyard.connect()
+
+    df = pd.DataFrame({'a': ["1", "2", "3", "4"], 'b': ["5", "6", "7", "8"]})
+    object_id = int(client.put(df))
+    ```
+
+- Rust
+
+    ```rust
+    let mut client = IPCClient::default()?;
+    let dataframe = client.get::<DataFrame>(object_id)?;
+    assert_that!(dataframe.num_columns()).is_equal_to(2);
+    assert_that!(dataframe.names().to_vec()).is_equal_to(vec!["a".into(), "b".into()]);
+    for index in 0..dataframe.num_columns() {
+        let column = dataframe.column(index);
+        assert_that!(column.len()).is_equal_to(4);
+    }
+    ```
+
+Inter-op with Python: `pyarrow.RecordBatch`
+-------------------------------------
+
+- Python
+
+    ```python
+    import pandas as pd
+    import pyarrow as pa
+    import vineyard
+
+    client = vineyard.connect()
+
+    arrays = [
+        pa.array([1, 2, 3, 4]),
+        pa.array(["foo", "bar", "baz", "qux"]),
+        pa.array([3.0, 5.0, 7.0, 9.0]),
+    ]
+    batch = pa.RecordBatch.from_arrays(arrays, ["f0", "f1", "f2"])
+    object_id = int(client.put(batch))
+    ```
+
+- Rust
+
+    ```rust
+    let batch = client.get::<RecordBatch>(object_id)?;
+    assert_that!(batch.num_columns()).is_equal_to(3);
+    assert_that!(batch.num_rows()).is_equal_to(4);
+    let schema = batch.schema();
+    let names = ["f0", "f1", "f2"];
+    let recordbatch = batch.as_ref().as_ref();
+    ```
+
+Inter-op with Python: `pyarrow.Table`
+-------------------------------------
+
+- Python
+
+    ```python
+    batches = [batch] * 5
+    table = pa.Table.from_batches(batches)
+    object_id = int(client.put(table))
+    ```
+
+- Rust
+
+    ```rust
+    let mut client = IPCClient::default()?;
+    let table = client.get::<Table>(object_id)?;
+    assert_that!(table.num_batches()).is_equal_to(5);
+    for batch in table.batches().iter() {
+        // ...
+    }
+    ```
+
+Inter-op with Python: `polars.DataFrame`
+----------------------------------------
+
+- Python
+
+    ```python
+    import polars
+
+    dataframe = polars.DataFrame(table)
+    object_id = int(client.put(dataframe))
+    ```
+
+- Rust
+
+    ```rust
+    let mut client = IPCClient::default()?;
+    let batch = client.get::<DataFrame>(object_id)?;
+    let dataframe = batch.as_ref().as_ref();
+    assert_that!(dataframe.width()).is_equal_to(3);
+    for column in dataframe.get_columns() {
+        // ...
+    }
+    ```

rust/vineyard-integration-testing/Cargo.toml

@@ -0,0 +1,29 @@
+[package]
+name = "vineyard-integration-testing"
+version = { workspace = true }
+description = "Vineyard Rust SDK: integration testing"
+homepage = { workspace = true }
+repository = { workspace = true }
+license = { workspace = true }
+keywords = { workspace = true }
+include = { workspace = true }
+edition = { workspace = true }
+readme = { workspace = true }
+
+[features]
+default = ["nightly"]
+nightly = []
+
+[lib]
+name = "vineyard_integration_testing"
+path = "src/lib.rs"
+
+[dependencies]
+
+[dev-dependencies]
+arrow-array = { workspace = true }
+inline-python = { workspace = true }
+polars-core = { workspace = true }
+spectral = { workspace = true }
+vineyard = { workspace = true }
+vineyard-polars = { workspace = true }