From c6417055539e83f452f5e1715764504fa33146f4 Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Sun, 29 Sep 2024 15:20:52 -0600 Subject: [PATCH] WIP Signed-off-by: Andrew Stein --- .github/actions/install-deps/action.yaml | 2 +- .github/workflows/build.yaml | 90 ++++++++++++++++--- examples/python-aiohttp/package.json | 2 +- examples/python-starlette/package.json | 2 +- .../python-tornado-streaming/package.json | 2 +- examples/python-tornado/package.json | 2 +- examples/workspace/package.json | 2 +- pnpm-lock.yaml | 35 +++----- pnpm-workspace.yaml | 1 + rust/perspective-client/build.rs | 2 +- rust/perspective-client/package.json | 18 ++++ rust/perspective-js/package.json | 2 +- rust/perspective-python/docs/lib.md | 24 ++--- rust/perspective-server/build/psp.rs | 9 +- tools/perspective-scripts/clean.mjs | 13 ++- tools/perspective-scripts/lint.mjs | 4 +- 16 files changed, 144 insertions(+), 66 deletions(-) create mode 100644 rust/perspective-client/package.json diff --git a/.github/actions/install-deps/action.yaml b/.github/actions/install-deps/action.yaml index de7816b83a..d0a48b0b67 100644 --- a/.github/actions/install-deps/action.yaml +++ b/.github/actions/install-deps/action.yaml @@ -48,7 +48,7 @@ runs: using: "composite" steps: - name: Clean System - uses: AdityaGarg8/remove-unwanted-software@v3 + uses: AdityaGarg8/remove-unwanted-software@v4.1 if: ${{ inputs.clean == 'true' && runner.os != 'Windows' }} with: remove-android: "true" diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index c8b75bf37e..2e07eea2c1 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -59,6 +59,55 @@ concurrency: cancel-in-progress: true jobs: + # , . . .-,--. + # ) . ,-. |- ,-. ,-. ,-| ' | \ ,-. ,-. ,-. + # / | | | | ,-| | | | | , | / | | | `-. + # `--' ' ' ' `' `-^ ' ' `-^ `-^--' `-' `-' `-' + # + lint_and_docs: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: + - ubuntu-22.04 + python-version: + - 3.9 + node-version: [20.x] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Config + id: config-step + uses: ./.github/actions/config + + - name: Initialize Build + id: init-step + uses: ./.github/actions/install-deps + with: + skip_cache: ${{ steps.config-step.outputs.SKIP_CACHE }} + + - name: Metadata Build + run: pnpm run build --ci + env: + PACKAGE: "perspective-metadata" + + # - name: Lint + # run: pnpm run lint + + # - name: Docs Build + # run: pnpm run docs + + - uses: actions/upload-artifact@v4 + with: + name: perspective-metadata + path: | + rust/perspective-client/src/ + rust/perspective-client/docs/ + rust/perspective-js/src/ + # ,-,---. . . ,-_/ .---. . # '|___/ . . . | ,-| ' | ,-. . , ,-. \___ ,-. ,-. . ,-. |- # ,| \ | | | | | | | ,-| | / ,-| \ | | | | | | @@ -67,6 +116,7 @@ jobs: # `--' ' build_js: runs-on: ${{ matrix.os }} + needs: [lint_and_docs] strategy: fail-fast: false matrix: @@ -84,6 +134,11 @@ jobs: id: config-step uses: ./.github/actions/config + - uses: actions/download-artifact@v4 + with: + name: perspective-metadata + path: rust/ + - name: Initialize Build id: init-step uses: ./.github/actions/install-deps @@ -95,11 +150,11 @@ jobs: - name: WebAssembly Build run: pnpm run build --ci env: - PACKAGE: "perspective-cpp,perspective,perspective-metadata,perspective-viewer,perspective-viewer-datagrid,perspective-viewer-d3fc,perspective-viewer-openlayers,perspective-workspace,perspective-cli" + PACKAGE: "perspective-cpp,perspective,perspective-viewer,perspective-viewer-datagrid,perspective-viewer-d3fc,perspective-viewer-openlayers,perspective-workspace,perspective-cli" # PSP_USE_CCACHE: 1 - - name: Lint - run: pnpm run lint + # - name: Lint + # run: pnpm run lint # - name: Docs Build # run: pnpm run docs @@ -130,6 +185,7 @@ jobs: # `-' build_python: runs-on: ${{ matrix.os }} + needs: [lint_and_docs] container: ${{ matrix.container }} strategy: fail-fast: false @@ -169,6 +225,11 @@ jobs: id: config-step uses: ./.github/actions/config + - uses: actions/download-artifact@v4 + with: + name: perspective-metadata + path: rust/ + - name: Initialize Build id: init-step uses: ./.github/actions/install-deps @@ -192,7 +253,7 @@ jobs: run: pnpm run build if: ${{ !contains(matrix.os, 'windows') }} env: - PACKAGE: "perspective-metadata,perspective-python" + PACKAGE: "perspective-python" PSP_ARCH: ${{ matrix.arch }} PSP_ROOT_DIR: ${{ github.workspace }} PSP_BUILD_WHEEL: 1 @@ -235,6 +296,7 @@ jobs: # build_and_test_rust: runs-on: ${{ matrix.os }} + needs: [lint_and_docs] strategy: fail-fast: false matrix: @@ -257,6 +319,11 @@ jobs: id: config-step uses: ./.github/actions/config + - uses: actions/download-artifact@v4 + with: + name: perspective-metadata + path: rust/ + - name: Initialize Build id: init-step uses: ./.github/actions/install-deps @@ -273,7 +340,7 @@ jobs: run: pnpm run build if: ${{ !contains(matrix.os, 'windows') }} env: - PACKAGE: "perspective-metadata,perspective-rs" + PACKAGE: "perspective-rs" PSP_ROOT_DIR: ${{ github.workspace }} - name: Python Build (Windows) @@ -323,6 +390,7 @@ jobs: # `-' build_emscripten_wheel: runs-on: ${{ matrix.os }} + needs: [lint_and_docs] # if: ${{ startsWith(github.ref, 'refs/tags/v') }} strategy: fail-fast: false @@ -342,6 +410,11 @@ jobs: id: config-step uses: ./.github/actions/config + - uses: actions/download-artifact@v4 + with: + name: perspective-metadata + path: rust/ + - name: Initialize Build id: init-step uses: ./.github/actions/install-deps @@ -355,7 +428,7 @@ jobs: - name: Python Build Pyodide run: pnpm install && pnpm run build env: - PACKAGE: "perspective-metadata,perspective-pyodide" + PACKAGE: "perspective-pyodide" - name: "Test Pyodide" run: pnpm run test @@ -700,6 +773,7 @@ jobs: benchmark_python, build_emscripten_wheel, build_and_test_rust, + lint_and_docs, ] if: startsWith(github.ref, 'refs/tags/v') strategy: @@ -763,10 +837,6 @@ jobs: with: name: perspective-rust - # - uses: actions/download-artifact@v4 - # with: - # name: perspective-python-benchmarks - - run: pnpm pack --pack-destination=../.. working-directory: ./rust/perspective-js diff --git a/examples/python-aiohttp/package.json b/examples/python-aiohttp/package.json index cb20e9374e..36044f3a92 100644 --- a/examples/python-aiohttp/package.json +++ b/examples/python-aiohttp/package.json @@ -19,6 +19,6 @@ "devDependencies": { "@finos/perspective-webpack-plugin": "workspace:^", "npm-run-all": "^4.1.3", - "rimraf": "^2.5.2" + "rimraf": "^6" } } diff --git a/examples/python-starlette/package.json b/examples/python-starlette/package.json index b9ac8c0fee..fc3449c06e 100644 --- a/examples/python-starlette/package.json +++ b/examples/python-starlette/package.json @@ -19,6 +19,6 @@ "devDependencies": { "@finos/perspective-webpack-plugin": "workspace:^", "npm-run-all": "^4.1.3", - "rimraf": "^2.5.2" + "rimraf": "^6" } } diff --git a/examples/python-tornado-streaming/package.json b/examples/python-tornado-streaming/package.json index 406fb3a62e..aa079a56ed 100644 --- a/examples/python-tornado-streaming/package.json +++ b/examples/python-tornado-streaming/package.json @@ -19,6 +19,6 @@ "devDependencies": { "@finos/perspective-webpack-plugin": "workspace:^", "npm-run-all": "^4.1.3", - "rimraf": "^2.5.2" + "rimraf": "^6" } } diff --git a/examples/python-tornado/package.json b/examples/python-tornado/package.json index 756b992ff1..848fd5cba3 100644 --- a/examples/python-tornado/package.json +++ b/examples/python-tornado/package.json @@ -19,6 +19,6 @@ "devDependencies": { "@finos/perspective-webpack-plugin": "workspace:^", "npm-run-all": "^4.1.3", - "rimraf": "^2.5.2" + "rimraf": "^6" } } diff --git a/examples/workspace/package.json b/examples/workspace/package.json index b3525bc452..418f44b652 100644 --- a/examples/workspace/package.json +++ b/examples/workspace/package.json @@ -20,6 +20,6 @@ "@finos/perspective-webpack-plugin": "workspace:^", "http-server": "^14.1.1", "npm-run-all": "^4.1.3", - "rimraf": "^2.5.2" + "rimraf": "^6" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 64971cc291..331dfa4591 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -316,8 +316,8 @@ importers: specifier: ^4.1.3 version: 4.1.5 rimraf: - specifier: ^2.5.2 - version: 2.7.1 + specifier: ^6 + version: 6.0.1 examples/python-starlette: dependencies: @@ -347,8 +347,8 @@ importers: specifier: ^4.1.3 version: 4.1.5 rimraf: - specifier: ^2.5.2 - version: 2.7.1 + specifier: ^6 + version: 6.0.1 examples/python-tornado: dependencies: @@ -378,8 +378,8 @@ importers: specifier: ^4.1.3 version: 4.1.5 rimraf: - specifier: ^2.5.2 - version: 2.7.1 + specifier: ^6 + version: 6.0.1 examples/python-tornado-streaming: dependencies: @@ -409,8 +409,8 @@ importers: specifier: ^4.1.3 version: 4.1.5 rimraf: - specifier: ^2.5.2 - version: 2.7.1 + specifier: ^6 + version: 6.0.1 examples/react-example: dependencies: @@ -516,8 +516,8 @@ importers: specifier: ^4.1.3 version: 4.1.5 rimraf: - specifier: ^2.5.2 - version: 2.7.1 + specifier: ^6 + version: 6.0.1 packages/perspective-cli: dependencies: @@ -767,7 +767,7 @@ importers: specifier: ^0.1.16 version: 0.1.16 - rust/metadata: {} + rust/generate-metadata: {} rust/perspective: {} @@ -788,7 +788,7 @@ importers: version: link:../../packages/perspective-esbuild-plugin '@finos/perspective-metadata': specifier: workspace:^ - version: link:../metadata + version: link:../generate-metadata '@finos/perspective-test': specifier: workspace:^ version: link:../../tools/perspective-test @@ -838,7 +838,7 @@ importers: version: link:../../packages/perspective-esbuild-plugin '@finos/perspective-metadata': specifier: workspace:^ - version: link:../metadata + version: link:../generate-metadata '@finos/perspective-test': specifier: workspace:^ version: link:../../tools/perspective-test @@ -7555,11 +7555,6 @@ packages: resolution: {integrity: sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==} engines: {iojs: '>=1.0.0', node: '>=0.10.0'} - rimraf@2.7.1: - resolution: {integrity: sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==} - deprecated: Rimraf versions prior to v4 are no longer supported - hasBin: true - rimraf@3.0.2: resolution: {integrity: sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA==} deprecated: Rimraf versions prior to v4 are no longer supported @@ -17650,10 +17645,6 @@ snapshots: reusify@1.0.4: {} - rimraf@2.7.1: - dependencies: - glob: 7.2.3 - rimraf@3.0.2: dependencies: glob: 7.2.3 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index ef5d168d5a..09201bb8d3 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -3,6 +3,7 @@ packages: - "tools/perspective-scripts" - "tools/perspective-bench" - "cpp/perspective" + - "packages/perspective-client" - "packages/perspective-esbuild-plugin" - "packages/perspective-viewer-datagrid" - "packages/perspective-viewer-d3fc" diff --git a/rust/perspective-client/build.rs b/rust/perspective-client/build.rs index adf3ae7944..ce9761e390 100644 --- a/rust/perspective-client/build.rs +++ b/rust/perspective-client/build.rs @@ -24,7 +24,7 @@ fn prost_build() -> Result<()> { // This source file is included at `publish` time, but not `sbuild` time // because it is initially generated from the `perspective.proto` definition // in the C++ source. - if std::env::var("CARGO_FEATURE_EXTERNAL_PROTO").is_ok() { + if std::env::var("CARGO_FEATURE_EXTERNAL_PROTO").is_ok() || cfg!(clippy) { println!("cargo:warning=MESSAGE Building in development mode"); let root_dir_env = std::env::var("PSP_ROOT_DIR").expect("Must set PSP_ROOT_DIR"); let root_dir = Path::new(root_dir_env.as_str()); diff --git a/rust/perspective-client/package.json b/rust/perspective-client/package.json new file mode 100644 index 0000000000..fa0c0e0ef0 --- /dev/null +++ b/rust/perspective-client/package.json @@ -0,0 +1,18 @@ +{ + "name": "@finos/perspective-client", + "version": "3.1.0", + "description": "", + "private": true, + "repository": { + "type": "git", + "url": "https://github.com/finos/perspective" + }, + "type": "module", + "license": "Apache-2.0", + "scripts": { + "clean": "rimraf src/rust/proto.rs" + }, + "devDependencies": { + "rimraf": "^6" + } +} diff --git a/rust/perspective-js/package.json b/rust/perspective-js/package.json index 13de7e0410..293125fa9a 100644 --- a/rust/perspective-js/package.json +++ b/rust/perspective-js/package.json @@ -37,7 +37,7 @@ "types": "./dist/esm/perspective.d.ts", "scripts": { "build": "node ./build.js", - "clean": "rimraf dist && rimraf build", + "clean": "rimraf dist && rimraf build && rimraf src/ts/ts-rs", "docs": "cargo doc" }, "publishConfig": { diff --git a/rust/perspective-python/docs/lib.md b/rust/perspective-python/docs/lib.md index 27d079f954..4af904fafc 100644 --- a/rust/perspective-python/docs/lib.md +++ b/rust/perspective-python/docs/lib.md @@ -165,7 +165,7 @@ treat the index as a primary key: ```python data.set_index("datetime") -table = perspective.Table(data, index="index") +table = perspective.table(data, index="index") ``` #### Time Zone Handling @@ -281,7 +281,8 @@ of clients and large datasets. _*server.py*_ ```python -from perspective import Server, PerspectiveTornadoHandler +from perspective import Server +from perspective.hadnlers.tornado import PerspectiveTornadoHandler # Create an instance of Server, and host a Table SERVER = Server() @@ -371,21 +372,15 @@ Building on top of the API provided by `perspective.Table`, the of Perspective within the Jupyter environment. It supports the same API semantics of ``, along with the additional data types supported by `perspective.Table`. `PerspectiveWidget` takes keyword arguments -for the managed `View`; additioanl arguments `index` and `limit` will be passed -to the `Table`. For convenience are the -[`Aggregate`](https://github.com/finos/perspective/blob/master/python/perspective/perspective/core/aggregate.py), -[`Sort`](https://github.com/finos/perspective/blob/master/python/perspective/perspective/core/sort.py), -and -[`Plugin`](https://github.com/finos/perspective/blob/master/python/perspective/perspective/core/plugin.py) -enums, which can be used as replacements to string values in the API: +for the managed `View`: ```python -from perspective import PerspectiveWidget, Aggregate, Sort, Plugin +from perspective import PerspectiveWidget w = perspective.PerspectiveWidget( data, - plugin=Plugin.XBAR, - aggregates={"datetime": Aggregate.ANY}, - sort=[["date", Sort.DESC]] + plugin="X Bar", + aggregates={"datetime": "any"}, + sort=[["date", "desc"]] ) ``` @@ -413,8 +408,7 @@ PerspectiveWidget({"a": int, "b": str}) .. or an instance of a `perspective.Table`: ```python -client = perspective.Server().new_local_client() -table = client.table(data) +table = perspective.table(data) PerspectiveWidget(table) ``` diff --git a/rust/perspective-server/build/psp.rs b/rust/perspective-server/build/psp.rs index 92e8fd5069..385f153b3a 100644 --- a/rust/perspective-server/build/psp.rs +++ b/rust/perspective-server/build/psp.rs @@ -50,7 +50,6 @@ pub fn cmake_build() -> Result, std::io::Error> { ); } - let mut dst = Config::new("cpp/perspective"); if cfg!(windows) && std::option_env!("CI").is_some() { std::fs::create_dir_all("D:\\psp-build")?; @@ -103,11 +102,17 @@ pub fn cmake_build() -> Result, std::io::Error> { } if !cfg!(windows) { + println!( + "cargo:warning=MESSAGE Building parallel {}", + num_cpus::get() + ); + dst.build_arg(format!("-j{}", num_cpus::get())); } println!("cargo:warning=MESSAGE Building cmake {}", profile); - if std::env::var("PSP_BUILD_VERBOSE").unwrap_or_default() != "" { // checks non-empty env var + if std::env::var("PSP_BUILD_VERBOSE").unwrap_or_default() != "" { + // checks non-empty env var dst.very_verbose(true); } let artifact_dir = dst.build(); diff --git a/tools/perspective-scripts/clean.mjs b/tools/perspective-scripts/clean.mjs index 012b5b370e..4e8175b202 100644 --- a/tools/perspective-scripts/clean.mjs +++ b/tools/perspective-scripts/clean.mjs @@ -31,14 +31,13 @@ for (const pkg of PACKAGES) { } } -if (JS_PKGS.length > 0) { +if (JS_PKGS.length > 0 || RUST_PKGS.length > 0) { console.log(`-- Cleaning ${JS_PKGS.join(", ")} via pnpm`); - execSync( - `pnpm run ${JS_PKGS.map((x) => `--filter ${x} --if-present`).join( - " " - )} clean`, - { stdio: "inherit" } - ); + const flags = JS_PKGS.concat(RUST_PKGS) + .map((x) => `--filter @finos/${x} --if-present`) + .join(" "); + + execSync(`pnpm run ${flags} clean`, { stdio: "inherit" }); } if (RUST_PKGS.length > 0) { diff --git a/tools/perspective-scripts/lint.mjs b/tools/perspective-scripts/lint.mjs index 8142016538..36ba92053c 100644 --- a/tools/perspective-scripts/lint.mjs +++ b/tools/perspective-scripts/lint.mjs @@ -30,8 +30,8 @@ export function lint_js(is_fix = false) { const fix = is_fix ? "--fix" : undefined; cmd.sh`cd rust/perspective-viewer`; cmd.sh`cargo build -p perspective-lint`; - cmd.sh`cargo clippy ${fix} ${dirty} ${staged} -- -Dwarnings`; - cmd.sh`RUSTFMT="../target/debug/lint" cargo fmt ${check}`; + cmd.sh`PSP_ROOT_DIR=../.. cargo clippy ${fix} ${dirty} ${staged} -- -Dwarnings`; + // cmd.sh`RUSTFMT="../target/debug/lint" cargo fmt ${check}`; cmd.runSync(); }