build_test.yml

name: build_test
on:
  pull_request:
    types: [ opened, synchronize ]
jobs:
  check-rust:
    runs-on: ubuntu-20.04
    steps:
      - name: Check out repository code
        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2
      - uses: prefix-dev/setup-pixi@v0.8.1
        with:
          pixi-version: v0.30.0
          cache: true
      - name: Cache
        uses: actions/cache@v3
        with:
          key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock', 'vegafusion-python/pyproject.toml') }}-check-rust
          path: |
            ~/.cargo
            target
            .pixi
      - name: Check fmt
        run: pixi run check-rs-fmt
      - name: Check warnings
        run: pixi run check-rs-warnings
      - name: Check clippy
        run: pixi run check-rs-clippy

  test-rust:
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os:
          - ubuntu-20.04
          - windows-2022
          - macos-14
    steps:
      - name: Check out repository code
        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2
      - uses: prefix-dev/setup-pixi@v0.8.1
        with:
          pixi-version: v0.30.0
      - name: Cache
        uses: actions/cache@v3
        with:
          key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock', 'vegafusion-python/pyproject.toml') }}-test-rust2
          path: |
            ~/.cargo
            target
            .pixi
      - name: start minio server in the background
        run: pixi run start-minio &
      - name: Test rust
        run: |
          pixi run test-rs --release
      - name: Upload test artifacts
        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2
        if: always()
        with:
          name: vegafusion-rt-test-images
          path: |
            vegafusion-runtime/tests/output

  build-vegafusion-wasm:
    runs-on: ubuntu-20.04
    steps:
      - name: Check out repository code
        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2
      - uses: prefix-dev/setup-pixi@v0.8.1
        with:
          pixi-version: v0.30.0
      - name: Cache
        uses: actions/cache@v3
        with:
          key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock', 'vegafusion-python/pyproject.toml') }}-build-vegafusion-wasm
          path: |
            ~/.cargo
            target
            .pixi
      - name: Build package
        run: |
          pixi run pack-wasm
      - name: Upload artifacts
        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2
        with:
          name: vegafusion-wasm-packages
          path: vegafusion-wasm/pkg/vegafusion-wasm-*.tgz

  check-vegafusion-python:
    runs-on: ubuntu-20.04
    steps:
      - name: Check out repository code
        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2
      - uses: prefix-dev/setup-pixi@v0.8.1
        with:
          pixi-version: v0.30.0
      - name: Cache
        uses: actions/cache@v3
        with:
          key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock', 'vegafusion-python/pyproject.toml') }}-build-vegafusion-wasm
          path: |
            ~/.cargo
            target
            .pixi
      - name: Dev python
        run: |
          pixi run dev-py
      - name: Check format and lint
        run: |
          pixi run lint-check-py
      - name: Type check
        run: |
          pixi run type-check-py

  # Use maturin action to build linux wheels within proper manylinux compatible containers
  # (This is why we don't use the pixi "build-py" action)
  build-vegafusion-python-linux-64:
    runs-on: ubuntu-latest
    steps:
      - name: Check out repository code
        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2
      - name: Install latest stable Rust toolchain
        uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # pin@v1
        with:
          toolchain: stable
          override: true
      - name: Cache rust dependencies
        uses: Swatinem/rust-cache@81d053bdb0871dcd3f10763c8cc60d0adc41762b # pin@v1
        with:
          cache-on-failure: True
      - name: Clear target/wheels
        run: rm -rf target/wheels
      - name: Build wheels (Linux)
        uses: messense/maturin-action@60d11847b29f81ca5375519a8eb33cc336ba4bfa # pin@v1.41.1
        with:
          command: build
          manylinux: 2014
          rust-toolchain: stable
          args: --release -m vegafusion-python/Cargo.toml --features=protobuf-src --strip
      - name: Upload artifacts
        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2
        with:
          name: python-wheels
          path: |
            target/wheels/*.tar.gz
            target/wheels/*.whl

  build-vegafusion-python-linux-arm64:
    runs-on: ubuntu-latest
    steps:
      - name: Check out repository code
        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2
      - name: Install latest stable Rust toolchain
        uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # pin@v1
        with:
          toolchain: stable
          override: true
      - name: Cache rust dependencies
        uses: Swatinem/rust-cache@81d053bdb0871dcd3f10763c8cc60d0adc41762b # pin@v1
        with:
          cache-on-failure: True
      - name: Download arm64 toolchain
        run: |
          rustup target add aarch64-unknown-linux-gnu
      - name: Clear target/wheels
        run: rm -rf target/wheels
      - name: Build arm64 wheels
        uses: messense/maturin-action@60d11847b29f81ca5375519a8eb33cc336ba4bfa # pin@v1.41.1
        with:
          command: build
          manylinux: 2_28
          rust-toolchain: stable
          args: --release -m vegafusion-python/Cargo.toml --features=protobuf-src --strip --target aarch64-unknown-linux-gnu
      - name: Upload artifacts
        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2
        with:
          name: python-wheels
          path: |
            target/wheels/*.tar.gz
            target/wheels/*.whl

  build-vegafusion-python-win-64:
    runs-on: windows-2022
    steps:
      - name: Check out repository code
        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2
      - name: Setup Python
        uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # pin@4.7
        with:
          python-version: '3.11'
      - uses: prefix-dev/setup-pixi@v0.8.1
        with:
          pixi-version: v0.30.0
      - name: Cache
        uses: actions/cache@v3
        with:
          key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock', 'vegafusion-python/pyproject.toml') }}-build-vegafusion-python-win-64
          path: |
            ~/.cargo
            target
            .pixi
      - name: Build vegafusion-python
        run: |
          pixi run build-py
      - name: Upload artifacts
        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2
        with:
          name: python-wheels
          path: |
            target/wheels/*.tar.gz
            target/wheels/*.whl

  build-vegafusion-python-osx-64:
    runs-on: macos-12
    steps:
      - name: Check out repository code
        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2
      - uses: prefix-dev/setup-pixi@v0.8.1
        with:
          pixi-version: v0.30.0
      - name: Cache
        uses: actions/cache@v3
        with:
          key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock', 'vegafusion-python/pyproject.toml') }}-build-vegafusion-python-osx-64
          path: |
            ~/.cargo
            target
            .pixi
      - name: Build vegafusion-python
        run: |
          pixi run build-py
      - name: Upload artifacts
        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2
        with:
          name: python-wheels
          path: |
            target/wheels/*.tar.gz
            target/wheels/*.whl

  build-vegafusion-python-osx-arm64:
    runs-on: macos-14
    steps:
      - name: Check out repository code
        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2
      - uses: prefix-dev/setup-pixi@v0.8.1
        with:
          pixi-version: v0.30.0
      - name: Cache
        uses: actions/cache@v3
        with:
          key: osx-arm64-${{ hashFiles('pixi.lock', 'Cargo.lock', 'vegafusion-python/pyproject.toml') }}-build-vegafusion-python-osx-arm64
          path: |
            ~/.cargo
            target
            .pixi
      - name: Build vegafusion-python
        run: |
          pixi run build-py --target aarch64-apple-darwin
      - name: Upload artifacts
        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2
        with:
          name: python-wheels
          path: |
            target/wheels/*.tar.gz
            target/wheels/*.whl

  test-vegafusion-python-linux-64:
    runs-on: ubuntu-20.04
    needs: [build-vegafusion-python-linux-64]
    steps:
      - name: Check out repository code
        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2
      - name: Setup Python
        uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # pin@4.7
        with:
          python-version: '3.11'
      - name: Install Chrome
        uses: browser-actions/setup-chrome@f0ff752add8c926994566c80b3ceadfd03f24d12 # pin@latest
        with:
          chrome-version: stable
      - name: Install fonts on Linux
        if: runner.os == 'Linux'
        run: |
          echo ttf-mscorefonts-installer msttcorefonts/accepted-mscorefonts-eula select true | sudo debconf-set-selections
          sudo apt-get install ttf-mscorefonts-installer
      - name: Download vegafusion-python wheel
        uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 # pin@v2
        with:
          name: python-wheels
          path: target/wheels/
      - name: install wheels
        working-directory: target/wheels/
        run: |
          ls -la
          python -m pip install vegafusion-*manylinux_2_17_x86_64*.whl

          # Optional dependencies
          python -m pip install pyarrow pandas polars-lts-cpu "duckdb>=1.0" "vl-convert-python>=1.0.1rc1" scikit-image "pandas>=2.2" jupytext voila anywidget ipywidgets chromedriver-binary-auto

          # Test dependencies
          python -m pip install pytest altair vega-datasets scikit-image jupytext voila ipykernel anywidget ipywidgets selenium flaky tenacity chromedriver-binary-auto 
      - name: Test lazy imports
        working-directory: vegafusion-python/
        run: python checks/check_lazy_imports.py
      - name: Test vegafusion
        working-directory: vegafusion-python/
        env:
          VEGAFUSION_TEST_HEADLESS: 1
        run: pytest
      - name: Upload test artifacts
        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2
        if: always()
        with:
          name: vegafusion-python-test-failures
          path: |
            vegafusion-python/tests/output/*

  test-vegafusion-python-osx-arm64:
    runs-on: macos-14
    needs: [build-vegafusion-python-osx-arm64]
    steps:
      - name: Check out repository code
        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2
      - name: Setup Python
        uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # pin@4.7
        with:
          python-version: '3.10'
      - name: Download vegafusion-python wheel
        uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 # pin@v2
        with:
          name: python-wheels
          path: target/wheels/
      - name: install wheels
        working-directory: target/wheels/
        run: |
          ls -la
          python -m pip install vegafusion-*macosx_11_*_arm64.whl

          # Optional dependencies
          python -m pip install pyarrow pandas polars "duckdb>=1.0" vl-convert-python "pandas>=2.2"

          # Test dependencies
          python -m pip install pytest altair vega-datasets scikit-image

      - name: Test vegafusion
        working-directory: vegafusion-python/
        run: pytest --ignore=tests/test_jupyter_widget.py

  test-vegafusion-python-win-64:
    runs-on: windows-2022
    needs: [build-vegafusion-python-win-64]
    steps:
      - name: Check out repository code
        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2
      - name: Setup Python
        uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # pin@4.7
        with:
          python-version: '3.9'
      - name: Install Chrome
        uses: browser-actions/setup-chrome@f0ff752add8c926994566c80b3ceadfd03f24d12 # pin@latest
        with:
          chrome-version: stable
      - name: Download vegafusion-python wheel
        uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 # pin@v2
        with:
          name: python-wheels
          path: target/wheels/
      - name: install wheels
        working-directory: target/wheels/
        shell: powershell
        run: |
          Get-ChildItem -Force
          $vegafusion = Get-ChildItem -Name "vegafusion-*win_amd64.whl" | Select-Object -First 1

          python -m pip install $vegafusion

          # Optional dependencies
          python -m pip install pyarrow pandas "numpy<2" polars[timezone] "duckdb>=1.0" vl-convert-python 
          
          # Test dependencies
          python -m pip install pytest altair vega-datasets scikit-image
      - name: Test vegafusion
        working-directory: vegafusion-python/
        run: pytest --ignore=tests/test_jupyter_widget.py


  build-vegafusion-server-linux-64:
    runs-on: ubuntu-20.04
    steps:
      - name: Check out repository code
        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2
      - uses: prefix-dev/setup-pixi@v0.8.1
        with:
          pixi-version: v0.30.0
      - name: Cache
        uses: actions/cache@v3
        with:
          key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock', 'vegafusion-python/pyproject.toml') }}-build-vegafusion-server-linux-64
          path: |
            ~/.cargo
            target
            .pixi
      - name: Build vegafusion-server
        run: |
          pixi run build-rs-server
      - name: zip executable
        uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # pin@v1.1
        with:
          files: target/release/vegafusion-server
          dest: vegafusion-server-linux-64.zip
      - name: Upload artifacts
        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2
        with:
          name: vegafusion-server
          path: |
            vegafusion-server-*

# # Below is an Attempt to cross compile to linux aarch64 within conda environment.
# # It fails in compiling protobuf-src. So instead we fall back to the direct use of
# # cargo (without involving conda).
#
#  build-vegafusion-server-linux-arm64:
#    runs-on: ubuntu-20.04
#    steps:
#      - name: Check out repository code
#        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2
#      - name: Install pixi
#        run: curl -fsSL https://pixi.sh/install.sh | bash && echo "${HOME}/.pixi/bin" >> $GITHUB_PATH
#      - name: Cache
#        uses: actions/cache@v3
#        with:
#          key: linux-arm64-${{ hashFiles('pixi.lock', 'Cargo.lock', 'vegafusion-python/pyproject.toml') }}-server
#          path: |
#            ~/.cargo
#            target
#            .pixi
#      - name: Build vegafusion-server
#        run: |
#          pixi run python automation/download_rust_target.py aarch64-unknown-linux-gnu
#          pixi add gcc_linux-aarch64 -p linux-64
#          export RUSTFLAGS="-C linker=aarch64-linux-gnu-gcc"
#          pixi run build-rs-server --features=protobuf-src --target aarch64-unknown-linux-gnu
#      - name: zip executable
#        uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # pin@v1.1
#        with:
#          files: target/aarch64-unknown-linux-gnu/release/vegafusion-server
#          dest: vegafusion-server-linux-arm64.zip
#      - name: Upload artifacts
#        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2
#        with:
#          name: vegafusion-server
#          path: |
#            vegafusion-server-*


  build-vegafusion-server-linux-arm64:
    runs-on: ubuntu-latest
    steps:
      - name: Check out repository code
        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2
      - name: Install latest stable Rust toolchain
        uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # pin@v1
        with:
          toolchain: stable
          override: true
      - name: Cache rust dependencies
        uses: Swatinem/rust-cache@81d053bdb0871dcd3f10763c8cc60d0adc41762b # pin@v1
        with:
          cache-on-failure: True
      - name: Download arm64 toolchain
        run: |
          rustup target add aarch64-unknown-linux-gnu
      - name: Download gcc for cross compilation
        run: |
          sudo apt-get update
          sudo apt-get install gcc-aarch64-linux-gnu
      - name: Build vegafusion-server
        uses: actions-rs/cargo@844f36862e911db73fe0815f00a4a2602c279505 # pin@v1
        env:
          RUSTFLAGS: "-C linker=aarch64-linux-gnu-gcc"
        with:
          command: build
          args: -p vegafusion-server --release --features=protobuf-src --target=aarch64-unknown-linux-gnu
      - name: zip executable
        uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # pin@v1.1
        with:
          files: target/aarch64-unknown-linux-gnu/release/vegafusion-server
          dest: vegafusion-server-linux-arm64.zip
      - name: Upload artifacts
        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2
        with:
          name: vegafusion-server
          path: |
            vegafusion-server-*

  build-vegafusion-server-win-64:
    runs-on: windows-2022
    steps:
      - name: Check out repository code
        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2
      - uses: prefix-dev/setup-pixi@v0.8.1
        with:
          pixi-version: v0.30.0
      - name: Cache
        uses: actions/cache@v3
        with:
          key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock', 'vegafusion-python/pyproject.toml') }}-build-vegafusion-server-win-64b
          path: |
            ~/.cargo
            target
            .pixi
      - name: Build vegafusion-server
        run: |
          pixi install -vvv
          pixi run build-rs-server
      - name: zip executable
        uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # pin@v1.1
        with:
          files: target/release/vegafusion-server.exe
          dest: vegafusion-server-win-64.zip
      - name: Upload artifacts
        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2
        with:
          name: vegafusion-server
          path: |
            vegafusion-server-*

  build-vegafusion-server-osx-64:
    runs-on: macos-13
    steps:
      - name: Check out repository code
        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2
      - uses: prefix-dev/setup-pixi@v0.8.1
        with:
          pixi-version: v0.30.0
      - name: Cache
        uses: actions/cache@v3
        with:
          key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock', 'vegafusion-python/pyproject.toml') }}-build-vegafusion-server-osx-64
          path: |
            ~/.cargo
            target
            .pixi
      - name: Build vegafusion-server
        run: |
          pixi run build-rs-server
      - name: zip executable
        uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # pin@v1.1
        with:
          files: target/release/vegafusion-server
          dest: vegafusion-server-osx-64.zip
      - name: Upload artifacts
        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2
        with:
          name: vegafusion-server
          path: |
            vegafusion-server-*

  build-vegafusion-server-osx-arm64:
    runs-on: macos-14
    steps:
      - name: Check out repository code
        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # pin@v3.5.2
      - uses: prefix-dev/setup-pixi@v0.8.1
        with:
          pixi-version: v0.30.0
      - name: Cache
        uses: actions/cache@v3
        with:
          key: osx-arm64-${{ hashFiles('pixi.lock', 'Cargo.lock', 'vegafusion-python/pyproject.toml') }}-build-vegafusion-server-osx-arm64
          path: |
            ~/.cargo
            target
            .pixi
      - name: Build vegafusion-server
        run: |
          pixi run build-rs-server --target aarch64-apple-darwin
      - name: zip executable
        uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # pin@v1.1
        with:
          files: target/aarch64-apple-darwin/release/vegafusion-server
          dest: vegafusion-server-osx-arm64.zip
      - name: Upload artifacts
        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # pin@v3.1.2
        with:
          name: vegafusion-server
          path: |
            vegafusion-server-*