build_test.yml

name: build_test
on:
  pull_request:
    types: [opened, synchronize]
jobs:
  check-rust:
    runs-on: ubuntu-20.04
    steps:
      - name: Check out repository code
        uses: actions/checkout@v4.2.2
      - uses: prefix-dev/setup-pixi@v0.8.1
        with:
          pixi-version: v0.34.0
          cache: true
      - name: Cache
        uses: actions/cache@v4.1.2
        with:
          key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock', 'vegafusion-python/pyproject.toml') }}-check-rust
          path: |
            ~/.cargo
            target
            .pixi
      - name: Check fmt
        run: pixi run check-rs-fmt
      - name: Check warnings
        run: pixi run check-rs-warnings
      - name: Check clippy
        run: pixi run check-rs-clippy

  test-rust:
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os:
          - ubuntu-20.04
          - windows-2022
          - macos-14
    steps:
      - name: Check out repository code
        uses: actions/checkout@v4.2.2
      - uses: prefix-dev/setup-pixi@v0.8.1
        with:
          pixi-version: v0.34.0
      - name: Cache
        uses: actions/cache@v4.1.2
        with:
          key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock', 'vegafusion-python/pyproject.toml') }}-test-rust2
          path: |
            ~/.cargo
            target
            .pixi
      - name: start minio server in the background
        run: pixi run start-minio &
      - name: Test rust
        run: |
          pixi run test-rs --release
      - name: Upload test artifacts
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: vegafusion-rt-test-images-${{ matrix.os }}
          path: |
            vegafusion-runtime/tests/output

  build-vegafusion-wasm:
    runs-on: ubuntu-20.04
    steps:
      - name: Check out repository code
        uses: actions/checkout@v4.2.2
      - uses: prefix-dev/setup-pixi@v0.8.1
        with:
          pixi-version: v0.34.0
      - name: Cache
        uses: actions/cache@v4.1.2
        with:
          key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock', 'vegafusion-python/pyproject.toml') }}-build-vegafusion-wasm
          path: |
            ~/.cargo
            target
            .pixi
      - name: Build package
        run: |
          pixi run pack-wasm
          mkdir -p vegafusion-wasm/package
          cp vegafusion-wasm/pkg/vegafusion-wasm-*.tgz vegafusion-wasm/package/
      - name: Upload artifacts
        uses: actions/upload-artifact@v4.4.3
        with:
          name: vegafusion-wasm-package
          path: vegafusion-wasm/package/

  check-vegafusion-python:
    runs-on: ubuntu-20.04
    steps:
      - name: Check out repository code
        uses: actions/checkout@v4.2.2
      - uses: prefix-dev/setup-pixi@v0.8.1
        with:
          pixi-version: v0.34.0
      - name: Cache
        uses: actions/cache@v4.1.2
        with:
          key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock', 'vegafusion-python/pyproject.toml') }}-build-vegafusion-wasm
          path: |
            ~/.cargo
            target
            .pixi
      - name: Dev python
        run: |
          pixi run dev-py
      - name: Check format and lint
        run: |
          pixi run lint-check-py
      - name: Type check
        run: |
          pixi run type-check-py

  # Use maturin action to build linux wheels within proper manylinux compatible containers
  # (This is why we don't use the pixi "build-py" action)
  build-vegafusion-python-linux-64:
    runs-on: ubuntu-latest
    steps:
      - name: Check out repository code
        uses: actions/checkout@v4.2.2
      - name: Clear target/wheels
        run: rm -rf target/wheels
      - name: Build wheels (Linux)
        uses: messense/maturin-action@v1.45
        with:
          command: build
          manylinux: 2014
          rust-toolchain: stable
          args: -m vegafusion-python/Cargo.toml --profile release-opt --features=protobuf-src --strip
      - name: Upload artifacts
        uses: actions/upload-artifact@v4.4.3
        with:
          name: python-wheels-linux-64
          path: |
            target/wheels/*.tar.gz
            target/wheels/*.whl
          retention-days: 1
          if-no-files-found: error

  build-vegafusion-python-linux-arm64:
    runs-on: ubuntu-latest
    steps:
      - name: Check out repository code
        uses: actions/checkout@v4.2.2
      - name: Clear target/wheels
        run: rm -rf target/wheels
      - name: Build arm64 wheels
        uses: messense/maturin-action@v1.45
        with:
          command: build
          manylinux: 2_28
          rust-toolchain: stable
          args: -m vegafusion-python/Cargo.toml --profile release-opt --features=protobuf-src --strip --target aarch64-unknown-linux-gnu
      - name: Upload artifacts
        uses: actions/upload-artifact@v4.4.3
        with:
          name: python-wheels-linux-arm64
          path: |
            target/wheels/*.tar.gz
            target/wheels/*.whl
          retention-days: 1
          if-no-files-found: error

  build-vegafusion-python-win-64:
    runs-on: windows-2022
    steps:
      - name: Check out repository code
        uses: actions/checkout@v4.2.2
      - uses: prefix-dev/setup-pixi@v0.8.1
        with:
          pixi-version: v0.34.0
      - name: Cache
        uses: actions/cache@v4.1.2
        with:
          key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock', 'vegafusion-python/pyproject.toml') }}-build-vegafusion-python-win-64
          path: |
            ~/.cargo
            target
            .pixi
      - name: Build vegafusion-python
        run: |
          pixi run build-py
      - name: Upload artifacts
        uses: actions/upload-artifact@v4.4.3
        with:
          name: python-wheels-win-64
          path: |
            target/wheels/*.tar.gz
            target/wheels/*.whl
          retention-days: 1
          if-no-files-found: error

  build-vegafusion-python-osx-64:
    runs-on: macos-12
    steps:
      - name: Check out repository code
        uses: actions/checkout@v4.2.2
      - uses: prefix-dev/setup-pixi@v0.8.1
        with:
          pixi-version: v0.34.0
      - name: Cache
        uses: actions/cache@v4.1.2
        with:
          key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock', 'vegafusion-python/pyproject.toml') }}-build-vegafusion-python-osx-64
          path: |
            ~/.cargo
            target
            .pixi
      - name: Build vegafusion-python
        run: |
          pixi run build-py
      - name: Upload artifacts
        uses: actions/upload-artifact@v4.4.3
        with:
          name: python-wheels-osx-64
          path: |
            target/wheels/*.tar.gz
            target/wheels/*.whl
          retention-days: 1
          if-no-files-found: error

  build-vegafusion-python-osx-arm64:
    runs-on: macos-14
    steps:
      - name: Check out repository code
        uses: actions/checkout@v4.2.2
      - uses: prefix-dev/setup-pixi@v0.8.1
        with:
          pixi-version: v0.34.0
      - name: Cache
        uses: actions/cache@v4.1.2
        with:
          key: osx-arm64-${{ hashFiles('pixi.lock', 'Cargo.lock', 'vegafusion-python/pyproject.toml') }}-build-vegafusion-python-osx-arm64
          path: |
            ~/.cargo
            target
            .pixi
      - name: Build vegafusion-python
        run: |
          pixi run build-py --target aarch64-apple-darwin
      - name: Upload artifacts
        uses: actions/upload-artifact@v4.4.3
        with:
          name: python-wheels-osx-arm64
          path: |
            target/wheels/*.tar.gz
            target/wheels/*.whl
          retention-days: 1
          if-no-files-found: error

  test-vegafusion-python-linux-64:
    runs-on: ubuntu-20.04
    needs: [build-vegafusion-python-linux-64]
    steps:
      - name: Check out repository code
        uses: actions/checkout@v4.2.2
      - name: Setup Python
        uses: actions/setup-python@v5.3.0
        with:
          python-version: "3.11"
      - name: Install Chrome
        uses: browser-actions/setup-chrome@v1
        with:
          chrome-version: stable
      - name: Install fonts on Linux
        if: runner.os == 'Linux'
        run: |
          echo ttf-mscorefonts-installer msttcorefonts/accepted-mscorefonts-eula select true | sudo debconf-set-selections
          sudo apt-get install ttf-mscorefonts-installer
      - name: Download vegafusion-python wheel
        uses: actions/download-artifact@v4.1.8
        with:
          name: python-wheels-linux-64
          path: target/wheels/
      - name: install wheels
        working-directory: target/wheels/
        run: |
          ls -la
          python -m pip install vegafusion-*manylinux_2_17_x86_64*.whl

          # Optional dependencies
          python -m pip install pyarrow pandas polars-lts-cpu "duckdb>=1.0" "vl-convert-python>=1.0.1rc1" scikit-image "pandas>=2.2" jupytext voila anywidget ipywidgets chromedriver-binary-auto

          # Test dependencies
          python -m pip install pytest altair vega-datasets scikit-image jupytext voila ipykernel anywidget ipywidgets selenium flaky tenacity chromedriver-binary-auto
      - name: Test lazy imports
        working-directory: vegafusion-python/
        run: python checks/check_lazy_imports.py
      - name: Test vegafusion
        working-directory: vegafusion-python/
        env:
          VEGAFUSION_TEST_HEADLESS: 1
        run: pytest
      - name: Upload test artifacts
        uses: actions/upload-artifact@v4.4.3
        if: always()
        with:
          name: vegafusion-python-test-failures
          path: |
            vegafusion-python/tests/output/*

  test-vegafusion-python-osx-arm64:
    runs-on: macos-14
    needs: [build-vegafusion-python-osx-arm64]
    steps:
      - name: Check out repository code
        uses: actions/checkout@v4.2.2
      - name: Setup Python
        uses: actions/setup-python@v5.3.0
        with:
          python-version: "3.10"
      - name: Download vegafusion-python wheel
        uses: actions/download-artifact@v4.1.8
        with:
          name: python-wheels-osx-64
          path: target/wheels/
      - name: install wheels
        working-directory: target/wheels/
        run: |
          ls -la
          python -m pip install vegafusion-*macosx_11_*_arm64.whl

          # Optional dependencies
          python -m pip install pyarrow pandas polars "duckdb>=1.0" vl-convert-python "pandas>=2.2"

          # Test dependencies
          python -m pip install pytest altair vega-datasets scikit-image

      - name: Test vegafusion
        working-directory: vegafusion-python/
        run: pytest --ignore=tests/test_jupyter_widget.py

  test-vegafusion-python-win-64:
    runs-on: windows-2022
    needs: [build-vegafusion-python-win-64]
    steps:
      - name: Check out repository code
        uses: actions/checkout@v4.2.2
      - name: Setup Python
        uses: actions/setup-python@v5.3.0
        with:
          python-version: "3.9"
      - name: Download vegafusion-python wheel
        uses: actions/download-artifact@v4.1.8
        with:
          name: python-wheels-win-64
          path: target/wheels/
      - name: install wheels
        working-directory: target/wheels/
        shell: powershell
        run: |
          Get-ChildItem -Force
          $vegafusion = Get-ChildItem -Name "vegafusion-*win_amd64.whl" | Select-Object -First 1

          python -m pip install $vegafusion

          # Optional dependencies
          python -m pip install pyarrow pandas "numpy<2" polars[timezone] "duckdb>=1.0" vl-convert-python 

          # Test dependencies
          python -m pip install pytest altair vega-datasets scikit-image
      - name: Test vegafusion
        working-directory: vegafusion-python/
        run: pytest --ignore=tests/test_jupyter_widget.py

  build-vegafusion-server-linux-64:
    runs-on: ubuntu-20.04
    steps:
      - name: Check out repository code
        uses: actions/checkout@v4.2.2
      - uses: prefix-dev/setup-pixi@v0.8.1
        with:
          pixi-version: v0.34.0
      - name: Cache
        uses: actions/cache@v4.1.2
        with:
          key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock', 'vegafusion-python/pyproject.toml') }}-build-vegafusion-server-linux-64
          path: |
            ~/.cargo
            target
            .pixi
      - name: Build vegafusion-server
        run: |
          pixi run build-rs-server
      - name: zip executable
        uses: vimtor/action-zip@v1.2
        with:
          files: target/release/vegafusion-server
          dest: vegafusion-server-linux-64.zip
      - name: Upload artifacts
        uses: actions/upload-artifact@v4.4.3
        with:
          name: vegafusion-server-linux-64
          path: |
            vegafusion-server-*
          retention-days: 1
          if-no-files-found: error

  # # Below is an Attempt to cross compile to linux aarch64 within conda environment.
  # # It fails in compiling protobuf-src. So instead we fall back to the direct use of
  # # cargo (without involving conda).
  #
  #  build-vegafusion-server-linux-arm64:
  #    runs-on: ubuntu-20.04
  #    steps:
  #      - name: Check out repository code
  #        uses: actions/checkout@v4.2.2
  #      - name: Install pixi
  #        run: curl -fsSL https://pixi.sh/install.sh | bash && echo "${HOME}/.pixi/bin" >> $GITHUB_PATH
  #      - name: Cache
  #        uses: actions/cache@v4.1.2
  #        with:
  #          key: linux-arm64-${{ hashFiles('pixi.lock', 'Cargo.lock', 'vegafusion-python/pyproject.toml') }}-server
  #          path: |
  #            ~/.cargo
  #            target
  #            .pixi
  #      - name: Build vegafusion-server
  #        run: |
  #          pixi run python automation/download_rust_target.py aarch64-unknown-linux-gnu
  #          pixi add gcc_linux-aarch64 -p linux-64
  #          export RUSTFLAGS="-C linker=aarch64-linux-gnu-gcc"
  #          pixi run build-rs-server --features=protobuf-src --target aarch64-unknown-linux-gnu
  #      - name: zip executable
  #        uses: vimtor/action-zip@v1.2
  #        with:
  #          files: target/aarch64-unknown-linux-gnu/release/vegafusion-server
  #          dest: vegafusion-server-linux-arm64.zip
  #      - name: Upload artifacts
  #        uses: actions/upload-artifact@v4.4.3
  #        with:
  #          name: vegafusion-server-linux-arm64
  #          path: |
  #            vegafusion-server-*

  build-vegafusion-server-linux-arm64:
    runs-on: ubuntu-latest
    steps:
      - name: Check out repository code
        uses: actions/checkout@v4.2.2
      - name: Install latest stable Rust toolchain
        uses: dtolnay/rust-toolchain@stable
        with:
          toolchain: stable
      - name: Cache rust dependencies
        uses: Swatinem/rust-cache@v2.7.5
        with:
          cache-on-failure: True
      - name: Download arm64 toolchain
        run: |
          rustup target add aarch64-unknown-linux-gnu
      - name: Download gcc for cross compilation
        run: |
          sudo apt-get update
          sudo apt-get install gcc-aarch64-linux-gnu
      - name: Build vegafusion-server
        env:
          RUSTFLAGS: "-C linker=aarch64-linux-gnu-gcc"
        run: |
          cargo build -p vegafusion-server --features=protobuf-src --profile release-opt --target=aarch64-unknown-linux-gnu
      - name: zip executable
        uses: vimtor/action-zip@v1.2
        with:
          files: target/aarch64-unknown-linux-gnu/release/vegafusion-server
          dest: vegafusion-server-linux-arm64.zip
      - name: Upload artifacts
        uses: actions/upload-artifact@v4.4.3
        with:
          name: vegafusion-server-linux-arm64
          path: |
            vegafusion-server-*
          retention-days: 1
          if-no-files-found: error

  build-vegafusion-server-win-64:
    runs-on: windows-2022
    steps:
      - name: Check out repository code
        uses: actions/checkout@v4.2.2
      - uses: prefix-dev/setup-pixi@v0.8.1
        with:
          pixi-version: v0.34.0
      - name: Cache
        uses: actions/cache@v4.1.2
        with:
          key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock', 'vegafusion-python/pyproject.toml') }}-build-vegafusion-server-win-64b
          path: |
            ~/.cargo
            target
            .pixi
      - name: Build vegafusion-server
        run: |
          pixi install -vvv
          pixi run build-rs-server
      - name: zip executable
        uses: vimtor/action-zip@v1.2
        with:
          files: target/release/vegafusion-server.exe
          dest: vegafusion-server-win-64.zip
      - name: Upload artifacts
        uses: actions/upload-artifact@v4.4.3
        with:
          name: vegafusion-server-win-64
          path: |
            vegafusion-server-*
          retention-days: 1
          if-no-files-found: error

  build-vegafusion-server-osx-64:
    runs-on: macos-13
    steps:
      - name: Check out repository code
        uses: actions/checkout@v4.2.2
      - uses: prefix-dev/setup-pixi@v0.8.1
        with:
          pixi-version: v0.34.0
      - name: Cache
        uses: actions/cache@v4.1.2
        with:
          key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock', 'vegafusion-python/pyproject.toml') }}-build-vegafusion-server-osx-64
          path: |
            ~/.cargo
            target
            .pixi
      - name: Build vegafusion-server
        run: |
          pixi run build-rs-server
      - name: zip executable
        uses: vimtor/action-zip@v1.2
        with:
          files: target/release/vegafusion-server
          dest: vegafusion-server-osx-64.zip
      - name: Upload artifacts
        uses: actions/upload-artifact@v4.4.3
        with:
          name: vegafusion-server-osx-64
          path: |
            vegafusion-server-*
          retention-days: 1
          if-no-files-found: error

  build-vegafusion-server-osx-arm64:
    runs-on: macos-14
    steps:
      - name: Check out repository code
        uses: actions/checkout@v4.2.2
      - uses: prefix-dev/setup-pixi@v0.8.1
        with:
          pixi-version: v0.34.0
      - name: Cache
        uses: actions/cache@v4.1.2
        with:
          key: osx-arm64-${{ hashFiles('pixi.lock', 'Cargo.lock', 'vegafusion-python/pyproject.toml') }}-build-vegafusion-server-osx-arm64
          path: |
            ~/.cargo
            target
            .pixi
      - name: Build vegafusion-server
        run: |
          pixi run build-rs-server --target aarch64-apple-darwin
      - name: zip executable
        uses: vimtor/action-zip@v1.2
        with:
          files: target/aarch64-apple-darwin/release/vegafusion-server
          dest: vegafusion-server-osx-arm64.zip
      - name: Upload artifacts
        uses: actions/upload-artifact@v4.4.3
        with:
          name: vegafusion-server-osx-arm64
          path: |
            vegafusion-server-*
          retention-days: 1
          if-no-files-found: error

  # Bundle all server builds into a single artifact for easier downloading
  bundle-vegafusion-server-builds:
    needs:
      [
        build-vegafusion-server-linux-64,
        build-vegafusion-server-linux-arm64,
        build-vegafusion-server-win-64,
        build-vegafusion-server-osx-64,
        build-vegafusion-server-osx-arm64,
      ]
    runs-on: ubuntu-latest
    steps:
      - name: Merge Server Artifacts
        uses: actions/upload-artifact/merge@v4.4.3
        with:
          name: vegafusion-server-all
          pattern: vegafusion-server-*
          delete-merged: true
          separate-directories: false

  # Bundle all python wheels into a single artifact for easier downloading
  bundle-vegafusion-python-wheels:
    needs: [
        # jobs that build but don't have test dependencies
        build-vegafusion-python-linux-arm64,
        build-vegafusion-python-osx-64,

        # jobs that test the wheels
        test-vegafusion-python-linux-64,
        test-vegafusion-python-osx-arm64,
        test-vegafusion-python-win-64,
      ]
    runs-on: ubuntu-latest
    steps:
      - name: Merge Wheel Artifacts
        uses: actions/upload-artifact/merge@v4.4.3
        with:
          name: vegafusion-python-wheels-all
          pattern: python-wheels-*
          delete-merged: true
          separate-directories: false