v2-chore: Update DataFusion to 43 final and Arrow 52.3 #1042

Workflow file for this run

.github/workflows/build_test.yml at bb0e636

	name: build_test
	on:
	pull_request:
	types: [opened, synchronize]
	jobs:
	check-rust:
	runs-on: ubuntu-20.04
	steps:
	- name: Check out repository code
	uses: actions/[email protected]
	- uses: prefix-dev/[email protected]
	with:
	pixi-version: v0.34.0
	cache: true
	- name: Cache
	uses: actions/[email protected]
	with:
	key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock', 'vegafusion-python/pyproject.toml') }}-check-rust
	path: \|
	~/.cargo
	target
	.pixi
	- name: Check fmt
	run: pixi run check-rs-fmt
	- name: Check warnings
	run: pixi run check-rs-warnings
	- name: Check clippy
	run: pixi run check-rs-clippy

	test-rust:
	runs-on: ${{ matrix.os }}
	strategy:
	fail-fast: false
	matrix:
	os:
	- ubuntu-20.04
	- windows-2022
	- macos-14
	steps:
	- name: Check out repository code
	uses: actions/[email protected]
	- uses: prefix-dev/[email protected]
	with:
	pixi-version: v0.34.0
	- name: Cache
	uses: actions/[email protected]
	with:
	key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock', 'vegafusion-python/pyproject.toml') }}-test-rust2
	path: \|
	~/.cargo
	target
	.pixi
	- name: start minio server in the background
	run: pixi run start-minio &
	- name: Test rust
	run: \|
	pixi run test-rs --release
	- name: Upload test artifacts
	uses: actions/upload-artifact@v4
	if: failure()
	with:
	name: vegafusion-rt-test-images-${{ matrix.os }}
	path: \|
	vegafusion-runtime/tests/output

	build-vegafusion-wasm:
	runs-on: ubuntu-20.04
	steps:
	- name: Check out repository code
	uses: actions/[email protected]
	- uses: prefix-dev/[email protected]
	with:
	pixi-version: v0.34.0
	- name: Cache
	uses: actions/[email protected]
	with:
	key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock', 'vegafusion-python/pyproject.toml') }}-build-vegafusion-wasm
	path: \|
	~/.cargo
	target
	.pixi
	- name: Build package
	run: \|
	pixi run pack-wasm
	mkdir -p package
	cp vegafusion-wasm/pkg/vegafusion-wasm-*.tgz package/
	echo "timestamp: $(date)" > package/info.txt
	- name: Upload artifacts
	uses: actions/[email protected]
	with:
	name: vegafusion-wasm-package
	path: package

	check-vegafusion-python:
	runs-on: ubuntu-20.04
	steps:
	- name: Check out repository code
	uses: actions/[email protected]
	- uses: prefix-dev/[email protected]
	with:
	pixi-version: v0.34.0
	- name: Cache
	uses: actions/[email protected]
	with:
	key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock', 'vegafusion-python/pyproject.toml') }}-build-vegafusion-wasm
	path: \|
	~/.cargo
	target
	.pixi
	- name: Dev python
	run: \|
	pixi run dev-py
	- name: Check format and lint
	run: \|
	pixi run lint-check-py
	- name: Type check
	run: \|
	pixi run type-check-py

	# Use maturin action to build linux wheels within proper manylinux compatible containers
	# (This is why we don't use the pixi "build-py" action)
	build-vegafusion-python-linux-64:
	runs-on: ubuntu-latest
	steps:
	- name: Check out repository code
	uses: actions/[email protected]
	- name: Clear target/wheels
	run: rm -rf target/wheels
	- name: Build wheels (Linux)
	uses: messense/[email protected]
	with:
	command: build
	manylinux: 2014
	rust-toolchain: stable
	args: -m vegafusion-python/Cargo.toml --profile release-opt --features=protobuf-src --strip
	- name: Upload artifacts
	uses: actions/[email protected]
	with:
	name: python-wheels-linux-64
	path: \|
	target/wheels/*.tar.gz
	target/wheels/*.whl
	retention-days: 1
	if-no-files-found: error

	build-vegafusion-python-linux-arm64:
	runs-on: ubuntu-latest
	steps:
	- name: Check out repository code
	uses: actions/[email protected]
	- name: Clear target/wheels
	run: rm -rf target/wheels
	- name: Build arm64 wheels
	uses: messense/[email protected]
	with:
	command: build
	manylinux: 2_28
	rust-toolchain: stable
	args: -m vegafusion-python/Cargo.toml --profile release-opt --features=protobuf-src --strip --target aarch64-unknown-linux-gnu
	- name: Upload artifacts
	uses: actions/[email protected]
	with:
	name: python-wheels-linux-arm64
	path: \|
	target/wheels/*.tar.gz
	target/wheels/*.whl
	retention-days: 1
	if-no-files-found: error

	build-vegafusion-python-win-64:
	runs-on: windows-2022
	steps:
	- name: Check out repository code
	uses: actions/[email protected]
	- uses: prefix-dev/[email protected]
	with:
	pixi-version: v0.34.0
	- name: Cache
	uses: actions/[email protected]
	with:
	key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock', 'vegafusion-python/pyproject.toml') }}-build-vegafusion-python-win-64
	path: \|
	~/.cargo
	target
	.pixi
	- name: Build vegafusion-python
	run: \|
	pixi run build-py
	- name: Upload artifacts
	uses: actions/[email protected]
	with:
	name: python-wheels-win-64
	path: \|
	target/wheels/*.tar.gz
	target/wheels/*.whl
	retention-days: 1
	if-no-files-found: error

	build-vegafusion-python-osx-64:
	runs-on: macos-13
	steps:
	- name: Check out repository code
	uses: actions/[email protected]
	- uses: prefix-dev/[email protected]
	with:
	pixi-version: v0.34.0
	- name: Cache
	uses: actions/[email protected]
	with:
	key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock', 'vegafusion-python/pyproject.toml') }}-build-vegafusion-python-osx-64
	path: \|
	~/.cargo
	target
	.pixi
	- name: Build vegafusion-python
	run: \|
	pixi run build-py
	- name: Upload artifacts
	uses: actions/[email protected]
	with:
	name: python-wheels-osx-64
	path: \|
	target/wheels/*.tar.gz
	target/wheels/*.whl
	retention-days: 1
	if-no-files-found: error

	build-vegafusion-python-osx-arm64:
	runs-on: macos-14
	steps:
	- name: Check out repository code
	uses: actions/[email protected]
	- uses: prefix-dev/[email protected]
	with:
	pixi-version: v0.34.0
	- name: Cache
	uses: actions/[email protected]
	with:
	key: osx-arm64-${{ hashFiles('pixi.lock', 'Cargo.lock', 'vegafusion-python/pyproject.toml') }}-build-vegafusion-python-osx-arm64
	path: \|
	~/.cargo
	target
	.pixi
	- name: Build vegafusion-python
	run: \|
	pixi run build-py --target aarch64-apple-darwin
	- name: Upload artifacts
	uses: actions/[email protected]
	with:
	name: python-wheels-osx-arm64
	path: \|
	target/wheels/*.tar.gz
	target/wheels/*.whl
	retention-days: 1
	if-no-files-found: error

	test-vegafusion-python-linux-64:
	runs-on: ubuntu-20.04
	needs: [build-vegafusion-python-linux-64]
	steps:
	- name: Check out repository code
	uses: actions/[email protected]
	- name: Setup Python
	uses: actions/[email protected]
	with:
	python-version: "3.11"
	- name: Install Chrome
	uses: browser-actions/setup-chrome@v1
	with:
	chrome-version: stable
	- name: Install fonts on Linux
	if: runner.os == 'Linux'
	run: \|
	echo ttf-mscorefonts-installer msttcorefonts/accepted-mscorefonts-eula select true \| sudo debconf-set-selections
	sudo apt-get install ttf-mscorefonts-installer
	- name: Download vegafusion-python wheel
	uses: actions/[email protected]
	with:
	name: python-wheels-linux-64
	path: target/wheels/
	- name: install wheels
	working-directory: target/wheels/
	run: \|
	ls -la
	python -m pip install vegafusion-manylinux_2_17_x86_64.whl

	# Optional dependencies
	python -m pip install pyarrow pandas polars-lts-cpu "duckdb>=1.0" "vl-convert-python>=1.0.1rc1" scikit-image "pandas>=2.2" jupytext voila anywidget ipywidgets chromedriver-binary-auto

	# Test dependencies
	python -m pip install pytest altair vega-datasets scikit-image jupytext voila ipykernel anywidget ipywidgets selenium flaky tenacity chromedriver-binary-auto
	- name: Test lazy imports
	working-directory: vegafusion-python/
	run: python checks/check_lazy_imports.py
	- name: Test vegafusion
	working-directory: vegafusion-python/
	env:
	VEGAFUSION_TEST_HEADLESS: 1
	run: pytest
	- name: Upload test artifacts
	uses: actions/[email protected]
	if: failure()
	with:
	name: vegafusion-python-test-failures
	path: \|
	vegafusion-python/tests/output/*

	test-vegafusion-python-osx-arm64:
	runs-on: macos-14
	needs: [build-vegafusion-python-osx-arm64]
	steps:
	- name: Check out repository code
	uses: actions/[email protected]
	- name: Setup Python
	uses: actions/[email protected]
	with:
	python-version: "3.10"
	- name: Download vegafusion-python wheel
	uses: actions/[email protected]
	with:
	name: python-wheels-osx-arm64
	path: target/wheels/
	- name: install wheels
	working-directory: target/wheels/
	run: \|
	ls -la
	python -m pip install vegafusion-macosx_11__arm64.whl

	# Optional dependencies
	python -m pip install pyarrow pandas polars "duckdb>=1.0" vl-convert-python "pandas>=2.2"

	# Test dependencies
	python -m pip install pytest altair vega-datasets scikit-image

	- name: Test vegafusion
	working-directory: vegafusion-python/
	run: pytest --ignore=tests/test_jupyter_widget.py

	test-vegafusion-python-win-64:
	runs-on: windows-2022
	needs: [build-vegafusion-python-win-64]
	steps:
	- name: Check out repository code
	uses: actions/[email protected]
	- name: Setup Python
	uses: actions/[email protected]
	with:
	python-version: "3.9"
	- name: Download vegafusion-python wheel
	uses: actions/[email protected]
	with:
	name: python-wheels-win-64
	path: target/wheels/
	- name: install wheels
	working-directory: target/wheels/
	shell: powershell
	run: \|
	Get-ChildItem -Force
	$vegafusion = Get-ChildItem -Name "vegafusion-*win_amd64.whl" \| Select-Object -First 1

	python -m pip install $vegafusion

	# Optional dependencies
	python -m pip install pyarrow pandas "numpy<2" polars[timezone] "duckdb>=1.0" vl-convert-python

	# Test dependencies
	python -m pip install pytest altair vega-datasets scikit-image
	- name: Test vegafusion
	working-directory: vegafusion-python/
	run: pytest --ignore=tests/test_jupyter_widget.py

	build-vegafusion-server-linux-64:
	runs-on: ubuntu-20.04
	steps:
	- name: Check out repository code
	uses: actions/[email protected]
	- uses: prefix-dev/[email protected]
	with:
	pixi-version: v0.34.0
	- name: Cache
	uses: actions/[email protected]
	with:
	key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock', 'vegafusion-python/pyproject.toml') }}-build-vegafusion-server-linux-64
	path: \|
	~/.cargo
	target
	.pixi
	- name: Build vegafusion-server
	run: \|
	pixi run build-rs-server
	- name: zip executable
	uses: vimtor/[email protected]
	with:
	files: target/release-opt/vegafusion-server
	dest: vegafusion-server-linux-64.zip
	- name: Upload artifacts
	uses: actions/[email protected]
	with:
	name: vegafusion-server-linux-64
	path: \|
	vegafusion-server-*
	retention-days: 1
	if-no-files-found: error

	# # Below is an Attempt to cross compile to linux aarch64 within conda environment.
	# # It fails in compiling protobuf-src. So instead we fall back to the direct use of
	# # cargo (without involving conda).
	#
	# build-vegafusion-server-linux-arm64:
	# runs-on: ubuntu-20.04
	# steps:
	# - name: Check out repository code
	# uses: actions/[email protected]
	# - name: Install pixi
	# run: curl -fsSL https://pixi.sh/install.sh \| bash && echo "${HOME}/.pixi/bin" >> $GITHUB_PATH
	# - name: Cache
	# uses: actions/[email protected]
	# with:
	# key: linux-arm64-${{ hashFiles('pixi.lock', 'Cargo.lock', 'vegafusion-python/pyproject.toml') }}-server
	# path: \|
	# ~/.cargo
	# target
	# .pixi
	# - name: Build vegafusion-server
	# run: \|
	# pixi run python automation/download_rust_target.py aarch64-unknown-linux-gnu
	# pixi add gcc_linux-aarch64 -p linux-64
	# export RUSTFLAGS="-C linker=aarch64-linux-gnu-gcc"
	# pixi run build-rs-server --features=protobuf-src --target aarch64-unknown-linux-gnu
	# - name: zip executable
	# uses: vimtor/[email protected]
	# with:
	# files: target/aarch64-unknown-linux-gnu/release/vegafusion-server
	# dest: vegafusion-server-linux-arm64.zip
	# - name: Upload artifacts
	# uses: actions/[email protected]
	# with:
	# name: vegafusion-server-linux-arm64
	# path: \|
	# vegafusion-server-*

	build-vegafusion-server-linux-arm64:
	runs-on: ubuntu-latest
	steps:
	- name: Check out repository code
	uses: actions/[email protected]
	- name: Install latest stable Rust toolchain
	uses: dtolnay/rust-toolchain@stable
	with:
	toolchain: stable
	- name: Cache rust dependencies
	uses: Swatinem/[email protected]
	with:
	cache-on-failure: True
	- name: Download arm64 toolchain
	run: \|
	rustup target add aarch64-unknown-linux-gnu
	- name: Download gcc for cross compilation
	run: \|
	sudo apt-get update
	sudo apt-get install gcc-aarch64-linux-gnu
	- name: Build vegafusion-server
	env:
	RUSTFLAGS: "-C linker=aarch64-linux-gnu-gcc"
	run: \|
	cargo build -p vegafusion-server --features=protobuf-src --profile release-opt --target=aarch64-unknown-linux-gnu
	- name: zip executable
	uses: vimtor/[email protected]
	with:
	files: target/aarch64-unknown-linux-gnu/release-opt/vegafusion-server
	dest: vegafusion-server-linux-arm64.zip
	- name: Upload artifacts
	uses: actions/[email protected]
	with:
	name: vegafusion-server-linux-arm64
	path: \|
	vegafusion-server-*
	retention-days: 1
	if-no-files-found: error

	build-vegafusion-server-win-64:
	runs-on: windows-2022
	steps:
	- name: Check out repository code
	uses: actions/[email protected]
	- uses: prefix-dev/[email protected]
	with:
	pixi-version: v0.34.0
	- name: Cache
	uses: actions/[email protected]
	with:
	key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock', 'vegafusion-python/pyproject.toml') }}-build-vegafusion-server-win-64b
	path: \|
	~/.cargo
	target
	.pixi
	- name: Build vegafusion-server
	run: \|
	pixi install -vvv
	pixi run build-rs-server
	- name: zip executable
	uses: vimtor/[email protected]
	with:
	files: target/release-opt/vegafusion-server.exe
	dest: vegafusion-server-win-64.zip
	- name: Upload artifacts
	uses: actions/[email protected]
	with:
	name: vegafusion-server-win-64
	path: \|
	vegafusion-server-*
	retention-days: 1
	if-no-files-found: error

	build-vegafusion-server-osx-64:
	runs-on: macos-13
	steps:
	- name: Check out repository code
	uses: actions/[email protected]
	- uses: prefix-dev/[email protected]
	with:
	pixi-version: v0.34.0
	- name: Cache
	uses: actions/[email protected]
	with:
	key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock', 'vegafusion-python/pyproject.toml') }}-build-vegafusion-server-osx-64
	path: \|
	~/.cargo
	target
	.pixi
	- name: Build vegafusion-server
	run: \|
	pixi run build-rs-server
	- name: zip executable
	uses: vimtor/[email protected]
	with:
	files: target/release-opt/vegafusion-server
	dest: vegafusion-server-osx-64.zip
	- name: Upload artifacts
	uses: actions/[email protected]
	with:
	name: vegafusion-server-osx-64
	path: \|
	vegafusion-server-*
	retention-days: 1
	if-no-files-found: error

	build-vegafusion-server-osx-arm64:
	runs-on: macos-14
	steps:
	- name: Check out repository code
	uses: actions/[email protected]
	- uses: prefix-dev/[email protected]
	with:
	pixi-version: v0.34.0
	- name: Cache
	uses: actions/[email protected]
	with:
	key: osx-arm64-${{ hashFiles('pixi.lock', 'Cargo.lock', 'vegafusion-python/pyproject.toml') }}-build-vegafusion-server-osx-arm64
	path: \|
	~/.cargo
	target
	.pixi
	- name: Build vegafusion-server
	run: \|
	pixi run build-rs-server --target aarch64-apple-darwin
	- name: zip executable
	uses: vimtor/[email protected]
	with:
	files: target/aarch64-apple-darwin/release-opts/vegafusion-server
	dest: vegafusion-server-osx-arm64.zip
	- name: Upload artifacts
	uses: actions/[email protected]
	with:
	name: vegafusion-server-osx-arm64
	path: \|
	vegafusion-server-*
	retention-days: 1
	if-no-files-found: error

	# Bundle all server builds into a single artifact for easier downloading
	bundle-vegafusion-server-builds:
	needs:
	[
	build-vegafusion-server-linux-64,
	build-vegafusion-server-linux-arm64,
	build-vegafusion-server-win-64,
	build-vegafusion-server-osx-64,
	build-vegafusion-server-osx-arm64,
	]
	runs-on: ubuntu-latest
	steps:
	- name: Merge Server Artifacts
	uses: actions/upload-artifact/[email protected]
	with:
	name: vegafusion-server-all
	pattern: vegafusion-server-*
	delete-merged: true
	separate-directories: false

	# Bundle all python wheels into a single artifact for easier downloading
	bundle-vegafusion-python-wheels:
	needs: [
	# jobs that build but don't have test dependencies
	build-vegafusion-python-linux-arm64,
	build-vegafusion-python-osx-64,

	# jobs that test the wheels
	test-vegafusion-python-linux-64,
	test-vegafusion-python-osx-arm64,
	test-vegafusion-python-win-64,
	]
	runs-on: ubuntu-latest
	steps:
	- name: Merge Wheel Artifacts
	uses: actions/upload-artifact/[email protected]
	with:
	name: vegafusion-python-wheels-all
	pattern: python-wheels-*
	delete-merged: true
	separate-directories: false

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

v2-chore: Update DataFusion to 43 final and Arrow 52.3 #1042

Workflow file

v2-chore: Update DataFusion to 43 final and Arrow 52.3 #1042

Jobs

Run details

Workflow file for this run