Skip to content

Add initial Arrow PyCapsule support #986

Add initial Arrow PyCapsule support

Add initial Arrow PyCapsule support #986

Workflow file for this run

name: build_test
on:
pull_request:
types: [ opened, synchronize ]
jobs:
check-rust:
runs-on: ubuntu-20.04
steps:
- name: Check out repository code
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
- uses: prefix-dev/[email protected]
with:
pixi-version: v0.30.0
cache: true
- name: Cache
uses: actions/cache@v3
with:
key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-check-rust
path: |
~/.cargo
target
.pixi
- name: Check fmt
run: pixi run check-rs-fmt
- name: Check warnings
run: pixi run check-rs-warnings
- name: Check clippy
run: pixi run check-rs-clippy
test-rust:
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
matrix:
os:
- ubuntu-20.04
- windows-2022
- macos-14
steps:
- name: Check out repository code
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
- uses: prefix-dev/[email protected]
with:
pixi-version: v0.30.0
- name: Cache
uses: actions/cache@v3
with:
key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-test-rust2
path: |
~/.cargo
target
.pixi
- name: start minio server in the background
run: pixi run start-minio &
- name: Test rust
run: |
pixi run test-rs --release
- name: Upload test artifacts
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # [email protected]
if: always()
with:
name: vegafusion-rt-test-images
path: |
vegafusion-runtime/tests/output
build-vegafusion-wasm:
runs-on: ubuntu-20.04
steps:
- name: Check out repository code
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
- uses: prefix-dev/[email protected]
with:
pixi-version: v0.30.0
- name: Cache
uses: actions/cache@v3
with:
key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-wasm
path: |
~/.cargo
target
.pixi
- name: Build package
run: |
pixi run pack-wasm
- name: Upload artifacts
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # [email protected]
with:
name: vegafusion-wasm-packages
path: vegafusion-wasm/pkg/vegafusion-wasm-*.tgz
check-vegafusion-python:
runs-on: ubuntu-20.04
steps:
- name: Check out repository code
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
- uses: prefix-dev/[email protected]
with:
pixi-version: v0.30.0
- name: Cache
uses: actions/cache@v3
with:
key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-wasm
path: |
~/.cargo
target
.pixi
- name: Dev python
run: |
pixi run dev-py
- name: Check format and lint
run: |
pixi run lint-check-py
- name: Type check
run: |
pixi run type-check-py
# Use maturin action to build linux wheels within proper manylinux compatible containers
# (This is why we don't use the pixi "build-py" action)
build-vegafusion-python-linux-64:
runs-on: ubuntu-latest
steps:
- name: Check out repository code
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
- name: Install latest stable Rust toolchain
uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # pin@v1
with:
toolchain: stable
override: true
- name: Cache rust dependencies
uses: Swatinem/rust-cache@81d053bdb0871dcd3f10763c8cc60d0adc41762b # pin@v1
with:
cache-on-failure: True
- name: Build wheels (Linux)
uses: messense/maturin-action@60d11847b29f81ca5375519a8eb33cc336ba4bfa # [email protected]
with:
command: build
manylinux: 2014
rust-toolchain: stable
args: --release -m vegafusion-python/Cargo.toml --features=protobuf-src --strip
- name: Upload artifacts
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # [email protected]
with:
name: vegafusion-python-wheels
path: |
target/wheels/*.tar.gz
target/wheels/*.whl
build-vegafusion-python-linux-arm64:
runs-on: ubuntu-latest
steps:
- name: Check out repository code
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
- name: Install latest stable Rust toolchain
uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # pin@v1
with:
toolchain: stable
override: true
- name: Cache rust dependencies
uses: Swatinem/rust-cache@81d053bdb0871dcd3f10763c8cc60d0adc41762b # pin@v1
with:
cache-on-failure: True
- name: Download arm64 toolchain
run: |
rustup target add aarch64-unknown-linux-gnu
- name: Build arm64 wheels
uses: messense/maturin-action@60d11847b29f81ca5375519a8eb33cc336ba4bfa # [email protected]
with:
command: build
manylinux: 2_28
rust-toolchain: stable
args: --release -m vegafusion-python/Cargo.toml --features=protobuf-src --strip --target aarch64-unknown-linux-gnu
- name: Upload artifacts
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # [email protected]
with:
name: vegafusion-python-wheels
path: |
target/wheels/*.tar.gz
target/wheels/*.whl
build-vegafusion-python-win-64:
runs-on: windows-2022
steps:
- name: Check out repository code
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
- name: Setup Python
uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # [email protected]
with:
python-version: '3.11'
- uses: prefix-dev/[email protected]
with:
pixi-version: v0.30.0
- name: Cache
uses: actions/cache@v3
with:
key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-python-win-64
path: |
~/.cargo
target
.pixi
- name: Build vegafusion-python
run: |
pixi run build-py
- name: Upload artifacts
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # [email protected]
with:
name: vegafusion-python-wheels
path: |
target/wheels/*.tar.gz
target/wheels/*.whl
build-vegafusion-python-osx-64:
runs-on: macos-12
steps:
- name: Check out repository code
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
- uses: prefix-dev/[email protected]
with:
pixi-version: v0.30.0
- name: Cache
uses: actions/cache@v3
with:
key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-python-osx-64
path: |
~/.cargo
target
.pixi
- name: Build vegafusion-python
run: |
pixi run build-py
- name: Upload artifacts
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # [email protected]
with:
name: vegafusion-python-wheels
path: |
target/wheels/*.tar.gz
target/wheels/*.whl
build-vegafusion-python-osx-arm64:
runs-on: macos-14
steps:
- name: Check out repository code
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
- uses: prefix-dev/[email protected]
with:
pixi-version: v0.30.0
- name: Cache
uses: actions/cache@v3
with:
key: osx-arm64-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-python-osx-arm64
path: |
~/.cargo
target
.pixi
- name: Build vegafusion-python
run: |
pixi run build-py --target aarch64-apple-darwin
- name: Upload artifacts
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # [email protected]
with:
name: vegafusion-python-wheels
path: |
target/wheels/*.tar.gz
target/wheels/*.whl
test-vegafusion-python-linux-64:
runs-on: ubuntu-20.04
needs: [build-vegafusion-python-linux-64]
steps:
- name: Check out repository code
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
- name: Setup Python
uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # [email protected]
with:
python-version: '3.11'
- name: Install Chrome
uses: browser-actions/setup-chrome@f0ff752add8c926994566c80b3ceadfd03f24d12 # pin@latest
with:
chrome-version: stable
- name: Install fonts on Linux
if: runner.os == 'Linux'
run: |
echo ttf-mscorefonts-installer msttcorefonts/accepted-mscorefonts-eula select true | sudo debconf-set-selections
sudo apt-get install ttf-mscorefonts-installer
- name: Download vegafusion-python wheel
uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 # pin@v2
with:
name: vegafusion-python-wheels
path: target/wheels/
- name: install wheels
working-directory: target/wheels/
run: |
ls -la
python -m pip install vegafusion-*manylinux_2_17_x86_64*.whl
# Optional dependencies
python -m pip install polars-lts-cpu "duckdb>=1.0" "vl-convert-python>=1.0.1rc1" scikit-image "pandas>=2.2" jupytext voila anywidget ipywidgets chromedriver-binary-auto
# Test dependencies
python -m pip install pytest altair vega-datasets scikit-image jupytext voila ipykernel anywidget ipywidgets selenium flaky tenacity chromedriver-binary-auto
- name: Test lazy imports
working-directory: vegafusion-python/
run: python checks/check_lazy_imports.py
- name: Test vegafusion
working-directory: vegafusion-python/
env:
VEGAFUSION_TEST_HEADLESS: 1
run: pytest
- name: Upload test artifacts
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # [email protected]
if: always()
with:
name: vegafusion-python-test-failures
path: |
vegafusion-python/tests/output/*
test-vegafusion-python-osx-arm64:
runs-on: macos-14
needs: [build-vegafusion-python-osx-arm64]
steps:
- name: Check out repository code
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
- name: Setup Python
uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # [email protected]
with:
python-version: '3.10'
- name: Download vegafusion-python wheel
uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 # pin@v2
with:
name: vegafusion-python-wheels
path: target/wheels/
- name: install wheels
working-directory: target/wheels/
run: |
ls -la
python -m pip install vegafusion-*macosx_11_*_arm64.whl
# Optional dependencies
python -m pip install polars "duckdb>=1.0" vl-convert-python "pandas>=2.2"
# Test dependencies
python -m pip install pytest altair vega-datasets scikit-image
- name: Test vegafusion
working-directory: vegafusion-python/
run: pytest --ignore=tests/test_jupyter_widget.py
test-vegafusion-python-win-64:
runs-on: windows-2022
needs: [build-vegafusion-python-win-64]
steps:
- name: Check out repository code
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
- name: Setup Python
uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # [email protected]
with:
python-version: '3.9'
- name: Install Chrome
uses: browser-actions/setup-chrome@f0ff752add8c926994566c80b3ceadfd03f24d12 # pin@latest
with:
chrome-version: stable
- name: Download vegafusion-python wheel
uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 # pin@v2
with:
name: vegafusion-python-wheels
path: target/wheels/
- name: install wheels
working-directory: target/wheels/
shell: powershell
run: |
Get-ChildItem -Force
$vegafusion = Get-ChildItem -Name "vegafusion-*win_amd64.whl" | Select-Object -First 1
python -m pip install $vegafusion
# Optional dependencies
python -m pip install "numpy<2" polars[timezone] "duckdb>=1.0" vl-convert-python
# Test dependencies
python -m pip install pytest altair vega-datasets scikit-image
- name: Test vegafusion
working-directory: vegafusion-python/
run: pytest --ignore=tests/test_jupyter_widget.py
build-vegafusion-server-linux-64:
runs-on: ubuntu-20.04
steps:
- name: Check out repository code
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
- uses: prefix-dev/[email protected]
with:
pixi-version: v0.30.0
- name: Cache
uses: actions/cache@v3
with:
key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-server-linux-64
path: |
~/.cargo
target
.pixi
- name: Build vegafusion-server
run: |
pixi run build-rs-vegafusion-server
- name: zip executable
uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # [email protected]
with:
files: target/release/vegafusion-server
dest: vegafusion-server-linux-64.zip
- name: Upload artifacts
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # [email protected]
with:
name: vegafusion-server
path: |
vegafusion-server-*
# # Below is an Attempt to cross compile to linux aarch64 within conda environment.
# # It fails in compiling protobuf-src. So instead we fall back to the direct use of
# # cargo (without involving conda).
#
# build-vegafusion-server-linux-arm64:
# runs-on: ubuntu-20.04
# steps:
# - name: Check out repository code
# uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
# - name: Install pixi
# run: curl -fsSL https://pixi.sh/install.sh | bash && echo "${HOME}/.pixi/bin" >> $GITHUB_PATH
# - name: Cache
# uses: actions/cache@v3
# with:
# key: linux-arm64-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-server
# path: |
# ~/.cargo
# target
# .pixi
# - name: Build vegafusion-server
# run: |
# pixi run python automation/download_rust_target.py aarch64-unknown-linux-gnu
# pixi add gcc_linux-aarch64 -p linux-64
# export RUSTFLAGS="-C linker=aarch64-linux-gnu-gcc"
# pixi run build-rs-vegafusion-server --features=protobuf-src --target aarch64-unknown-linux-gnu
# - name: zip executable
# uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # [email protected]
# with:
# files: target/aarch64-unknown-linux-gnu/release/vegafusion-server
# dest: vegafusion-server-linux-arm64.zip
# - name: Upload artifacts
# uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # [email protected]
# with:
# name: vegafusion-server
# path: |
# vegafusion-server-*
build-vegafusion-server-linux-arm64:
runs-on: ubuntu-latest
steps:
- name: Check out repository code
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
- name: Install latest stable Rust toolchain
uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af # pin@v1
with:
toolchain: stable
override: true
- name: Cache rust dependencies
uses: Swatinem/rust-cache@81d053bdb0871dcd3f10763c8cc60d0adc41762b # pin@v1
with:
cache-on-failure: True
- name: Download arm64 toolchain
run: |
rustup target add aarch64-unknown-linux-gnu
- name: Download gcc for cross compilation
run: |
sudo apt-get update
sudo apt-get install gcc-aarch64-linux-gnu
- name: Build vegafusion-server
uses: actions-rs/cargo@844f36862e911db73fe0815f00a4a2602c279505 # pin@v1
env:
RUSTFLAGS: "-C linker=aarch64-linux-gnu-gcc"
with:
command: build
args: -p vegafusion-server --release --features=protobuf-src --target=aarch64-unknown-linux-gnu
- name: zip executable
uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # [email protected]
with:
files: target/aarch64-unknown-linux-gnu/release/vegafusion-server
dest: vegafusion-server-linux-arm64.zip
- name: Upload artifacts
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # [email protected]
with:
name: vegafusion-server
path: |
vegafusion-server-*
build-vegafusion-server-win-64:
runs-on: windows-2022
steps:
- name: Check out repository code
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
- uses: prefix-dev/[email protected]
with:
pixi-version: v0.30.0
- name: Cache
uses: actions/cache@v3
with:
key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-server-win-64b
path: |
~/.cargo
target
.pixi
- name: Build vegafusion-server
run: |
pixi install -vvv
pixi run build-rs-vegafusion-server
- name: zip executable
uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # [email protected]
with:
files: target/release/vegafusion-server.exe
dest: vegafusion-server-win-64.zip
- name: Upload artifacts
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # [email protected]
with:
name: vegafusion-server
path: |
vegafusion-server-*
build-vegafusion-server-osx-64:
runs-on: macos-13
steps:
- name: Check out repository code
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
- uses: prefix-dev/[email protected]
with:
pixi-version: v0.30.0
- name: Cache
uses: actions/cache@v3
with:
key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-server-osx-64
path: |
~/.cargo
target
.pixi
- name: Build vegafusion-server
run: |
pixi run build-rs-vegafusion-server
- name: zip executable
uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # [email protected]
with:
files: target/release/vegafusion-server
dest: vegafusion-server-osx-64.zip
- name: Upload artifacts
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # [email protected]
with:
name: vegafusion-server
path: |
vegafusion-server-*
build-vegafusion-server-osx-arm64:
runs-on: macos-14
steps:
- name: Check out repository code
uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
- uses: prefix-dev/[email protected]
with:
pixi-version: v0.30.0
- name: Cache
uses: actions/cache@v3
with:
key: osx-arm64-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-server-osx-arm64
path: |
~/.cargo
target
.pixi
- name: Build vegafusion-server
run: |
pixi run build-rs-vegafusion-server --target aarch64-apple-darwin
- name: zip executable
uses: vimtor/action-zip@26a249fb00d43ca98dad77a4b3838025fc226aa1 # [email protected]
with:
files: target/aarch64-apple-darwin/release/vegafusion-server
dest: vegafusion-server-osx-arm64.zip
- name: Upload artifacts
uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # [email protected]
with:
name: vegafusion-server
path: |
vegafusion-server-*