diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 000000000..0f4a190b6 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,11 @@ +[target.x86_64-apple-darwin] +rustflags = [ + "-C", "link-arg=-undefined", + "-C", "link-arg=dynamic_lookup", +] + +[target.aarch64-apple-darwin] +rustflags = [ + "-C", "link-arg=-undefined", + "-C", "link-arg=dynamic_lookup", +] \ No newline at end of file diff --git a/.github/workflows/binaries.yml b/.github/workflows/binaries.yml index 86b6b7de0..f6d20f2db 100644 --- a/.github/workflows/binaries.yml +++ b/.github/workflows/binaries.yml @@ -17,8 +17,8 @@ jobs: matrix: # see https://docs.github.com/en/actions/using-github-hosted-runners/using-github-hosted-runners/about-github-hosted-runners#standard-github-hosted-runners-for-public-repositories config: - - { runner: buildjet-32vcpu-ubuntu-2204, protoc: linux-x86_64, artifact: linux-x86_64 } - - { runner: buildjet-32vcpu-ubuntu-2204-arm, protoc: linux-aarch_64, artifact: linux-arm64 } + - { runner: buildjet-32vcpu-ubuntu-2204, protoc: linux-x86_64, pyarch: x86_64, artifact: linux-x86_64 } + - { runner: buildjet-32vcpu-ubuntu-2204-arm, protoc: linux-aarch_64, pyarch: aarch64, artifact: linux-arm64 } runs-on: ${{ matrix.config.runner }} services: postgres: @@ -47,6 +47,15 @@ jobs: - name: Update rust run: | rustup update + - name: Install Python 3.12 + run: | + curl -OL https://github.com/indygreg/python-build-standalone/releases/download/20240814/cpython-3.12.5+20240814-${{ matrix.config.pyarch }}-unknown-linux-gnu-install_only.tar.gz + tar xvfz cpython*.tar.gz + sudo cp -r python/bin/* /usr/local/bin/ + sudo cp -r python/include/* /usr/local/include/ + sudo cp -r python/lib/* /usr/local/lib/ + sudo cp -r python/share/* /usr/local/share/ + sudo ldconfig - name: Run DB migrations run: | @@ -62,41 +71,41 @@ jobs: path: target/release/arroyo if-no-files-found: error - macos: - strategy: - fail-fast: true - matrix: - # see https://docs.github.com/en/actions/using-github-hosted-runners/using-github-hosted-runners/about-github-hosted-runners#standard-github-hosted-runners-for-public-repositories - config: - - { runner: macos-13, protoc: osx-x86_64, artifact: macos-x86_64 } - - { runner: macos-14, protoc: osx-aarch_64, artifact: macos-m1 } - runs-on: ${{ matrix.config.runner }} - steps: - - name: Check out - uses: actions/checkout@v4 - - name: Setup pnpm - uses: pnpm/action-setup@v4 - with: - version: 9.7.1 - - name: Install protoc compiler - run: | - wget https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOC_VERSION/protoc-$PROTOC_VERSION-${{ matrix.config.protoc }}.zip - unzip protoc*.zip && sudo mv bin/protoc /usr/local/bin - - name: Install Postgres and prepare DB - run: | - brew install postgresql@14 && brew services start postgresql && sleep 10 - psql postgres -c "CREATE USER arroyo WITH PASSWORD 'arroyo' SUPERUSER;" - createdb arroyo - - name: Run DB migrations - run: | - cargo install --debug refinery_cli --version $REFINERY_VERSION - refinery migrate -e REFINERY_CONFIG -p crates/arroyo-api/migrations - - name: Run frontend build - run: cd webui && pnpm install && pnpm build - - name: Build Arroyo - run: cargo build --release --package arroyo && strip target/release/arroyo - - uses: actions/upload-artifact@v4 - with: - name: arroyo-${{ matrix.config.artifact }} - path: target/release/arroyo - if-no-files-found: error +# macos: +# strategy: +# fail-fast: true +# matrix: +# # see https://docs.github.com/en/actions/using-github-hosted-runners/using-github-hosted-runners/about-github-hosted-runners#standard-github-hosted-runners-for-public-repositories +# config: +# - { runner: macos-13, protoc: osx-x86_64, artifact: macos-x86_64 } +# - { runner: macos-14, protoc: osx-aarch_64, artifact: macos-m1 } +# runs-on: ${{ matrix.config.runner }} +# steps: +# - name: Check out +# uses: actions/checkout@v4 +# - name: Setup pnpm +# uses: pnpm/action-setup@v4 +# with: +# version: 9.7.1 +# - name: Install protoc compiler +# run: | +# wget https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOC_VERSION/protoc-$PROTOC_VERSION-${{ matrix.config.protoc }}.zip +# unzip protoc*.zip && sudo mv bin/protoc /usr/local/bin +# - name: Install Postgres and prepare DB +# run: | +# brew install postgresql@14 && brew services start postgresql && sleep 10 +# psql postgres -c "CREATE USER arroyo WITH PASSWORD 'arroyo' SUPERUSER;" +# createdb arroyo +# - name: Run DB migrations +# run: | +# cargo install --debug refinery_cli --version $REFINERY_VERSION +# refinery migrate -e REFINERY_CONFIG -p crates/arroyo-api/migrations +# - name: Run frontend build +# run: cd webui && pnpm install && pnpm build +# - name: Build Arroyo +# run: cargo build --release --package arroyo && strip target/release/arroyo +# - uses: actions/upload-artifact@v4 +# with: +# name: arroyo-${{ matrix.config.artifact }} +# path: target/release/arroyo +# if-no-files-found: error diff --git a/Cargo.toml b/Cargo.toml index b28fee177..5224829a2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -65,7 +65,6 @@ debug = 1 [profile.dev] split-debuginfo = "unpacked" - [patch.crates-io] deltalake = { git = 'https://github.com/delta-io/delta-rs', rev = 'e75a0b49b40f35ed361444bbea0e5720f359d732' } typify = { git = 'https://github.com/ArroyoSystems/typify.git', branch = 'arroyo' } diff --git a/crates/arroyo-udf/arroyo-udf-python/Cargo.toml b/crates/arroyo-udf/arroyo-udf-python/Cargo.toml index 3acf4282c..04df5b865 100644 --- a/crates/arroyo-udf/arroyo-udf-python/Cargo.toml +++ b/crates/arroyo-udf/arroyo-udf-python/Cargo.toml @@ -7,7 +7,7 @@ edition = "2021" arroyo-udf-common = { path = "../arroyo-udf-common" } arrow = { workspace = true, features = ["ffi"] } datafusion = { workspace = true } -pyo3 = { version = "0.21"} +pyo3 = { version = "0.21", features = ["auto-initialize"]} anyhow = "1" tokio = { version = "1", features = ["full"] } itertools = "0.13.0" \ No newline at end of file