From f011a9636ae4b9a66dc09d2a2e7b85e444a28f18 Mon Sep 17 00:00:00 2001 From: Lei Xu Date: Sun, 23 Jun 2024 17:12:22 -0700 Subject: [PATCH 01/10] build --- .github/workflows/java-publish.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 .github/workflows/java-publish.yml diff --git a/.github/workflows/java-publish.yml b/.github/workflows/java-publish.yml new file mode 100644 index 0000000000..1f0f6e181d --- /dev/null +++ b/.github/workflows/java-publish.yml @@ -0,0 +1,21 @@ +name: Build and publish Java packages +on: + release: + types: [released] + +jobs: + linux-x86: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - uses: Swatinem/rust-cache@v2 + with: + workspaces: java/core/lance-jni + - name: Set up Java 8 + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: 8 + cache: "maven" From 12303fd3f7f17a5030aaa4cebcd0fa4c30b7874f Mon Sep 17 00:00:00 2001 From: Lei Xu Date: Sun, 23 Jun 2024 17:18:07 -0700 Subject: [PATCH 02/10] use ubuntu 22.04 --- .github/workflows/java-publish.yml | 85 ++++++++++++++++++++++++++++-- java/core/pom.xml | 2 +- java/pom.xml | 9 +++- 3 files changed, 91 insertions(+), 5 deletions(-) diff --git a/.github/workflows/java-publish.yml b/.github/workflows/java-publish.yml index 1f0f6e181d..97efa29695 100644 --- a/.github/workflows/java-publish.yml +++ b/.github/workflows/java-publish.yml @@ -2,20 +2,99 @@ name: Build and publish Java packages on: release: types: [released] + pull_request: + paths: + - .github/workflows/java-publish.yml jobs: - linux-x86: - runs-on: ubuntu-latest + macos-arm64: + name: Build on MacOS Arm64 + runs-on: macos-14 + timeout-minutes: 30 + defaults: + run: + working-directory: ./java + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - uses: Swatinem/rust-cache@v2 + - name: Install dependencies + run: | + brew install protobuf + - name: Build release + run: | + cargo build --release + - uses: actions/upload-artifact@v4 + with: + name: liblance_jni_darwin_aarch64.zip + path: /home/runner/work/lance/lance/target/release/liblance_jni.dylib + retention-days: 1 + if-no-files-found: error + linux-arm64: + name: Build on Linux Arm64 + runs-on: warp-ubuntu-2204-x64-8x timeout-minutes: 30 + defaults: + run: + working-directory: ./java steps: - name: Checkout repository uses: actions/checkout@v4 - uses: Swatinem/rust-cache@v2 with: - workspaces: java/core/lance-jni + workspaces: rust + - uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + toolchain: "1.79.0" + cache-workspaces: "src/rust" + # Disable full debug symbol generation to speed up CI build and keep memory down + # "1" means line tables only, which is useful for panic tracebacks. + rustflags: "-C debuginfo=1" + components: "rustfmt,clippy" + - name: Install dependencies + run: | + sudo apt -y -qq update + sudo apt install -y protobuf-compiler libssl-dev pkg-config + - name: Build release + run: | + cargo build --release + - uses: actions/upload-artifact@v4 + with: + name: liblance_jni_linux_aarch64.zip + path: /home/runner/work/lance/lance/target/release/liblance_jni.so + retention-days: 1 + if-no-files-found: error + linux-x86: + runs-on: ubuntu-22.04 + timeout-minutes: 30 + needs: [macos-arm64, linux-arm64] + defaults: + run: + working-directory: ./java + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - uses: Swatinem/rust-cache@v2 - name: Set up Java 8 uses: actions/setup-java@v4 with: distribution: temurin java-version: 8 cache: "maven" + server-id: ossrh + server-username: ${{ secrets.SONATYPE_USER }} + server-password: ${{ secrets.SONATYPE_TOKEN }} + - name: Install dependencies + run: | + sudo apt update + sudo apt install -y protobuf-compiler libssl-dev + - name: Download artifact + uses: actions/download-artifact@v4 + - name: Copy native libs + run: | + mkdir -p ./core/target/classes/nativelib/darwin-aarch64 ./core/target/classes/nativelib/linux-aarch64 + cp ../liblance_jni_darwin_aarch64.zip/liblance_jni.dylib ./core/target/classes/nativelib/darwin-aarch64/liblance_jni.dylib + cp ../liblance_jni_linux_aarch64.zip/liblance_jni.so ./core/target/classes/nativelib/linux-aarch64/liblance_jni.so + - name: Build and publish with Java 8 + run: | + mvn release:prepare -DdryRun=true diff --git a/java/core/pom.xml b/java/core/pom.xml index 8d8c74b0d3..00323c278e 100644 --- a/java/core/pom.xml +++ b/java/core/pom.xml @@ -68,7 +68,7 @@ lance-jni - + true ${project.build.directory}/classes/nativelib true diff --git a/java/pom.xml b/java/pom.xml index a4dd4f3dd3..6bb5e5240c 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -22,6 +22,12 @@ spark + + scm:git:https://github.com/lancedb/lance.git + scm:git:ssh://git@github.com/lancedb/lance.git + https://github.com/lancedb/lance + + @@ -182,7 +188,8 @@ 3.2.5 --add-opens=java.base/java.nio=ALL-UNNAMED - + false From 36181b03e0cf575242557b6d42df31f226aaea87 Mon Sep 17 00:00:00 2001 From: Lei Xu Date: Mon, 24 Jun 2024 10:17:36 -0700 Subject: [PATCH 03/10] show pwd --- .github/workflows/java-publish.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/java-publish.yml b/.github/workflows/java-publish.yml index 97efa29695..b19386b626 100644 --- a/.github/workflows/java-publish.yml +++ b/.github/workflows/java-publish.yml @@ -24,6 +24,7 @@ jobs: - name: Build release run: | cargo build --release + pwd - uses: actions/upload-artifact@v4 with: name: liblance_jni_darwin_aarch64.zip From 69b7655bb48bc3e73b219617edaeada76257eb09 Mon Sep 17 00:00:00 2001 From: Lei Xu Date: Mon, 24 Jun 2024 10:18:24 -0700 Subject: [PATCH 04/10] copy lib --- .github/workflows/java-publish.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/java-publish.yml b/.github/workflows/java-publish.yml index b19386b626..f31f744e96 100644 --- a/.github/workflows/java-publish.yml +++ b/.github/workflows/java-publish.yml @@ -24,11 +24,11 @@ jobs: - name: Build release run: | cargo build --release - pwd + cp ../target/release/liblance_jni.dylib liblance_jni.dylib - uses: actions/upload-artifact@v4 with: name: liblance_jni_darwin_aarch64.zip - path: /home/runner/work/lance/lance/target/release/liblance_jni.dylib + path: liblance_jni.dylib retention-days: 1 if-no-files-found: error linux-arm64: @@ -59,10 +59,11 @@ jobs: - name: Build release run: | cargo build --release + cp ../target/release/liblance_jni.so liblance_jni.so - uses: actions/upload-artifact@v4 with: name: liblance_jni_linux_aarch64.zip - path: /home/runner/work/lance/lance/target/release/liblance_jni.so + path: liblance_jni.so retention-days: 1 if-no-files-found: error linux-x86: From 32b6f29ee15ccbe6fc543f0f876e22da7cf03efe Mon Sep 17 00:00:00 2001 From: Lei Xu Date: Mon, 24 Jun 2024 10:29:12 -0700 Subject: [PATCH 05/10] related path --- .github/workflows/java-publish.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/java-publish.yml b/.github/workflows/java-publish.yml index f31f744e96..f22d9fce36 100644 --- a/.github/workflows/java-publish.yml +++ b/.github/workflows/java-publish.yml @@ -24,11 +24,10 @@ jobs: - name: Build release run: | cargo build --release - cp ../target/release/liblance_jni.dylib liblance_jni.dylib - uses: actions/upload-artifact@v4 with: name: liblance_jni_darwin_aarch64.zip - path: liblance_jni.dylib + path: target/release/liblance_jni.dylib retention-days: 1 if-no-files-found: error linux-arm64: @@ -63,7 +62,7 @@ jobs: - uses: actions/upload-artifact@v4 with: name: liblance_jni_linux_aarch64.zip - path: liblance_jni.so + path: target/release/liblance_jni.so retention-days: 1 if-no-files-found: error linux-x86: From b4ce8bf08de51a293c7758c19f53fafd2fa401c4 Mon Sep 17 00:00:00 2001 From: Lu Qiu Date: Mon, 24 Jun 2024 21:57:18 -0700 Subject: [PATCH 06/10] Try to revert to nodejs 16 --- .github/workflows/java-publish.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/java-publish.yml b/.github/workflows/java-publish.yml index f22d9fce36..d3e19d2881 100644 --- a/.github/workflows/java-publish.yml +++ b/.github/workflows/java-publish.yml @@ -1,4 +1,4 @@ -name: Build and publish Java packages +name: Build and publish Java packages Lu debug on: release: types: [released] @@ -40,6 +40,10 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4 + - name: Set up Node.js 16 + uses: actions/setup-node@v4 + with: + node-version: '16' - uses: Swatinem/rust-cache@v2 with: workspaces: rust From 8f1ae9d7da61e5d494c14fbcc7137aab8245bc3d Mon Sep 17 00:00:00 2001 From: Lu Qiu Date: Mon, 24 Jun 2024 22:34:48 -0700 Subject: [PATCH 07/10] Build spark jar --- .github/workflows/java-publish.yml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/.github/workflows/java-publish.yml b/.github/workflows/java-publish.yml index d3e19d2881..d5c91f76d4 100644 --- a/.github/workflows/java-publish.yml +++ b/.github/workflows/java-publish.yml @@ -100,6 +100,20 @@ jobs: mkdir -p ./core/target/classes/nativelib/darwin-aarch64 ./core/target/classes/nativelib/linux-aarch64 cp ../liblance_jni_darwin_aarch64.zip/liblance_jni.dylib ./core/target/classes/nativelib/darwin-aarch64/liblance_jni.dylib cp ../liblance_jni_linux_aarch64.zip/liblance_jni.so ./core/target/classes/nativelib/linux-aarch64/liblance_jni.so - - name: Build and publish with Java 8 + #- name: Build and publish with Java 8 + # run: | + # mvn release:prepare -DdryRun=true + + - name: Create Lance Spark Jar with scala 2.13 run: | - mvn release:prepare -DdryRun=true + mvn clean install -Pscala-2.13 + JAR_PATH=$(find spark/target -name 'lance-spark-2.13-*-jar-with-dependencies.jar') + JAR_NAME=$(basename $JAR_PATH) + echo "JAR_PATH=$JAR_PATH" >> $GITHUB_ENV + echo "JAR_NAME=$JAR_NAME" >> $GITHUB_ENV + - uses: actions/upload-artifact@v4 + with: + name: ${{ env.JAR_NAME }} + path: ${{ env.JAR_PATH }} + retention-days: 1 + if-no-files-found: error From 5ae37bd2709b24ad030423c9946eab66f2cee59c Mon Sep 17 00:00:00 2001 From: Lu Qiu Date: Tue, 25 Jun 2024 10:02:39 -0700 Subject: [PATCH 08/10] upload-artifacts do not respect working directory config --- .github/workflows/java-publish.yml | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/.github/workflows/java-publish.yml b/.github/workflows/java-publish.yml index d5c91f76d4..96d13c28ac 100644 --- a/.github/workflows/java-publish.yml +++ b/.github/workflows/java-publish.yml @@ -80,6 +80,10 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - uses: Swatinem/rust-cache@v2 + - name: Set up Node.js 16 + uses: actions/setup-node@v4 + with: + node-version: '16' - name: Set up Java 8 uses: actions/setup-java@v4 with: @@ -97,18 +101,19 @@ jobs: uses: actions/download-artifact@v4 - name: Copy native libs run: | - mkdir -p ./core/target/classes/nativelib/darwin-aarch64 ./core/target/classes/nativelib/linux-aarch64 - cp ../liblance_jni_darwin_aarch64.zip/liblance_jni.dylib ./core/target/classes/nativelib/darwin-aarch64/liblance_jni.dylib - cp ../liblance_jni_linux_aarch64.zip/liblance_jni.so ./core/target/classes/nativelib/linux-aarch64/liblance_jni.so - #- name: Build and publish with Java 8 + mkdir -p ./core/src/main/resources/nativelib/darwin-aarch64 ./core/src/main/resources/nativelib/linux-aarch64 + cp ../liblance_jni_darwin_aarch64.zip/liblance_jni.dylib ./core/src/main/resources/nativelib/darwin-aarch64/liblance_jni.dylib + cp ../liblance_jni_linux_aarch64.zip/liblance_jni.so ./core/src/main/resources/nativelib/linux-aarch64/liblance_jni.so + - name: Build and publish with Java 8 # run: | # mvn release:prepare -DdryRun=true - - - name: Create Lance Spark Jar with scala 2.13 run: | mvn clean install -Pscala-2.13 - JAR_PATH=$(find spark/target -name 'lance-spark-2.13-*-jar-with-dependencies.jar') - JAR_NAME=$(basename $JAR_PATH) + - name: Find Spark Jar + run: | + RELATIVE_JAR_PATH=$(find spark/target -name 'lance-spark-2.13-*-jar-with-dependencies.jar') + JAR_NAME=$(basename $RELATIVE_JAR_PATH) + JAR_PATH=$(readlink -f "$RELATIVE_JAR_PATH") echo "JAR_PATH=$JAR_PATH" >> $GITHUB_ENV echo "JAR_NAME=$JAR_NAME" >> $GITHUB_ENV - uses: actions/upload-artifact@v4 From ee66a0810a6eb8350e1edf5a312e12ec583e8ccd Mon Sep 17 00:00:00 2001 From: Lu Qiu Date: Tue, 25 Jun 2024 20:03:25 -0700 Subject: [PATCH 09/10] cleanup --- .github/workflows/java-publish.yml | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/.github/workflows/java-publish.yml b/.github/workflows/java-publish.yml index 96d13c28ac..73eeddcc25 100644 --- a/.github/workflows/java-publish.yml +++ b/.github/workflows/java-publish.yml @@ -1,4 +1,4 @@ -name: Build and publish Java packages Lu debug +name: Build and publish Java packages on: release: types: [released] @@ -18,6 +18,10 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - uses: Swatinem/rust-cache@v2 + - name: Set up Node.js 16 + uses: actions/setup-node@v4 + with: + node-version: '16' - name: Install dependencies run: | brew install protobuf @@ -40,13 +44,13 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4 + - uses: Swatinem/rust-cache@v2 + with: + workspaces: rust - name: Set up Node.js 16 uses: actions/setup-node@v4 with: node-version: '16' - - uses: Swatinem/rust-cache@v2 - with: - workspaces: rust - uses: actions-rust-lang/setup-rust-toolchain@v1 with: toolchain: "1.79.0" @@ -105,13 +109,12 @@ jobs: cp ../liblance_jni_darwin_aarch64.zip/liblance_jni.dylib ./core/src/main/resources/nativelib/darwin-aarch64/liblance_jni.dylib cp ../liblance_jni_linux_aarch64.zip/liblance_jni.so ./core/src/main/resources/nativelib/linux-aarch64/liblance_jni.so - name: Build and publish with Java 8 - # run: | - # mvn release:prepare -DdryRun=true run: | - mvn clean install -Pscala-2.13 + mvn release:prepare -DdryRun=true + mvn clean install - name: Find Spark Jar run: | - RELATIVE_JAR_PATH=$(find spark/target -name 'lance-spark-2.13-*-jar-with-dependencies.jar') + RELATIVE_JAR_PATH=$(find spark/target -name 'lance-spark-2.12-*-jar-with-dependencies.jar') JAR_NAME=$(basename $RELATIVE_JAR_PATH) JAR_PATH=$(readlink -f "$RELATIVE_JAR_PATH") echo "JAR_PATH=$JAR_PATH" >> $GITHUB_ENV From 8a889d318b2f4f2f33af9dcadd37c690a3672fc1 Mon Sep 17 00:00:00 2001 From: Lu Qiu Date: Wed, 26 Jun 2024 10:02:53 -0700 Subject: [PATCH 10/10] Node js 18 --- .github/workflows/java-publish.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/java-publish.yml b/.github/workflows/java-publish.yml index 73eeddcc25..59bdbf1810 100644 --- a/.github/workflows/java-publish.yml +++ b/.github/workflows/java-publish.yml @@ -18,10 +18,10 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - uses: Swatinem/rust-cache@v2 - - name: Set up Node.js 16 + - name: Set up Node.js 18 uses: actions/setup-node@v4 with: - node-version: '16' + node-version: '18' - name: Install dependencies run: | brew install protobuf @@ -47,10 +47,10 @@ jobs: - uses: Swatinem/rust-cache@v2 with: workspaces: rust - - name: Set up Node.js 16 + - name: Set up Node.js 18 uses: actions/setup-node@v4 with: - node-version: '16' + node-version: '18' - uses: actions-rust-lang/setup-rust-toolchain@v1 with: toolchain: "1.79.0" @@ -84,10 +84,10 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - uses: Swatinem/rust-cache@v2 - - name: Set up Node.js 16 + - name: Set up Node.js 18 uses: actions/setup-node@v4 with: - node-version: '16' + node-version: '18' - name: Set up Java 8 uses: actions/setup-java@v4 with: @@ -110,7 +110,7 @@ jobs: cp ../liblance_jni_linux_aarch64.zip/liblance_jni.so ./core/src/main/resources/nativelib/linux-aarch64/liblance_jni.so - name: Build and publish with Java 8 run: | - mvn release:prepare -DdryRun=true + # mvn release:prepare -DdryRun=true mvn clean install - name: Find Spark Jar run: |