[CELEBORN] Upgrade celeborn to 0.4.1 to support scala 2.13-based comp…

…ilation (#6226)
apache · Jun 27, 2024 · 32808dd · 32808dd
1 parent ac227de
commit 32808dd
Show file tree

Hide file tree

Showing 6 changed files with 31 additions and 15 deletions.
diff --git a/.github/workflows/velox_docker.yml b/.github/workflows/velox_docker.yml
@@ -521,7 +521,7 @@ jobs:
       fail-fast: false
       matrix:
         spark: ["spark-3.2"]
-        celeborn: ["celeborn-0.4.0", "celeborn-0.3.2"]
+        celeborn: ["celeborn-0.4.1", "celeborn-0.3.2-incubating"]
     runs-on: ubuntu-20.04
     container: ubuntu:22.04
     steps:
@@ -557,8 +557,8 @@ jobs:
           fi
           echo "EXTRA_PROFILE: ${EXTRA_PROFILE}"
           cd /opt && mkdir -p celeborn && \
-          wget https://archive.apache.org/dist/incubator/celeborn/${{ matrix.celeborn }}-incubating/apache-${{ matrix.celeborn }}-incubating-bin.tgz && \
-          tar xzf apache-${{ matrix.celeborn }}-incubating-bin.tgz -C /opt/celeborn --strip-components=1 && cd celeborn && \
+          wget https://archive.apache.org/dist/celeborn/${{ matrix.celeborn }}/apache-${{ matrix.celeborn }}-bin.tgz && \
+          tar xzf apache-${{ matrix.celeborn }}-bin.tgz -C /opt/celeborn --strip-components=1 && cd celeborn && \
           mv ./conf/celeborn-env.sh.template ./conf/celeborn-env.sh && \
           bash -c "echo -e 'CELEBORN_MASTER_MEMORY=4g\nCELEBORN_WORKER_MEMORY=4g\nCELEBORN_WORKER_OFFHEAP_MEMORY=8g' > ./conf/celeborn-env.sh" && \
           bash -c "echo -e 'celeborn.worker.commitFiles.threads 128\nceleborn.worker.sortPartition.threads 64' > ./conf/celeborn-defaults.conf" && \

diff --git a/docs/get-started/ClickHouse.md b/docs/get-started/ClickHouse.md
@@ -679,13 +679,13 @@ spark.shuffle.manager=org.apache.spark.shuffle.gluten.celeborn.CelebornShuffleMa
 
 quickly start a celeborn cluster
 ```shell
-wget https://archive.apache.org/dist/incubator/celeborn/celeborn-0.3.0-incubating/apache-celeborn-0.3.0-incubating-bin.tgz && \
-tar -zxvf apache-celeborn-0.3.0-incubating-bin.tgz && \
-mv apache-celeborn-0.3.0-incubating-bin/conf/celeborn-defaults.conf.template apache-celeborn-0.3.0-incubating-bin/conf/celeborn-defaults.conf && \
-mv apache-celeborn-0.3.0-incubating-bin/conf/log4j2.xml.template apache-celeborn-0.3.0-incubating-bin/conf/log4j2.xml && \
+wget https://archive.apache.org/dist/celeborn/celeborn-0.3.2-incubating/apache-celeborn-0.3.2-incubating-bin.tgz && \
+tar -zxvf apache-celeborn-0.3.2-incubating-bin.tgz && \
+mv apache-celeborn-0.3.2-incubating-bin/conf/celeborn-defaults.conf.template apache-celeborn-0.3.2-incubating-bin/conf/celeborn-defaults.conf && \
+mv apache-celeborn-0.3.2-incubating-bin/conf/log4j2.xml.template apache-celeborn-0.3.2-incubating-bin/conf/log4j2.xml && \
 mkdir /opt/hadoop && chmod 777 /opt/hadoop && \
-echo -e "celeborn.worker.flusher.threads 4\nceleborn.worker.storage.dirs /tmp\nceleborn.worker.monitor.disk.enabled false" > apache-celeborn-0.3.0-incubating-bin/conf/celeborn-defaults.conf && \
-bash apache-celeborn-0.3.0-incubating-bin/sbin/start-master.sh && bash apache-celeborn-0.3.0-incubating-bin/sbin/start-worker.sh
+echo -e "celeborn.worker.flusher.threads 4\nceleborn.worker.storage.dirs /tmp\nceleborn.worker.monitor.disk.enabled false" > apache-celeborn-0.3.2-incubating-bin/conf/celeborn-defaults.conf && \
+bash apache-celeborn-0.3.2-incubating-bin/sbin/start-master.sh && bash apache-celeborn-0.3.2-incubating-bin/sbin/start-worker.sh
 ```
 
 ### Columnar shuffle mode

diff --git a/...common/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java b/...common/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornShuffleManager.java
@@ -217,7 +217,13 @@ public boolean unregisterShuffle(int shuffleId) {
       }
     }
     return CelebornUtils.unregisterShuffle(
-        lifecycleManager, shuffleClient, shuffleIdTracker, shuffleId, appUniqueId, isDriver());
+        lifecycleManager,
+        shuffleClient,
+        shuffleIdTracker,
+        shuffleId,
+        appUniqueId,
+        throwsFetchFailure,
+        isDriver());
   }
 
   @Override

diff --git a/...celeborn/common/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornUtils.java b/...celeborn/common/src/main/java/org/apache/spark/shuffle/gluten/celeborn/CelebornUtils.java
@@ -49,11 +49,21 @@ public static boolean unregisterShuffle(
       Object shuffleIdTracker,
       int appShuffleId,
       String appUniqueId,
+      boolean throwsFetchFailure,
       boolean isDriver) {
     try {
-      // for Celeborn 0.4.0
       try {
-        if (lifecycleManager != null) {
+        try {
+          // for Celeborn 0.4.1
+          if (lifecycleManager != null) {
+            Method unregisterAppShuffle =
+                lifecycleManager
+                    .getClass()
+                    .getMethod("unregisterAppShuffle", int.class, boolean.class);
+            unregisterAppShuffle.invoke(lifecycleManager, appShuffleId, throwsFetchFailure);
+          }
+        } catch (NoSuchMethodException ex) {
+          // for Celeborn 0.4.0
           Method unregisterAppShuffle =
               lifecycleManager.getClass().getMethod("unregisterAppShuffle", int.class);
           unregisterAppShuffle.invoke(lifecycleManager, appShuffleId);

diff --git a/pom.xml b/pom.xml
@@ -53,7 +53,7 @@
     <delta.package.name>delta-core</delta.package.name>
     <delta.version>2.4.0</delta.version>
     <delta.binary.version>24</delta.binary.version>
-    <celeborn.version>0.3.2-incubating</celeborn.version>
+    <celeborn.version>0.4.1</celeborn.version>
     <uniffle.version>0.8.0</uniffle.version>
     <arrow.version>15.0.0</arrow.version>
     <arrow-gluten.version>15.0.0-gluten</arrow-gluten.version>

diff --git a/tools/gluten-it/pom.xml b/tools/gluten-it/pom.xml
@@ -21,7 +21,7 @@
     <spark.version>3.4.2</spark.version>
     <scala.binary.version>2.12</scala.binary.version>
     <spark.major.version>3</spark.major.version>
-    <celeborn.version>0.3.0-incubating</celeborn.version>
+    <celeborn.version>0.3.2-incubating</celeborn.version>
     <uniffle.version>0.8.0</uniffle.version>
     <gluten.version>1.2.0-SNAPSHOT</gluten.version>
     <guava.version>32.0.1-jre</guava.version>
@@ -167,7 +167,7 @@
     <profile>
       <id>celeborn-0.4</id>
       <properties>
-        <celeborn.version>0.4.0-incubating</celeborn.version>
+        <celeborn.version>0.4.1</celeborn.version>
       </properties>
     </profile>
   </profiles>