update spark to 3.5.4 matching pyspark

mspass-team · Jan 9, 2025 · c1d61aa · c1d61aa
1 parent f07f06e
commit c1d61aa
Showing 1 changed file with 6 additions and 6 deletions.
diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml
@@ -40,14 +40,14 @@ jobs:
     - name: Install Apache Spark
       run: |
         mkdir -p /opt
-        wget -q -O /opt/spark.tgz https://archive.apache.org/dist/spark/spark-3.0.0/spark-3.0.0-bin-hadoop2.7.tgz
+        wget -q -O /opt/spark.tgz https://archive.apache.org/dist/spark/spark-3.5.4/spark-3.5.4-bin-hadoop3.tgz
         tar xzf /opt/spark.tgz -C /opt/
         rm /opt/spark.tgz
-        export SPARK_HOME=/opt/spark-3.0.0-bin-hadoop2.7
-        echo "SPARK_HOME=/opt/spark-3.0.0-bin-hadoop2.7" >> $GITHUB_ENV
-        export PATH=$PATH:/opt/spark-3.0.0-bin-hadoop2.7/bin
-        echo "/opt/spark-3.0.0-bin-hadoop2.7/bin" >> $GITHUB_PATH
-        python -m pip install --upgrade --upgrade-strategy eager pyspark pytest-spark
+        export SPARK_HOME=/opt/spark-3.5.4-bin-hadoop3
+        echo "SPARK_HOME=/opt/spark-3.5.4-bin-hadoop3" >> $GITHUB_ENV
+        export PATH=$PATH:/opt/spark-3.5.4-bin-hadoop3/bin
+        echo "/opt/spark-3.5.4-bin-hadoop3/bin" >> $GITHUB_PATH
+        PYSPARK_HADOOP_VERSION=3 python -m pip install --upgrade --upgrade-strategy eager pyspark pytest-spark
         sudo apt-get update && sudo apt-get install openjdk-8-jdk
     - name: Install
       run: python -m pip install -C--global-option=build -C--global-option=--debug -v '.[seisbench]'