From c1d61aa45c858b2ef7f78bde493a34de601f85eb Mon Sep 17 00:00:00 2001 From: wangyinz Date: Thu, 9 Jan 2025 12:21:59 +0800 Subject: [PATCH] update spark to 3.5.4 matching pyspark --- .github/workflows/python-package.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 220871d2d..6e7bb245a 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -40,14 +40,14 @@ jobs: - name: Install Apache Spark run: | mkdir -p /opt - wget -q -O /opt/spark.tgz https://archive.apache.org/dist/spark/spark-3.0.0/spark-3.0.0-bin-hadoop2.7.tgz + wget -q -O /opt/spark.tgz https://archive.apache.org/dist/spark/spark-3.5.4/spark-3.5.4-bin-hadoop3.tgz tar xzf /opt/spark.tgz -C /opt/ rm /opt/spark.tgz - export SPARK_HOME=/opt/spark-3.0.0-bin-hadoop2.7 - echo "SPARK_HOME=/opt/spark-3.0.0-bin-hadoop2.7" >> $GITHUB_ENV - export PATH=$PATH:/opt/spark-3.0.0-bin-hadoop2.7/bin - echo "/opt/spark-3.0.0-bin-hadoop2.7/bin" >> $GITHUB_PATH - python -m pip install --upgrade --upgrade-strategy eager pyspark pytest-spark + export SPARK_HOME=/opt/spark-3.5.4-bin-hadoop3 + echo "SPARK_HOME=/opt/spark-3.5.4-bin-hadoop3" >> $GITHUB_ENV + export PATH=$PATH:/opt/spark-3.5.4-bin-hadoop3/bin + echo "/opt/spark-3.5.4-bin-hadoop3/bin" >> $GITHUB_PATH + PYSPARK_HADOOP_VERSION=3 python -m pip install --upgrade --upgrade-strategy eager pyspark pytest-spark sudo apt-get update && sudo apt-get install openjdk-8-jdk - name: Install run: python -m pip install -C--global-option=build -C--global-option=--debug -v '.[seisbench]'