Increase spark memory

dmwm · Jul 16, 2024 · bba8fa3 · bba8fa3
1 parent 7b0c51b
commit bba8fa3
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/cpueff-goweb/spark/cron4cpueff_goweb.sh b/cpueff-goweb/spark/cron4cpueff_goweb.sh
@@ -80,7 +80,7 @@ function run_spark() {
         --conf spark.shuffle.useOldFetchProtocol=true
         --conf "spark.driver.bindAddress=0.0.0.0" --conf "spark.driver.host=${K8SHOST}"
         --conf "spark.driver.port=${PORT1}" --conf "spark.driver.blockManager.port=${PORT2}"
-        --driver-memory=8g --executor-memory=8g --packages org.apache.spark:spark-avro_2.12:3.4.0
+        --driver-memory=16g --executor-memory=16g --packages org.apache.spark:spark-avro_2.12:3.4.0
     )
 
     py_input_args=(--hdfs_out_dir "$hdfs_out_dir" --start_date "$start_date" --end_date "$end_date")