change spark version to 3.3

Azure · Aug 6, 2024 · e99f877 · e99f877
1 parent 1a086da
commit e99f877
Show file tree

Hide file tree

Showing 14 changed files with 23 additions and 23 deletions.
diff --git a/cli/jobs/pipelines-with-components/shakespear_sample_and_word_count_using_spark/pipeline.yml b/cli/jobs/pipelines-with-components/shakespear_sample_and_word_count_using_spark/pipeline.yml
@@ -32,7 +32,7 @@ jobs:
 
     resources:
       instance_type: Standard_E8S_V3
-      runtime_version: 3.1.0
+      runtime_version: 3.3.0
 
   count_word:
     type: spark
@@ -44,6 +44,6 @@ jobs:
 
     resources:
       instance_type: Standard_E8S_V3
-      runtime_version: 3.1.0
+      runtime_version: 3.3.0
 
 
diff --git a/cli/jobs/pipelines/add-column-and-word-count-using-spark/pipeline.yml b/cli/jobs/pipelines/add-column-and-word-count-using-spark/pipeline.yml
@@ -36,7 +36,7 @@ jobs:
 
     resources:
       instance_type: Standard_E8S_V3
-      runtime_version: 3.1.0
+      runtime_version: 3.3.0
 
     conf:
       spark.driver.cores: 2
@@ -71,7 +71,7 @@ jobs:
 
     resources:
       instance_type: Standard_E8S_V3
-      runtime_version: 3.1.0
+      runtime_version: 3.3.0
 
     conf:
       spark.driver.cores: 2

diff --git a/cli/jobs/spark/serverless-spark-pipeline-default-identity.yml b/cli/jobs/spark/serverless-spark-pipeline-default-identity.yml
@@ -22,4 +22,4 @@ jobs:
 
     resources:
       instance_type: standard_e8s_v3
-      runtime_version: "3.2"
+      runtime_version: "3.3"
diff --git a/cli/jobs/spark/serverless-spark-pipeline-managed-identity.yml b/cli/jobs/spark/serverless-spark-pipeline-managed-identity.yml
@@ -25,4 +25,4 @@ jobs:
 
     resources:
       instance_type: standard_e8s_v3
-      runtime_version: "3.2"
+      runtime_version: "3.3"
diff --git a/cli/jobs/spark/serverless-spark-pipeline-user-identity.yml b/cli/jobs/spark/serverless-spark-pipeline-user-identity.yml
@@ -25,4 +25,4 @@ jobs:
 
     resources:
       instance_type: standard_e8s_v3
-      runtime_version: "3.2"
+      runtime_version: "3.3"
diff --git a/cli/jobs/spark/serverless-spark-standalone-default-identity.yml b/cli/jobs/spark/serverless-spark-standalone-default-identity.yml
@@ -31,5 +31,5 @@ args: >-
 
 resources:
   instance_type: standard_e4s_v3
-  runtime_version: "3.2"
+  runtime_version: "3.3"
 
diff --git a/cli/jobs/spark/serverless-spark-standalone-managed-identity.yml b/cli/jobs/spark/serverless-spark-standalone-managed-identity.yml
@@ -34,5 +34,5 @@ identity:
 
 resources:
   instance_type: standard_e4s_v3
-  runtime_version: "3.2"
+  runtime_version: "3.3"
 
diff --git a/cli/jobs/spark/serverless-spark-standalone-user-identity.yml b/cli/jobs/spark/serverless-spark-standalone-user-identity.yml
@@ -34,5 +34,5 @@ identity:
 
 resources:
   instance_type: standard_e4s_v3
-  runtime_version: "3.2"
+  runtime_version: "3.3"
 
diff --git a/sdk/python/jobs/pipelines/1i_pipeline_with_spark_nodes/pipeline_with_spark_nodes.ipynb b/sdk/python/jobs/pipelines/1i_pipeline_with_spark_nodes/pipeline_with_spark_nodes.ipynb
@@ -171,7 +171,7 @@
     "    kmeans_clustering = spark_kmeans(file_input=train_data)\n",
     "    kmeans_clustering.resources = {\n",
     "        \"instance_type\": \"Standard_E8S_V3\",\n",
-    "        \"runtime_version\": \"3.2.0\",\n",
+    "        \"runtime_version\": \"3.3.0\",\n",
     "    }\n",
     "    kmeans_clustering.outputs.output.mode = InputOutputModes.DIRECT\n",
     "\n",

diff --git a/...d_forecast_pipeline/aml-demand-forecast-mm-pipeline/aml-demand-forecast-mm-pipeline.ipynb b/...d_forecast_pipeline/aml-demand-forecast-mm-pipeline/aml-demand-forecast-mm-pipeline.ipynb
@@ -601,7 +601,7 @@
     "|Property|Description|\n",
     "|:-|:-|\n",
     "| **instance_type**            | A key that defines the compute instance type to be used for the serverless Spark compute. The following instance types are currently supported:<ul><li>`Standard_E4S_V3`</li><li>`Standard_E8S_V3`</li><li>`Standard_E16S_V3`</li><li>`Standard_E32S_V3`</li><li>`Standard_E64S_V3`</li></ul>|\n",
-    "| **runtime_version**          | A key that defines the Spark runtime version. The following Spark runtime versions are currently supported:<ul><li>`3.1.0`</li><li>`3.2.0`</li></ul> |\n",
+    "| **runtime_version**          | A key that defines the Spark runtime version. The following Spark runtime versions are currently supported:<ul><li>`3.3.0`</li></ul> |\n",
     "| **driver_cores**       | The he number of cores allocated for the Spark driver. |\n",
     "| **driver_memory**      | The allocated memory for the Spark exedriver, with a size unit suffix `k`, `m`, `g` or `t` (for example, `512m`, `2g`). |\n",
     "| **executor_cores**     | The number of cores allocated for the Spark executor. |\n",
@@ -639,7 +639,7 @@
     "if USE_PARTITIONING_COMPONENT:\n",
     "    spark_parameters = dict(\n",
     "        instance_type=\"Standard_E4S_V3\",\n",
-    "        runtime_version=\"3.2.0\",\n",
+    "        runtime_version=\"3.3.0\",\n",
     "        driver_cores=1,\n",
     "        driver_memory=\"2g\",\n",
     "        executor_cores=2,\n",
@@ -704,7 +704,7 @@
     "\n",
     "        partition_step.resources = {\n",
     "            \"instance_type\": spark_parameters.get(\"instance_type\", \"Standard_E4S_V3\"),\n",
-    "            \"runtime_version\": str(spark_parameters.get(\"runtime_version\", \"3.2.0\")),\n",
+    "            \"runtime_version\": str(spark_parameters.get(\"runtime_version\", \"3.3.0\")),\n",
     "        }\n",
     "        partition_step.conf = {\n",
     "            \"spark.driver.cores\": spark_parameters.get(\"driver_cores\", 1),\n",

diff --git a/sdk/python/jobs/spark/automation/run_interactive_session_notebook.ipynb b/sdk/python/jobs/spark/automation/run_interactive_session_notebook.ipynb
@@ -45,7 +45,7 @@
     "    executor_instances=2,\n",
     "    resources={\n",
     "        \"instance_type\": \"Standard_E8S_V3\",\n",
-    "        \"runtime_version\": \"3.2.0\",\n",
+    "        \"runtime_version\": \"3.3.0\",\n",
     "    },\n",
     ")\n",
     "\n",

diff --git a/sdk/python/jobs/spark/submit_spark_pipeline_jobs.ipynb b/sdk/python/jobs/spark/submit_spark_pipeline_jobs.ipynb
@@ -342,7 +342,7 @@
     "    spark_step.identity = ManagedIdentityConfiguration()\n",
     "    spark_step.resources = {\n",
     "        \"instance_type\": \"Standard_E8S_V3\",\n",
-    "        \"runtime_version\": \"3.2.0\",\n",
+    "        \"runtime_version\": \"3.3.0\",\n",
     "    }\n",
     "\n",
     "\n",
@@ -422,7 +422,7 @@
     "    spark_step.identity = UserIdentityConfiguration()\n",
     "    spark_step.resources = {\n",
     "        \"instance_type\": \"Standard_E8S_V3\",\n",
-    "        \"runtime_version\": \"3.2.0\",\n",
+    "        \"runtime_version\": \"3.3.0\",\n",
     "    }\n",
     "\n",
     "\n",
@@ -501,7 +501,7 @@
     "    spark_step.outputs.wrangled_data.mode = InputOutputModes.DIRECT\n",
     "    spark_step.resources = {\n",
     "        \"instance_type\": \"Standard_E8S_V3\",\n",
-    "        \"runtime_version\": \"3.2.0\",\n",
+    "        \"runtime_version\": \"3.3.0\",\n",
     "    }\n",
     "\n",
     "\n",

diff --git a/sdk/python/jobs/spark/submit_spark_standalone_jobs.ipynb b/sdk/python/jobs/spark/submit_spark_standalone_jobs.ipynb
@@ -267,7 +267,7 @@
     "    executor_instances=2,\n",
     "    resources={\n",
     "        \"instance_type\": \"Standard_E8S_V3\",\n",
-    "        \"runtime_version\": \"3.2.0\",\n",
+    "        \"runtime_version\": \"3.3.0\",\n",
     "    },\n",
     "    inputs={\n",
     "        \"titanic_data\": Input(\n",
@@ -329,7 +329,7 @@
     "    executor_instances=2,\n",
     "    resources={\n",
     "        \"instance_type\": \"Standard_E8S_V3\",\n",
-    "        \"runtime_version\": \"3.2.0\",\n",
+    "        \"runtime_version\": \"3.3.0\",\n",
     "    },\n",
     "    inputs={\n",
     "        \"titanic_data\": Input(\n",
@@ -391,7 +391,7 @@
     "    executor_instances=2,\n",
     "    resources={\n",
     "        \"instance_type\": \"Standard_E8S_V3\",\n",
-    "        \"runtime_version\": \"3.2.0\",\n",
+    "        \"runtime_version\": \"3.3.0\",\n",
     "    },\n",
     "    inputs={\n",
     "        \"titanic_data\": Input(\n",

diff --git a/sdk/python/jobs/spark/submit_spark_standalone_jobs_managed_vnet.ipynb b/sdk/python/jobs/spark/submit_spark_standalone_jobs_managed_vnet.ipynb
@@ -531,7 +531,7 @@
     "    executor_instances=2,\n",
     "    resources={\n",
     "        \"instance_type\": \"Standard_E8S_V3\",\n",
-    "        \"runtime_version\": \"3.2.0\",\n",
+    "        \"runtime_version\": \"3.3.0\",\n",
     "    },\n",
     "    inputs={\n",
     "        \"titanic_data\": Input(\n",
@@ -765,7 +765,7 @@
     "    executor_instances=2,\n",
     "    resources={\n",
     "        \"instance_type\": \"Standard_E8S_V3\",\n",
-    "        \"runtime_version\": \"3.2.0\",\n",
+    "        \"runtime_version\": \"3.3.0\",\n",
     "    },\n",
     "    inputs={\n",
     "        \"titanic_data\": Input(\n",