Merge branch 'main' into ayushmishra/add_deployment_for_automl_pipeline

Azure · Aug 10, 2023 · 96b1a6c · 96b1a6c
2 parents 7e73ab0 + ce72472
commit 96b1a6c
Show file tree

Hide file tree

Showing 32 changed files with 55 additions and 851 deletions.
diff --git a/.github/workflows/cli-jobs-spark-attached-spark-pipeline-default-identity.yml b/.github/workflows/cli-jobs-spark-attached-spark-pipeline-default-identity.yml
@@ -52,7 +52,7 @@ jobs:
       working-directory: cli
       continue-on-error: true
       run: |
-          bash -x jobs/spark/setup-attached-resources.sh resources/compute/attached-spark.yml
+          bash -x jobs/spark/setup-attached-resources.sh resources/compute/attached-spark.yml jobs/spark/attached-spark-pipeline-default-identity.yml
     - name: run job
       run: |
           source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh";

diff --git a/.github/workflows/cli-jobs-spark-attached-spark-pipeline-managed-identity.yml b/.github/workflows/cli-jobs-spark-attached-spark-pipeline-managed-identity.yml
@@ -57,7 +57,7 @@ jobs:
       working-directory: cli
       continue-on-error: true
       run: |
-          bash -x jobs/spark/setup-attached-resources.sh resources/compute/attached-spark-system-identity.yml
+          bash -x jobs/spark/setup-attached-resources.sh resources/compute/attached-spark-system-identity.yml jobs/spark/attached-spark-pipeline-managed-identity.yml
     - name: run job
       run: |
           source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh";

diff --git a/.github/workflows/cli-jobs-spark-attached-spark-pipeline-user-identity.yml b/.github/workflows/cli-jobs-spark-attached-spark-pipeline-user-identity.yml
@@ -52,7 +52,7 @@ jobs:
       working-directory: cli
       continue-on-error: true
       run: |
-          bash -x jobs/spark/setup-attached-resources.sh resources/compute/attached-spark-user-identity.yml
+          bash -x jobs/spark/setup-attached-resources.sh resources/compute/attached-spark-user-identity.yml jobs/spark/attached-spark-pipeline-user-identity.yml
     - name: run job
       run: |
           source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh";

diff --git a/.github/workflows/cli-jobs-spark-attached-spark-standalone-default-identity.yml b/.github/workflows/cli-jobs-spark-attached-spark-standalone-default-identity.yml
@@ -52,7 +52,7 @@ jobs:
       working-directory: cli
       continue-on-error: true
       run: |
-          bash -x jobs/spark/setup-attached-resources.sh resources/compute/attached-spark.yml
+          bash -x jobs/spark/setup-attached-resources.sh resources/compute/attached-spark.yml jobs/spark/attached-spark-standalone-default-identity.yml
     - name: run job
       run: |
           source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh";

diff --git a/.github/workflows/cli-jobs-spark-attached-spark-standalone-managed-identity.yml b/.github/workflows/cli-jobs-spark-attached-spark-standalone-managed-identity.yml
@@ -57,7 +57,7 @@ jobs:
       working-directory: cli
       continue-on-error: true
       run: |
-          bash -x jobs/spark/setup-attached-resources.sh resources/compute/attached-spark-system-identity.yml
+          bash -x jobs/spark/setup-attached-resources.sh resources/compute/attached-spark-system-identity.yml jobs/spark/attached-spark-standalone-managed-identity.yml
     - name: run job
       run: |
           source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh";

diff --git a/.github/workflows/cli-jobs-spark-attached-spark-standalone-user-identity.yml b/.github/workflows/cli-jobs-spark-attached-spark-standalone-user-identity.yml
@@ -52,7 +52,7 @@ jobs:
       working-directory: cli
       continue-on-error: true
       run: |
-          bash -x jobs/spark/setup-attached-resources.sh resources/compute/attached-spark-user-identity.yml
+          bash -x jobs/spark/setup-attached-resources.sh resources/compute/attached-spark-user-identity.yml jobs/spark/attached-spark-standalone-user-identity.yml
     - name: run job
       run: |
           source "${{ github.workspace }}/infra/bootstrapping/sdk_helpers.sh";

diff --git a/cli/jobs/spark/attached-spark-pipeline-default-identity.yml b/cli/jobs/spark/attached-spark-pipeline-default-identity.yml
@@ -20,4 +20,4 @@ jobs:
         path: azureml://datastores/workspaceblobstore/paths/data/wrangled/
         mode: direct
 
-    compute: mysparkcompute
+    compute: <ATTACHED_SPARK_POOL_NAME>
diff --git a/cli/jobs/spark/attached-spark-pipeline-managed-identity.yml b/cli/jobs/spark/attached-spark-pipeline-managed-identity.yml
@@ -23,4 +23,4 @@ jobs:
     identity:
       type: managed
 
-    compute: mysparkcompute
+    compute: <ATTACHED_SPARK_POOL_NAME>
diff --git a/cli/jobs/spark/attached-spark-pipeline-user-identity.yml b/cli/jobs/spark/attached-spark-pipeline-user-identity.yml
@@ -23,4 +23,4 @@ jobs:
     identity:
       type: user_identity
 
-    compute: mysparkcompute
+    compute: <ATTACHED_SPARK_POOL_NAME>
diff --git a/cli/jobs/spark/attached-spark-standalone-default-identity.yml b/cli/jobs/spark/attached-spark-standalone-default-identity.yml
@@ -29,4 +29,4 @@ args: >-
   --titanic_data ${{inputs.titanic_data}}
   --wrangled_data ${{outputs.wrangled_data}}
 
-compute: mysparkcompute
+compute: <ATTACHED_SPARK_POOL_NAME>
diff --git a/cli/jobs/spark/attached-spark-standalone-managed-identity.yml b/cli/jobs/spark/attached-spark-standalone-managed-identity.yml
@@ -32,4 +32,4 @@ args: >-
 identity:
   type: managed
 
-compute: mysparkcompute
+compute: <ATTACHED_SPARK_POOL_NAME>
diff --git a/cli/jobs/spark/attached-spark-standalone-user-identity.yml b/cli/jobs/spark/attached-spark-standalone-user-identity.yml
@@ -32,4 +32,4 @@ args: >-
 identity:
   type: user_identity
 
-compute: mysparkcompute
+compute: <ATTACHED_SPARK_POOL_NAME>
diff --git a/cli/jobs/spark/setup-attached-resources.sh b/cli/jobs/spark/setup-attached-resources.sh
@@ -30,15 +30,24 @@ az role assignment create --role "Storage Blob Data Owner" --assignee $AML_USER_
 az synapse spark pool create --name $SPARK_POOL_NAME --workspace-name $SYNAPSE_WORKSPACE_NAME --resource-group $RESOURCE_GROUP --spark-version 3.2 --node-count 3 --node-size Medium --min-node-count 3 --max-node-count 10 --enable-auto-scale true
 az synapse workspace firewall-rule create --name allowAll --workspace-name $SYNAPSE_WORKSPACE_NAME --resource-group $RESOURCE_GROUP --start-ip-address 0.0.0.0 --end-ip-address 255.255.255.255
 
-TEMP_COMPUTE_FILE="temp-compute-setup.yml"
-cp $1 $TEMP_COMPUTE_FILE
+if [[ "$2" == *"managed-identity"* ]]
+then
+	ATTACHED_COMPUTE_NAME=${ATTACHED_COMPUTE_NAME}mi
+elif [[ "$2" == *"user-identity"* ]]
+then
+	ATTACHED_COMPUTE_NAME=${ATTACHED_COMPUTE_NAME}ui
+fi
+
 sed -i "s/<SUBSCRIPTION_ID>/$SUBSCRIPTION_ID/g;
 		s/<RESOURCE_GROUP>/$RESOURCE_GROUP/g;
 		s/<SYNAPSE_WORKSPACE_NAME>/$SYNAPSE_WORKSPACE_NAME/g;
 		s/<SPARK_POOL_NAME>/$SPARK_POOL_NAME/g;
-		s/<AML_USER_MANAGED_ID>/$AML_USER_MANAGED_ID/g;" $TEMP_COMPUTE_FILE
+		s/<AML_USER_MANAGED_ID>/$AML_USER_MANAGED_ID/g;
+		s/<ATTACHED_SPARK_POOL_NAME>/$ATTACHED_COMPUTE_NAME/g;" $1
+
+sed -i "s/<ATTACHED_SPARK_POOL_NAME>/$ATTACHED_COMPUTE_NAME/g;" $2
 
-az ml compute attach --file $TEMP_COMPUTE_FILE --subscription $SUBSCRIPTION_ID --resource-group $RESOURCE_GROUP --workspace-name $AML_WORKSPACE_NAME
+az ml compute attach --file $1 --subscription $SUBSCRIPTION_ID --resource-group $RESOURCE_GROUP --workspace-name $AML_WORKSPACE_NAME
 az synapse role assignment create --workspace-name $SYNAPSE_WORKSPACE_NAME --role $SPARK_POOL_ADMIN_ROLE_ID --assignee $AML_USER_MANAGED_ID_OID
 
 COMPUTE_MANAGED_IDENTITY=$(az ml compute show --name $ATTACHED_COMPUTE_NAME --resource-group $RESOURCE_GROUP --workspace-name $AML_WORKSPACE_NAME --query identity.principal_id --out tsv)

diff --git a/cli/readme.py b/cli/readme.py
@@ -471,7 +471,7 @@ def write_job_workflow(job):
       working-directory: cli
       continue-on-error: true\n"""
     if is_spark_sample:
-        workflow_yaml += get_spark_setup_workflow(job)
+        workflow_yaml += get_spark_setup_workflow(job, posix_project_dir, filename)
     workflow_yaml += f"""    - name: run job
       run: |
           source "{GITHUB_WORKSPACE}/infra/bootstrapping/sdk_helpers.sh";
@@ -862,7 +862,7 @@ def get_endpoint_name(filename, hyphenated):
     return endpoint_name
 
 
-def get_spark_setup_workflow(job):
+def get_spark_setup_workflow(job, posix_project_dir, filename):
     is_attached = "attached-spark" in job
     is_user_identity = "user-identity" in job
     is_managed_identity = "managed-identity" in job
@@ -876,7 +876,7 @@ def get_spark_setup_workflow(job):
         workflow += f"""    - name: setup identities
       run: |
           bash -x setup-identities.sh
-      working-directory: cli/jobs/spark
+      working-directory: cli/{posix_project_dir}
       continue-on-error: true\n"""
     if is_attached:
         workflow += f"""    - name: setup attached spark
@@ -885,15 +885,15 @@ def get_spark_setup_workflow(job):
     if is_attached and is_user_identity:
         workflow += f"""
       run: |
-          bash -x jobs/spark/setup-attached-resources.sh resources/compute/attached-spark-user-identity.yml\n"""
+          bash -x {posix_project_dir}/setup-attached-resources.sh resources/compute/attached-spark-user-identity.yml {posix_project_dir}/{filename}.yml\n"""
     if is_attached and is_managed_identity:
         workflow += f"""
       run: |
-          bash -x jobs/spark/setup-attached-resources.sh resources/compute/attached-spark-system-identity.yml\n"""
+          bash -x {posix_project_dir}/setup-attached-resources.sh resources/compute/attached-spark-system-identity.yml {posix_project_dir}/{filename}.yml\n"""
     if is_attached and is_default_identity:
         workflow += f"""
       run: |
-          bash -x jobs/spark/setup-attached-resources.sh resources/compute/attached-spark.yml\n"""
+          bash -x {posix_project_dir}/setup-attached-resources.sh resources/compute/attached-spark.yml {posix_project_dir}/{filename}.yml\n"""
 
     return workflow
 

diff --git a/cli/resources/compute/attached-spark-system-identity.yml b/cli/resources/compute/attached-spark-system-identity.yml
@@ -1,5 +1,5 @@
 # attached-spark-system-identity.yaml
-name: mysparkcompute
+name: <ATTACHED_SPARK_POOL_NAME>
 
 type: synapsespark
 

diff --git a/cli/resources/compute/attached-spark-user-identity.yml b/cli/resources/compute/attached-spark-user-identity.yml
@@ -1,5 +1,5 @@
 # attached-spark-user-identity.yml
-name: mysparkcompute
+name: <ATTACHED_SPARK_POOL_NAME>
 
 type: synapsespark
 

diff --git a/cli/resources/compute/attached-spark.yml b/cli/resources/compute/attached-spark.yml
@@ -1,5 +1,5 @@
 # attached-spark.yaml
-name: mysparkcompute
+name: <ATTACHED_SPARK_POOL_NAME>
 
 type: synapsespark
 

diff --git a/...undation-models/system/inference/text-generation/llama-files/hf_tgi_docker_env/Dockerfile b/...undation-models/system/inference/text-generation/llama-files/hf_tgi_docker_env/Dockerfile
diff --git a/...on-models/system/inference/text-generation/llama-files/hf_tgi_docker_env/requirements.txt b/...on-models/system/inference/text-generation/llama-files/hf_tgi_docker_env/requirements.txt