Switched to new GPU SKU because NC6 is deprecated (#2462)

* Switched to new GPU SKU because NC6 is deprecated * Updated credentials for remaining V1 notebooks * Updated gpu-cluster in bootstrap.sh
Azure · Jul 18, 2023 · df7837c · df7837c
1 parent 6a89a6a
commit df7837c
Show file tree

Hide file tree

Showing 45 changed files with 55 additions and 85 deletions.
diff --git a/.github/workflows/python-sdk-deploy-pytorch-mnist-job.yml b/.github/workflows/python-sdk-deploy-pytorch-mnist-job.yml
@@ -29,7 +29,7 @@ jobs:
     - name: azure login
       uses: azure/login@v1
       with:
-        creds: ${{secrets.AZ_CREDS}}
+        creds: ${{secrets.AZUREML_CREDENTIALS}}
     - name: Run update-azure-extensions
       run: |
          chmod +x ./v1/scripts/update-azure-extensions.sh

diff --git a/.github/workflows/python-sdk-deploy-scikit-learn-diabetes-job.yml b/.github/workflows/python-sdk-deploy-scikit-learn-diabetes-job.yml
@@ -29,7 +29,7 @@ jobs:
     - name: azure login
       uses: azure/login@v1
       with:
-        creds: ${{secrets.AZ_CREDS}}
+        creds: ${{secrets.AZUREML_CREDENTIALS}}
     - name: Run update-azure-extensions
       run: |
          chmod +x ./v1/scripts/update-azure-extensions.sh

diff --git a/.github/workflows/python-sdk-experimental-automl-model-testing-classification-tsi.yml b/.github/workflows/python-sdk-experimental-automl-model-testing-classification-tsi.yml
@@ -43,14 +43,14 @@ jobs:
     - name: azure login
       uses: azure/login@v1
       with:
-        creds: ${{secrets.AZ_AE_CREDS}}
+        creds: ${{secrets.AZUREML_CREDENTIALS}}
     - name: Run update-azure-extensions
       run: |
          chmod +x ./v1/scripts/update-azure-extensions.sh
          ./v1/scripts/update-azure-extensions.sh
       shell: bash
     - name: attach to workspace
-      run: az ml folder attach -w main-python-sdk -g azureml-examples-rg
+      run: az ml folder attach -w main -g azureml-examples
     - name: run classification-TSI.ipynb
       run: papermill -k python classification-TSI.ipynb classification-TSI.output.ipynb
       working-directory: v1/python-sdk/experimental/automl-model-testing/classification

diff --git a/.github/workflows/python-sdk-experimental-automl-model-testing-forecasting-tsi.yml b/.github/workflows/python-sdk-experimental-automl-model-testing-forecasting-tsi.yml
@@ -43,14 +43,14 @@ jobs:
     - name: azure login
       uses: azure/login@v1
       with:
-        creds: ${{secrets.AZ_AE_CREDS}}
+        creds: ${{secrets.AZUREML_CREDENTIALS}}
     - name: Run update-azure-extensions
       run: |
          chmod +x ./v1/scripts/update-azure-extensions.sh
          ./v1/scripts/update-azure-extensions.sh
       shell: bash
     - name: attach to workspace
-      run: az ml folder attach -w main-python-sdk -g azureml-examples-rg
+      run: az ml folder attach -w main -g azureml-examples
     - name: run forecasting-TSI.ipynb
       run: papermill -k python forecasting-TSI.ipynb forecasting-TSI.output.ipynb
       working-directory: v1/python-sdk/experimental/automl-model-testing/forecasting

diff --git a/.github/workflows/python-sdk-experimental-automl-model-testing-regression-tsi.yml b/.github/workflows/python-sdk-experimental-automl-model-testing-regression-tsi.yml
@@ -43,14 +43,14 @@ jobs:
     - name: azure login
       uses: azure/login@v1
       with:
-        creds: ${{secrets.AZ_AE_CREDS}}
+        creds: ${{secrets.AZUREML_CREDENTIALS}}
     - name: Run update-azure-extensions
       run: |
          chmod +x ./v1/scripts/update-azure-extensions.sh
          ./v1/scripts/update-azure-extensions.sh
       shell: bash
     - name: attach to workspace
-      run: az ml folder attach -w main-python-sdk -g azureml-examples-rg
+      run: az ml folder attach -w main -g azureml-examples
     - name: run regression-TSI.ipynb
       run: papermill -k python regression-TSI.ipynb regression-TSI.output.ipynb
       working-directory: v1/python-sdk/experimental/automl-model-testing/regression

diff --git a/.github/workflows/python-sdk-train-deepspeed-cifar-job.yml b/.github/workflows/python-sdk-train-deepspeed-cifar-job.yml
@@ -29,7 +29,7 @@ jobs:
     - name: azure login
       uses: azure/login@v1
       with:
-        creds: ${{secrets.AZ_CREDS}}
+        creds: ${{secrets.AZUREML_CREDENTIALS}}
     - name: Run update-azure-extensions
       run: |
          chmod +x ./v1/scripts/update-azure-extensions.sh

diff --git a/.github/workflows/python-sdk-train-deepspeed-transformers-job.yml b/.github/workflows/python-sdk-train-deepspeed-transformers-job.yml
@@ -29,7 +29,7 @@ jobs:
     - name: azure login
       uses: azure/login@v1
       with:
-        creds: ${{secrets.AZ_CREDS}}
+        creds: ${{secrets.AZUREML_CREDENTIALS}}
     - name: Run update-azure-extensions
       run: |
          chmod +x ./v1/scripts/update-azure-extensions.sh

diff --git a/.github/workflows/python-sdk-train-fastai-mnist-job.yml b/.github/workflows/python-sdk-train-fastai-mnist-job.yml
@@ -29,7 +29,7 @@ jobs:
     - name: azure login
       uses: azure/login@v1
       with:
-        creds: ${{secrets.AZ_CREDS}}
+        creds: ${{secrets.AZUREML_CREDENTIALS}}
     - name: Run update-azure-extensions
       run: |
          chmod +x ./v1/scripts/update-azure-extensions.sh

diff --git a/.github/workflows/python-sdk-train-fastai-mnist-mlproject-job.yml b/.github/workflows/python-sdk-train-fastai-mnist-mlproject-job.yml
@@ -29,7 +29,7 @@ jobs:
     - name: azure login
       uses: azure/login@v1
       with:
-        creds: ${{secrets.AZ_CREDS}}
+        creds: ${{secrets.AZUREML_CREDENTIALS}}
     - name: Run update-azure-extensions
       run: |
          chmod +x ./v1/scripts/update-azure-extensions.sh

diff --git a/.github/workflows/python-sdk-train-lightgbm-iris-job.yml b/.github/workflows/python-sdk-train-lightgbm-iris-job.yml
@@ -29,7 +29,7 @@ jobs:
     - name: azure login
       uses: azure/login@v1
       with:
-        creds: ${{secrets.AZ_CREDS}}
+        creds: ${{secrets.AZUREML_CREDENTIALS}}
     - name: Run update-azure-extensions
       run: |
          chmod +x ./v1/scripts/update-azure-extensions.sh

diff --git a/.github/workflows/python-sdk-train-pytorch-cifar-distributed-job.yml b/.github/workflows/python-sdk-train-pytorch-cifar-distributed-job.yml
@@ -29,7 +29,7 @@ jobs:
     - name: azure login
       uses: azure/login@v1
       with:
-        creds: ${{secrets.AZ_CREDS}}
+        creds: ${{secrets.AZUREML_CREDENTIALS}}
     - name: Run update-azure-extensions
       run: |
          chmod +x ./v1/scripts/update-azure-extensions.sh

diff --git a/.github/workflows/python-sdk-train-pytorch-mnist-job.yml b/.github/workflows/python-sdk-train-pytorch-mnist-job.yml
@@ -29,7 +29,7 @@ jobs:
     - name: azure login
       uses: azure/login@v1
       with:
-        creds: ${{secrets.AZ_CREDS}}
+        creds: ${{secrets.AZUREML_CREDENTIALS}}
     - name: Run update-azure-extensions
       run: |
          chmod +x ./v1/scripts/update-azure-extensions.sh

diff --git a/.github/workflows/python-sdk-train-pytorch-mnist-mlproject-job.yml b/.github/workflows/python-sdk-train-pytorch-mnist-mlproject-job.yml
@@ -29,7 +29,7 @@ jobs:
     - name: azure login
       uses: azure/login@v1
       with:
-        creds: ${{secrets.AZ_CREDS}}
+        creds: ${{secrets.AZUREML_CREDENTIALS}}
     - name: Run update-azure-extensions
       run: |
          chmod +x ./v1/scripts/update-azure-extensions.sh

diff --git a/.github/workflows/python-sdk-train-scikit-learn-diabetes-job.yml b/.github/workflows/python-sdk-train-scikit-learn-diabetes-job.yml
@@ -29,7 +29,7 @@ jobs:
     - name: azure login
       uses: azure/login@v1
       with:
-        creds: ${{secrets.AZ_CREDS}}
+        creds: ${{secrets.AZUREML_CREDENTIALS}}
     - name: Run update-azure-extensions
       run: |
          chmod +x ./v1/scripts/update-azure-extensions.sh

diff --git a/.github/workflows/python-sdk-train-scikit-learn-diabetes-mlproject-job.yml b/.github/workflows/python-sdk-train-scikit-learn-diabetes-mlproject-job.yml
@@ -29,7 +29,7 @@ jobs:
     - name: azure login
       uses: azure/login@v1
       with:
-        creds: ${{secrets.AZ_CREDS}}
+        creds: ${{secrets.AZUREML_CREDENTIALS}}
     - name: Run update-azure-extensions
       run: |
          chmod +x ./v1/scripts/update-azure-extensions.sh

diff --git a/.github/workflows/python-sdk-train-tensorflow-mnist-distributed-horovod-job.yml b/.github/workflows/python-sdk-train-tensorflow-mnist-distributed-horovod-job.yml
@@ -29,7 +29,7 @@ jobs:
     - name: azure login
       uses: azure/login@v1
       with:
-        creds: ${{secrets.AZ_CREDS}}
+        creds: ${{secrets.AZUREML_CREDENTIALS}}
     - name: Run update-azure-extensions
       run: |
          chmod +x ./v1/scripts/update-azure-extensions.sh

diff --git a/.github/workflows/python-sdk-train-tensorflow-mnist-distributed-job.yml b/.github/workflows/python-sdk-train-tensorflow-mnist-distributed-job.yml
@@ -29,7 +29,7 @@ jobs:
     - name: azure login
       uses: azure/login@v1
       with:
-        creds: ${{secrets.AZ_CREDS}}
+        creds: ${{secrets.AZUREML_CREDENTIALS}}
     - name: Run update-azure-extensions
       run: |
          chmod +x ./v1/scripts/update-azure-extensions.sh

diff --git a/.github/workflows/python-sdk-train-tensorflow-mnist-job.yml b/.github/workflows/python-sdk-train-tensorflow-mnist-job.yml
@@ -29,7 +29,7 @@ jobs:
     - name: azure login
       uses: azure/login@v1
       with:
-        creds: ${{secrets.AZ_CREDS}}
+        creds: ${{secrets.AZUREML_CREDENTIALS}}
     - name: Run update-azure-extensions
       run: |
          chmod +x ./v1/scripts/update-azure-extensions.sh

diff --git a/.github/workflows/python-sdk-train-transformers-glue-1-aml-finetune-job.yml b/.github/workflows/python-sdk-train-transformers-glue-1-aml-finetune-job.yml
@@ -29,7 +29,7 @@ jobs:
     - name: azure login
       uses: azure/login@v1
       with:
-        creds: ${{secrets.AZ_CREDS}}
+        creds: ${{secrets.AZUREML_CREDENTIALS}}
     - name: Run update-azure-extensions
       run: |
          chmod +x ./v1/scripts/update-azure-extensions.sh

diff --git a/.github/workflows/python-sdk-train-transformers-glue-2-aml-comparison-of-sku-job.yml b/.github/workflows/python-sdk-train-transformers-glue-2-aml-comparison-of-sku-job.yml
@@ -29,7 +29,7 @@ jobs:
     - name: azure login
       uses: azure/login@v1
       with:
-        creds: ${{secrets.AZ_CREDS}}
+        creds: ${{secrets.AZUREML_CREDENTIALS}}
     - name: Run update-azure-extensions
       run: |
          chmod +x ./v1/scripts/update-azure-extensions.sh

diff --git a/.github/workflows/python-sdk-train-transformers-glue-3-aml-hyperdrive-job.yml b/.github/workflows/python-sdk-train-transformers-glue-3-aml-hyperdrive-job.yml
@@ -29,7 +29,7 @@ jobs:
     - name: azure login
       uses: azure/login@v1
       with:
-        creds: ${{secrets.AZ_CREDS}}
+        creds: ${{secrets.AZUREML_CREDENTIALS}}
     - name: Run update-azure-extensions
       run: |
          chmod +x ./v1/scripts/update-azure-extensions.sh

diff --git a/.github/workflows/python-sdk-train-xgboost-iris-job.yml b/.github/workflows/python-sdk-train-xgboost-iris-job.yml
@@ -29,7 +29,7 @@ jobs:
     - name: azure login
       uses: azure/login@v1
       with:
-        creds: ${{secrets.AZ_CREDS}}
+        creds: ${{secrets.AZUREML_CREDENTIALS}}
     - name: Run update-azure-extensions
       run: |
          chmod +x ./v1/scripts/update-azure-extensions.sh

diff --git a/.github/workflows/python-sdk-tutorial-automl-with-pycaret.yml b/.github/workflows/python-sdk-tutorial-automl-with-pycaret.yml
diff --git a/.github/workflows/python-sdk-tutorial-dataset-uploads.yml b/.github/workflows/python-sdk-tutorial-dataset-uploads.yml
@@ -32,7 +32,7 @@ jobs:
     - name: azure login
       uses: azure/login@v1
       with:
-        creds: ${{secrets.AZ_CREDS}}
+        creds: ${{secrets.AZUREML_CREDENTIALS}}
     - name: install azmlcli
       run: az extension add -n azure-cli-ml -y
     - name: attach to workspace

diff --git a/.github/workflows/python-sdk-tutorial-deploy-local.yml b/.github/workflows/python-sdk-tutorial-deploy-local.yml
@@ -26,7 +26,7 @@ jobs:
     - name: azure login
       uses: azure/login@v1
       with:
-        creds: ${{secrets.AZ_CREDS}}
+        creds: ${{secrets.AZUREML_CREDENTIALS}}
     - name: install azmlcli
       run: az extension add -n azure-cli-ml -y
     - name: attach to workspace

diff --git a/.github/workflows/python-sdk-tutorial-deploy-triton.yml b/.github/workflows/python-sdk-tutorial-deploy-triton.yml
@@ -26,7 +26,7 @@ jobs:
     - name: azure login
       uses: azure/login@v1
       with:
-        creds: ${{secrets.AZ_CREDS}}
+        creds: ${{secrets.AZUREML_CREDENTIALS}}
     - name: install azmlcli
       run: az extension add -n azure-cli-ml -y
     - name: attach to workspace

diff --git a/...lows/python-sdk-tutorial-metrics-binary-classification-metric-and-confidence-interval.yml b/...lows/python-sdk-tutorial-metrics-binary-classification-metric-and-confidence-interval.yml
@@ -43,14 +43,14 @@ jobs:
     - name: azure login
       uses: azure/login@v1
       with:
-        creds: ${{secrets.AZ_AE_CREDS}}
+        creds: ${{secrets.AZUREML_CREDENTIALS}}
     - name: Run update-azure-extensions
       run: |
          chmod +x ./v1/scripts/update-azure-extensions.sh
          ./v1/scripts/update-azure-extensions.sh
       shell: bash
     - name: attach to workspace
-      run: az ml folder attach -w main-python-sdk -g azureml-examples-rg
+      run: az ml folder attach -w main -g azureml-examples
     - name: run binary-classification-metric-and-confidence-interval.ipynb
       run: papermill -k python binary-classification-metric-and-confidence-interval.ipynb binary-classification-metric-and-confidence-interval.output.ipynb
       working-directory: v1/python-sdk/tutorials/automl-with-azureml/metrics

diff --git a/.github/workflows/python-sdk-tutorial-train-on-low-priority-aml-compute.yml b/.github/workflows/python-sdk-tutorial-train-on-low-priority-aml-compute.yml
@@ -32,14 +32,14 @@ jobs:
     - name: azure login
       uses: azure/login@v1
       with:
-        creds: ${{secrets.AZ_AE_CREDS}}
+        creds: ${{secrets.AZUREML_CREDENTIALS}}
     - name: Run update-azure-extensions
       run: |
          chmod +x ./v1/scripts/update-azure-extensions.sh
          ./v1/scripts/update-azure-extensions.sh
       shell: bash
     - name: attach to workspace
-      run: az ml folder attach -w main-python-sdk -g azureml-examples-rg
+      run: az ml folder attach -w main -g azureml-examples
     - name: run train-on-low-priority-aml-compute
       run: papermill -k python train-on-low-priority-aml-compute.ipynb train-on-low-priority-aml-compute.output.ipynb
       working-directory: v1/python-sdk/tutorials/train-on-low-priority-aml-compute

diff --git a/.github/workflows/python-sdk-tutorial-using-pipelines.yml b/.github/workflows/python-sdk-tutorial-using-pipelines.yml
@@ -32,7 +32,7 @@ jobs:
     - name: azure login
       uses: azure/login@v1
       with:
-        creds: ${{secrets.AZ_CREDS}}
+        creds: ${{secrets.AZUREML_CREDENTIALS}}
     - name: install azmlcli
       run: az extension add -n azure-cli-ml -y
     - name: attach to workspace

diff --git a/.github/workflows/python-sdk-tutorial-using-pytorch-lightning.yml b/.github/workflows/python-sdk-tutorial-using-pytorch-lightning.yml
@@ -26,7 +26,7 @@ jobs:
     - name: azure login
       uses: azure/login@v1
       with:
-        creds: ${{secrets.AZ_CREDS}}
+        creds: ${{secrets.AZUREML_CREDENTIALS}}
     - name: install azmlcli
       run: az extension add -n azure-cli-ml -y
     - name: attach to workspace

diff --git a/.github/workflows/python-sdk-tutorial-using-rapids.yml b/.github/workflows/python-sdk-tutorial-using-rapids.yml
@@ -26,7 +26,7 @@ jobs:
     - name: azure login
       uses: azure/login@v1
       with:
-        creds: ${{secrets.AZ_CREDS}}
+        creds: ${{secrets.AZUREML_CREDENTIALS}}
     - name: install azmlcli
       run: az extension add -n azure-cli-ml -y
     - name: attach to workspace

diff --git a/cli/endpoints/batch/deploy-models/imagenet-classifier/deploy-and-run.sh b/cli/endpoints/batch/deploy-models/imagenet-classifier/deploy-and-run.sh
@@ -26,7 +26,7 @@ az ml model create --name $MODEL_NAME --path "model"
 
 echo "Creating compute with GPU"
 # <create_compute>
-az ml compute create -n gpu-cluster --type amlcompute --size STANDARD_NC6 --min-instances 0 --max-instances 2
+az ml compute create -n gpu-cluster --type amlcompute --size STANDARD_NC6s_v3 --min-instances 0 --max-instances 2
 # </create_compute>
 
 echo "Creating batch endpoint $ENDPOINT_NAME"

diff --git a/infra/bootstrapping/bootstrap.sh b/infra/bootstrapping/bootstrap.sh
@@ -100,8 +100,8 @@ if [[ ! -z "${RUN_BOOTSTRAP:-}" ]]; then
     "$SCRIPT_DIR"/sdk_helpers.sh ensure_aml_compute "cpu-cluster-lg" 0 4 "Standard_DS15_v2"
 
     echo_title "Ensuring GPU compute"
-    "$SCRIPT_DIR"/sdk_helpers.sh ensure_aml_compute "gpu-cluster" 0 20 "Standard_NC6"
-    "$SCRIPT_DIR"/sdk_helpers.sh ensure_aml_compute "automl-gpu-cluster" 0 4 "STANDARD_NC6"
+    "$SCRIPT_DIR"/sdk_helpers.sh ensure_aml_compute "gpu-cluster" 0 20 "STANDARD_NC6s_v3"
+    "$SCRIPT_DIR"/sdk_helpers.sh ensure_aml_compute "automl-gpu-cluster" 0 4 "STANDARD_NC6s_v3"
     # v100 single GPU cluster for pytorch 2.0 based notebooks
     "$SCRIPT_DIR"/sdk_helpers.sh ensure_aml_compute "gpu-v100-1GPU-cluster" 0 4 "Standard_NC6s_v3"
     # v100 GPU cluster for deepspeed cli examples

diff --git a/sdk/dotnet/AzureML-Samples-CSharp/Resources/Compute/AMLComputeOperations.cs b/sdk/dotnet/AzureML-Samples-CSharp/Resources/Compute/AMLComputeOperations.cs
@@ -48,7 +48,7 @@ public static async Task<MachineLearningComputeResource> GetOrCreateAMLComputeAs
                     {
                         ScaleSettings = new ScaleSettings(4),
                         VmSize = "Standard_DS3_v2",
-                        // VmSize = "STANDARD_NC6",
+                        // VmSize = "STANDARD_NC6s_v3",
                     }
                 },
             };

diff --git a/sdk/python/jobs/configuration.ipynb b/sdk/python/jobs/configuration.ipynb
@@ -173,7 +173,10 @@
     "except Exception:\n",
     "    print(\"Creating a new gpu compute target...\")\n",
     "    compute = AmlCompute(\n",
-    "        name=gpu_compute_target, size=\"STANDARD_NC6\", min_instances=0, max_instances=4\n",
+    "        name=gpu_compute_target,\n",
+    "        size=\"STANDARD_NC6s_v3\",\n",
+    "        min_instances=0,\n",
+    "        max_instances=4,\n",
     "    )\n",
     "    ml_client.compute.begin_create_or_update(compute).result()"
    ]