cherry-pick: cleanup + refactor for test infra migration (#5252)

Co-authored-by: Cameron Meissner <[email protected]>

.github/workflows/check-coverage.yml

@@ -6,13 +6,6 @@ permissions:
   id-token: write
   contents: read
 
-env:
-  SUBSCRIPTION_ID: "8ecadfc9-d1a3-4ea4-b844-0d9f87e4d7c8"
-  RESOURCE_GROUP_NAME: "agentbaker-e2e-tests"
-  LOCATION: "eastus"
-  CLUSTER_NAME: "agentbaker-e2e-test-cluster"
-  AZURE_TENANT_ID: "72f988bf-86f1-41af-91ab-2d7cd011db47"
-
 jobs:
   unit_tests:
     runs-on: ubuntu-latest

.pipelines/.vsts-vhd-builder-release.yaml

@@ -1308,7 +1308,7 @@ stages:
               useOverrides: ${{ parameters.useOverrides }}
               overrideBranch: ${{ parameters.overrideBranch }}
               artifactName: 2204-minimal-gen2-containerd
-  - stage: Run_E2E_Tests
+  - stage: e2e
     condition: and(ne(variables.SKIP_E2E_TESTS, 'true'), eq('${{ parameters.dryrun }}', false))
     variables:
       VHD_BUILD_ID: $(Build.BuildId)

.pipelines/.vsts-vhd-builder.yaml

@@ -1,4 +1,4 @@
-name: $(Date:yyyyMMdd)$(Rev:.r)_$(OS_SKU)$(OS_VERSION)_$(HYPERV_GENERATION)_$(FEATURE_FLAGS)_$(Build.SourceBranchName)
+name: $(Date:yyyyMMdd)$(Rev:.r)_$(Build.SourceBranchName)_$(BuildID)
 trigger: none
 pr:
   branches:
@@ -97,7 +97,7 @@ stages:
         - template: ./templates/.builder-release-template.yaml
           parameters:
             artifactName: marinerv2-gen2
-  - stage: Run_E2E_Tests
+  - stage: e2e
     condition: ne(variables.SKIP_E2E_TESTS, 'true')
     variables:
       VHD_BUILD_ID: $(Build.BuildId)

.pipelines/templates/e2e-template.yaml

@@ -1,7 +1,7 @@
 jobs:
-  - job: E2E_Tests
+  - job: e2e
     pool:
-      name: nodesigtest-pool
+      name: $(POOL_NAME)
     timeoutInMinutes: 90
     displayName: Run AgentBaker E2E
     variables:
@@ -10,7 +10,7 @@ jobs:
       - bash: |
           set -ex
           az login --identity
-          az account set -s $(AZURE_SUBSCRIPTION_ID)
+          az account set -s $(SUBSCRIPTION_ID)
         displayName: Azure login
       - bash: bash .pipelines/scripts/setup_go.sh
         displayName: Setup go
@@ -32,7 +32,7 @@ jobs:
             export SIG_VERSION_TAG_VALUE=$VHD_BUILD_ID
             export IGNORE_SCENARIOS_WITH_MISSING_VHD=true
           else
-            echo "Build.BuildId is not specified. Running default e2e tests."
+            echo "VHD_BUILD_ID is not specified. Running tests with default SIG version tag selectors."
           fi
           
           cd e2e

e2e/config/azure.go

@@ -57,16 +57,16 @@ type AzureClient struct {
 	VirutalNetworkLinksClient *armprivatedns.VirtualNetworkLinksClient
 }
 
-func mustNewAzureClient(subscription string) *AzureClient {
-	client, err := NewAzureClient(subscription)
+func mustNewAzureClient() *AzureClient {
+	client, err := NewAzureClient()
 	if err != nil {
 		panic(err)
 	}
 	return client
 
 }
 
-func NewAzureClient(subscription string) (*AzureClient, error) {
+func NewAzureClient() (*AzureClient, error) {
 	httpClient := &http.Client{
 		// use a bunch of connections for load balancing
 		// ensure all timeouts are defined and reasonable
@@ -128,87 +128,87 @@ func NewAzureClient(subscription string) (*AzureClient, error) {
 		return nil, fmt.Errorf("create core client: %w", err)
 	}
 
-	cloud.RegistriesClient, err = armcontainerregistry.NewRegistriesClient(subscription, credential, opts)
+	cloud.RegistriesClient, err = armcontainerregistry.NewRegistriesClient(Config.SubscriptionID, credential, opts)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create registry client: %w", err)
 	}
 
-	cloud.CacheRulesClient, err = armcontainerregistry.NewCacheRulesClient(subscription, credential, opts)
+	cloud.CacheRulesClient, err = armcontainerregistry.NewCacheRulesClient(Config.SubscriptionID, credential, opts)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create cache rules client: %w", err)
 	}
 
-	cloud.PrivateEndpointClient, err = armnetwork.NewPrivateEndpointsClient(subscription, credential, opts)
+	cloud.PrivateEndpointClient, err = armnetwork.NewPrivateEndpointsClient(Config.SubscriptionID, credential, opts)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create private endpoint client: %w", err)
 	}
 
-	cloud.PrivateZonesClient, err = armprivatedns.NewPrivateZonesClient(subscription, credential, opts)
+	cloud.PrivateZonesClient, err = armprivatedns.NewPrivateZonesClient(Config.SubscriptionID, credential, opts)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create private dns zones client: %w", err)
 	}
 
-	cloud.VirutalNetworkLinksClient, err = armprivatedns.NewVirtualNetworkLinksClient(subscription, credential, opts)
+	cloud.VirutalNetworkLinksClient, err = armprivatedns.NewVirtualNetworkLinksClient(Config.SubscriptionID, credential, opts)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create virtual network links client: %w", err)
 	}
 
-	cloud.RecordSetClient, err = armprivatedns.NewRecordSetsClient(subscription, credential, opts)
+	cloud.RecordSetClient, err = armprivatedns.NewRecordSetsClient(Config.SubscriptionID, credential, opts)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create record set client: %w", err)
 	}
 
-	cloud.PrivateDNSZoneGroup, err = armnetwork.NewPrivateDNSZoneGroupsClient(subscription, credential, opts)
+	cloud.PrivateDNSZoneGroup, err = armnetwork.NewPrivateDNSZoneGroupsClient(Config.SubscriptionID, credential, opts)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create private dns zone group client: %w", err)
 	}
 
-	cloud.SecurityGroup, err = armnetwork.NewSecurityGroupsClient(subscription, credential, opts)
+	cloud.SecurityGroup, err = armnetwork.NewSecurityGroupsClient(Config.SubscriptionID, credential, opts)
 	if err != nil {
 		return nil, fmt.Errorf("create security group client: %w", err)
 	}
 
-	cloud.Subnet, err = armnetwork.NewSubnetsClient(subscription, credential, opts)
+	cloud.Subnet, err = armnetwork.NewSubnetsClient(Config.SubscriptionID, credential, opts)
 	if err != nil {
 		return nil, fmt.Errorf("create subnet client: %w", err)
 	}
 
-	cloud.AKS, err = armcontainerservice.NewManagedClustersClient(subscription, credential, opts)
+	cloud.AKS, err = armcontainerservice.NewManagedClustersClient(Config.SubscriptionID, credential, opts)
 	if err != nil {
 		return nil, fmt.Errorf("create aks client: %w", err)
 	}
 
-	cloud.Maintenance, err = armcontainerservice.NewMaintenanceConfigurationsClient(subscription, credential, opts)
+	cloud.Maintenance, err = armcontainerservice.NewMaintenanceConfigurationsClient(Config.SubscriptionID, credential, opts)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create maintenance client: %w", err)
 	}
 
-	cloud.VMSS, err = armcompute.NewVirtualMachineScaleSetsClient(subscription, credential, opts)
+	cloud.VMSS, err = armcompute.NewVirtualMachineScaleSetsClient(Config.SubscriptionID, credential, opts)
 	if err != nil {
 		return nil, fmt.Errorf("create vmss client: %w", err)
 	}
 
-	cloud.VMSSVM, err = armcompute.NewVirtualMachineScaleSetVMsClient(subscription, credential, opts)
+	cloud.VMSSVM, err = armcompute.NewVirtualMachineScaleSetVMsClient(Config.SubscriptionID, credential, opts)
 	if err != nil {
 		return nil, fmt.Errorf("create vmss vm client: %w", err)
 	}
 
-	cloud.Resource, err = armresources.NewClient(subscription, credential, opts)
+	cloud.Resource, err = armresources.NewClient(Config.SubscriptionID, credential, opts)
 	if err != nil {
 		return nil, fmt.Errorf("create resource client: %w", err)
 	}
 
-	cloud.ResourceGroup, err = armresources.NewResourceGroupsClient(subscription, credential, opts)
+	cloud.ResourceGroup, err = armresources.NewResourceGroupsClient(Config.SubscriptionID, credential, opts)
 	if err != nil {
 		return nil, fmt.Errorf("create resource group client: %w", err)
 	}
 
-	cloud.VNet, err = armnetwork.NewVirtualNetworksClient(subscription, credential, opts)
+	cloud.VNet, err = armnetwork.NewVirtualNetworksClient(Config.SubscriptionID, credential, opts)
 	if err != nil {
 		return nil, fmt.Errorf("create vnet client: %w", err)
 	}
 
-	cloud.GalleryImageVersion, err = armcompute.NewGalleryImageVersionsClient(subscription, credential, opts)
+	cloud.GalleryImageVersion, err = armcompute.NewGalleryImageVersionsClient(Config.GallerySubscriptionID, credential, opts)
 	if err != nil {
 		return nil, fmt.Errorf("create a new images client: %v", err)
 	}

e2e/config/config.go

@@ -1,6 +1,8 @@
 package config
 
 import (
+	"encoding/json"
+	"fmt"
 	"time"
 
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime"
@@ -10,34 +12,37 @@ import (
 
 var (
 	Config            = mustLoadConfig()
-	Azure             = mustNewAzureClient(Config.SubscriptionID)
+	Azure             = mustNewAzureClient()
 	ResourceGroupName = "abe2e-" + Config.Location
 	VMIdentityName    = "abe2e-vm-identity"
-	PrivateACRName    = "privateacre2e"
+	PrivateACRName    = "privateacre2e" + Config.Location
 
 	DefaultPollUntilDoneOptions = &runtime.PollUntilDoneOptions{
 		Frequency: time.Second,
 	}
 )
 
 type Configuration struct {
-	AirgapNSGName                 string        `env:"AIRGAP_NSG_NAME" envDefault:"abe2e-airgap-securityGroup"`
-	DefaultSubnetName             string        `env:"DEFAULT_SUBNET_NAME" envDefault:"aks-subnet"`
-	BuildID                       string        `env:"BUILD_ID" envDefault:"local"`
-	Location                      string        `env:"LOCATION" envDefault:"westus3"`
-	SubscriptionID                string        `env:"SUBSCRIPTION_ID" envDefault:"8ecadfc9-d1a3-4ea4-b844-0d9f87e4d7c8"`
-	SIGVersionTagName             string        `env:"SIG_VERSION_TAG_NAME" envDefault:"branch"`
-	SIGVersionTagValue            string        `env:"SIG_VERSION_TAG_VALUE" envDefault:"refs/heads/master"`
-	TagsToRun                     string        `env:"TAGS_TO_RUN"`
-	TagsToSkip                    string        `env:"TAGS_TO_SKIP"`
-	TestTimeout                   time.Duration `env:"TEST_TIMEOUT" envDefault:"35m"`
-	E2ELoggingDir                 string        `env:"LOGGING_DIR" envDefault:"scenario-logs"`
-	IgnoreScenariosWithMissingVHD bool          `env:"IGNORE_SCENARIOS_WITH_MISSING_VHD"`
-	SkipTestsWithSKUCapacityIssue bool          `env:"SKIP_TESTS_WITH_SKU_CAPACITY_ISSUE"`
-	KeepVMSS                      bool          `env:"KEEP_VMSS"`
-	BlobStorageAccountPrefix      string        `env:"BLOB_STORAGE_ACCOUNT_PREFIX" envDefault:"abe2e"`
-	BlobContainer                 string        `env:"BLOB_CONTAINER" envDefault:"abe2e"`
-	EnableNodeBootstrapperTest    bool          `env:"ENABLE_NODE_BOOTSTRAPPER_TEST"`
+	AirgapNSGName                 string        `env:"AIRGAP_NSG_NAME" envDefault:"abe2e-airgap-securityGroup" json:"airgapNSGName"`
+	DefaultSubnetName             string        `env:"DEFAULT_SUBNET_NAME" envDefault:"aks-subnet" json:"defaultSubnetName"`
+	BuildID                       string        `env:"BUILD_ID" envDefault:"local" json:"buildID"`
+	Location                      string        `env:"LOCATION" envDefault:"westus3" json:"location"`
+	SubscriptionID                string        `env:"SUBSCRIPTION_ID" envDefault:"8ecadfc9-d1a3-4ea4-b844-0d9f87e4d7c8" json:"subscriptionID"`
+	GallerySubscriptionID         string        `env:"GALLERY_SUBSCRIPTION_ID" envDefault:"c4c3550e-a965-4993-a50c-628fd38cd3e1" json:"gallerySubscriptionID"`
+	GalleryResourceGroupName      string        `env:"GALLERY_RESOURCE_GROUP_NAME" envDefault:"aksvhdtestbuildrg" json:"galleryResourceGroupName"`
+	GalleryName                   string        `env:"GALLERY_NAME" envDefault:"PackerSigGalleryEastUS" json:"galleryName"`
+	SIGVersionTagName             string        `env:"SIG_VERSION_TAG_NAME" envDefault:"branch" json:"sigVersionTagName"`
+	SIGVersionTagValue            string        `env:"SIG_VERSION_TAG_VALUE" envDefault:"refs/heads/master" json:"sigVersionTagValue"`
+	TagsToRun                     string        `env:"TAGS_TO_RUN" json:"tagsToRun"`
+	TagsToSkip                    string        `env:"TAGS_TO_SKIP" json:"tagsToSkip"`
+	TestTimeout                   time.Duration `env:"TEST_TIMEOUT" envDefault:"35m" json:"testTimeout"`
+	E2ELoggingDir                 string        `env:"LOGGING_DIR" envDefault:"scenario-logs" json:"e2eLoggingDir"`
+	IgnoreScenariosWithMissingVHD bool          `env:"IGNORE_SCENARIOS_WITH_MISSING_VHD" json:"ignoreScenariosWithMissingVHD"`
+	SkipTestsWithSKUCapacityIssue bool          `env:"SKIP_TESTS_WITH_SKU_CAPACITY_ISSUE" json:"skipTestsWithSKUCapacityIssue"`
+	KeepVMSS                      bool          `env:"KEEP_VMSS" json:"keepVMSS"`
+	BlobStorageAccountPrefix      string        `env:"BLOB_STORAGE_ACCOUNT_PREFIX" envDefault:"abe2e" json:"blobStorageAccountPrefix"`
+	BlobContainer                 string        `env:"BLOB_CONTAINER" envDefault:"abe2e" json:"blobContainer"`
+	EnableNodeBootstrapperTest    bool          `env:"ENABLE_NODE_BOOTSTRAPPER_TEST" json:"enableNodeBootstrapperTest"`
 }
 
 func (c *Configuration) BlobStorageAccount() string {
@@ -48,6 +53,18 @@ func (c *Configuration) BlobStorageAccountURL() string {
 	return "https://" + c.BlobStorageAccount() + ".blob.core.windows.net"
 }
 
+func (c *Configuration) GalleryResourceID() string {
+	return fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Compute/galleries/%s", c.GallerySubscriptionID, c.GalleryResourceGroupName, c.GalleryName)
+}
+
+func (c Configuration) String() string {
+	content, err := json.MarshalIndent(c, "", "	")
+	if err != nil {
+		panic(err)
+	}
+	return string(content)
+}
+
 func mustLoadConfig() Configuration {
 	_ = godotenv.Load(".env")
 	cfg := Configuration{}

e2e/config/vhd.go

@@ -14,7 +14,6 @@ import (
 )
 
 const (
-	imageGallery       = "/subscriptions/8ecadfc9-d1a3-4ea4-b844-0d9f87e4d7c8/resourceGroups/aksvhdtestbuildrg/providers/Microsoft.Compute/galleries/PackerSigGalleryEastUS/images/"
 	noSelectionTagName = "abe2e-ignore"
 )
 
@@ -64,6 +63,7 @@ var (
 	// this is a particular 2204gen2containerd image originally built with private packages,
 	// if we ever want to update this then we'd need to run a new VHD build using private package overrides
 	VHDUbuntu2204Gen2ContainerdPrivateKubePkg = &Image{
+		// 2204Gen2 is a special image definition holding historical VHDs used by agentbaker e2e's.
 		Name:    "2204Gen2",
 		OS:      "ubuntu",
 		Arch:    "amd64",
@@ -73,7 +73,7 @@ var (
 
 	// without kubelet, kubectl, credential-provider and wasm
 	VHDUbuntu2204Gen2ContainerdAirgapped = &Image{
-		Name:    "2204gen2containerd",
+		Name:    "2204Gen2",
 		OS:      "ubuntu",
 		Arch:    "amd64",
 		Version: "1.1725612526.29638",
@@ -102,7 +102,7 @@ func (i *Image) String() string {
 
 func (i *Image) VHDResourceID(ctx context.Context, t *testing.T) (VHDResourceID, error) {
 	i.vhdOnce.Do(func() {
-		imageDefinitionResourceID := imageGallery + i.Name
+		imageDefinitionResourceID := fmt.Sprintf("%s/images/%s", Config.GalleryResourceID(), i.Name)
 		if i.Version != "" {
 			i.vhd, i.vhdErr = ensureStaticSIGImageVersion(ctx, t, imageDefinitionResourceID+"/versions/"+i.Version)
 		} else {

e2e/scenario_test.go

@@ -16,12 +16,14 @@ import (
 )
 
 func TestMain(m *testing.M) {
+	fmt.Printf("using E2E environment configuration:\n%s\n", config.Config)
 	// delete scenario-logs folder if it exists
 	if _, err := os.Stat("scenario-logs"); err == nil {
 		_ = os.RemoveAll("scenario-logs")
 	}
 	m.Run()
 }
+
 func Test_azurelinuxv2(t *testing.T) {
 	RunScenario(t, &Scenario{
 		Description: "Tests that a node using a AzureLinuxV2 (CgroupV2) VHD can be properly bootstrapped",

vhdbuilder/packer/init-variables.sh

@@ -81,7 +81,7 @@ fi
 
 if [ -z "${VNET_RG_NAME}" ]; then
 	if [ "$MODE" == "linuxVhdMode" ]; then
-		VNET_RG_NAME="nodesig-${ENVIRONMENT}-${PACKER_BUILD_LOCATION}-agent-pool"
+		VNET_RG_NAME="nodesig-${ENVIRONMENT}-${PACKER_BUILD_LOCATION}-pool-vnet-rg"
 	fi
 	if [ "$MODE" == "windowsVhdMode" ]; then
 		if [[ "${POOL_NAME}" == *nodesigprod* ]]; then

vhdbuilder/packer/test-scan-and-cleanup.sh

@@ -10,6 +10,7 @@ required_env_vars=(
   "SIG_IMAGE_NAME"
   "UMSI_RESOURCE_ID"
   "UMSI_PRINCIPAL_ID"
+  "AZURE_MSI_RESOURCE_STRING"
   "UMSI_CLIENT_ID"
   "BUILD_RUN_NUMBER"
 )

vhdbuilder/packer/trivy-scan.sh

@@ -31,13 +31,14 @@ SEVERITY=${17}
 MODULE_VERSION=${18}
 UMSI_PRINCIPAL_ID=${19}
 UMSI_CLIENT_ID=${20}
-BUILD_RUN_NUMBER=${21}
-export BUILD_REPOSITORY_NAME=${22}
-export BUILD_SOURCEBRANCH=${23}
-export BUILD_SOURCEVERSION=${24}
-export SYSTEM_COLLECTIONURI=${25}
-export SYSTEM_TEAMPROJECT=${26}
-export BUILD_BUILDID=${27}
+AZURE_MSI_RESOURCE_STRING=${21}
+BUILD_RUN_NUMBER=${22}
+export BUILD_REPOSITORY_NAME=${23}
+export BUILD_SOURCEBRANCH=${24}
+export BUILD_SOURCEVERSION=${25}
+export SYSTEM_COLLECTIONURI=${26}
+export SYSTEM_TEAMPROJECT=${27}
+export BUILD_BUILDID=${28}
 
 retrycmd_if_failure() {
     retries=$1; wait_sleep=$2; timeout=$3; shift && shift && shift
@@ -93,13 +94,21 @@ install_azure_cli() {
     fi
 }
 
+login_with_user_assigned_managed_identity() {
+    local USERNAME=$1
+
+    LOGIN_FLAGS="--identity --username $USERNAME"
+    if [ "${ENABLE_TRUSTED_LAUNCH,,}" == "true" ]; then
+        LOGIN_FLAGS="$LOGIN_FLAGS --allow-no-subscriptions"
+    fi
+
+   echo "logging into azure with flags: $LOGIN_FLAGS"
+   az login $LOGIN_FLAGS
+}
+
 install_azure_cli $OS_SKU $OS_VERSION $ARCHITECTURE $TEST_VM_ADMIN_USERNAME
 
-if [[ "${ENABLE_TRUSTED_LAUNCH}" == "True" ]]; then
-    az login --identity --allow-no-subscriptions --username ${UMSI_PRINCIPAL_ID}
-else
-    az login --identity
-fi
+login_with_user_assigned_managed_identity ${UMSI_PRINCIPAL_ID}
 
 arch="$(uname -m)"
 if [ "${arch,,}" == "arm64" ] || [ "${arch,,}" == "aarch64" ]; then
@@ -182,6 +191,8 @@ rm ./trivy
 chmod a+r "${TRIVY_REPORT_ROOTFS_JSON_PATH}"
 chmod a+r "${TRIVY_REPORT_IMAGE_TABLE_PATH}"
 
+login_with_user_assigned_managed_identity ${AZURE_MSI_RESOURCE_STRING}
+
 az storage blob upload --file ${TRIVY_REPORT_ROOTFS_JSON_PATH} \
     --container-name ${SIG_CONTAINER_NAME} \
     --name ${TRIVY_UPLOAD_REPORT_NAME} \