Merge branch 'main' into zb/regression2

.pipelines/.vsts-vhd-builder.yaml

@@ -104,3 +104,18 @@ stages:
       LOCATION: $(PACKER_BUILD_LOCATION)
     jobs:
       - template: ./templates/e2e-template.yaml
+  - stage: Generate_and_Publish_Testdata
+    condition: succeeded()
+    jobs:
+      - job: Generate_Publish_Testdata
+        timeoutInMinutes: 10
+        steps:
+        - bash: |
+            make generate
+          displayName: Generate Testdata
+        - task: PublishPipelineArtifact@1
+          inputs:
+            targetPath: $(Pipeline.Workspace)/pkg/agent/testdata
+            artifact: testdata
+            publishLocation: pipeline
+

.pipelines/scripts/verify_shell.sh

@@ -25,7 +25,7 @@ else
     echo "shellcheck installed"
 fi
 
-filesToCheck=$(find . -type f -name "*.sh" -not -path './parts/linux/cloud-init/artifacts/*' -not -path './pkg/agent/testdata/*' -not -path './vendor/*' -not -path './hack/tools/vendor/*' -not -path './.git/*' -not -path './self-contained/*' -not -path './hack/tools/bin/shellspecsrc/*')
+filesToCheck=$(find . -type f -name "*.sh" -not -path './parts/linux/cloud-init/artifacts/*' -not -path './pkg/agent/testdata/*' -not -path './vendor/*' -not -path './hack/tools/vendor/*' -not -path './.git/*' -not -path './hack/tools/bin/shellspecsrc/*')
 
 # also shell-check generated test data
 generatedTestData=$(find ./pkg/agent/testdata -type f -name "*.sh" )

.pipelines/templates/.builder-release-template-windows.yaml

@@ -69,21 +69,6 @@ steps:
       echo "##vso[task.setvariable variable=BUILD_DATE]$BUILD_DATE"
     displayName: Get Build Mode
 
-  - bash: |
-      if [[ ${{ parameters.artifactName }} =~ "2019" ]]; then
-        WINDOWS_VERSION="$(cat vhdbuilder/packer/windows-image.env | grep -a "WINDOWS_2019_BASE_IMAGE_VERSION" | cut -d "=" -f 2 | cut -d "." -f 1,2)"
-      elif [[ ${{ parameters.artifactName }} =~ "2022" ]]; then
-        WINDOWS_VERSION="$(cat vhdbuilder/packer/windows-image.env | grep -a "WINDOWS_2022_BASE_IMAGE_VERSION" | cut -d "=" -f 2 | cut -d "." -f 1,2)"
-      elif [[ ${{ parameters.artifactName }} =~ "23H2" ]]; then
-        WINDOWS_VERSION="$(cat vhdbuilder/packer/windows-image.env | grep -a "WINDOWS_23H2_BASE_IMAGE_VERSION" | cut -d "=" -f 2 | cut -d "." -f 1,2)"
-      else
-        echo "Current distro is not supported to get image version. You need to update related code."
-        exit 1
-      fi
-      AKS_WINDOWS_IMAGE_VERSION="$WINDOWS_VERSION.${BUILD_DATE}"
-      echo "##vso[task.setvariable variable=AKS_WINDOWS_IMAGE_VERSION]$AKS_WINDOWS_IMAGE_VERSION"
-    displayName: Get Image Version
-
   - script: |
       branch=$(Build.SourceBranch)
       branch=$(echo "${branch}" | sed 's/refs\/heads\///g')
@@ -116,7 +101,7 @@ steps:
       -e WINDOWS_CORE_IMAGE_URL=${WINDOWS_CORE_IMAGE_URL} \
       -e WINDOWS_PRIVATE_PACKAGES_URL=${WINDOWS_PRIVATE_PACKAGES_URL} \
       -e AZURE_MSI_RESOURCE_STRING=${AZURE_MSI_RESOURCE_STRING} \
-      -e AKS_WINDOWS_IMAGE_VERSION=${AKS_WINDOWS_IMAGE_VERSION} \
+      -e BUILD_DATE=${BUILD_DATE} \
       ${AZURE_CONTAINER_IMAGE} make -f packer.mk run-packer-windows
     displayName: Building windows VHD
 
@@ -156,6 +141,12 @@ steps:
   - bash: |
       sudo chmod 777 image-bom.json
       jq . image-bom.json > tmp.json
+
+      echo "Reading image version from image-bom.json"
+      AKS_WINDOWS_IMAGE_VERSION=$(cat image-bom.json | jq -r '.imageVersion')
+      echo "##vso[task.setvariable variable=AKS_WINDOWS_IMAGE_VERSION]$AKS_WINDOWS_IMAGE_VERSION"
+      echo "Image version: $AKS_WINDOWS_IMAGE_VERSION"
+
       mv tmp.json ${AKS_WINDOWS_IMAGE_VERSION}-image-list.json
       cp release-notes.txt ${AKS_WINDOWS_IMAGE_VERSION}.txt
     displayName: Reformat image-bom.json and rename release-notes.txt

.pipelines/templates/.builder-release-template.yaml

@@ -135,7 +135,7 @@ steps:
       echo "##vso[task.setvariable variable=OS_DISK_URI]$(cat packer-output | grep "OSDiskUri:" | cut -d " " -f 2)" && \
       echo "##vso[task.setvariable variable=MANAGED_SIG_ID]$(cat packer-output | grep "ManagedImageSharedImageGalleryId:" | cut -d " " -f 2)" && \
       echo "##vso[task.setvariable variable=SIG_GALLERY_NAME]$(cat vhdbuilder/packer/settings.json | grep "sig_gallery_name" | awk -F':' '{print $2}' | awk -F'"' '{print $2}')" && \
-      echo "##vso[task.setvariable variable=BUILD_PERF_DATA_FILE]vhd-build-performance-data.json" && \
+      echo "##vso[task.setvariable variable=PERFORMANCE_DATA_FILE]vhd-build-performance-data.json" && \
       echo "##vso[task.setvariable variable=PKR_RG_NAME]$(cat packer-output | grep "ResourceGroupName" | cut -d "'" -f 2 | head -1)" && \
       echo "##vso[task.setvariable variable=IS_NOT_1804]$( [[ "${OS_VERSION}" != "18.04" ]] && echo true || echo false )" && \
       echo "##vso[task.setvariable variable=OS_NAME]Linux" && \
@@ -160,9 +160,20 @@ steps:
       SYSTEM_TEAMPROJECT: $(System.TeamProject)
       BUILD_RUN_NUMBER: $(Build.BuildNumber)
 
-  - bash: make -f packer.mk evaluate-build-performance
+  - task: PublishPipelineArtifact@0
     condition: always()
-    displayName: Check Build Performance
+    displayName: Publish BCC Tools Installation Log
+    inputs:
+      artifactName: 'bcc-tools-installation-log-${{ parameters.artifactName }}'
+      targetPath: 'bcc-tools-installation.log'
+
+  - task: CopyFiles@2
+    condition: always()
+    displayName: Copy BCC Tools Installation Log
+    inputs:
+      SourceFolder: '$(System.DefaultWorkingDirectory)'
+      Contents: 'bcc-tools-installation.log'
+      TargetFolder: '$(Build.ArtifactStagingDirectory)'
 
   - task: CopyFiles@2
     condition: eq(variables['IS_NOT_1804'], 'true')
@@ -213,6 +224,10 @@ steps:
     env:
       RESOURCE_GROUP_NAME: $(AZURE_RESOURCE_GROUP_NAME)
 
+  - bash: make -f packer.mk evaluate-build-performance
+    condition: always()
+    displayName: Check Build Performance
+
   - bash: make -f packer.mk generate-sas
     displayName: Getting Shared Access Signature URI
     condition: and(succeeded(), eq(variables.DRY_RUN, 'False'))

e2e/aks_model.go

@@ -150,7 +150,17 @@ func airGapSecurityGroup(location, clusterFQDN string) (armnetwork.SecurityGroup
 func addPrivateEndpointForACR(ctx context.Context, t *testing.T, nodeResourceGroup string, vnet VNet) error {
 	t.Logf("Adding private endpoint for ACR in rg %s\n", nodeResourceGroup)
 
-	peResp, err := createPrivateEndpoint(ctx, t, nodeResourceGroup, vnet)
+	privateEndpointName := "PE-for-ABE2ETests"
+	exists, err := privateEndpointExists(ctx, t, nodeResourceGroup, privateEndpointName)
+	if err != nil {
+		return err
+	}
+	if exists {
+		t.Logf("Private Endpoint already exists, skipping creation")
+		return nil
+	}
+
+	peResp, err := createPrivateEndpoint(ctx, t, nodeResourceGroup, privateEndpointName, vnet)
 	if err != nil {
 		return err
 	}
@@ -179,8 +189,19 @@ func addPrivateEndpointForACR(ctx context.Context, t *testing.T, nodeResourceGro
 	return nil
 }
 
-func createPrivateEndpoint(ctx context.Context, t *testing.T, nodeResourceGroup string, vnet VNet) (armnetwork.PrivateEndpointsClientCreateOrUpdateResponse, error) {
-	endpointName := "PE-for-ABE2ETests"
+func privateEndpointExists(ctx context.Context, t *testing.T, nodeResourceGroup, privateEndpointName string) (bool, error) {
+	existingPE, err := config.Azure.PrivateEndpointClient.Get(ctx, nodeResourceGroup, privateEndpointName, nil)
+	if err == nil && existingPE.ID != nil {
+		t.Logf("Private Endpoint already exists with ID: %s\n", *existingPE.ID)
+		return true, nil
+	}
+	if err != nil && !strings.Contains(err.Error(), "ResourceNotFound") {
+		return false, fmt.Errorf("failed to get private endpoint: %w", err)
+	}
+	return false, nil
+}
+
+func createPrivateEndpoint(ctx context.Context, t *testing.T, nodeResourceGroup, privateEndpointName string, vnet VNet) (armnetwork.PrivateEndpointsClientCreateOrUpdateResponse, error) {
 	peParams := armnetwork.PrivateEndpoint{
 		Location: to.Ptr(config.Config.Location),
 		Properties: &armnetwork.PrivateEndpointProperties{
@@ -189,7 +210,7 @@ func createPrivateEndpoint(ctx context.Context, t *testing.T, nodeResourceGroup
 			},
 			PrivateLinkServiceConnections: []*armnetwork.PrivateLinkServiceConnection{
 				{
-					Name: to.Ptr(endpointName),
+					Name: to.Ptr(privateEndpointName),
 					Properties: &armnetwork.PrivateLinkServiceConnectionProperties{
 						PrivateLinkServiceID: to.Ptr("/subscriptions/8ecadfc9-d1a3-4ea4-b844-0d9f87e4d7c8/resourceGroups/aksvhdtestbuildrg/providers/Microsoft.ContainerRegistry/registries/aksvhdtestcr"),
 						GroupIDs:             []*string{to.Ptr("registry")},
@@ -202,7 +223,7 @@ func createPrivateEndpoint(ctx context.Context, t *testing.T, nodeResourceGroup
 	poller, err := config.Azure.PrivateEndpointClient.BeginCreateOrUpdate(
 		ctx,
 		nodeResourceGroup,
-		endpointName,
+		privateEndpointName,
 		peParams,
 		nil,
 	)
@@ -213,7 +234,7 @@ func createPrivateEndpoint(ctx context.Context, t *testing.T, nodeResourceGroup
 	if err != nil {
 		return armnetwork.PrivateEndpointsClientCreateOrUpdateResponse{}, fmt.Errorf("failed to create private endpoint in polling: %w", err)
 	}
-	
+
 	t.Logf("Private Endpoint created or updated with ID: %s\n", *resp.ID)
 	return resp, nil
 }
@@ -346,7 +367,6 @@ func getRequiredSecurityRules(clusterFQDN string) ([]*armnetwork.SecurityRule, e
 	// https://learn.microsoft.com/en-us/azure/aks/outbound-rules-control-egress#azure-global-required-fqdn--application-rules
 	// note that we explicitly exclude packages.microsoft.com
 	requiredDNSNames := []string{
-		"mcr.microsoft.com",
 		"management.azure.com",
 		clusterFQDN,
 	}

e2e/cluster.go

@@ -62,14 +62,14 @@ func (c *Cluster) MaxPodsPerNode() (int, error) {
 // sync.Once is used to ensure that only one cluster for the set of tests is created
 func ClusterKubenet(ctx context.Context, t *testing.T) (*Cluster, error) {
 	clusterKubenetOnce.Do(func() {
-		clusterKubenet, clusterKubenetError = prepareCluster(ctx, t, getKubenetClusterModel("abe2e-kubenet"))
+		clusterKubenet, clusterKubenetError = prepareCluster(ctx, t, getKubenetClusterModel("abe2e-kubenet"), false)
 	})
 	return clusterKubenet, clusterKubenetError
 }
 
 func ClusterKubenetAirgap(ctx context.Context, t *testing.T) (*Cluster, error) {
 	clusterKubenetAirgapOnce.Do(func() {
-		cluster, err := prepareCluster(ctx, t, getKubenetClusterModel("abe2e-kubenet-airgap"))
+		cluster, err := prepareCluster(ctx, t, getKubenetClusterModel("abe2e-kubenet-airgap"), true)
 		if err == nil {
 			err = addAirgapNetworkSettings(ctx, t, cluster)
 		}
@@ -80,7 +80,7 @@ func ClusterKubenetAirgap(ctx context.Context, t *testing.T) (*Cluster, error) {
 
 func ClusterAzureNetwork(ctx context.Context, t *testing.T) (*Cluster, error) {
 	clusterAzureNetworkOnce.Do(func() {
-		clusterAzureNetwork, clusterAzureNetworkError = prepareCluster(ctx, t, getAzureNetworkClusterModel("abe2e-azure-network"))
+		clusterAzureNetwork, clusterAzureNetworkError = prepareCluster(ctx, t, getAzureNetworkClusterModel("abe2e-azure-network"), false)
 	})
 	return clusterAzureNetwork, clusterAzureNetworkError
 }
@@ -99,7 +99,7 @@ func nodeBootsrappingConfig(ctx context.Context, t *testing.T, kube *Kubeclient)
 	return baseNodeBootstrappingConfig, nil
 }
 
-func prepareCluster(ctx context.Context, t *testing.T, cluster *armcontainerservice.ManagedCluster) (*Cluster, error) {
+func prepareCluster(ctx context.Context, t *testing.T, cluster *armcontainerservice.ManagedCluster, isAirgap bool) (*Cluster, error) {
 	cluster.Name = to.Ptr(fmt.Sprintf("%s-%s", *cluster.Name, hash(cluster)))
 
 	cluster, err := getOrCreateCluster(ctx, t, cluster)
@@ -123,10 +123,11 @@ func prepareCluster(ctx context.Context, t *testing.T, cluster *armcontainerserv
 		return nil, fmt.Errorf("get kube client using cluster %q: %w", *cluster.Name, err)
 	}
 
-	if err := ensureDebugDaemonsets(ctx, kube); err != nil {
+	if err := ensureDebugDaemonsets(ctx, kube, isAirgap); err != nil {
 		return nil, fmt.Errorf("ensure debug daemonsets for %q: %w", *cluster.Name, err)
 	}
 
+	t.Logf("node resource group: %s", *cluster.Properties.NodeResourceGroup)
 	subnetID, err := getClusterSubnetID(ctx, *cluster.Properties.NodeResourceGroup)
 	if err != nil {
 		return nil, fmt.Errorf("get cluster subnet: %w", err)
@@ -163,6 +164,7 @@ func getOrCreateCluster(ctx context.Context, t *testing.T, cluster *armcontainer
 	if err != nil {
 		return nil, fmt.Errorf("failed to get cluster %q: %w", *cluster.Name, err)
 	}
+	t.Logf("cluster %s already exists in rg %s\n", *cluster.Name, config.ResourceGroupName)
 	switch *existingCluster.Properties.ProvisioningState {
 	case "Succeeded":
 		return &existingCluster.ManagedCluster, nil

e2e/kube.go

@@ -76,19 +76,24 @@ func getClusterKubeconfigBytes(ctx context.Context, resourceGroupName, clusterNa
 }
 
 // this is a bit ugly, but we don't want to execute this piece concurrently with other tests
-func ensureDebugDaemonsets(ctx context.Context, kube *Kubeclient) error {
-	hostDS := getDebugDaemonsetTemplate(hostNetworkDebugAppLabel, "nodepool1", true)
+func ensureDebugDaemonsets(ctx context.Context, kube *Kubeclient, isAirgap bool) error {
+	hostDS := getDebugDaemonsetTemplate(hostNetworkDebugAppLabel, "nodepool1", true, isAirgap)
 	if err := createDebugDaemonset(ctx, kube, hostDS); err != nil {
 		return err
 	}
-	nonHostDS := getDebugDaemonsetTemplate(podNetworkDebugAppLabel, "nodepool2", false)
+	nonHostDS := getDebugDaemonsetTemplate(podNetworkDebugAppLabel, "nodepool2", false, isAirgap)
 	if err := createDebugDaemonset(ctx, kube, nonHostDS); err != nil {
 		return err
 	}
 	return nil
 }
 
-func getDebugDaemonsetTemplate(deploymentName, targetNodeLabel string, isHostNetwork bool) string {
+func getDebugDaemonsetTemplate(deploymentName, targetNodeLabel string, isHostNetwork, isAirgap bool) string {
+	image := "mcr.microsoft.com/cbl-mariner/base/core:2.0"
+	if isAirgap {
+		image = "aksvhdtestcr.azurecr.io/aks/cbl-mariner/base/core:2.0"
+	}
+
 	return fmt.Sprintf(`apiVersion: apps/v1
 kind: DaemonSet
 metadata:
@@ -111,7 +116,7 @@ spec:
         kubernetes.azure.com/agentpool: %[3]s 
       hostPID: true
       containers:
-      - image: mcr.microsoft.com/cbl-mariner/base/core:2.0
+      - image: %[4]s
         name: mariner
         command: ["sleep", "infinity"]
         resources:
@@ -121,7 +126,7 @@ spec:
           privileged: true
           capabilities:
             add: ["SYS_PTRACE", "SYS_RAWIO"]
-`, deploymentName, isHostNetwork, targetNodeLabel)
+`, deploymentName, isHostNetwork, targetNodeLabel, image)
 }
 
 func createDebugDaemonset(ctx context.Context, kube *Kubeclient, manifest string) error {

e2e/scenario_helpers_test.go

@@ -123,7 +123,7 @@ func executeScenario(ctx context.Context, t *testing.T, opts *scenarioRunOpts) {
 	createVMSS(ctx, t, vmssName, opts, privateKeyBytes, publicKeyBytes)
 
 	t.Logf("vmss %s creation succeeded, proceeding with node readiness and pod checks...", vmssName)
-	nodeName := validateNodeHealth(ctx, t, opts.clusterConfig.Kube, vmssName)
+	nodeName := validateNodeHealth(ctx, t, opts.clusterConfig.Kube, vmssName, opts.scenario.Tags.Airgap)
 
 	// skip when outbound type is block as the wasm will create pod from gcr, however, network isolated cluster scenario will block egress traffic of gcr.
 	// TODO(xinhl): add another way to validate

e2e/scenario_test.go

@@ -43,7 +43,6 @@ func Test_azurelinuxv2AirGap(t *testing.T) {
 			BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) {
 				nbc.ContainerService.Properties.AgentPoolProfiles[0].Distro = "aks-azurelinux-v2-gen2"
 				nbc.AgentPoolProfile.Distro = "aks-azurelinux-v2-gen2"
-
 				nbc.OutboundType = datamodel.OutboundTypeBlock
 				nbc.ContainerService.Properties.SecurityProfile = &datamodel.SecurityProfile{
 					PrivateEgress: &datamodel.PrivateEgress{

e2e/template.go

@@ -448,15 +448,20 @@ func baseTemplate(location string) *datamodel.NodeBootstrappingConfiguration {
 	}
 }
 
-func getHTTPServerTemplate(podName, nodeName string) string {
+func getHTTPServerTemplate(podName, nodeName string, isAirgap bool) string {
+	image := "mcr.microsoft.com/cbl-mariner/busybox:2.0"
+	if isAirgap {
+		image = "aksvhdtestcr.azurecr.io/aks/cbl-mariner/busybox:2.0"
+	}
+
 	return fmt.Sprintf(`apiVersion: v1
 kind: Pod
 metadata:
   name: %s
 spec:
   containers:
   - name: mariner
-    image: mcr.microsoft.com/cbl-mariner/busybox:2.0
+    image: %s
     imagePullPolicy: IfNotPresent
     command: ["sh", "-c"]
     args:
@@ -473,7 +478,7 @@ spec:
       httpGet:
         path: /
         port: 80
-`, podName, nodeName)
+`, podName, image, nodeName)
 }
 
 func getWasmSpinPodTemplate(podName, nodeName string) string {

e2e/validation.go

@@ -11,10 +11,10 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func validateNodeHealth(ctx context.Context, t *testing.T, kube *Kubeclient, vmssName string) string {
+func validateNodeHealth(ctx context.Context, t *testing.T, kube *Kubeclient, vmssName string, isAirgap bool) string {
 	nodeName := waitUntilNodeReady(ctx, t, kube, vmssName)
 	testPodName := fmt.Sprintf("test-pod-%s", nodeName)
-	testPodManifest := getHTTPServerTemplate(testPodName, nodeName)
+	testPodManifest := getHTTPServerTemplate(testPodName, nodeName, isAirgap)
 	err := ensurePod(ctx, t, defaultNamespace, kube, testPodName, testPodManifest)
 	require.NoError(t, err, "failed to validate node health, unable to ensure test pod on node %q", nodeName)
 	return nodeName

parts/linux/cloud-init/artifacts/components.json

@@ -219,7 +219,7 @@
       "multiArchVersionsV2": [
         {
           "renovateTag": "registry=https://mcr.microsoft.com, name=azuremonitor/containerinsights/ciprod",
-          "latestVersion": "3.1.23"
+          "latestVersion": "3.1.24"
         }
       ]
     },