Merge branch 'master' into djsly/uncached-metrics-server

.golangci.yaml

@@ -148,7 +148,7 @@ linters-settings:
       shadow:
         # Whether to be strict about shadowing; can be noisy.
         # Default: false
-        strict: true
+        strict: false
 
   nakedret:
     # Make an issue if func has more lines of code than this setting, and it has naked returns.

.pipelines/.vsts-vhd-builder.yaml

@@ -104,18 +104,4 @@ stages:
       LOCATION: $(PACKER_BUILD_LOCATION)
     jobs:
       - template: ./templates/e2e-template.yaml
-  - stage: Generate_and_Publish_Testdata
-    condition: succeeded()
-    jobs:
-      - job: Generate_Publish_Testdata
-        timeoutInMinutes: 10
-        steps:
-        - bash: |
-            make generate
-          displayName: Generate Testdata
-        - task: PublishPipelineArtifact@1
-          inputs:
-            targetPath: $(Pipeline.Workspace)/pkg/agent/testdata
-            artifact: testdata
-            publishLocation: pipeline
 

.pipelines/.vsts-vhd-refresh-testdata.yaml

@@ -0,0 +1,38 @@
+name: $(Date:yyyyMMdd)$(Rev:.r)_$(Build.SourceBranchName)_$(BuildID)
+trigger:
+  branches:
+    include:
+    - master
+  paths:
+    include:
+    - schemas
+    - vhdbuilder/packer
+    - vhdbuilder/scripts/linux
+    - .pipelines/.vsts-vhd-refresh-testdata.yaml
+    - parts/linux/*
+    - packer.mk
+    exclude:
+    - vhdbuilder/packer/*.ps1
+    - vhdbuilder/packer/**/*.ps1
+    - vhdbuilder/packer/*windows*
+    - vhdbuilder/packer/**/*windows*
+
+pool:
+  name: $(POOL_NAME)
+
+stages:
+  - stage: Generate_and_Publish_Testdata
+    condition: succeeded()
+    jobs:
+      - job: Generate_Publish_Testdata
+        timeoutInMinutes: 10
+        steps:
+        - bash: |
+            make generate
+          displayName: Generate Testdata
+        - task: PublishPipelineArtifact@1
+          inputs:
+            targetPath: $(Pipeline.Workspace)/s/pkg/agent/testdata
+            artifact: testdata
+            publishLocation: pipeline
+

.pipelines/e2e.yaml

@@ -1,8 +1,8 @@
 name: $(Date:yyyyMMdd)$(Rev:.r)
-trigger: 
+trigger:
   branches:
-    include: 
-    - master
+    include:
+      - master
 pr:
   branches:
     include:

.pipelines/scripts/verify_shell.sh

@@ -25,7 +25,7 @@ else
     echo "shellcheck installed"
 fi
 
-filesToCheck=$(find . -type f -name "*.sh" -not -path './parts/linux/cloud-init/artifacts/*' -not -path './pkg/agent/testdata/*' -not -path './vendor/*' -not -path './hack/tools/vendor/*' -not -path './.git/*' -not -path './self-contained/*' -not -path './hack/tools/bin/shellspecsrc/*')
+filesToCheck=$(find . -type f -name "*.sh" -not -path './parts/linux/cloud-init/artifacts/*' -not -path './pkg/agent/testdata/*' -not -path './vendor/*' -not -path './hack/tools/vendor/*' -not -path './.git/*' -not -path './hack/tools/bin/shellspecsrc/*')
 
 # also shell-check generated test data
 generatedTestData=$(find ./pkg/agent/testdata -type f -name "*.sh" )

.pipelines/templates/.builder-release-template.yaml

@@ -40,12 +40,12 @@ steps:
         exit 1
       fi
       echo "Found source components.json to use for overrides: ${COMPONENT_JSON_SRC}"
-      
+
       echo "Overwriting component.json from ${COMPONENT_JSON_SRC} -> ${COMPONENT_JSON_DEST}"
       cp -af "${COMPONENT_JSON_SRC}" "${COMPONENT_JSON_DEST}"
     condition: eq('${{ parameters.useOverrides }}', true)
     displayName: Apply Overrides
-  
+
   - bash: |
       m="linuxVhdMode" && \
       echo "Set build mode to $m" && \
@@ -112,7 +112,10 @@ steps:
       echo "##vso[task.setvariable variable=SKU_NAME]$SKU_NAME"
       echo "Set SKU_NAME to $SKU_NAME"
     displayName: Set SKU Name
-    
+  - bash: |
+      ./build.sh
+    displayName: Build node-bootstrapper
+    workingDirectory: node-bootstrapper
   - bash: make -f packer.mk run-packer
     displayName: Build VHD
     retryCountOnTaskFailure: 3
@@ -123,7 +126,7 @@ steps:
       BUILD_NUMBER: $(Build.BuildNumber)
       BUILD_ID: $(Build.BuildId)
       BUILD_DEFINITION_NAME: $(Build.DefinitionName)
-  
+
   - bash: |
       CAPTURED_SIG_VERSION="$(cat vhdbuilder/packer/settings.json | grep "captured_sig_version" | awk -F':' '{print $2}' | awk -F'"' '{print $2}')" && \
       SIG_IMAGE_NAME="$(cat vhdbuilder/packer/settings.json | grep "sig_image_name" | awk -F':' '{print $2}' | awk -F'"' '{print $2}')" && \
@@ -135,7 +138,7 @@ steps:
       echo "##vso[task.setvariable variable=OS_DISK_URI]$(cat packer-output | grep "OSDiskUri:" | cut -d " " -f 2)" && \
       echo "##vso[task.setvariable variable=MANAGED_SIG_ID]$(cat packer-output | grep "ManagedImageSharedImageGalleryId:" | cut -d " " -f 2)" && \
       echo "##vso[task.setvariable variable=SIG_GALLERY_NAME]$(cat vhdbuilder/packer/settings.json | grep "sig_gallery_name" | awk -F':' '{print $2}' | awk -F'"' '{print $2}')" && \
-      echo "##vso[task.setvariable variable=BUILD_PERF_DATA_FILE]vhd-build-performance-data.json" && \
+      echo "##vso[task.setvariable variable=PERFORMANCE_DATA_FILE]vhd-build-performance-data.json" && \
       echo "##vso[task.setvariable variable=PKR_RG_NAME]$(cat packer-output | grep "ResourceGroupName" | cut -d "'" -f 2 | head -1)" && \
       echo "##vso[task.setvariable variable=IS_NOT_1804]$( [[ "${OS_VERSION}" != "18.04" ]] && echo true || echo false )" && \
       echo "##vso[task.setvariable variable=OS_NAME]Linux" && \
@@ -160,10 +163,6 @@ steps:
       SYSTEM_TEAMPROJECT: $(System.TeamProject)
       BUILD_RUN_NUMBER: $(Build.BuildNumber)
 
-  - bash: make -f packer.mk evaluate-build-performance
-    condition: always()
-    displayName: Check Build Performance
-
   - task: PublishPipelineArtifact@0
     condition: always()
     displayName: Publish BCC Tools Installation Log
@@ -228,6 +227,10 @@ steps:
     env:
       RESOURCE_GROUP_NAME: $(AZURE_RESOURCE_GROUP_NAME)
 
+  - bash: make -f packer.mk evaluate-build-performance
+    condition: always()
+    displayName: Check Build Performance
+
   - bash: make -f packer.mk generate-sas
     displayName: Getting Shared Access Signature URI
     condition: and(succeeded(), eq(variables.DRY_RUN, 'False'))

e2e/cluster.go

@@ -85,7 +85,7 @@ func ClusterAzureNetwork(ctx context.Context, t *testing.T) (*Cluster, error) {
 	return clusterAzureNetwork, clusterAzureNetworkError
 }
 
-func nodeBootsrappingConfig(ctx context.Context, t *testing.T, kube *Kubeclient) (*datamodel.NodeBootstrappingConfiguration, error) {
+func nodeBootstrappingConfig(ctx context.Context, t *testing.T, kube *Kubeclient) (*datamodel.NodeBootstrappingConfiguration, error) {
 	clusterParams, err := extractClusterParameters(ctx, t, kube)
 	if err != nil {
 		return nil, fmt.Errorf("extract cluster parameters: %w", err)
@@ -133,7 +133,7 @@ func prepareCluster(ctx context.Context, t *testing.T, cluster *armcontainerserv
 		return nil, fmt.Errorf("get cluster subnet: %w", err)
 	}
 
-	nbc, err := nodeBootsrappingConfig(ctx, t, kube)
+	nbc, err := nodeBootstrappingConfig(ctx, t, kube)
 	if err != nil {
 		return nil, fmt.Errorf("get node bootstrapping configuration: %w", err)
 	}

e2e/config/azure.go

@@ -1,42 +1,50 @@
 package config
 
 import (
+	"context"
 	"crypto/tls"
 	"fmt"
 	"net"
 	"net/http"
+	"os"
 	"time"
 
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/arm"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime"
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
 	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v6"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service"
 	"github.com/Azure/go-armbalancer"
 )
 
 type AzureClient struct {
+	AKS                       *armcontainerservice.ManagedClustersClient
+	Blob                      *azblob.Client
 	Core                      *azcore.Client
-	VMSS                      *armcompute.VirtualMachineScaleSetsClient
-	VMSSVM                    *armcompute.VirtualMachineScaleSetVMsClient
-	VNet                      *armnetwork.VirtualNetworksClient
+	Credential                *azidentity.DefaultAzureCredential
+	GalleryImageVersion       *armcompute.GalleryImageVersionsClient
+	Maintenance               *armcontainerservice.MaintenanceConfigurationsClient
+	PrivateDNSZoneGroup       *armnetwork.PrivateDNSZoneGroupsClient
+	PrivateEndpointClient     *armnetwork.PrivateEndpointsClient
+	PrivateZonesClient        *armprivatedns.PrivateZonesClient
+	RecordSetClient           *armprivatedns.RecordSetsClient
 	Resource                  *armresources.Client
 	ResourceGroup             *armresources.ResourceGroupsClient
-	AKS                       *armcontainerservice.ManagedClustersClient
-	Maintenance               *armcontainerservice.MaintenanceConfigurationsClient
 	SecurityGroup             *armnetwork.SecurityGroupsClient
 	Subnet                    *armnetwork.SubnetsClient
-	GalleryImageVersionClient *armcompute.GalleryImageVersionsClient
-	PrivateEndpointClient     *armnetwork.PrivateEndpointsClient
-	PrivateZonesClient        *armprivatedns.PrivateZonesClient
+	VMSS                      *armcompute.VirtualMachineScaleSetsClient
+	VMSSVM                    *armcompute.VirtualMachineScaleSetVMsClient
+	VNet                      *armnetwork.VirtualNetworksClient
 	VirutalNetworkLinksClient *armprivatedns.VirtualNetworkLinksClient
-	RecordSetClient           *armprivatedns.RecordSetsClient
-	PrivateDNSZoneGroup       *armnetwork.PrivateDNSZoneGroupsClient
 }
 
 func mustNewAzureClient(subscription string) *AzureClient {
@@ -107,7 +115,7 @@ func NewAzureClient(subscription string) (*AzureClient, error) {
 	// purely for telemetry, entirely unused today
 	cloud.Core, err = azcore.NewClient("agentbakere2e.e2e_test", "v0.0.0", plOpts, clOpts)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create core client: %w", err)
+		return nil, fmt.Errorf("create core client: %w", err)
 	}
 
 	cloud.PrivateEndpointClient, err = armnetwork.NewPrivateEndpointsClient(subscription, credential, opts)
@@ -137,17 +145,17 @@ func NewAzureClient(subscription string) (*AzureClient, error) {
 
 	cloud.SecurityGroup, err = armnetwork.NewSecurityGroupsClient(subscription, credential, opts)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create security group client: %w", err)
+		return nil, fmt.Errorf("create security group client: %w", err)
 	}
 
 	cloud.Subnet, err = armnetwork.NewSubnetsClient(subscription, credential, opts)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create subnet client: %w", err)
+		return nil, fmt.Errorf("create subnet client: %w", err)
 	}
 
 	cloud.AKS, err = armcontainerservice.NewManagedClustersClient(subscription, credential, opts)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create aks client: %w", err)
+		return nil, fmt.Errorf("create aks client: %w", err)
 	}
 
 	cloud.Maintenance, err = armcontainerservice.NewMaintenanceConfigurationsClient(subscription, credential, opts)
@@ -157,37 +165,74 @@ func NewAzureClient(subscription string) (*AzureClient, error) {
 
 	cloud.VMSS, err = armcompute.NewVirtualMachineScaleSetsClient(subscription, credential, opts)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create vmss client: %w", err)
+		return nil, fmt.Errorf("create vmss client: %w", err)
 	}
 
 	cloud.VMSSVM, err = armcompute.NewVirtualMachineScaleSetVMsClient(subscription, credential, opts)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create vmss vm client: %w", err)
+		return nil, fmt.Errorf("create vmss vm client: %w", err)
 	}
 
 	cloud.Resource, err = armresources.NewClient(subscription, credential, opts)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create resource client: %w", err)
+		return nil, fmt.Errorf("create resource client: %w", err)
 	}
 
 	cloud.ResourceGroup, err = armresources.NewResourceGroupsClient(subscription, credential, opts)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create resource group client: %w", err)
+		return nil, fmt.Errorf("create resource group client: %w", err)
 	}
 
 	cloud.VNet, err = armnetwork.NewVirtualNetworksClient(subscription, credential, opts)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create vnet client: %w", err)
+		return nil, fmt.Errorf("create vnet client: %w", err)
+	}
+
+	cloud.GalleryImageVersion, err = armcompute.NewGalleryImageVersionsClient(subscription, credential, opts)
+	if err != nil {
+		return nil, fmt.Errorf("create a new images client: %v", err)
 	}
 
-	cloud.GalleryImageVersionClient, err = armcompute.NewGalleryImageVersionsClient(subscription, credential, opts)
+	cloud.Blob, err = azblob.NewClient(Config.BlobStorageAccount, credential, nil)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create a new images client: %v", err)
+		return nil, fmt.Errorf("create blob container client: %w", err)
 	}
 
+	cloud.Credential = credential
+
 	return cloud, nil
 }
 
+// UploadAndGetLink uploads the data to the blob storage and returns the signed link to download the blob
+// If the blob already exists, it will be overwritten
+func (a *AzureClient) UploadAndGetLink(ctx context.Context, blobName string, file *os.File) (string, error) {
+	_, err := a.Blob.UploadFile(ctx, Config.BlobContainer, blobName, file, nil)
+	if err != nil {
+		return "", fmt.Errorf("upload blob: %w", err)
+	}
+
+	udc, err := a.Blob.ServiceClient().GetUserDelegationCredential(ctx, service.KeyInfo{
+		Expiry: to.Ptr(time.Now().Add(time.Hour).UTC().Format(sas.TimeFormat)),
+		Start:  to.Ptr(time.Now().UTC().Format(sas.TimeFormat)),
+	}, nil)
+	if err != nil {
+		return "", fmt.Errorf("get user delegation credential: %w", err)
+	}
+
+	sig, err := sas.BlobSignatureValues{
+		Protocol:      sas.ProtocolHTTPS,
+		ExpiryTime:    time.Now().Add(time.Hour),
+		Permissions:   to.Ptr(sas.BlobPermissions{Read: true}).String(),
+		ContainerName: Config.BlobContainer,
+		BlobName:      blobName,
+	}.SignWithUserDelegation(udc)
+	if err != nil {
+		return "", fmt.Errorf("sign blob: %w", err)
+	}
+
+	return fmt.Sprintf("%s/%s/%s?%s", Config.BlobStorageAccount, Config.BlobContainer, blobName, sig.Encode()), nil
+}
+
 func DefaultRetryOpts() policy.RetryOptions {
 	return policy.RetryOptions{
 		MaxRetries: 3,

e2e/config/config.go

@@ -32,6 +32,9 @@ type Configuration struct {
 	IgnoreScenariosWithMissingVHD bool          `env:"IGNORE_SCENARIOS_WITH_MISSING_VHD"`
 	SkipTestsWithSKUCapacityIssue bool          `env:"SKIP_TESTS_WITH_SKU_CAPACITY_ISSUE"`
 	KeepVMSS                      bool          `env:"KEEP_VMSS"`
+	BlobStorageAccount            string        `env:"BLOB_STORAGE_ACCOUNT" envDefault:"https://abe2e.blob.core.windows.net"`
+	BlobContainer                 string        `env:"BLOB_CONTAINER" envDefault:"abe2e"`
+	EnableNodeBootstrapperTest    bool          `env:"ENABLE_NODE_BOOTSTRAPPER_TEST"`
 }
 
 func mustLoadConfig() Configuration {

e2e/config/vhd.go

@@ -153,7 +153,7 @@ func ensureStaticSIGImageVersion(ctx context.Context, t *testing.T, imageVersion
 	}
 	version := newSIGImageVersionFromResourceID(rid)
 
-	resp, err := Azure.GalleryImageVersionClient.Get(ctx, version.resourceGroup, version.gallery, version.definition, version.version, nil)
+	resp, err := Azure.GalleryImageVersion.Get(ctx, version.resourceGroup, version.gallery, version.definition, version.version, nil)
 	if err != nil {
 		return "", fmt.Errorf("getting live image version info: %w", err)
 	}
@@ -177,7 +177,7 @@ func findLatestSIGImageVersionWithTag(ctx context.Context, t *testing.T, imageDe
 	}
 	definition := newSIGImageDefinitionFromResourceID(rid)
 
-	pager := Azure.GalleryImageVersionClient.NewListByGalleryImagePager(definition.resourceGroup, definition.gallery, definition.definition, nil)
+	pager := Azure.GalleryImageVersion.NewListByGalleryImagePager(definition.resourceGroup, definition.gallery, definition.definition, nil)
 	var latestVersion *armcompute.GalleryImageVersion
 	for pager.More() {
 		page, err := pager.NextPage(ctx)
@@ -241,7 +241,7 @@ func replicateToCurrentRegion(ctx context.Context, t *testing.T, definition sigI
 		StorageAccountType:   to.Ptr(armcompute.StorageAccountTypeStandardLRS),
 	})
 
-	resp, err := Azure.GalleryImageVersionClient.BeginCreateOrUpdate(ctx, definition.resourceGroup, definition.gallery, definition.definition, *version.Name, *version, nil)
+	resp, err := Azure.GalleryImageVersion.BeginCreateOrUpdate(ctx, definition.resourceGroup, definition.gallery, definition.definition, *version.Name, *version, nil)
 	if err != nil {
 		return fmt.Errorf("begin updating image version target regions: %w", err)
 	}