node-bootstrapper (#4891)

Co-authored-by: Tim Wright <[email protected]>

.golangci.yaml

@@ -148,7 +148,7 @@ linters-settings:
       shadow:
         # Whether to be strict about shadowing; can be noisy.
         # Default: false
-        strict: true
+        strict: false
 
   nakedret:
     # Make an issue if func has more lines of code than this setting, and it has naked returns.

.pipelines/e2e.yaml

@@ -1,8 +1,8 @@
 name: $(Date:yyyyMMdd)$(Rev:.r)
-trigger: 
+trigger:
   branches:
-    include: 
-    - master
+    include:
+      - master
 pr:
   branches:
     include:

.pipelines/templates/.builder-release-template.yaml

@@ -40,12 +40,12 @@ steps:
         exit 1
       fi
       echo "Found source components.json to use for overrides: ${COMPONENT_JSON_SRC}"
-      
+
       echo "Overwriting component.json from ${COMPONENT_JSON_SRC} -> ${COMPONENT_JSON_DEST}"
       cp -af "${COMPONENT_JSON_SRC}" "${COMPONENT_JSON_DEST}"
     condition: eq('${{ parameters.useOverrides }}', true)
     displayName: Apply Overrides
-  
+
   - bash: |
       m="linuxVhdMode" && \
       echo "Set build mode to $m" && \
@@ -112,7 +112,10 @@ steps:
       echo "##vso[task.setvariable variable=SKU_NAME]$SKU_NAME"
       echo "Set SKU_NAME to $SKU_NAME"
     displayName: Set SKU Name
-    
+  - bash: |
+      ./build.sh
+    displayName: Build node-bootstrapper
+    workingDirectory: node-bootstrapper
   - bash: make -f packer.mk run-packer
     displayName: Build VHD
     retryCountOnTaskFailure: 3
@@ -123,7 +126,7 @@ steps:
       BUILD_NUMBER: $(Build.BuildNumber)
       BUILD_ID: $(Build.BuildId)
       BUILD_DEFINITION_NAME: $(Build.DefinitionName)
-  
+
   - bash: |
       CAPTURED_SIG_VERSION="$(cat vhdbuilder/packer/settings.json | grep "captured_sig_version" | awk -F':' '{print $2}' | awk -F'"' '{print $2}')" && \
       SIG_IMAGE_NAME="$(cat vhdbuilder/packer/settings.json | grep "sig_image_name" | awk -F':' '{print $2}' | awk -F'"' '{print $2}')" && \

e2e/cluster.go

@@ -85,7 +85,7 @@ func ClusterAzureNetwork(ctx context.Context, t *testing.T) (*Cluster, error) {
 	return clusterAzureNetwork, clusterAzureNetworkError
 }
 
-func nodeBootsrappingConfig(ctx context.Context, t *testing.T, kube *Kubeclient) (*datamodel.NodeBootstrappingConfiguration, error) {
+func nodeBootstrappingConfig(ctx context.Context, t *testing.T, kube *Kubeclient) (*datamodel.NodeBootstrappingConfiguration, error) {
 	clusterParams, err := extractClusterParameters(ctx, t, kube)
 	if err != nil {
 		return nil, fmt.Errorf("extract cluster parameters: %w", err)
@@ -133,7 +133,7 @@ func prepareCluster(ctx context.Context, t *testing.T, cluster *armcontainerserv
 		return nil, fmt.Errorf("get cluster subnet: %w", err)
 	}
 
-	nbc, err := nodeBootsrappingConfig(ctx, t, kube)
+	nbc, err := nodeBootstrappingConfig(ctx, t, kube)
 	if err != nil {
 		return nil, fmt.Errorf("get node bootstrapping configuration: %w", err)
 	}

e2e/config/azure.go

@@ -1,42 +1,50 @@
 package config
 
 import (
+	"context"
 	"crypto/tls"
 	"fmt"
 	"net"
 	"net/http"
+	"os"
 	"time"
 
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/arm"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
 	"github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime"
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
 	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v6"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns"
 	"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service"
 	"github.com/Azure/go-armbalancer"
 )
 
 type AzureClient struct {
+	AKS                       *armcontainerservice.ManagedClustersClient
+	Blob                      *azblob.Client
 	Core                      *azcore.Client
-	VMSS                      *armcompute.VirtualMachineScaleSetsClient
-	VMSSVM                    *armcompute.VirtualMachineScaleSetVMsClient
-	VNet                      *armnetwork.VirtualNetworksClient
+	Credential                *azidentity.DefaultAzureCredential
+	GalleryImageVersion       *armcompute.GalleryImageVersionsClient
+	Maintenance               *armcontainerservice.MaintenanceConfigurationsClient
+	PrivateDNSZoneGroup       *armnetwork.PrivateDNSZoneGroupsClient
+	PrivateEndpointClient     *armnetwork.PrivateEndpointsClient
+	PrivateZonesClient        *armprivatedns.PrivateZonesClient
+	RecordSetClient           *armprivatedns.RecordSetsClient
 	Resource                  *armresources.Client
 	ResourceGroup             *armresources.ResourceGroupsClient
-	AKS                       *armcontainerservice.ManagedClustersClient
-	Maintenance               *armcontainerservice.MaintenanceConfigurationsClient
 	SecurityGroup             *armnetwork.SecurityGroupsClient
 	Subnet                    *armnetwork.SubnetsClient
-	GalleryImageVersionClient *armcompute.GalleryImageVersionsClient
-	PrivateEndpointClient     *armnetwork.PrivateEndpointsClient
-	PrivateZonesClient        *armprivatedns.PrivateZonesClient
+	VMSS                      *armcompute.VirtualMachineScaleSetsClient
+	VMSSVM                    *armcompute.VirtualMachineScaleSetVMsClient
+	VNet                      *armnetwork.VirtualNetworksClient
 	VirutalNetworkLinksClient *armprivatedns.VirtualNetworkLinksClient
-	RecordSetClient           *armprivatedns.RecordSetsClient
-	PrivateDNSZoneGroup       *armnetwork.PrivateDNSZoneGroupsClient
 }
 
 func mustNewAzureClient(subscription string) *AzureClient {
@@ -107,7 +115,7 @@ func NewAzureClient(subscription string) (*AzureClient, error) {
 	// purely for telemetry, entirely unused today
 	cloud.Core, err = azcore.NewClient("agentbakere2e.e2e_test", "v0.0.0", plOpts, clOpts)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create core client: %w", err)
+		return nil, fmt.Errorf("create core client: %w", err)
 	}
 
 	cloud.PrivateEndpointClient, err = armnetwork.NewPrivateEndpointsClient(subscription, credential, opts)
@@ -137,17 +145,17 @@ func NewAzureClient(subscription string) (*AzureClient, error) {
 
 	cloud.SecurityGroup, err = armnetwork.NewSecurityGroupsClient(subscription, credential, opts)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create security group client: %w", err)
+		return nil, fmt.Errorf("create security group client: %w", err)
 	}
 
 	cloud.Subnet, err = armnetwork.NewSubnetsClient(subscription, credential, opts)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create subnet client: %w", err)
+		return nil, fmt.Errorf("create subnet client: %w", err)
 	}
 
 	cloud.AKS, err = armcontainerservice.NewManagedClustersClient(subscription, credential, opts)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create aks client: %w", err)
+		return nil, fmt.Errorf("create aks client: %w", err)
 	}
 
 	cloud.Maintenance, err = armcontainerservice.NewMaintenanceConfigurationsClient(subscription, credential, opts)
@@ -157,37 +165,74 @@ func NewAzureClient(subscription string) (*AzureClient, error) {
 
 	cloud.VMSS, err = armcompute.NewVirtualMachineScaleSetsClient(subscription, credential, opts)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create vmss client: %w", err)
+		return nil, fmt.Errorf("create vmss client: %w", err)
 	}
 
 	cloud.VMSSVM, err = armcompute.NewVirtualMachineScaleSetVMsClient(subscription, credential, opts)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create vmss vm client: %w", err)
+		return nil, fmt.Errorf("create vmss vm client: %w", err)
 	}
 
 	cloud.Resource, err = armresources.NewClient(subscription, credential, opts)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create resource client: %w", err)
+		return nil, fmt.Errorf("create resource client: %w", err)
 	}
 
 	cloud.ResourceGroup, err = armresources.NewResourceGroupsClient(subscription, credential, opts)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create resource group client: %w", err)
+		return nil, fmt.Errorf("create resource group client: %w", err)
 	}
 
 	cloud.VNet, err = armnetwork.NewVirtualNetworksClient(subscription, credential, opts)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create vnet client: %w", err)
+		return nil, fmt.Errorf("create vnet client: %w", err)
+	}
+
+	cloud.GalleryImageVersion, err = armcompute.NewGalleryImageVersionsClient(subscription, credential, opts)
+	if err != nil {
+		return nil, fmt.Errorf("create a new images client: %v", err)
 	}
 
-	cloud.GalleryImageVersionClient, err = armcompute.NewGalleryImageVersionsClient(subscription, credential, opts)
+	cloud.Blob, err = azblob.NewClient(Config.BlobStorageAccount, credential, nil)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create a new images client: %v", err)
+		return nil, fmt.Errorf("create blob container client: %w", err)
 	}
 
+	cloud.Credential = credential
+
 	return cloud, nil
 }
 
+// UploadAndGetLink uploads the data to the blob storage and returns the signed link to download the blob
+// If the blob already exists, it will be overwritten
+func (a *AzureClient) UploadAndGetLink(ctx context.Context, blobName string, file *os.File) (string, error) {
+	_, err := a.Blob.UploadFile(ctx, Config.BlobContainer, blobName, file, nil)
+	if err != nil {
+		return "", fmt.Errorf("upload blob: %w", err)
+	}
+
+	udc, err := a.Blob.ServiceClient().GetUserDelegationCredential(ctx, service.KeyInfo{
+		Expiry: to.Ptr(time.Now().Add(time.Hour).UTC().Format(sas.TimeFormat)),
+		Start:  to.Ptr(time.Now().UTC().Format(sas.TimeFormat)),
+	}, nil)
+	if err != nil {
+		return "", fmt.Errorf("get user delegation credential: %w", err)
+	}
+
+	sig, err := sas.BlobSignatureValues{
+		Protocol:      sas.ProtocolHTTPS,
+		ExpiryTime:    time.Now().Add(time.Hour),
+		Permissions:   to.Ptr(sas.BlobPermissions{Read: true}).String(),
+		ContainerName: Config.BlobContainer,
+		BlobName:      blobName,
+	}.SignWithUserDelegation(udc)
+	if err != nil {
+		return "", fmt.Errorf("sign blob: %w", err)
+	}
+
+	return fmt.Sprintf("%s/%s/%s?%s", Config.BlobStorageAccount, Config.BlobContainer, blobName, sig.Encode()), nil
+}
+
 func DefaultRetryOpts() policy.RetryOptions {
 	return policy.RetryOptions{
 		MaxRetries: 3,

e2e/config/config.go

@@ -32,6 +32,9 @@ type Configuration struct {
 	IgnoreScenariosWithMissingVHD bool          `env:"IGNORE_SCENARIOS_WITH_MISSING_VHD"`
 	SkipTestsWithSKUCapacityIssue bool          `env:"SKIP_TESTS_WITH_SKU_CAPACITY_ISSUE"`
 	KeepVMSS                      bool          `env:"KEEP_VMSS"`
+	BlobStorageAccount            string        `env:"BLOB_STORAGE_ACCOUNT" envDefault:"https://abe2e.blob.core.windows.net"`
+	BlobContainer                 string        `env:"BLOB_CONTAINER" envDefault:"abe2e"`
+	EnableNodeBootstrapperTest    bool          `env:"ENABLE_NODE_BOOTSTRAPPER_TEST"`
 }
 
 func mustLoadConfig() Configuration {

e2e/config/vhd.go

@@ -153,7 +153,7 @@ func ensureStaticSIGImageVersion(ctx context.Context, t *testing.T, imageVersion
 	}
 	version := newSIGImageVersionFromResourceID(rid)
 
-	resp, err := Azure.GalleryImageVersionClient.Get(ctx, version.resourceGroup, version.gallery, version.definition, version.version, nil)
+	resp, err := Azure.GalleryImageVersion.Get(ctx, version.resourceGroup, version.gallery, version.definition, version.version, nil)
 	if err != nil {
 		return "", fmt.Errorf("getting live image version info: %w", err)
 	}
@@ -177,7 +177,7 @@ func findLatestSIGImageVersionWithTag(ctx context.Context, t *testing.T, imageDe
 	}
 	definition := newSIGImageDefinitionFromResourceID(rid)
 
-	pager := Azure.GalleryImageVersionClient.NewListByGalleryImagePager(definition.resourceGroup, definition.gallery, definition.definition, nil)
+	pager := Azure.GalleryImageVersion.NewListByGalleryImagePager(definition.resourceGroup, definition.gallery, definition.definition, nil)
 	var latestVersion *armcompute.GalleryImageVersion
 	for pager.More() {
 		page, err := pager.NextPage(ctx)
@@ -241,7 +241,7 @@ func replicateToCurrentRegion(ctx context.Context, t *testing.T, definition sigI
 		StorageAccountType:   to.Ptr(armcompute.StorageAccountTypeStandardLRS),
 	})
 
-	resp, err := Azure.GalleryImageVersionClient.BeginCreateOrUpdate(ctx, definition.resourceGroup, definition.gallery, definition.definition, *version.Name, *version, nil)
+	resp, err := Azure.GalleryImageVersion.BeginCreateOrUpdate(ctx, definition.resourceGroup, definition.gallery, definition.definition, *version.Name, *version, nil)
 	if err != nil {
 		return fmt.Errorf("begin updating image version target regions: %w", err)
 	}

e2e/exec.go

@@ -53,15 +53,16 @@ func (r podExecResult) dumpStderr(t *testing.T) {
 	}
 }
 
-func extractLogsFromVM(ctx context.Context, t *testing.T, vmssName, privateIP, sshPrivateKey string, opts *scenarioRunOpts) (map[string]string, error) {
+func extractLogsFromVM(ctx context.Context, t *testing.T, vmssName, privateIP, sshPrivateKey string, cluster *Cluster) (map[string]string, error) {
 	commandList := map[string]string{
-		"/var/log/azure/cluster-provision": "cat /var/log/azure/cluster-provision.log",
-		"kubelet":                          "journalctl -u kubelet",
-		"/var/log/azure/cluster-provision-cse-output": "cat /var/log/azure/cluster-provision-cse-output.log",
-		"sysctl-out": "sysctl -a",
+		"cluster-provision":            "cat /var/log/azure/cluster-provision.log",
+		"kubelet":                      "journalctl -u kubelet",
+		"cluster-provision-cse-output": "cat /var/log/azure/cluster-provision-cse-output.log",
+		"sysctl-out":                   "sysctl -a",
+		"node-bootstrapper":            "cat /var/log/azure/node-bootstrapper.log",
 	}
 
-	podName, err := getHostNetworkDebugPodName(ctx, opts.clusterConfig.Kube)
+	podName, err := getHostNetworkDebugPodName(ctx, cluster.Kube)
 	if err != nil {
 		return nil, fmt.Errorf("unable to get debug pod name: %w", err)
 	}
@@ -70,12 +71,11 @@ func extractLogsFromVM(ctx context.Context, t *testing.T, vmssName, privateIP, s
 	for file, sourceCmd := range commandList {
 		t.Logf("executing command on remote VM at %s of VMSS %s: %q", privateIP, vmssName, sourceCmd)
 
-		execResult, err := execOnVM(ctx, opts.clusterConfig.Kube, privateIP, podName, sshPrivateKey, sourceCmd, false)
+		execResult, err := execOnVM(ctx, cluster.Kube, privateIP, podName, sshPrivateKey, sourceCmd, false)
 		if err != nil {
 			t.Logf("error executing command on remote VM at %s of VMSS %s: %s", privateIP, vmssName, err)
 			return nil, err
 		}
-
 		if execResult.stdout != nil {
 			out := execResult.stdout.String()
 			if out != "" {

e2e/go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v6 v6.0.0
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6 v6.0.0
 	github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.0
 	github.com/Azure/go-armbalancer v0.0.2
 	github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df
 	github.com/caarlos0/env/v11 v11.2.2

e2e/go.sum

@@ -26,6 +26,10 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns v
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns v1.3.0/go.mod h1:GE4m0rnnfwLGX0Y9A9A25Zx5N/90jneT5ABevqzhuFQ=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 h1:Dd+RhdJn0OTtVGaeDLZpcumkIVCtA/3/Fo42+eoYvVM=
 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0/go.mod h1:5kakwfW5CjC9KK+Q4wjXAg+ShuIm2mBMua0ZFj2C8PE=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 h1:PiSrjRPpkQNjrM8H0WwKMnZUdu1RGMtd/LdGKUrOo+c=
+github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0/go.mod h1:oDrbWx4ewMylP7xHivfgixbfGBT6APAwsSoHRKotnIc=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.0 h1:Be6KInmFEKV81c0pOAEbRYehLMwmmGI1exuFj248AMk=
+github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.0/go.mod h1:WCPBHsOXfBVnivScjs2ypRfimjEW0qPVLGgJkZlrIOA=
 github.com/Azure/go-armbalancer v0.0.2 h1:NVnxsTWHI5/fEzL6k6TjxPUfcB/3Si3+HFOZXOu0QtA=
 github.com/Azure/go-armbalancer v0.0.2/go.mod h1:yTg7MA/8YnfKQc9o97tzAJ7fbdVkod1xGsIvKmhYPRE=
 github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs=

e2e/log.go

@@ -6,12 +6,19 @@ import (
 	"testing"
 
 	"github.com/Azure/agentbakere2e/config"
+	"github.com/stretchr/testify/require"
 )
 
 func testDir(t *testing.T) string {
 	return filepath.Join(config.Config.E2ELoggingDir, t.Name())
 }
 
+func cleanTestDir(t *testing.T) {
+	dirPath := testDir(t)
+	err := os.RemoveAll(dirPath)
+	require.NoError(t, err)
+}
+
 func writeToFile(t *testing.T, fileName, content string) error {
 	dirPath := testDir(t)
 	// Create the directory if it doesn't exist