From 082fa66756974d354c133d8446dec6835f741be1 Mon Sep 17 00:00:00 2001 From: r2k1 Date: Wed, 30 Oct 2024 15:12:25 +1300 Subject: [PATCH] test node-bootstrapper using managed identity instead of tokens --- e2e/config/azure.go | 120 +++++++++++++++++++++++++++++----- e2e/config/config.go | 11 +++- e2e/go.mod | 9 ++- e2e/go.sum | 4 ++ e2e/node_bootstrapper_test.go | 58 ++++++---------- 5 files changed, 142 insertions(+), 60 deletions(-) diff --git a/e2e/config/azure.go b/e2e/config/azure.go index 2e83f8d8553..b7c35470303 100644 --- a/e2e/config/azure.go +++ b/e2e/config/azure.go @@ -3,6 +3,7 @@ package config import ( "context" "crypto/tls" + "errors" "fmt" "net" "net/http" @@ -15,21 +16,25 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v6" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns" "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" - "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas" - "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/service" "github.com/Azure/go-armbalancer" + "github.com/google/uuid" ) type AzureClient struct { AKS *armcontainerservice.ManagedClustersClient Blob *azblob.Client + StorageContainers *armstorage.BlobContainersClient + CacheRulesClient *armcontainerregistry.CacheRulesClient Core *azcore.Client Credential *azidentity.DefaultAzureCredential GalleryImageVersion *armcompute.GalleryImageVersionsClient @@ -38,16 +43,18 @@ type AzureClient struct { PrivateEndpointClient *armnetwork.PrivateEndpointsClient PrivateZonesClient *armprivatedns.PrivateZonesClient RecordSetClient *armprivatedns.RecordSetsClient + RegistriesClient *armcontainerregistry.RegistriesClient Resource *armresources.Client ResourceGroup *armresources.ResourceGroupsClient + RoleAssignments *armauthorization.RoleAssignmentsClient SecurityGroup *armnetwork.SecurityGroupsClient + StorageAccounts *armstorage.AccountsClient Subnet *armnetwork.SubnetsClient + UserAssignedIdentities *armmsi.UserAssignedIdentitiesClient VMSS *armcompute.VirtualMachineScaleSetsClient VMSSVM *armcompute.VirtualMachineScaleSetVMsClient VNet *armnetwork.VirtualNetworksClient VirutalNetworkLinksClient *armprivatedns.VirtualNetworkLinksClient - RegistriesClient *armcontainerregistry.RegistriesClient - CacheRulesClient *armcontainerregistry.CacheRulesClient } func mustNewAzureClient(subscription string) *AzureClient { @@ -206,11 +213,31 @@ func NewAzureClient(subscription string) (*AzureClient, error) { return nil, fmt.Errorf("create a new images client: %v", err) } - cloud.Blob, err = azblob.NewClient(Config.BlobStorageAccount, credential, nil) + cloud.Blob, err = azblob.NewClient(Config.BlobStorageAccountURL(), credential, nil) if err != nil { return nil, fmt.Errorf("create blob container client: %w", err) } + cloud.StorageContainers, err = armstorage.NewBlobContainersClient(Config.SubscriptionID, credential, opts) + if err != nil { + return nil, fmt.Errorf("create blob container client: %w", err) + } + + cloud.RoleAssignments, err = armauthorization.NewRoleAssignmentsClient(Config.SubscriptionID, credential, opts) + if err != nil { + return nil, fmt.Errorf("create role assignment client: %w", err) + } + + cloud.UserAssignedIdentities, err = armmsi.NewUserAssignedIdentitiesClient(Config.SubscriptionID, credential, nil) + if err != nil { + return nil, fmt.Errorf("create user assigned identities client: %w", err) + } + + cloud.StorageAccounts, err = armstorage.NewAccountsClient(Config.SubscriptionID, credential, nil) + if err != nil { + return nil, fmt.Errorf("create storage accounts client: %w", err) + } + cloud.Credential = credential return cloud, nil @@ -224,26 +251,83 @@ func (a *AzureClient) UploadAndGetLink(ctx context.Context, blobName string, fil return "", fmt.Errorf("upload blob: %w", err) } - udc, err := a.Blob.ServiceClient().GetUserDelegationCredential(ctx, service.KeyInfo{ - Expiry: to.Ptr(time.Now().Add(time.Hour).UTC().Format(sas.TimeFormat)), - Start: to.Ptr(time.Now().UTC().Format(sas.TimeFormat)), + // is there a better way? + return fmt.Sprintf("%s/%s/%s", Config.BlobStorageAccountURL(), Config.BlobContainer, blobName), nil +} + +func (a *AzureClient) CreateVMManagedIdentity(ctx context.Context) (string, error) { + identity, err := a.UserAssignedIdentities.CreateOrUpdate(ctx, ResourceGroupName, VMIdentityName, armmsi.Identity{ + Location: to.Ptr(Config.Location), }, nil) if err != nil { - return "", fmt.Errorf("get user delegation credential: %w", err) + return "", fmt.Errorf("create managed identity: %w", err) + } + err = a.createBlobStorageAccount(ctx) + if err != nil { + return "", err } + err = a.createBlobStorageContainer(ctx) + if err != nil { + return "", err + } + + if err := a.assignReaderRoleToBlobStorage(ctx, identity.Properties.PrincipalID); err != nil { + return "", err + } + return *identity.Properties.ClientID, nil +} + +func (a *AzureClient) createBlobStorageAccount(ctx context.Context) error { + poller, err := a.StorageAccounts.BeginCreate(ctx, ResourceGroupName, Config.BlobStorageAccount(), armstorage.AccountCreateParameters{ + Kind: to.Ptr(armstorage.KindStorageV2), + Location: &Config.Location, + SKU: &armstorage.SKU{ + Name: to.Ptr(armstorage.SKUNameStandardLRS), + }, + Properties: &armstorage.AccountPropertiesCreateParameters{ + AllowBlobPublicAccess: to.Ptr(false), + AllowSharedKeyAccess: to.Ptr(false), + }, + }, nil) + if err != nil { + return fmt.Errorf("create storage account: %w", err) + } + _, err = poller.PollUntilDone(ctx, DefaultPollUntilDoneOptions) + if err != nil { + return fmt.Errorf("create storage account: %w", err) + } + return nil +} + +func (a *AzureClient) createBlobStorageContainer(ctx context.Context) error { + _, err := a.StorageContainers.Create(ctx, ResourceGroupName, Config.BlobStorageAccount(), Config.BlobContainer, armstorage.BlobContainer{}, nil) + if err != nil { + return fmt.Errorf("create blob container: %w", err) + } + return nil +} - sig, err := sas.BlobSignatureValues{ - Protocol: sas.ProtocolHTTPS, - ExpiryTime: time.Now().Add(time.Hour), - Permissions: to.Ptr(sas.BlobPermissions{Read: true}).String(), - ContainerName: Config.BlobContainer, - BlobName: blobName, - }.SignWithUserDelegation(udc) +func (a *AzureClient) assignReaderRoleToBlobStorage(ctx context.Context, principalID *string) error { + scope := fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Storage/storageAccounts/%s", Config.SubscriptionID, ResourceGroupName, Config.BlobStorageAccount()) + // Role assignment requires uid to be provided + uid := uuid.New().String() + _, err := a.RoleAssignments.Create(ctx, scope, uid, armauthorization.RoleAssignmentCreateParameters{ + Properties: &armauthorization.RoleAssignmentProperties{ + PrincipalID: principalID, + // built-in "Storage Blob Data Reader" role + RoleDefinitionID: to.Ptr("/providers/Microsoft.Authorization/roleDefinitions/2a2b9908-6ea1-4ae2-8e65-a410df84e7d1"), + }, + }, nil) + var respError *azcore.ResponseError if err != nil { - return "", fmt.Errorf("sign blob: %w", err) + // if the role assignment already exists, ignore the error + if errors.As(err, &respError) && respError.StatusCode == http.StatusConflict { + return nil + } + return fmt.Errorf("assign reader role: %w", err) } + return nil - return fmt.Sprintf("%s/%s/%s?%s", Config.BlobStorageAccount, Config.BlobContainer, blobName, sig.Encode()), nil } func DefaultRetryOpts() policy.RetryOptions { diff --git a/e2e/config/config.go b/e2e/config/config.go index f106534fd01..628cc1a63c9 100644 --- a/e2e/config/config.go +++ b/e2e/config/config.go @@ -12,6 +12,7 @@ var ( Config = mustLoadConfig() Azure = mustNewAzureClient(Config.SubscriptionID) ResourceGroupName = "abe2e-" + Config.Location + VMIdentityName = "abe2e-vm-identity" PrivateACRName = "privateacre2e" DefaultPollUntilDoneOptions = &runtime.PollUntilDoneOptions{ @@ -34,11 +35,19 @@ type Configuration struct { IgnoreScenariosWithMissingVHD bool `env:"IGNORE_SCENARIOS_WITH_MISSING_VHD"` SkipTestsWithSKUCapacityIssue bool `env:"SKIP_TESTS_WITH_SKU_CAPACITY_ISSUE"` KeepVMSS bool `env:"KEEP_VMSS"` - BlobStorageAccount string `env:"BLOB_STORAGE_ACCOUNT" envDefault:"https://abe2e.blob.core.windows.net"` + BlobStorageAccountPrefix string `env:"BLOB_STORAGE_ACCOUNT_PREFIX" envDefault:"abe2e"` BlobContainer string `env:"BLOB_CONTAINER" envDefault:"abe2e"` EnableNodeBootstrapperTest bool `env:"ENABLE_NODE_BOOTSTRAPPER_TEST"` } +func (c *Configuration) BlobStorageAccount() string { + return c.BlobStorageAccountPrefix + c.Location +} + +func (c *Configuration) BlobStorageAccountURL() string { + return "https://" + c.BlobStorageAccount() + ".blob.core.windows.net" +} + func mustLoadConfig() Configuration { _ = godotenv.Load(".env") cfg := Configuration{} diff --git a/e2e/go.mod b/e2e/go.mod index 0c01cf6a6ed..2a1ef098c84 100644 --- a/e2e/go.mod +++ b/e2e/go.mod @@ -6,10 +6,15 @@ require ( github.com/Azure/agentbaker v0.0.0-00010101000000-000000000000 github.com/Azure/azure-sdk-for-go/sdk/azcore v1.13.0 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2 v2.2.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6 v6.1.0 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry v1.3.0-beta.1 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerservice/armcontainerservice/v6 v6.0.0 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.2.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6 v6.0.0 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns v1.3.0 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.0 github.com/Azure/go-armbalancer v0.0.2 github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df @@ -28,10 +33,8 @@ require ( require ( github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry v1.3.0-beta.1 // indirect - github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns v1.3.0 github.com/Azure/go-autorest v14.2.0+incompatible // indirect - github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect + github.com/Azure/go-autorest/autorest/to v0.4.0 github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect github.com/Masterminds/semver/v3 v3.2.1 // indirect github.com/blang/semver v3.5.1+incompatible // indirect diff --git a/e2e/go.sum b/e2e/go.sum index 2d70372e9af..2002fe97147 100644 --- a/e2e/go.sum +++ b/e2e/go.sum @@ -4,6 +4,8 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 h1:tfLQ34V6F7tVSwoTf/4lH github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg= github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY= github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2 v2.2.0 h1:Hp+EScFOu9HeCbeW8WU2yQPJd4gGwhMgKxWe+G6jNzw= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/authorization/armauthorization/v2 v2.2.0/go.mod h1:/pz8dyNQe+Ey3yBp/XuYz7oqX8YDNWVpPB0hH3XWfbc= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6 v6.1.0 h1:zDeQI/PaWztI2tcrGO/9RIMey9NvqYbnyttf/0P3QWM= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v6 v6.1.0/go.mod h1:zflC9v4VfViJrSvcvplqws/yGXVbUEMZi/iHpZdSPWA= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/containerregistry/armcontainerregistry v1.3.0-beta.1 h1:VNLfijkPSLB25P1l52CXGyeaD8Aj0gcsigmbOpJKwhk= @@ -18,6 +20,8 @@ github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.1.0 h1:2qsI github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.1.0/go.mod h1:AW8VEadnhw9xox+VaVd9sP7NjzOAnaZBLRH6Tq3cJ38= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/managementgroups/armmanagementgroups v1.0.0 h1:pPvTJ1dY0sA35JOeFq6TsY2xj6Z85Yo23Pj4wCCvu4o= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/managementgroups/armmanagementgroups v1.0.0/go.mod h1:mLfWfj8v3jfWKsL9G4eoBoXVcsqcIUTapmdKy7uGOp0= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.2.0 h1:z4YeiSXxnUI+PqB46Yj6MZA3nwb1CcJIkEMDrzUd8Cs= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/msi/armmsi v1.2.0/go.mod h1:rko9SzMxcMk0NJsNAxALEGaTYyy79bNRwxgJfrH0Spw= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6 v6.0.0 h1:6gbgo57khn0HUCcozxGgDodl7HPH0wr9x3QPt1uJSMM= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6 v6.0.0/go.mod h1:ulHyBFJOI0ONiRL4vcJTmS7rx18jQQlEPmAgo80cRdM= github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns v1.3.0 h1:yzrctSl9GMIQ5lHu7jc8olOsGjWDCsBpJhWqfGa/YIM= diff --git a/e2e/node_bootstrapper_test.go b/e2e/node_bootstrapper_test.go index 3a2ec9bdb37..8829649a4c2 100644 --- a/e2e/node_bootstrapper_test.go +++ b/e2e/node_bootstrapper_test.go @@ -1,17 +1,14 @@ package e2e import ( - "context" - "crypto/sha256" - "encoding/base32" "encoding/base64" "encoding/json" "fmt" - "io" "os" "os/exec" "path/filepath" "testing" + "time" "github.com/Azure/agentbaker/pkg/agent" "github.com/Azure/agentbaker/pkg/agent/datamodel" @@ -44,6 +41,12 @@ func Test_ubuntu2204NodeBootstrapper(t *testing.T) { } t.Logf("node-bootstrapper log: %s", string(log)) }) + identity, err := config.Azure.CreateVMManagedIdentity(ctx) + require.NoError(t, err) + binary := compileNodeBootstrapper(t) + url, err := config.Azure.UploadAndGetLink(ctx, time.Now().Format("2006-01-02-15-04-05")+"/node-bootstrapper", binary) + require.NoError(t, err) + RunScenario(t, &Scenario{ Description: "Tests that a node using the Ubuntu 2204 VHD can be properly bootstrapped", Config: Config{ @@ -51,7 +54,12 @@ func Test_ubuntu2204NodeBootstrapper(t *testing.T) { Cluster: ClusterKubenet, VHD: config.VHDUbuntu2204Gen2Containerd, VMConfigMutator: func(model *armcompute.VirtualMachineScaleSet) { - model.Properties.VirtualMachineProfile.OSProfile.CustomData = nil + model.Identity = &armcompute.VirtualMachineScaleSetIdentity{ + Type: to.Ptr(armcompute.ResourceIdentityTypeSystemAssignedUserAssigned), + UserAssignedIdentities: map[string]*armcompute.UserAssignedIdentitiesValue{ + fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.ManagedIdentity/userAssignedIdentities/%s", config.Config.SubscriptionID, config.ResourceGroupName, config.VMIdentityName): {}, + }, + } model.Properties.VirtualMachineProfile.ExtensionProfile = &armcompute.VirtualMachineScaleSetExtensionProfile{ Extensions: []*armcompute.VirtualMachineScaleSetExtension{ { @@ -63,11 +71,11 @@ func Test_ubuntu2204NodeBootstrapper(t *testing.T) { AutoUpgradeMinorVersion: to.Ptr(true), Settings: map[string]any{}, ProtectedSettings: map[string]any{ - //"fileUris": []string{}, - "commandToExecute": CSENodeBootstrapper(ctx, t, cluster), - //"managedIdentity": map[string]any{ - // "objectId": "", - //}, + "fileUris": []string{url}, + "commandToExecute": CSENodeBootstrapper(t, cluster), + "managedIdentity": map[string]any{ + "clientId": identity, + }, }, }, }, @@ -84,7 +92,7 @@ func Test_ubuntu2204NodeBootstrapper(t *testing.T) { }) } -func CSENodeBootstrapper(ctx context.Context, t *testing.T, cluster *Cluster) string { +func CSENodeBootstrapper(t *testing.T, cluster *Cluster) string { nbcAny, err := deepcopy.Anything(cluster.NodeBootstrappingConfiguration) require.NoError(t, err) nbc := nbcAny.(*datamodel.NodeBootstrappingConfiguration) @@ -95,10 +103,7 @@ func CSENodeBootstrapper(ctx context.Context, t *testing.T, cluster *Cluster) st configJSON, err := json.Marshal(configContent) require.NoError(t, err) - binary := compileNodeBootstrapper(t) - url, err := config.Azure.UploadAndGetLink(ctx, "node-bootstrapper-"+hashFile(t, binary.Name()), binary) - require.NoError(t, err) - return fmt.Sprintf(`sh -c "(mkdir -p /etc/node-bootstrapper && echo '%s' | base64 -d > /etc/node-bootstrapper/config.json && curl -L -o ./node-bootstrapper '%s' && chmod +x ./node-bootstrapper && ./node-bootstrapper provision --provision-config=/etc/node-bootstrapper/config.json)"`, base64.StdEncoding.EncodeToString(configJSON), url) + return fmt.Sprintf(`sh -c "(mkdir -p /etc/node-bootstrapper && echo '%s' | base64 -d > /etc/node-bootstrapper/config.json && ./node-bootstrapper provision --provision-config=/etc/node-bootstrapper/config.json)"`, base64.StdEncoding.EncodeToString(configJSON)) } func compileNodeBootstrapper(t *testing.T) *os.File { @@ -116,26 +121,3 @@ func compileNodeBootstrapper(t *testing.T) *os.File { require.NoError(t, err) return f } - -func hashFile(t *testing.T, filePath string) string { - // Open the file - file, err := os.Open(filePath) - require.NoError(t, err) - defer file.Close() - - // Create a SHA-256 hasher - hasher := sha256.New() - - // Copy the file content to the hasher - _, err = io.Copy(hasher, file) - require.NoError(t, err) - - // Compute the hash - hashSum := hasher.Sum(nil) - - // Encode the hash using base32 - encodedHash := base32.StdEncoding.EncodeToString(hashSum) - - // Return the first 5 characters of the encoded hash - return encodedHash[:5] -}