test: add e2e test for kubelet configs (#5180)

Co-authored-by: Cameron Meissner <[email protected]>

.devcontainer/devcontainer.json

@@ -3,9 +3,10 @@
 {
 	"name": "Go",
 	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
-	"image": "mcr.microsoft.com/devcontainers/go:1-1.20-bullseye",
+	"image": "mcr.microsoft.com/devcontainers/go:1-1.22-bullseye",
 	"features": {
-		"ghcr.io/devcontainers-contrib/features/protoc:1": {}
+		"ghcr.io/devcontainers-contrib/features/protoc:1": {},
+		"ghcr.io/devcontainers/features/azure-cli:1": {}
 	},
 
 	// Features to add to the dev container. More info: https://containers.dev/features.

e2e/README.md

@@ -8,7 +8,7 @@ AgentBaker E2E tests verify that node bootstrapping artifacts generated by the A
 
 From a high-level, each E2E scenario makes a call out to the primary node-bootstrapping
 API [GetLatestNodeBootstrapping](https://github.com/Azure/AgentBaker/blob/2e730b5a498c5be9b082d912fd08ac9346582db9/pkg/agent/bakerapi.go#L14)
-with a set of parameters (represented by a NodeBootstrappingConfiugration) which define the given scenario to generate
+with a set of parameters (represented by a `NodeBootstrappingConfiugration`) which define the given scenario to generate
 CSE and custom data. A new VMSS containing a single VM will then be created and associated with an AKS cluster that is
 already running in Azure. The CSE and custom data generated by AgentBaker will then be applied to the new VM so it can
 bootstrap and register itself with the apiserver of the running cluster. Liveness and health checks and then run to make

e2e/scenario_test.go

@@ -1254,3 +1254,31 @@ func Test_AzureLinuxV2MessageOfTheDay(t *testing.T) {
 		},
 	})
 }
+
+func Test_Ubuntu2204_KubeletCustomConfig_SeccompDefaultEnabled(t *testing.T) {
+	kubeletConfigFilePath := "/etc/default/kubeletconfig.json"
+	RunScenario(t, &Scenario{
+		Tags: Tags{
+			KubeletCustomConfig: true,
+			OS:                  "ubuntu",
+		},
+		Description: "tests that a node on ubuntu 2204 bootstrapped with kubelet custom config for seccomp set to non default values",
+		Config: Config{
+			Cluster: ClusterKubenet,
+			VHD:     config.VHDUbuntu2204Gen2Containerd,
+			BootstrapConfigMutator: func(nbc *datamodel.NodeBootstrappingConfiguration) {
+				nbc.ContainerService.Properties.AgentPoolProfiles[0].Distro = "aks-ubuntu-containerd-22.04-gen2"
+				nbc.AgentPoolProfile.Distro = "aks-ubuntu-containerd-22.04-gen2"
+				customKubeletConfig := &datamodel.CustomKubeletConfig{
+					SeccompDefault: to.Ptr(true),
+				}
+				nbc.AgentPoolProfile.CustomKubeletConfig = customKubeletConfig
+				nbc.ContainerService.Properties.AgentPoolProfiles[0].CustomKubeletConfig = customKubeletConfig
+			},
+			LiveVMValidators: []*LiveVMValidator{
+				KubeletHasConfigFlagsValidator(kubeletConfigFilePath),
+				FileHasContentsValidator(kubeletConfigFilePath, "\"seccompDefault\": true"),
+			},
+		},
+	})
+}

e2e/types.go

@@ -27,6 +27,7 @@ type Tags struct {
 	WASM                   bool
 	ServerTLSBootstrapping bool
 	Scriptless             bool
+	KubeletCustomConfig    bool
 }
 
 // MatchesFilters checks if the Tags struct matches all given filters.
@@ -75,7 +76,7 @@ func (t Tags) matchFilters(filters string, all bool) (bool, error) {
 			return false, fmt.Errorf("unknown filter key: %s", key)
 		}
 
-		match := false
+		var match bool
 		switch field.Kind() {
 		case reflect.String:
 			match = strings.EqualFold(field.String(), value)
@@ -100,7 +101,7 @@ func (t Tags) matchFilters(filters string, all bool) (bool, error) {
 	return all, nil
 }
 
-// Scenario represents an AgentBaker E2E scenario
+// Scenario represents an AgentBaker E2E scenario.
 type Scenario struct {
 	// Description is a short description of what the scenario does and tests for
 	Description string
@@ -121,7 +122,7 @@ type ScenarioRuntime struct {
 	Cluster       *Cluster
 }
 
-// Config represents the configuration of an AgentBaker E2E scenario
+// Config represents the configuration of an AgentBaker E2E scenario.
 type Config struct {
 	// Cluster creates, updates or re-uses an AKS cluster for the scenario
 	Cluster func(ctx context.Context, t *testing.T) (*Cluster, error)

e2e/validators.go

@@ -408,3 +408,21 @@ func KubeletHasNotStoppedValidator() *LiveVMValidator {
 		},
 	}
 }
+
+// KubeletHasConfigFlagsValidator checks kubelet is started with the right flags and configs.
+func KubeletHasConfigFlagsValidator(filePath string) *LiveVMValidator {
+	return &LiveVMValidator{
+		Description: "assert that kubelet service is properly configured",
+		Command:     "journalctl -u kubelet",
+		Asserter: func(code, stdout, stderr string) error {
+			if code != "0" {
+				return fmt.Errorf("validator command terminated with exit code %q but expected code 0", code)
+			}
+			configFileFlags := fmt.Sprintf("FLAG: --config=\"%s\"", filePath)
+			if !strings.Contains(stdout, configFileFlags) {
+				return fmt.Errorf(fmt.Sprintf("expected to find flag %s, but not found: %s", "config", stdout))
+			}
+			return nil
+		},
+	}
+}

pkg/agent/baker_test.go

@@ -54,8 +54,7 @@ type decodedValue struct {
 type cseVariableEncoding string
 
 const (
-	cseVariableEncodingBase64 cseVariableEncoding = "base64"
-	cseVariableEncodingGzip   cseVariableEncoding = "gzip"
+	cseVariableEncodingGzip cseVariableEncoding = "gzip"
 )
 
 type outputValidator func(*nodeBootstrappingOutput)
@@ -388,7 +387,6 @@ var _ = Describe("Assert generated customData and cseCmd", func() {
 			err = os.WriteFile(fmt.Sprintf("./testdata/%s/CSECommand", folder), []byte(cseCommand), 0644)
 			Expect(err).To(BeNil())
 		}
-
 		expectedCSECommand, err := os.ReadFile(fmt.Sprintf("./testdata/%s/CSECommand", folder))
 		Expect(err).To(BeNil())
 		Expect(cseCommand).To(Equal(string(expectedCSECommand)))
@@ -1533,6 +1531,7 @@ oom_score = 0
 					ContainerLogMaxSizeMB: to.Int32Ptr(1000),
 					ContainerLogMaxFiles:  to.Int32Ptr(99),
 					PodMaxPids:            to.Int32Ptr(12345),
+					SeccompDefault:        to.BoolPtr(true),
 				}
 				config.ContainerService.Properties.AgentPoolProfiles[0].CustomLinuxOSConfig = &datamodel.CustomLinuxOSConfig{
 					Sysctls: &datamodel.SysctlConfig{
@@ -1549,6 +1548,13 @@ oom_score = 0
 					SwapFileSizeMB:             &swapFileSizeMB,
 				}
 			}, func(o *nodeBootstrappingOutput) {
+				kubeletConfigFileContent, err := getBase64DecodedValue([]byte(o.vars["KUBELET_CONFIG_FILE_CONTENT"]))
+				Expect(err).To(BeNil())
+				var kubeletConfigFile datamodel.AKSKubeletConfiguration
+				err = json.Unmarshal([]byte(kubeletConfigFileContent), &kubeletConfigFile)
+				Expect(err).To(BeNil())
+				Expect(kubeletConfigFile.SeccompDefault).To(Equal(to.BoolPtr(true)))
+
 				sysctlContent, err := getBase64DecodedValue([]byte(o.vars["SYSCTL_CONTENT"]))
 				Expect(err).To(BeNil())
 				// assert defaults for gc_thresh2 and gc_thresh3