From 1ade642f9e680ed9918a0dc890043a6471b34d7c Mon Sep 17 00:00:00 2001 From: Jason Deal Date: Mon, 22 Apr 2024 00:50:00 -0700 Subject: [PATCH] test updates --- pkg/cloudprovider/suite_test.go | 13 ++-- pkg/controllers/nodeclass/status/ami_test.go | 12 +++- pkg/providers/amifamily/suite_test.go | 62 +++++++++++++++---- pkg/providers/instancetype/suite_test.go | 40 ++++++++++-- pkg/providers/launchtemplate/suite_test.go | 22 +++++++ pkg/test/nodeclass.go | 7 +++ test/pkg/environment/aws/environment.go | 5 ++ test/suites/integration/ami_test.go | 24 +++++++ .../integration/extended_resources_test.go | 9 +++ .../suites/integration/kubelet_config_test.go | 9 +++ test/suites/integration/scheduling_test.go | 3 + test/suites/scale/deprovisioning_test.go | 6 ++ 12 files changed, 189 insertions(+), 23 deletions(-) diff --git a/pkg/cloudprovider/suite_test.go b/pkg/cloudprovider/suite_test.go index b9d255ed3808..271fa8ee11ad 100644 --- a/pkg/cloudprovider/suite_test.go +++ b/pkg/cloudprovider/suite_test.go @@ -750,9 +750,14 @@ var _ = Describe("CloudProvider", func() { Tags: map[string]string{ "fakeKey": "fakeValue", }, - Context: lo.ToPtr("fake-context"), - DetailedMonitoring: lo.ToPtr(false), - AMIFamily: lo.ToPtr(v1beta1.AMIFamilyAL2023), + Context: lo.ToPtr("fake-context"), + DetailedMonitoring: lo.ToPtr(false), + AMIFamily: lo.ToPtr(v1beta1.AMIFamilyAL2023), + AMISelectorTerms: []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{ + Family: v1beta1.AMIFamilyAL2023, + }, + }}, AssociatePublicIPAddress: lo.ToPtr(false), MetadataOptions: &v1beta1.MetadataOptions{ HTTPEndpoint: lo.ToPtr("disabled"), @@ -800,7 +805,7 @@ var _ = Describe("CloudProvider", func() { Entry("Tags", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{Tags: map[string]string{"keyTag-test-3": "valueTag-test-3"}}}), Entry("Context", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{Context: lo.ToPtr("context-2")}}), Entry("DetailedMonitoring", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{DetailedMonitoring: aws.Bool(true)}}), - Entry("AMIFamily", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{AMIFamily: lo.ToPtr(v1beta1.AMIFamilyBottlerocket)}}), + Entry("AMIFamily", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{AMIFamily: lo.ToPtr(v1beta1.AMIFamilyCustom)}}), Entry("InstanceStorePolicy", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{InstanceStorePolicy: lo.ToPtr(v1beta1.InstanceStorePolicyRAID0)}}), Entry("AssociatePublicIPAddress", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{AssociatePublicIPAddress: lo.ToPtr(true)}}), Entry("MetadataOptions HTTPEndpoint", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{MetadataOptions: &v1beta1.MetadataOptions{HTTPEndpoint: lo.ToPtr("enabled")}}}), diff --git a/pkg/controllers/nodeclass/status/ami_test.go b/pkg/controllers/nodeclass/status/ami_test.go index 0e442cbf33a1..ab00ef97a22b 100644 --- a/pkg/controllers/nodeclass/status/ami_test.go +++ b/pkg/controllers/nodeclass/status/ami_test.go @@ -134,7 +134,11 @@ var _ = Describe("NodeClass AMI Status Controller", func() { }, }, }) - nodeClass.Spec.AMISelectorTerms = nil + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{ + Family: v1beta1.AMIFamilyAL2, + }, + }} ExpectApplied(ctx, env.Client, nodeClass) ExpectReconcileSucceeded(ctx, statusController, client.ObjectKeyFromObject(nodeClass)) nodeClass = ExpectExists(ctx, env.Client, nodeClass) @@ -237,7 +241,11 @@ var _ = Describe("NodeClass AMI Status Controller", func() { fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s/arm64/latest/image_id", version): "ami-id-456", } nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket - nodeClass.Spec.AMISelectorTerms = nil + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{ + Family: v1beta1.AMIFamilyBottlerocket, + }, + }} awsEnv.EC2API.DescribeImagesOutput.Set(&ec2.DescribeImagesOutput{ Images: []*ec2.Image{ { diff --git a/pkg/providers/amifamily/suite_test.go b/pkg/providers/amifamily/suite_test.go index 7dfc5ebe0d0b..9f43df46d013 100644 --- a/pkg/providers/amifamily/suite_test.go +++ b/pkg/providers/amifamily/suite_test.go @@ -130,7 +130,11 @@ var _ = Describe("AMIProvider", func() { nodeClass = test.EC2NodeClass() }) It("should succeed to resolve AMIs (AL2)", func() { - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{ + Family: v1beta1.AMIFamilyAL2, + }, + }} awsEnv.SSMAPI.Parameters = map[string]string{ fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2/recommended/image_id", version): amd64AMI, fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2-gpu/recommended/image_id", version): amd64NvidiaAMI, @@ -141,7 +145,11 @@ var _ = Describe("AMIProvider", func() { Expect(amis).To(HaveLen(4)) }) It("should succeed to resolve AMIs (AL2023)", func() { - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2023 + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{ + Family: v1beta1.AMIFamilyAL2023, + }, + }} awsEnv.SSMAPI.Parameters = map[string]string{ fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/x86_64/standard/recommended/image_id", version): amd64AMI, fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/arm64/standard/recommended/image_id", version): arm64AMI, @@ -151,7 +159,11 @@ var _ = Describe("AMIProvider", func() { Expect(amis).To(HaveLen(2)) }) It("should succeed to resolve AMIs (Bottlerocket)", func() { - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{ + Family: v1beta1.AMIFamilyBottlerocket, + }, + }} awsEnv.SSMAPI.Parameters = map[string]string{ fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s/x86_64/latest/image_id", version): amd64AMI, fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s-nvidia/x86_64/latest/image_id", version): amd64NvidiaAMI, @@ -163,7 +175,11 @@ var _ = Describe("AMIProvider", func() { Expect(amis).To(HaveLen(6)) }) It("should succeed to resolve AMIs (Ubuntu)", func() { - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyUbuntu + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{ + Family: v1beta1.AMIFamilyUbuntu, + }, + }} awsEnv.SSMAPI.Parameters = map[string]string{ fmt.Sprintf("/aws/service/canonical/ubuntu/eks/20.04/%s/stable/current/amd64/hvm/ebs-gp2/ami-id", version): amd64AMI, fmt.Sprintf("/aws/service/canonical/ubuntu/eks/20.04/%s/stable/current/arm64/hvm/ebs-gp2/ami-id", version): arm64AMI, @@ -173,7 +189,11 @@ var _ = Describe("AMIProvider", func() { Expect(amis).To(HaveLen(2)) }) It("should succeed to resolve AMIs (Windows2019)", func() { - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyWindows2019 + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{ + Family: v1beta1.AMIFamilyWindows2019, + }, + }} awsEnv.SSMAPI.Parameters = map[string]string{ fmt.Sprintf("/aws/service/ami-windows-latest/Windows_Server-2019-English-Core-EKS_Optimized-%s/image_id", version): amd64AMI, } @@ -182,7 +202,11 @@ var _ = Describe("AMIProvider", func() { Expect(amis).To(HaveLen(1)) }) It("should succeed to resolve AMIs (Windows2022)", func() { - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyWindows2022 + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{ + Family: v1beta1.AMIFamilyWindows2022, + }, + }} awsEnv.SSMAPI.Parameters = map[string]string{ fmt.Sprintf("/aws/service/ami-windows-latest/Windows_Server-2022-English-Core-EKS_Optimized-%s/image_id", version): amd64AMI, } @@ -191,14 +215,18 @@ var _ = Describe("AMIProvider", func() { Expect(amis).To(HaveLen(1)) }) It("should succeed to resolve AMIs (Custom)", func() { - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyCustom + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{} amis, err := awsEnv.AMIProvider.Get(ctx, nodeClass, &amifamily.Options{}) Expect(err).ToNot(HaveOccurred()) Expect(amis).To(HaveLen(0)) }) Context("SSM Alias Missing", func() { It("should succeed to partially resolve AMIs if all SSM aliases don't exist (Al2)", func() { - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{ + Family: v1beta1.AMIFamilyAL2, + }, + }} // No GPU AMI exists here awsEnv.SSMAPI.Parameters = map[string]string{ fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2/recommended/image_id", version): amd64AMI, @@ -210,7 +238,11 @@ var _ = Describe("AMIProvider", func() { Expect(amis).To(HaveLen(2)) }) It("should succeed to partially resolve AMIs if all SSM aliases don't exist (AL2023)", func() { - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2023 + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{ + Family: v1beta1.AMIFamilyAL2023, + }, + }} awsEnv.SSMAPI.Parameters = map[string]string{ fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/x86_64/standard/recommended/image_id", version): amd64AMI, } @@ -219,7 +251,11 @@ var _ = Describe("AMIProvider", func() { Expect(amis).To(HaveLen(1)) }) It("should succeed to partially resolve AMIs if all SSM aliases don't exist (Bottlerocket)", func() { - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{ + Family: v1beta1.AMIFamilyBottlerocket, + }, + }} // No GPU AMI exists for AM64 here awsEnv.SSMAPI.Parameters = map[string]string{ fmt.Sprintf("/aws/service/bottlerocket/aws-k8s-%s/x86_64/latest/image_id", version): amd64AMI, @@ -232,7 +268,11 @@ var _ = Describe("AMIProvider", func() { Expect(amis).To(HaveLen(4)) }) It("should succeed to partially resolve AMIs if all SSM aliases don't exist (Ubuntu)", func() { - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyUbuntu + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{ + Family: v1beta1.AMIFamilyUbuntu, + }, + }} // No AMD64 AMI exists here awsEnv.SSMAPI.Parameters = map[string]string{ fmt.Sprintf("/aws/service/canonical/ubuntu/eks/20.04/%s/stable/current/arm64/hvm/ebs-gp2/ami-id", version): arm64AMI, diff --git a/pkg/providers/instancetype/suite_test.go b/pkg/providers/instancetype/suite_test.go index 0e270e24a70b..4d9474a59812 100644 --- a/pkg/providers/instancetype/suite_test.go +++ b/pkg/providers/instancetype/suite_test.go @@ -133,6 +133,9 @@ var _ = Describe("InstanceTypeProvider", func() { windowsNodeClass = test.EC2NodeClass(v1beta1.EC2NodeClass{ Spec: v1beta1.EC2NodeClassSpec{ AMIFamily: &v1beta1.AMIFamilyWindows2022, + AMISelectorTerms: []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{Family: v1beta1.AMIFamilyWindows2022}, + }}, }, }) windowsNodePool = coretest.NodePool(corev1beta1.NodePool{ @@ -1200,6 +1203,9 @@ var _ = Describe("InstanceTypeProvider", func() { }) It("should ignore eviction threshold when using Bottlerocket AMI", func() { nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{Family: v1beta1.AMIFamilyBottlerocket}, + }} nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ SystemReserved: map[string]string{ string(v1.ResourceMemory): "20Gi", @@ -1566,6 +1572,9 @@ var _ = Describe("InstanceTypeProvider", func() { instanceInfo, err := awsEnv.EC2API.DescribeInstanceTypesWithContext(ctx, &ec2.DescribeInstanceTypesInput{}) Expect(err).To(BeNil()) nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{Family: v1beta1.AMIFamilyBottlerocket}, + }} nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ PodsPerCore: ptr.Int32(1), } @@ -2010,7 +2019,10 @@ var _ = Describe("InstanceTypeProvider", func() { }) Context("Ephemeral Storage", func() { BeforeEach(func() { - nodeClass.Spec.AMIFamily = aws.String(v1beta1.AMIFamilyAL2) + nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{Family: v1beta1.AMIFamilyAL2}, + }} nodeClass.Spec.BlockDeviceMappings = []*v1beta1.BlockDeviceMapping{ { DeviceName: aws.String("/dev/xvda"), @@ -2021,7 +2033,7 @@ var _ = Describe("InstanceTypeProvider", func() { } }) It("should default to EBS defaults when volumeSize is not defined in blockDeviceMappings for custom AMIs", func() { - nodeClass.Spec.AMIFamily = aws.String(v1beta1.AMIFamilyCustom) + nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyCustom nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ { Tags: map[string]string{ @@ -2055,7 +2067,10 @@ var _ = Describe("InstanceTypeProvider", func() { }) }) It("should default to EBS defaults when volumeSize is not defined in blockDeviceMappings for AL2023 Root volume", func() { - nodeClass.Spec.AMIFamily = aws.String(v1beta1.AMIFamilyAL2023) + nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2023 + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{Family: v1beta1.AMIFamilyAL2023}, + }} awsEnv.LaunchTemplateProvider.CABundle = lo.ToPtr("Y2EtYnVuZGxlCg==") awsEnv.LaunchTemplateProvider.ClusterCIDR.Store(lo.ToPtr("10.100.0.0/16")) ExpectApplied(ctx, env.Client, nodePool, nodeClass) @@ -2071,7 +2086,10 @@ var _ = Describe("InstanceTypeProvider", func() { }) }) It("should default to EBS defaults when volumeSize is not defined in blockDeviceMappings for Bottlerocket Root volume", func() { - nodeClass.Spec.AMIFamily = aws.String(v1beta1.AMIFamilyBottlerocket) + nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{Family: v1beta1.AMIFamilyBottlerocket}, + }} nodeClass.Spec.BlockDeviceMappings[0].DeviceName = aws.String("/dev/xvdb") ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod() @@ -2087,7 +2105,10 @@ var _ = Describe("InstanceTypeProvider", func() { }) }) It("should default to EBS defaults when volumeSize is not defined in blockDeviceMappings for Ubuntu Root volume", func() { - nodeClass.Spec.AMIFamily = aws.String(v1beta1.AMIFamilyUbuntu) + nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyUbuntu + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{Family: v1beta1.AMIFamilyUbuntu}, + }} nodeClass.Spec.BlockDeviceMappings[0].DeviceName = aws.String("/dev/sda1") ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod() @@ -2205,7 +2226,14 @@ var _ = Describe("InstanceTypeProvider", func() { nodeClassChanges := []*v1beta1.EC2NodeClass{ {}, // Testing the base case black EC2NodeClass {Spec: v1beta1.EC2NodeClassSpec{InstanceStorePolicy: lo.ToPtr(v1beta1.InstanceStorePolicyRAID0)}}, - {Spec: v1beta1.EC2NodeClassSpec{AMIFamily: &v1beta1.AMIFamilyUbuntu}}, + { + Spec: v1beta1.EC2NodeClassSpec{ + AMIFamily: &v1beta1.AMIFamilyUbuntu, + AMISelectorTerms: []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{Family: v1beta1.AMIFamilyUbuntu}, + }}, + }, + }, { Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{ { diff --git a/pkg/providers/launchtemplate/suite_test.go b/pkg/providers/launchtemplate/suite_test.go index 6799b9ed3ab8..00878bafcea0 100644 --- a/pkg/providers/launchtemplate/suite_test.go +++ b/pkg/providers/launchtemplate/suite_test.go @@ -531,6 +531,9 @@ var _ = Describe("LaunchTemplate Provider", func() { }) It("should default AL2023 block device mappings", func() { nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2023 + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{Family: v1beta1.AMIFamilyAL2023}, + }} awsEnv.LaunchTemplateProvider.CABundle = lo.ToPtr("Y2EtYnVuZGxlCg==") awsEnv.LaunchTemplateProvider.ClusterCIDR.Store(lo.ToPtr("10.100.0.0/16")) ExpectApplied(ctx, env.Client, nodePool, nodeClass) @@ -634,6 +637,9 @@ var _ = Describe("LaunchTemplate Provider", func() { }) It("should default bottlerocket second volume with root volume size", func() { nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{Family: v1beta1.AMIFamilyBottlerocket}, + }} ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) @@ -984,6 +990,9 @@ var _ = Describe("LaunchTemplate Provider", func() { })) nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{Family: v1beta1.AMIFamilyBottlerocket}, + }} amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) it := instancetype.NewInstanceType(ctx, info, @@ -1009,6 +1018,9 @@ var _ = Describe("LaunchTemplate Provider", func() { })) nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{Family: v1beta1.AMIFamilyBottlerocket}, + }} nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{MaxPods: lo.ToPtr[int32](110)} amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) it := instancetype.NewInstanceType(ctx, @@ -1291,6 +1303,9 @@ var _ = Describe("LaunchTemplate Provider", func() { Context("Bottlerocket", func() { BeforeEach(func() { nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{Family: v1beta1.AMIFamilyBottlerocket}, + }} nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{MaxPods: lo.ToPtr[int32](110)} }) It("should merge in custom user data", func() { @@ -1564,6 +1579,9 @@ var _ = Describe("LaunchTemplate Provider", func() { Context("AL2023", func() { BeforeEach(func() { nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2023 + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{Family: v1beta1.AMIFamilyAL2023}, + }} // base64 encoded version of "ca-bundle" to ensure the nodeadm bootstrap provider can decode successfully awsEnv.LaunchTemplateProvider.CABundle = lo.ToPtr("Y2EtYnVuZGxlCg==") @@ -1755,6 +1773,7 @@ var _ = Describe("LaunchTemplate Provider", func() { ) It("should fail to create launch templates if cluster CIDR is unresolved", func() { awsEnv.LaunchTemplateProvider.ClusterCIDR.Store(nil) + fmt.Printf("nodeClass: %s\n", string(lo.Must(json.Marshal(nodeClass)))) ExpectApplied(ctx, env.Client, nodeClass, nodePool) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) @@ -2004,6 +2023,9 @@ var _ = Describe("LaunchTemplate Provider", func() { BeforeEach(func() { nodePool.Spec.Template.Spec.Requirements = []corev1beta1.NodeSelectorRequirementWithMinValues{{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: v1.LabelOSStable, Operator: v1.NodeSelectorOpIn, Values: []string{string(v1.Windows)}}}} nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyWindows2022 + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{Family: v1beta1.AMIFamilyWindows2022}, + }} nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{MaxPods: lo.ToPtr[int32](110)} }) It("should merge and bootstrap with custom user data", func() { diff --git a/pkg/test/nodeclass.go b/pkg/test/nodeclass.go index 4aa58d4a75d3..448da15e7ff2 100644 --- a/pkg/test/nodeclass.go +++ b/pkg/test/nodeclass.go @@ -38,6 +38,13 @@ func EC2NodeClass(overrides ...v1beta1.EC2NodeClass) *v1beta1.EC2NodeClass { if options.Spec.AMIFamily == nil { options.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 } + if options.Spec.AMISelectorTerms == nil { + options.Spec.AMISelectorTerms = append(options.Spec.AMISelectorTerms, v1beta1.AMISelectorTerm{ + EKSOptimized: &v1beta1.EKSOptimized{ + Family: v1beta1.AMIFamilyAL2, + }, + }) + } if options.Spec.Role == "" { options.Spec.Role = "test-role" options.Status.InstanceProfile = "test-profile" diff --git a/test/pkg/environment/aws/environment.go b/test/pkg/environment/aws/environment.go index e874cf325ce1..c78547368f01 100644 --- a/test/pkg/environment/aws/environment.go +++ b/test/pkg/environment/aws/environment.go @@ -137,6 +137,11 @@ func GetTimeStreamAPI(session *session.Session) timestreamwriteiface.TimestreamW func (env *Environment) DefaultEC2NodeClass() *v1beta1.EC2NodeClass { nodeClass := test.EC2NodeClass() nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2023 + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{ + Family: v1beta1.AMIFamilyAL2023, + }, + }} nodeClass.Spec.Tags = map[string]string{ "testing/cluster": env.ClusterName, } diff --git a/test/suites/integration/ami_test.go b/test/suites/integration/ami_test.go index 96c3bdc57a1c..a5272250b25d 100644 --- a/test/suites/integration/ami_test.go +++ b/test/suites/integration/ami_test.go @@ -135,12 +135,18 @@ var _ = Describe("AMI", func() { It("should provision a node using the AL2 family", func() { pod := coretest.Pod() nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{Family: v1beta1.AMIFamilyAL2}, + }} env.ExpectCreated(nodeClass, nodePool, pod) env.EventuallyExpectHealthy(pod) env.ExpectCreatedNodeCount("==", 1) }) It("should provision a node using the AL2023 family", func() { nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2023 + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{Family: v1beta1.AMIFamilyAL2023}, + }} pod := coretest.Pod() env.ExpectCreated(nodeClass, nodePool, pod) env.EventuallyExpectHealthy(pod) @@ -148,6 +154,9 @@ var _ = Describe("AMI", func() { }) It("should provision a node using the Bottlerocket family", func() { nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{Family: v1beta1.AMIFamilyBottlerocket}, + }} pod := coretest.Pod() env.ExpectCreated(nodeClass, nodePool, pod) env.EventuallyExpectHealthy(pod) @@ -155,6 +164,9 @@ var _ = Describe("AMI", func() { }) It("should provision a node using the Ubuntu family", func() { nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyUbuntu + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{Family: v1beta1.AMIFamilyUbuntu}, + }} // TODO (jmdeal@): remove once 22.04 AMIs are supported if env.GetK8sVersion(0) == "1.29" { nodeClass.Spec.AMISelectorTerms = lo.Map([]string{ @@ -227,6 +239,9 @@ var _ = Describe("AMI", func() { content, err := os.ReadFile("testdata/al2_userdata_input.sh") Expect(err).ToNot(HaveOccurred()) nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{Family: v1beta1.AMIFamilyAL2}, + }} nodeClass.Spec.UserData = aws.String(string(content)) nodePool.Spec.Template.Spec.Taints = []v1.Taint{{Key: "example.com", Value: "value", Effect: "NoExecute"}} nodePool.Spec.Template.Spec.StartupTaints = []v1.Taint{{Key: "example.com", Value: "value", Effect: "NoSchedule"}} @@ -248,6 +263,9 @@ var _ = Describe("AMI", func() { content, err := os.ReadFile("testdata/al2_no_mime_userdata_input.sh") Expect(err).ToNot(HaveOccurred()) nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{Family: v1beta1.AMIFamilyAL2}, + }} nodeClass.Spec.UserData = aws.String(string(content)) nodePool.Spec.Template.Spec.Taints = []v1.Taint{{Key: "example.com", Value: "value", Effect: "NoExecute"}} nodePool.Spec.Template.Spec.StartupTaints = []v1.Taint{{Key: "example.com", Value: "value", Effect: "NoSchedule"}} @@ -269,6 +287,9 @@ var _ = Describe("AMI", func() { content, err := os.ReadFile("testdata/br_userdata_input.sh") Expect(err).ToNot(HaveOccurred()) nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{Family: v1beta1.AMIFamilyBottlerocket}, + }} nodeClass.Spec.UserData = aws.String(string(content)) nodePool.Spec.Template.Spec.Taints = []v1.Taint{{Key: "example.com", Value: "value", Effect: "NoExecute"}} nodePool.Spec.Template.Spec.StartupTaints = []v1.Taint{{Key: "example.com", Value: "value", Effect: "NoSchedule"}} @@ -293,6 +314,9 @@ var _ = Describe("AMI", func() { content, err := os.ReadFile("testdata/windows_userdata_input.ps1") Expect(err).ToNot(HaveOccurred()) nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyWindows2022 + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{Family: v1beta1.AMIFamilyWindows2022}, + }} nodeClass.Spec.UserData = aws.String(string(content)) nodePool.Spec.Template.Spec.Taints = []v1.Taint{{Key: "example.com", Value: "value", Effect: "NoExecute"}} nodePool.Spec.Template.Spec.StartupTaints = []v1.Taint{{Key: "example.com", Value: "value", Effect: "NoSchedule"}} diff --git a/test/suites/integration/extended_resources_test.go b/test/suites/integration/extended_resources_test.go index 49c2e8d3c7ae..ed333a46248c 100644 --- a/test/suites/integration/extended_resources_test.go +++ b/test/suites/integration/extended_resources_test.go @@ -47,6 +47,9 @@ var _ = Describe("Extended Resources", func() { ExpectNvidiaDevicePluginCreated() // TODO: jmdeal@ remove AL2 pin once AL2023 accelerated AMIs are available nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{Family: v1beta1.AMIFamilyAL2}, + }} numPods := 1 dep := test.Deployment(test.DeploymentOptions{ Replicas: int32(numPods), @@ -79,6 +82,9 @@ var _ = Describe("Extended Resources", func() { It("should provision nodes for a deployment that requests nvidia.com/gpu (Bottlerocket)", func() { // For Bottlerocket, we are testing that resources are initialized without needing a device plugin nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{Family: v1beta1.AMIFamilyBottlerocket}, + }} numPods := 1 dep := test.Deployment(test.DeploymentOptions{ Replicas: int32(numPods), @@ -240,6 +246,9 @@ var _ = Describe("Extended Resources", func() { nodeClass.Spec.SubnetSelectorTerms[0].Tags["Name"] = "*Private*" // TODO: jmdeal@ remove AL2 pin once AL2023 accelerated AMIs are available nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{Family: v1beta1.AMIFamilyAL2}, + }} numPods := 1 dep := test.Deployment(test.DeploymentOptions{ diff --git a/test/suites/integration/kubelet_config_test.go b/test/suites/integration/kubelet_config_test.go index a79804cdab0d..b9dc300ab7ad 100644 --- a/test/suites/integration/kubelet_config_test.go +++ b/test/suites/integration/kubelet_config_test.go @@ -89,6 +89,9 @@ var _ = Describe("KubeletConfiguration Overrides", func() { DescribeTable("Linux AMIFamilies", func(amiFamily *string) { nodeClass.Spec.AMIFamily = amiFamily + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{Family: *amiFamily}, + }} // TODO (jmdeal@): remove once 22.04 AMIs are supported if *amiFamily == v1beta1.AMIFamilyUbuntu && env.GetK8sVersion(0) == "1.29" { nodeClass.Spec.AMISelectorTerms = lo.Map([]string{ @@ -123,6 +126,9 @@ var _ = Describe("KubeletConfiguration Overrides", func() { }) nodeClass.Spec.AMIFamily = amiFamily + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{Family: *amiFamily}, + }} // Need to enable nodepool-level OS-scoping for now since DS evaluation is done off of the nodepool // requirements, not off of the instance type options so scheduling can fail if nodepool aren't // properly scoped @@ -230,6 +236,9 @@ var _ = Describe("KubeletConfiguration Overrides", func() { }) It("should ignore podsPerCore value when Bottlerocket is used", func() { nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{Family: v1beta1.AMIFamilyBottlerocket}, + }} // All pods should schedule to a single node since we are ignoring podsPerCore value // This would normally schedule to 3 nodes if not using Bottlerocket test.ReplaceRequirements(nodePool, diff --git a/test/suites/integration/scheduling_test.go b/test/suites/integration/scheduling_test.go index 9c9dba0f6a26..e88d6b7153ab 100644 --- a/test/suites/integration/scheduling_test.go +++ b/test/suites/integration/scheduling_test.go @@ -259,6 +259,9 @@ var _ = Describe("Scheduling", Ordered, ContinueOnFailure, func() { Image: aws.WindowsDefaultImage, }}) nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyWindows2022 + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{Family: v1beta1.Windows2022}, + }} // TODO: remove this requirement once VPC RC rolls out m7a.*, r7a.* ENI data (https://github.com/aws/karpenter-provider-aws/issues/4472) test.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ diff --git a/test/suites/scale/deprovisioning_test.go b/test/suites/scale/deprovisioning_test.go index cb7086bbee08..78681edfd7ac 100644 --- a/test/suites/scale/deprovisioning_test.go +++ b/test/suites/scale/deprovisioning_test.go @@ -241,6 +241,9 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), Label(debug.NoEvents), nodePoolMap[expirationValue].Spec.Limits = disableProvisioningLimits // Update the drift NodeClass to start drift on Nodes assigned to this NodeClass driftNodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket + driftNodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{Family: v1beta1.AMIFamilyBottlerocket}, + }} // Create test assertions to ensure during the multiple deprovisioner scale-downs type testAssertions struct { @@ -663,6 +666,9 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), Label(debug.NoEvents), env.MeasureDeprovisioningDurationFor(func() { By("kicking off deprovisioning drift by changing the nodeClass AMIFamily") nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{Family: v1beta1.AMIFamilyBottlerocket}, + }} env.ExpectCreatedOrUpdated(nodeClass) env.EventuallyExpectDeletedNodeCount("==", expectedNodeCount)