From f8ad58849a80f76efe220cf6f55fcc171ca5d9f7 Mon Sep 17 00:00:00 2001 From: Jason Deal Date: Tue, 23 Apr 2024 15:21:46 -0700 Subject: [PATCH] e2e test updates --- test/suites/integration/ami_test.go | 24 +++++++++++++++++++ .../integration/extended_resources_test.go | 9 +++++++ .../suites/integration/kubelet_config_test.go | 9 +++++++ test/suites/integration/scheduling_test.go | 3 +++ test/suites/scale/deprovisioning_test.go | 6 +++++ 5 files changed, 51 insertions(+) diff --git a/test/suites/integration/ami_test.go b/test/suites/integration/ami_test.go index 96c3bdc57a1c..a5272250b25d 100644 --- a/test/suites/integration/ami_test.go +++ b/test/suites/integration/ami_test.go @@ -135,12 +135,18 @@ var _ = Describe("AMI", func() { It("should provision a node using the AL2 family", func() { pod := coretest.Pod() nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{Family: v1beta1.AMIFamilyAL2}, + }} env.ExpectCreated(nodeClass, nodePool, pod) env.EventuallyExpectHealthy(pod) env.ExpectCreatedNodeCount("==", 1) }) It("should provision a node using the AL2023 family", func() { nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2023 + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{Family: v1beta1.AMIFamilyAL2023}, + }} pod := coretest.Pod() env.ExpectCreated(nodeClass, nodePool, pod) env.EventuallyExpectHealthy(pod) @@ -148,6 +154,9 @@ var _ = Describe("AMI", func() { }) It("should provision a node using the Bottlerocket family", func() { nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{Family: v1beta1.AMIFamilyBottlerocket}, + }} pod := coretest.Pod() env.ExpectCreated(nodeClass, nodePool, pod) env.EventuallyExpectHealthy(pod) @@ -155,6 +164,9 @@ var _ = Describe("AMI", func() { }) It("should provision a node using the Ubuntu family", func() { nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyUbuntu + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{Family: v1beta1.AMIFamilyUbuntu}, + }} // TODO (jmdeal@): remove once 22.04 AMIs are supported if env.GetK8sVersion(0) == "1.29" { nodeClass.Spec.AMISelectorTerms = lo.Map([]string{ @@ -227,6 +239,9 @@ var _ = Describe("AMI", func() { content, err := os.ReadFile("testdata/al2_userdata_input.sh") Expect(err).ToNot(HaveOccurred()) nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{Family: v1beta1.AMIFamilyAL2}, + }} nodeClass.Spec.UserData = aws.String(string(content)) nodePool.Spec.Template.Spec.Taints = []v1.Taint{{Key: "example.com", Value: "value", Effect: "NoExecute"}} nodePool.Spec.Template.Spec.StartupTaints = []v1.Taint{{Key: "example.com", Value: "value", Effect: "NoSchedule"}} @@ -248,6 +263,9 @@ var _ = Describe("AMI", func() { content, err := os.ReadFile("testdata/al2_no_mime_userdata_input.sh") Expect(err).ToNot(HaveOccurred()) nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{Family: v1beta1.AMIFamilyAL2}, + }} nodeClass.Spec.UserData = aws.String(string(content)) nodePool.Spec.Template.Spec.Taints = []v1.Taint{{Key: "example.com", Value: "value", Effect: "NoExecute"}} nodePool.Spec.Template.Spec.StartupTaints = []v1.Taint{{Key: "example.com", Value: "value", Effect: "NoSchedule"}} @@ -269,6 +287,9 @@ var _ = Describe("AMI", func() { content, err := os.ReadFile("testdata/br_userdata_input.sh") Expect(err).ToNot(HaveOccurred()) nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{Family: v1beta1.AMIFamilyBottlerocket}, + }} nodeClass.Spec.UserData = aws.String(string(content)) nodePool.Spec.Template.Spec.Taints = []v1.Taint{{Key: "example.com", Value: "value", Effect: "NoExecute"}} nodePool.Spec.Template.Spec.StartupTaints = []v1.Taint{{Key: "example.com", Value: "value", Effect: "NoSchedule"}} @@ -293,6 +314,9 @@ var _ = Describe("AMI", func() { content, err := os.ReadFile("testdata/windows_userdata_input.ps1") Expect(err).ToNot(HaveOccurred()) nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyWindows2022 + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{Family: v1beta1.AMIFamilyWindows2022}, + }} nodeClass.Spec.UserData = aws.String(string(content)) nodePool.Spec.Template.Spec.Taints = []v1.Taint{{Key: "example.com", Value: "value", Effect: "NoExecute"}} nodePool.Spec.Template.Spec.StartupTaints = []v1.Taint{{Key: "example.com", Value: "value", Effect: "NoSchedule"}} diff --git a/test/suites/integration/extended_resources_test.go b/test/suites/integration/extended_resources_test.go index 49c2e8d3c7ae..ed333a46248c 100644 --- a/test/suites/integration/extended_resources_test.go +++ b/test/suites/integration/extended_resources_test.go @@ -47,6 +47,9 @@ var _ = Describe("Extended Resources", func() { ExpectNvidiaDevicePluginCreated() // TODO: jmdeal@ remove AL2 pin once AL2023 accelerated AMIs are available nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{Family: v1beta1.AMIFamilyAL2}, + }} numPods := 1 dep := test.Deployment(test.DeploymentOptions{ Replicas: int32(numPods), @@ -79,6 +82,9 @@ var _ = Describe("Extended Resources", func() { It("should provision nodes for a deployment that requests nvidia.com/gpu (Bottlerocket)", func() { // For Bottlerocket, we are testing that resources are initialized without needing a device plugin nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{Family: v1beta1.AMIFamilyBottlerocket}, + }} numPods := 1 dep := test.Deployment(test.DeploymentOptions{ Replicas: int32(numPods), @@ -240,6 +246,9 @@ var _ = Describe("Extended Resources", func() { nodeClass.Spec.SubnetSelectorTerms[0].Tags["Name"] = "*Private*" // TODO: jmdeal@ remove AL2 pin once AL2023 accelerated AMIs are available nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{Family: v1beta1.AMIFamilyAL2}, + }} numPods := 1 dep := test.Deployment(test.DeploymentOptions{ diff --git a/test/suites/integration/kubelet_config_test.go b/test/suites/integration/kubelet_config_test.go index a79804cdab0d..b9dc300ab7ad 100644 --- a/test/suites/integration/kubelet_config_test.go +++ b/test/suites/integration/kubelet_config_test.go @@ -89,6 +89,9 @@ var _ = Describe("KubeletConfiguration Overrides", func() { DescribeTable("Linux AMIFamilies", func(amiFamily *string) { nodeClass.Spec.AMIFamily = amiFamily + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{Family: *amiFamily}, + }} // TODO (jmdeal@): remove once 22.04 AMIs are supported if *amiFamily == v1beta1.AMIFamilyUbuntu && env.GetK8sVersion(0) == "1.29" { nodeClass.Spec.AMISelectorTerms = lo.Map([]string{ @@ -123,6 +126,9 @@ var _ = Describe("KubeletConfiguration Overrides", func() { }) nodeClass.Spec.AMIFamily = amiFamily + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{Family: *amiFamily}, + }} // Need to enable nodepool-level OS-scoping for now since DS evaluation is done off of the nodepool // requirements, not off of the instance type options so scheduling can fail if nodepool aren't // properly scoped @@ -230,6 +236,9 @@ var _ = Describe("KubeletConfiguration Overrides", func() { }) It("should ignore podsPerCore value when Bottlerocket is used", func() { nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{Family: v1beta1.AMIFamilyBottlerocket}, + }} // All pods should schedule to a single node since we are ignoring podsPerCore value // This would normally schedule to 3 nodes if not using Bottlerocket test.ReplaceRequirements(nodePool, diff --git a/test/suites/integration/scheduling_test.go b/test/suites/integration/scheduling_test.go index 9c9dba0f6a26..e88d6b7153ab 100644 --- a/test/suites/integration/scheduling_test.go +++ b/test/suites/integration/scheduling_test.go @@ -259,6 +259,9 @@ var _ = Describe("Scheduling", Ordered, ContinueOnFailure, func() { Image: aws.WindowsDefaultImage, }}) nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyWindows2022 + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{Family: v1beta1.Windows2022}, + }} // TODO: remove this requirement once VPC RC rolls out m7a.*, r7a.* ENI data (https://github.com/aws/karpenter-provider-aws/issues/4472) test.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ diff --git a/test/suites/scale/deprovisioning_test.go b/test/suites/scale/deprovisioning_test.go index cb7086bbee08..78681edfd7ac 100644 --- a/test/suites/scale/deprovisioning_test.go +++ b/test/suites/scale/deprovisioning_test.go @@ -241,6 +241,9 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), Label(debug.NoEvents), nodePoolMap[expirationValue].Spec.Limits = disableProvisioningLimits // Update the drift NodeClass to start drift on Nodes assigned to this NodeClass driftNodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket + driftNodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{Family: v1beta1.AMIFamilyBottlerocket}, + }} // Create test assertions to ensure during the multiple deprovisioner scale-downs type testAssertions struct { @@ -663,6 +666,9 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), Label(debug.NoEvents), env.MeasureDeprovisioningDurationFor(func() { By("kicking off deprovisioning drift by changing the nodeClass AMIFamily") nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ + EKSOptimized: &v1beta1.EKSOptimized{Family: v1beta1.AMIFamilyBottlerocket}, + }} env.ExpectCreatedOrUpdated(nodeClass) env.EventuallyExpectDeletedNodeCount("==", expectedNodeCount)