From ef8b6da5d18ef46d9ad0bca1c14096322c547a48 Mon Sep 17 00:00:00 2001 From: Amanuel Engeda Date: Wed, 11 Sep 2024 15:37:24 -0700 Subject: [PATCH] Add e2e testing for deprecated ami --- test/pkg/environment/aws/environment.go | 6 ++++ test/pkg/environment/aws/expectations.go | 41 ++++++++++++++++++++++++ test/suites/ami/suite_test.go | 35 ++++++++++++++++++++ test/suites/drift/suite_test.go | 39 +++++++++++++--------- 4 files changed, 105 insertions(+), 16 deletions(-) diff --git a/test/pkg/environment/aws/environment.go b/test/pkg/environment/aws/environment.go index 8df6765a9915..b761fda5006c 100644 --- a/test/pkg/environment/aws/environment.go +++ b/test/pkg/environment/aws/environment.go @@ -36,6 +36,7 @@ import ( "github.com/aws/aws-sdk-go/service/timestreamwrite" "github.com/aws/aws-sdk-go/service/timestreamwrite/timestreamwriteiface" . "github.com/onsi/ginkgo/v2" + corecache "github.com/patrickmn/go-cache" "github.com/samber/lo" corev1 "k8s.io/api/core/v1" "k8s.io/utils/env" @@ -43,7 +44,9 @@ import ( karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1" v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" + cache "github.com/aws/karpenter-provider-aws/pkg/cache" "github.com/aws/karpenter-provider-aws/pkg/providers/sqs" + "github.com/aws/karpenter-provider-aws/pkg/providers/version" "github.com/aws/karpenter-provider-aws/pkg/test" "github.com/aws/karpenter-provider-aws/test/pkg/environment/common" ) @@ -72,6 +75,7 @@ type Environment struct { ClusterName string ClusterEndpoint string + ClusterVersion string InterruptionQueue string PrivateCluster bool ZoneInfo []ZoneInfo @@ -95,6 +99,7 @@ func NewEnvironment(t *testing.T) *Environment { }, )) + version := lo.Must(version.NewDefaultProvider(env.KubeClient, corecache.New(cache.DefaultTTL, cache.DefaultCleanupInterval)).Get(env.Context)) awsEnv := &Environment{ Region: *session.Config.Region, Environment: env, @@ -109,6 +114,7 @@ func NewEnvironment(t *testing.T) *Environment { ClusterName: lo.Must(os.LookupEnv("CLUSTER_NAME")), ClusterEndpoint: lo.Must(os.LookupEnv("CLUSTER_ENDPOINT")), + ClusterVersion: version, } if _, awsEnv.PrivateCluster = os.LookupEnv("PRIVATE_CLUSTER"); awsEnv.PrivateCluster { diff --git a/test/pkg/environment/aws/expectations.go b/test/pkg/environment/aws/expectations.go index b40a4c4ae545..e256da06e198 100644 --- a/test/pkg/environment/aws/expectations.go +++ b/test/pkg/environment/aws/expectations.go @@ -34,6 +34,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/karpenter/pkg/test" coretest "sigs.k8s.io/karpenter/pkg/test" v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1" @@ -362,6 +363,46 @@ func (env *Environment) GetAMIBySSMPath(ssmPath string) string { return *parameter.Parameter.Value } +func (env *Environment) GetDeprecatedAMI(amiId string, amifamily string) string { + out, err := env.EC2API.DescribeImages(&ec2.DescribeImagesInput{ + Filters: []*ec2.Filter{ + { + Name: lo.ToPtr(fmt.Sprintf("tag:%s", test.DiscoveryLabel)), + Values: []*string{lo.ToPtr(env.ClusterVersion)}, + }, + { + Name: lo.ToPtr("AMIFamily"), + Values: []*string{lo.ToPtr(amifamily)}, + }, + }, + IncludeDeprecated: lo.ToPtr(true), + }) + Expect(err).To(BeNil()) + if len(out.Images) == 1 { + return lo.FromPtr(out.Images[0].ImageId) + } + + input := &ec2.CopyImageInput{ + SourceImageId: lo.ToPtr(amiId), + TagSpecifications: []*ec2.TagSpecification{ + {ResourceType: lo.ToPtr(ec2.ResourceTypeImage), Tags: []*ec2.Tag{ + { + Key: lo.ToPtr(test.DiscoveryLabel), + Value: lo.ToPtr(env.ClusterVersion), + }, + { + Key: lo.ToPtr("AMIFamily"), + Value: lo.ToPtr(amifamily), + }, + }}, + }, + } + output, err := env.EC2API.CopyImage(input) + Expect(err).To(BeNil()) + + return lo.FromPtr(output.ImageId) +} + func (env *Environment) EventuallyExpectRunInstances(instanceInput *ec2.RunInstancesInput) *ec2.Reservation { GinkgoHelper() // implement IMDSv2 diff --git a/test/suites/ami/suite_test.go b/test/suites/ami/suite_test.go index 8ac10a874339..e0d1848a13a8 100644 --- a/test/suites/ami/suite_test.go +++ b/test/suites/ami/suite_test.go @@ -71,8 +71,10 @@ var _ = AfterEach(func() { env.AfterEach() }) var _ = Describe("AMI", func() { var customAMI string + var deprecatedAMI string BeforeEach(func() { customAMI = env.GetAMIBySSMPath(fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/x86_64/standard/recommended/image_id", env.K8sVersion())) + deprecatedAMI = "ami-0f5202a918cd98f37" }) It("should use the AMI defined by the AMI Selector Terms", func() { @@ -156,6 +158,39 @@ var _ = Describe("AMI", func() { env.ExpectInstance(pod.Spec.NodeName).To(HaveField("ImageId", HaveValue(Equal(customAMI)))) }) + FIt("should support launching nodes with a deprecated ami", func() { + nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyAL2023) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ + { + ID: deprecatedAMI, + }, + } + pod := coretest.Pod() + + env.ExpectCreated(pod, nodeClass, nodePool) + env.EventuallyExpectHealthy(pod) + env.ExpectCreatedNodeCount("==", 1) + + env.ExpectInstance(pod.Spec.NodeName).To(HaveField("ImageId", HaveValue(Equal(deprecatedAMI)))) + }) + FIt("should prioritize launch with non-deprecated AMIs", func() { + nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyAL2023) + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{ + { + ID: deprecatedAMI, + }, + { + ID: customAMI, + }, + } + pod := coretest.Pod() + + env.ExpectCreated(pod, nodeClass, nodePool) + env.EventuallyExpectHealthy(pod) + env.ExpectCreatedNodeCount("==", 1) + + env.ExpectInstance(pod.Spec.NodeName).To(HaveField("ImageId", HaveValue(Equal(customAMI)))) + }) Context("AMIFamily", func() { DescribeTable( diff --git a/test/suites/drift/suite_test.go b/test/suites/drift/suite_test.go index 02d3b75154ff..383f02610bd7 100644 --- a/test/suites/drift/suite_test.go +++ b/test/suites/drift/suite_test.go @@ -48,6 +48,7 @@ import ( var env *aws.Environment var amdAMI string +var deprecatedAMI string var nodeClass *v1.EC2NodeClass var nodePool *karpv1.NodePool @@ -76,6 +77,7 @@ var _ = Describe("Drift", func() { var numPods int BeforeEach(func() { amdAMI = env.GetAMIBySSMPath(fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/x86_64/standard/recommended/image_id", env.K8sVersion())) + deprecatedAMI = "ami-0f5202a918cd98f37" numPods = 1 // Add pods with a do-not-disrupt annotation so that we can check node metadata before we disrupt dep = coretest.Deployment(coretest.DeploymentOptions{ @@ -390,10 +392,9 @@ var _ = Describe("Drift", func() { env.EventuallyExpectNotFound(pod, nodeClaim, node) env.EventuallyExpectHealthyPodCount(selector, numPods) }) - It("should return drifted if the AMI no longer matches the existing NodeClaims instance type", func() { - armAMI := env.GetAMIBySSMPath(fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/arm64/standard/recommended/image_id", env.K8sVersion())) + It("should disrupt nodes for deprecated AMIs to non-deprecated AMIs", func() { nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyAL2023) - nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{ID: armAMI}} + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{ID: deprecatedAMI}} env.ExpectCreated(dep, nodeClass, nodePool) pod := env.EventuallyExpectHealthyPodCount(selector, numPods)[0] @@ -401,7 +402,7 @@ var _ = Describe("Drift", func() { nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0] node := env.EventuallyExpectNodeCount("==", 1)[0] - nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{ID: amdAMI}} + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{ID: amdAMI}, {ID: deprecatedAMI}} env.ExpectCreatedOrUpdated(nodeClass) env.EventuallyExpectDrifted(nodeClaim) @@ -410,26 +411,32 @@ var _ = Describe("Drift", func() { env.ExpectUpdated(pod) env.EventuallyExpectNotFound(pod, nodeClaim, node) env.EventuallyExpectHealthyPodCount(selector, numPods) - }) - It("should not disrupt nodes that have drifted without the featureGate enabled", func() { - env.ExpectSettingsOverridden(corev1.EnvVar{Name: "FEATURE_GATES", Value: "Drift=false"}) - oldCustomAMI := env.GetAMIBySSMPath(fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/x86_64/standard/recommended/image_id", env.K8sVersionWithOffset(1))) + // validate the AMI id matches the non-deprecated AMI + pod = env.EventuallyExpectHealthyPodCount(selector, numPods)[0] + env.ExpectInstance(pod.Spec.NodeName).To(HaveField("ImageId", HaveValue(Equal(amdAMI)))) + + }) + It("should return drifted if the AMI no longer matches the existing NodeClaims instance type", func() { + armAMI := env.GetAMIBySSMPath(fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/arm64/standard/recommended/image_id", env.K8sVersion())) nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyAL2023) - nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{ID: oldCustomAMI}} + nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{ID: armAMI}} env.ExpectCreated(dep, nodeClass, nodePool) - env.EventuallyExpectHealthyPodCount(selector, numPods) + pod := env.EventuallyExpectHealthyPodCount(selector, numPods)[0] env.ExpectCreatedNodeCount("==", 1) - node := env.Monitor.CreatedNodes()[0] + nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0] + node := env.EventuallyExpectNodeCount("==", 1)[0] nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{ID: amdAMI}} - env.ExpectUpdated(nodeClass) + env.ExpectCreatedOrUpdated(nodeClass) - // We should consistently get the same node existing for a minute - Consistently(func(g Gomega) { - g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(node), &corev1.Node{})).To(Succeed()) - }).WithTimeout(time.Minute).Should(Succeed()) + env.EventuallyExpectDrifted(nodeClaim) + + delete(pod.Annotations, karpv1.DoNotDisruptAnnotationKey) + env.ExpectUpdated(pod) + env.EventuallyExpectNotFound(pod, nodeClaim, node) + env.EventuallyExpectHealthyPodCount(selector, numPods) }) It("should disrupt nodes that have drifted due to securitygroup", func() { By("getting the cluster vpc id")