Add e2e testing for deprecated ami

test/pkg/environment/aws/environment.go

@@ -36,14 +36,17 @@ import (
 	"github.com/aws/aws-sdk-go/service/timestreamwrite"
 	"github.com/aws/aws-sdk-go/service/timestreamwrite/timestreamwriteiface"
 	. "github.com/onsi/ginkgo/v2"
+	corecache "github.com/patrickmn/go-cache"
 	"github.com/samber/lo"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/utils/env"
 
 	karpv1 "sigs.k8s.io/karpenter/pkg/apis/v1"
 
 	v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1"
+	cache "github.com/aws/karpenter-provider-aws/pkg/cache"
 	"github.com/aws/karpenter-provider-aws/pkg/providers/sqs"
+	"github.com/aws/karpenter-provider-aws/pkg/providers/version"
 	"github.com/aws/karpenter-provider-aws/pkg/test"
 	"github.com/aws/karpenter-provider-aws/test/pkg/environment/common"
 )
@@ -72,6 +75,7 @@ type Environment struct {
 
 	ClusterName       string
 	ClusterEndpoint   string
+	ClusterVersion    string
 	InterruptionQueue string
 	PrivateCluster    bool
 	ZoneInfo          []ZoneInfo
@@ -95,6 +99,7 @@ func NewEnvironment(t *testing.T) *Environment {
 		},
 	))
 
+	version := lo.Must(version.NewDefaultProvider(env.KubeClient, corecache.New(cache.DefaultTTL, cache.DefaultCleanupInterval)).Get(env.Context))
 	awsEnv := &Environment{
 		Region:      *session.Config.Region,
 		Environment: env,
@@ -109,6 +114,7 @@ func NewEnvironment(t *testing.T) *Environment {
 
 		ClusterName:     lo.Must(os.LookupEnv("CLUSTER_NAME")),
 		ClusterEndpoint: lo.Must(os.LookupEnv("CLUSTER_ENDPOINT")),
+		ClusterVersion:  version,
 	}
 
 	if _, awsEnv.PrivateCluster = os.LookupEnv("PRIVATE_CLUSTER"); awsEnv.PrivateCluster {

test/pkg/environment/aws/expectations.go

@@ -34,6 +34,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 
+	"sigs.k8s.io/karpenter/pkg/test"
 	coretest "sigs.k8s.io/karpenter/pkg/test"
 
 	v1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1"
@@ -362,6 +363,46 @@ func (env *Environment) GetAMIBySSMPath(ssmPath string) string {
 	return *parameter.Parameter.Value
 }
 
+func (env *Environment) GetDeprecatedAMI(amiId string, amifamily string) string {
+	out, err := env.EC2API.DescribeImages(&ec2.DescribeImagesInput{
+		Filters: []*ec2.Filter{
+			{
+				Name:   lo.ToPtr(fmt.Sprintf("tag:%s", test.DiscoveryLabel)),
+				Values: []*string{lo.ToPtr(env.ClusterVersion)},
+			},
+			{
+				Name:   lo.ToPtr("AMIFamily"),
+				Values: []*string{lo.ToPtr(amifamily)},
+			},
+		},
+		IncludeDeprecated: lo.ToPtr(true),
+	})
+	Expect(err).To(BeNil())
+	if len(out.Images) == 1 {
+		return lo.FromPtr(out.Images[0].ImageId)
+	}
+
+	input := &ec2.CopyImageInput{
+		SourceImageId: lo.ToPtr(amiId),
+		TagSpecifications: []*ec2.TagSpecification{
+			{ResourceType: lo.ToPtr(ec2.ResourceTypeImage), Tags: []*ec2.Tag{
+				{
+					Key:   lo.ToPtr(test.DiscoveryLabel),
+					Value: lo.ToPtr(env.ClusterVersion),
+				},
+				{
+					Key:   lo.ToPtr("AMIFamily"),
+					Value: lo.ToPtr(amifamily),
+				},
+			}},
+		},
+	}
+	output, err := env.EC2API.CopyImage(input)
+	Expect(err).To(BeNil())
+
+	return lo.FromPtr(output.ImageId)
+}
+
 func (env *Environment) EventuallyExpectRunInstances(instanceInput *ec2.RunInstancesInput) *ec2.Reservation {
 	GinkgoHelper()
 	// implement IMDSv2

test/suites/ami/suite_test.go

@@ -71,8 +71,10 @@ var _ = AfterEach(func() { env.AfterEach() })
 
 var _ = Describe("AMI", func() {
 	var customAMI string
+	var deprecatedAMI string
 	BeforeEach(func() {
 		customAMI = env.GetAMIBySSMPath(fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/x86_64/standard/recommended/image_id", env.K8sVersion()))
+		deprecatedAMI = "ami-0f5202a918cd98f37"
 	})
 
 	It("should use the AMI defined by the AMI Selector Terms", func() {
@@ -156,6 +158,39 @@ var _ = Describe("AMI", func() {
 
 		env.ExpectInstance(pod.Spec.NodeName).To(HaveField("ImageId", HaveValue(Equal(customAMI))))
 	})
+	FIt("should support launching nodes with a deprecated ami", func() {
+		nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyAL2023)
+		nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{
+			{
+				ID: deprecatedAMI,
+			},
+		}
+		pod := coretest.Pod()
+
+		env.ExpectCreated(pod, nodeClass, nodePool)
+		env.EventuallyExpectHealthy(pod)
+		env.ExpectCreatedNodeCount("==", 1)
+
+		env.ExpectInstance(pod.Spec.NodeName).To(HaveField("ImageId", HaveValue(Equal(deprecatedAMI))))
+	})
+	FIt("should prioritize launch with non-deprecated AMIs", func() {
+		nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyAL2023)
+		nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{
+			{
+				ID: deprecatedAMI,
+			},
+			{
+				ID: customAMI,
+			},
+		}
+		pod := coretest.Pod()
+
+		env.ExpectCreated(pod, nodeClass, nodePool)
+		env.EventuallyExpectHealthy(pod)
+		env.ExpectCreatedNodeCount("==", 1)
+
+		env.ExpectInstance(pod.Spec.NodeName).To(HaveField("ImageId", HaveValue(Equal(customAMI))))
+	})
 
 	Context("AMIFamily", func() {
 		DescribeTable(

test/suites/drift/suite_test.go

@@ -48,6 +48,7 @@ import (
 
 var env *aws.Environment
 var amdAMI string
+var deprecatedAMI string
 var nodeClass *v1.EC2NodeClass
 var nodePool *karpv1.NodePool
 
@@ -76,6 +77,7 @@ var _ = Describe("Drift", func() {
 	var numPods int
 	BeforeEach(func() {
 		amdAMI = env.GetAMIBySSMPath(fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/x86_64/standard/recommended/image_id", env.K8sVersion()))
+		deprecatedAMI = "ami-0f5202a918cd98f37"
 		numPods = 1
 		// Add pods with a do-not-disrupt annotation so that we can check node metadata before we disrupt
 		dep = coretest.Deployment(coretest.DeploymentOptions{
@@ -390,18 +392,17 @@ var _ = Describe("Drift", func() {
 		env.EventuallyExpectNotFound(pod, nodeClaim, node)
 		env.EventuallyExpectHealthyPodCount(selector, numPods)
 	})
-	It("should return drifted if the AMI no longer matches the existing NodeClaims instance type", func() {
-		armAMI := env.GetAMIBySSMPath(fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/arm64/standard/recommended/image_id", env.K8sVersion()))
+	It("should disrupt nodes for deprecated AMIs to non-deprecated AMIs", func() {
 		nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyAL2023)
-		nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{ID: armAMI}}
+		nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{ID: deprecatedAMI}}
 
 		env.ExpectCreated(dep, nodeClass, nodePool)
 		pod := env.EventuallyExpectHealthyPodCount(selector, numPods)[0]
 		env.ExpectCreatedNodeCount("==", 1)
 
 		nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0]
 		node := env.EventuallyExpectNodeCount("==", 1)[0]
-		nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{ID: amdAMI}}
+		nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{ID: amdAMI}, {ID: deprecatedAMI}}
 		env.ExpectCreatedOrUpdated(nodeClass)
 
 		env.EventuallyExpectDrifted(nodeClaim)
@@ -410,26 +411,32 @@ var _ = Describe("Drift", func() {
 		env.ExpectUpdated(pod)
 		env.EventuallyExpectNotFound(pod, nodeClaim, node)
 		env.EventuallyExpectHealthyPodCount(selector, numPods)
-	})
-	It("should not disrupt nodes that have drifted without the featureGate enabled", func() {
-		env.ExpectSettingsOverridden(corev1.EnvVar{Name: "FEATURE_GATES", Value: "Drift=false"})
 
-		oldCustomAMI := env.GetAMIBySSMPath(fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/x86_64/standard/recommended/image_id", env.K8sVersionWithOffset(1)))
+		// validate the AMI id matches the non-deprecated AMI
+		pod = env.EventuallyExpectHealthyPodCount(selector, numPods)[0]
+		env.ExpectInstance(pod.Spec.NodeName).To(HaveField("ImageId", HaveValue(Equal(amdAMI))))
+
+	})
+	It("should return drifted if the AMI no longer matches the existing NodeClaims instance type", func() {
+		armAMI := env.GetAMIBySSMPath(fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/arm64/standard/recommended/image_id", env.K8sVersion()))
 		nodeClass.Spec.AMIFamily = lo.ToPtr(v1.AMIFamilyAL2023)
-		nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{ID: oldCustomAMI}}
+		nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{ID: armAMI}}
 
 		env.ExpectCreated(dep, nodeClass, nodePool)
-		env.EventuallyExpectHealthyPodCount(selector, numPods)
+		pod := env.EventuallyExpectHealthyPodCount(selector, numPods)[0]
 		env.ExpectCreatedNodeCount("==", 1)
 
-		node := env.Monitor.CreatedNodes()[0]
+		nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0]
+		node := env.EventuallyExpectNodeCount("==", 1)[0]
 		nodeClass.Spec.AMISelectorTerms = []v1.AMISelectorTerm{{ID: amdAMI}}
-		env.ExpectUpdated(nodeClass)
+		env.ExpectCreatedOrUpdated(nodeClass)
 
-		// We should consistently get the same node existing for a minute
-		Consistently(func(g Gomega) {
-			g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(node), &corev1.Node{})).To(Succeed())
-		}).WithTimeout(time.Minute).Should(Succeed())
+		env.EventuallyExpectDrifted(nodeClaim)
+
+		delete(pod.Annotations, karpv1.DoNotDisruptAnnotationKey)
+		env.ExpectUpdated(pod)
+		env.EventuallyExpectNotFound(pod, nodeClaim, node)
+		env.EventuallyExpectHealthyPodCount(selector, numPods)
 	})
 	It("should disrupt nodes that have drifted due to securitygroup", func() {
 		By("getting the cluster vpc id")