From e86e82fcb5adaf9502b0e163fdfbfee32dc45a89 Mon Sep 17 00:00:00 2001 From: jigisha620 Date: Wed, 12 Jun 2024 11:30:48 -0700 Subject: [PATCH] chore: Wait for instance termination before deleting nodeclaim --- go.mod | 6 +++--- go.sum | 12 ++++++------ test/pkg/environment/common/environment.go | 2 +- test/pkg/environment/common/expectations.go | 2 +- test/suites/expiration/suite_test.go | 7 ------- test/suites/integration/termination_test.go | 2 +- test/suites/interruption/suite_test.go | 10 +++++----- test/suites/nodeclaim/garbage_collection_test.go | 2 +- test/suites/nodeclaim/nodeclaim_test.go | 10 +++++----- 9 files changed, 23 insertions(+), 30 deletions(-) diff --git a/go.mod b/go.mod index 121ce05e900a..23db04d0f884 100644 --- a/go.mod +++ b/go.mod @@ -30,7 +30,7 @@ require ( k8s.io/utils v0.0.0-20240102154912-e7106e64919e knative.dev/pkg v0.0.0-20231010144348-ca8c009405dd sigs.k8s.io/controller-runtime v0.18.4 - sigs.k8s.io/karpenter v0.37.1-0.20240610184513-261b55a31660 + sigs.k8s.io/karpenter v0.37.1-0.20240612182320-24c976186c48 sigs.k8s.io/yaml v1.4.0 ) @@ -95,9 +95,9 @@ require ( golang.org/x/oauth2 v0.18.0 // indirect golang.org/x/sys v0.20.0 // indirect golang.org/x/term v0.20.0 // indirect - golang.org/x/text v0.15.0 // indirect + golang.org/x/text v0.16.0 // indirect golang.org/x/time v0.5.0 // indirect - golang.org/x/tools v0.21.0 // indirect + golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/api v0.146.0 // indirect google.golang.org/appengine v1.6.8 // indirect diff --git a/go.sum b/go.sum index 532666e0cfc7..6016f56b519f 100644 --- a/go.sum +++ b/go.sum @@ -550,8 +550,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk= -golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -601,8 +601,8 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.21.0 h1:qc0xYgIbsSDt9EyWz05J5wfa7LOVW0YTLOXrqdLAWIw= -golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -761,8 +761,8 @@ sigs.k8s.io/controller-runtime v0.18.4 h1:87+guW1zhvuPLh1PHybKdYFLU0YJp4FhJRmiHv sigs.k8s.io/controller-runtime v0.18.4/go.mod h1:TVoGrfdpbA9VRFaRnKgk9P5/atA0pMwq+f+msb9M8Sg= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/karpenter v0.37.1-0.20240610184513-261b55a31660 h1:vYMlBlYIUpMaiuYdVXYVSjI4cNaadyHR0Y3IUqhyQMA= -sigs.k8s.io/karpenter v0.37.1-0.20240610184513-261b55a31660/go.mod h1:V/+zk8fRo3x0No93quokU87mUAYuYQw2s8QGkBth2cY= +sigs.k8s.io/karpenter v0.37.1-0.20240612182320-24c976186c48 h1:U/bzF0qgk1RMtpl97F+IltVE7caxTLoy7Pwk4NB+zIg= +sigs.k8s.io/karpenter v0.37.1-0.20240612182320-24c976186c48/go.mod h1:rkTEnr5zMsc5RLCcnSlnq6Uklgom7SKG7zyVk7+dtIU= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= diff --git a/test/pkg/environment/common/environment.go b/test/pkg/environment/common/environment.go index ef677cbbd8a7..bb3e6a8daab2 100644 --- a/test/pkg/environment/common/environment.go +++ b/test/pkg/environment/common/environment.go @@ -77,7 +77,7 @@ func NewEnvironment(t *testing.T) *Environment { ctx = context.WithValue(ctx, GitRefContextKey, val) } - gomega.SetDefaultEventuallyTimeout(5 * time.Minute) + gomega.SetDefaultEventuallyTimeout(16 * time.Minute) gomega.SetDefaultEventuallyPollingInterval(1 * time.Second) return &Environment{ Context: ctx, diff --git a/test/pkg/environment/common/expectations.go b/test/pkg/environment/common/expectations.go index bef21c33c657..b08b8e941ff3 100644 --- a/test/pkg/environment/common/expectations.go +++ b/test/pkg/environment/common/expectations.go @@ -778,7 +778,7 @@ func (env *Environment) EventuallyExpectAvgUtilization(resource v1.ResourceName, GinkgoHelper() Eventually(func(g Gomega) { g.Expect(env.Monitor.AvgUtilization(resource)).To(BeNumerically(comparator, value)) - }, 10*time.Minute).Should(Succeed()) + }, 12*time.Minute).Should(Succeed()) } func (env *Environment) ExpectDaemonSetEnvironmentVariableUpdated(obj client.ObjectKey, name, value string, containers ...string) { diff --git a/test/suites/expiration/suite_test.go b/test/suites/expiration/suite_test.go index ed6e884e4dcd..b1d8d7a32b16 100644 --- a/test/suites/expiration/suite_test.go +++ b/test/suites/expiration/suite_test.go @@ -586,13 +586,6 @@ var _ = Describe("Expiration", func() { // TODO: reduce timeouts when deprovisioning waits are factored out env.EventuallyExpectNodesUntaintedWithTimeout(11*time.Minute, taintedNodes...) - // The nodeclaims that never registers will be removed - Eventually(func(g Gomega) { - nodeClaims := &corev1beta1.NodeClaimList{} - g.Expect(env.Client.List(env, nodeClaims, client.HasLabels{coretest.DiscoveryLabel})).To(Succeed()) - g.Expect(len(nodeClaims.Items)).To(BeNumerically("==", int(numPods))) - }).WithTimeout(6 * time.Minute).Should(Succeed()) - // Expect all the NodeClaims that existed on the initial provisioning loop are not removed Consistently(func(g Gomega) { nodeClaims := &corev1beta1.NodeClaimList{} diff --git a/test/suites/integration/termination_test.go b/test/suites/integration/termination_test.go index cbd32fb51a27..9c0bb5b7a433 100644 --- a/test/suites/integration/termination_test.go +++ b/test/suites/integration/termination_test.go @@ -44,7 +44,7 @@ var _ = Describe("Termination", func() { env.ExpectDeleted(nodes[0]) env.EventuallyExpectNotFound(nodes[0]) Eventually(func(g Gomega) { - g.Expect(lo.FromPtr(env.GetInstanceByID(instanceID).State.Name)).To(Equal("shutting-down")) + g.Expect(lo.FromPtr(env.GetInstanceByID(instanceID).State.Name)).To(BeElementOf("terminated", "shutting-down")) }, time.Second*10).Should(Succeed()) }) }) diff --git a/test/suites/interruption/suite_test.go b/test/suites/interruption/suite_test.go index 1ca8cb565446..3b83e69e9257 100644 --- a/test/suites/interruption/suite_test.go +++ b/test/suites/interruption/suite_test.go @@ -129,8 +129,8 @@ var _ = Describe("Interruption", func() { node := env.Monitor.CreatedNodes()[0] By("Stopping the EC2 instance without the EKS cluster's knowledge") - env.ExpectInstanceStopped(node.Name) // Make a call to the EC2 api to stop the instance - env.EventuallyExpectNotFoundAssertion(node).WithTimeout(time.Minute).Should(Succeed()) // shorten the timeout since we should react faster + env.ExpectInstanceStopped(node.Name) // Make a call to the EC2 api to stop the instance + env.EventuallyExpectNotFoundAssertion(node).Should(Succeed()) env.EventuallyExpectHealthyPodCount(selector, 1) }) It("should terminate the node at the API server when the EC2 instance is terminated", func() { @@ -155,8 +155,8 @@ var _ = Describe("Interruption", func() { node := env.Monitor.CreatedNodes()[0] By("Terminating the EC2 instance without the EKS cluster's knowledge") - env.ExpectInstanceTerminated(node.Name) // Make a call to the EC2 api to stop the instance - env.EventuallyExpectNotFoundAssertion(node).WithTimeout(time.Minute).Should(Succeed()) // shorten the timeout since we should react faster + env.ExpectInstanceTerminated(node.Name) // Make a call to the EC2 api to stop the instance + env.EventuallyExpectNotFoundAssertion(node).Should(Succeed()) env.EventuallyExpectHealthyPodCount(selector, 1) }) It("should terminate the node when receiving a scheduled change health event", func() { @@ -184,7 +184,7 @@ var _ = Describe("Interruption", func() { By("Creating a scheduled change health event in the SQS message queue") env.ExpectMessagesCreated(scheduledChangeMessage(env.Region, "000000000000", instanceID)) - env.EventuallyExpectNotFoundAssertion(node).WithTimeout(time.Minute).Should(Succeed()) // shorten the timeout since we should react faster + env.EventuallyExpectNotFoundAssertion(node).Should(Succeed()) env.EventuallyExpectHealthyPodCount(selector, 1) }) }) diff --git a/test/suites/nodeclaim/garbage_collection_test.go b/test/suites/nodeclaim/garbage_collection_test.go index 0872116b00c4..78f7cca88f64 100644 --- a/test/suites/nodeclaim/garbage_collection_test.go +++ b/test/suites/nodeclaim/garbage_collection_test.go @@ -140,7 +140,7 @@ var _ = Describe("GarbageCollection", func() { // Eventually expect the node and the instance to be removed (shutting-down) env.EventuallyExpectNotFound(node) Eventually(func(g Gomega) { - g.Expect(lo.FromPtr(env.GetInstanceByID(aws.StringValue(out.Instances[0].InstanceId)).State.Name)).To(Equal("shutting-down")) + g.Expect(lo.FromPtr(env.GetInstanceByID(aws.StringValue(out.Instances[0].InstanceId)).State.Name)).To(BeElementOf("terminated", "shutting-down")) }, time.Second*10).Should(Succeed()) }) It("should succeed to garbage collect an Instance that was deleted without the cluster's knowledge", func() { diff --git a/test/suites/nodeclaim/nodeclaim_test.go b/test/suites/nodeclaim/nodeclaim_test.go index d4e81e4528cd..5e0693286536 100644 --- a/test/suites/nodeclaim/nodeclaim_test.go +++ b/test/suites/nodeclaim/nodeclaim_test.go @@ -19,15 +19,15 @@ import ( "os" "time" - "github.com/samber/lo" - v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/karpenter/pkg/utils/resources" + "github.com/samber/lo" + v1 "k8s.io/api/core/v1" corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" "sigs.k8s.io/karpenter/pkg/test" - "sigs.k8s.io/karpenter/pkg/utils/resources" "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" @@ -225,7 +225,7 @@ var _ = Describe("StandaloneNodeClaim", func() { env.EventuallyExpectNotFound(nodeClaim, node) Eventually(func(g Gomega) { - g.Expect(lo.FromPtr(env.GetInstanceByID(instanceID).State.Name)).To(Equal("shutting-down")) + g.Expect(lo.FromPtr(env.GetInstanceByID(instanceID).State.Name)).To(BeElementOf("terminated", "shutting-down")) }, time.Second*10).Should(Succeed()) }) It("should delete a NodeClaim from the node termination finalizer", func() { @@ -264,7 +264,7 @@ var _ = Describe("StandaloneNodeClaim", func() { env.EventuallyExpectNotFound(nodeClaim, node) Eventually(func(g Gomega) { - g.Expect(lo.FromPtr(env.GetInstanceByID(instanceID).State.Name)).To(Equal("shutting-down")) + g.Expect(lo.FromPtr(env.GetInstanceByID(instanceID).State.Name)).To(BeElementOf("terminated", "shutting-down")) }, time.Second*10).Should(Succeed()) }) It("should create a NodeClaim with custom labels passed through the userData", func() {