Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

chore: Wait for instance termination before deleting nodeclaim #5979

Merged
merged 1 commit into from
Jun 12, 2024
Merged

chore: Wait for instance termination before deleting nodeclaim #5979

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ require (
k8s.io/utils v0.0.0-20240102154912-e7106e64919e
knative.dev/pkg v0.0.0-20231010144348-ca8c009405dd
sigs.k8s.io/controller-runtime v0.18.4
sigs.k8s.io/karpenter v0.37.1-0.20240610184513-261b55a31660
sigs.k8s.io/karpenter v0.37.1-0.20240612182320-24c976186c48
sigs.k8s.io/yaml v1.4.0
)

Expand Down Expand Up @@ -95,9 +95,9 @@ require (
golang.org/x/oauth2 v0.18.0 // indirect
golang.org/x/sys v0.20.0 // indirect
golang.org/x/term v0.20.0 // indirect
golang.org/x/text v0.15.0 // indirect
golang.org/x/text v0.16.0 // indirect
golang.org/x/time v0.5.0 // indirect
golang.org/x/tools v0.21.0 // indirect
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
google.golang.org/api v0.146.0 // indirect
google.golang.org/appengine v1.6.8 // indirect
Expand Down
12 changes: 6 additions & 6 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -550,8 +550,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
Expand Down Expand Up @@ -601,8 +601,8 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.21.0 h1:qc0xYgIbsSDt9EyWz05J5wfa7LOVW0YTLOXrqdLAWIw=
golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
Expand Down Expand Up @@ -761,8 +761,8 @@ sigs.k8s.io/controller-runtime v0.18.4 h1:87+guW1zhvuPLh1PHybKdYFLU0YJp4FhJRmiHv
sigs.k8s.io/controller-runtime v0.18.4/go.mod h1:TVoGrfdpbA9VRFaRnKgk9P5/atA0pMwq+f+msb9M8Sg=
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
sigs.k8s.io/karpenter v0.37.1-0.20240610184513-261b55a31660 h1:vYMlBlYIUpMaiuYdVXYVSjI4cNaadyHR0Y3IUqhyQMA=
sigs.k8s.io/karpenter v0.37.1-0.20240610184513-261b55a31660/go.mod h1:V/+zk8fRo3x0No93quokU87mUAYuYQw2s8QGkBth2cY=
sigs.k8s.io/karpenter v0.37.1-0.20240612182320-24c976186c48 h1:U/bzF0qgk1RMtpl97F+IltVE7caxTLoy7Pwk4NB+zIg=
sigs.k8s.io/karpenter v0.37.1-0.20240612182320-24c976186c48/go.mod h1:rkTEnr5zMsc5RLCcnSlnq6Uklgom7SKG7zyVk7+dtIU=
sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4=
sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08=
sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
Expand Down
2 changes: 1 addition & 1 deletion test/pkg/environment/common/environment.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ func NewEnvironment(t *testing.T) *Environment {
ctx = context.WithValue(ctx, GitRefContextKey, val)
}

gomega.SetDefaultEventuallyTimeout(5 * time.Minute)
gomega.SetDefaultEventuallyTimeout(16 * time.Minute)
gomega.SetDefaultEventuallyPollingInterval(1 * time.Second)
return &Environment{
Context: ctx,
Expand Down
2 changes: 1 addition & 1 deletion test/pkg/environment/common/expectations.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -778,7 +778,7 @@ func (env *Environment) EventuallyExpectAvgUtilization(resource v1.ResourceName,
GinkgoHelper()
Eventually(func(g Gomega) {
g.Expect(env.Monitor.AvgUtilization(resource)).To(BeNumerically(comparator, value))
}, 10*time.Minute).Should(Succeed())
}, 12*time.Minute).Should(Succeed())
}

func (env *Environment) ExpectDaemonSetEnvironmentVariableUpdated(obj client.ObjectKey, name, value string, containers ...string) {
Expand Down
7 changes: 0 additions & 7 deletions test/suites/expiration/suite_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -586,13 +586,6 @@ var _ = Describe("Expiration", func() {
// TODO: reduce timeouts when deprovisioning waits are factored out
env.EventuallyExpectNodesUntaintedWithTimeout(11*time.Minute, taintedNodes...)

// The nodeclaims that never registers will be removed
jigisha620 marked this conversation as resolved.
Show resolved Hide resolved
Eventually(func(g Gomega) {
nodeClaims := &corev1beta1.NodeClaimList{}
g.Expect(env.Client.List(env, nodeClaims, client.HasLabels{coretest.DiscoveryLabel})).To(Succeed())
g.Expect(len(nodeClaims.Items)).To(BeNumerically("==", int(numPods)))
}).WithTimeout(6 * time.Minute).Should(Succeed())

// Expect all the NodeClaims that existed on the initial provisioning loop are not removed
Consistently(func(g Gomega) {
nodeClaims := &corev1beta1.NodeClaimList{}
Expand Down
2 changes: 1 addition & 1 deletion test/suites/integration/termination_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ var _ = Describe("Termination", func() {
env.ExpectDeleted(nodes[0])
env.EventuallyExpectNotFound(nodes[0])
Eventually(func(g Gomega) {
g.Expect(lo.FromPtr(env.GetInstanceByID(instanceID).State.Name)).To(Equal("shutting-down"))
g.Expect(lo.FromPtr(env.GetInstanceByID(instanceID).State.Name)).To(BeElementOf("terminated", "shutting-down"))
}, time.Second*10).Should(Succeed())
})
})
10 changes: 5 additions & 5 deletions test/suites/interruption/suite_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,8 +129,8 @@ var _ = Describe("Interruption", func() {
node := env.Monitor.CreatedNodes()[0]

By("Stopping the EC2 instance without the EKS cluster's knowledge")
env.ExpectInstanceStopped(node.Name) // Make a call to the EC2 api to stop the instance
env.EventuallyExpectNotFoundAssertion(node).WithTimeout(time.Minute).Should(Succeed()) // shorten the timeout since we should react faster
env.ExpectInstanceStopped(node.Name) // Make a call to the EC2 api to stop the instance
env.EventuallyExpectNotFoundAssertion(node).Should(Succeed())
env.EventuallyExpectHealthyPodCount(selector, 1)
})
It("should terminate the node at the API server when the EC2 instance is terminated", func() {
Expand All @@ -155,8 +155,8 @@ var _ = Describe("Interruption", func() {
node := env.Monitor.CreatedNodes()[0]

By("Terminating the EC2 instance without the EKS cluster's knowledge")
env.ExpectInstanceTerminated(node.Name) // Make a call to the EC2 api to stop the instance
env.EventuallyExpectNotFoundAssertion(node).WithTimeout(time.Minute).Should(Succeed()) // shorten the timeout since we should react faster
env.ExpectInstanceTerminated(node.Name) // Make a call to the EC2 api to stop the instance
env.EventuallyExpectNotFoundAssertion(node).Should(Succeed())
env.EventuallyExpectHealthyPodCount(selector, 1)
})
It("should terminate the node when receiving a scheduled change health event", func() {
Expand Down Expand Up @@ -184,7 +184,7 @@ var _ = Describe("Interruption", func() {

By("Creating a scheduled change health event in the SQS message queue")
env.ExpectMessagesCreated(scheduledChangeMessage(env.Region, "000000000000", instanceID))
env.EventuallyExpectNotFoundAssertion(node).WithTimeout(time.Minute).Should(Succeed()) // shorten the timeout since we should react faster
env.EventuallyExpectNotFoundAssertion(node).Should(Succeed())
env.EventuallyExpectHealthyPodCount(selector, 1)
})
})
Expand Down
2 changes: 1 addition & 1 deletion test/suites/nodeclaim/garbage_collection_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ var _ = Describe("GarbageCollection", func() {
// Eventually expect the node and the instance to be removed (shutting-down)
env.EventuallyExpectNotFound(node)
Eventually(func(g Gomega) {
g.Expect(lo.FromPtr(env.GetInstanceByID(aws.StringValue(out.Instances[0].InstanceId)).State.Name)).To(Equal("shutting-down"))
g.Expect(lo.FromPtr(env.GetInstanceByID(aws.StringValue(out.Instances[0].InstanceId)).State.Name)).To(BeElementOf("terminated", "shutting-down"))
}, time.Second*10).Should(Succeed())
})
It("should succeed to garbage collect an Instance that was deleted without the cluster's knowledge", func() {
Expand Down
10 changes: 5 additions & 5 deletions test/suites/nodeclaim/nodeclaim_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,15 @@ import (
"os"
"time"

"github.com/samber/lo"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/karpenter/pkg/utils/resources"

"github.com/samber/lo"
v1 "k8s.io/api/core/v1"
corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1"
"sigs.k8s.io/karpenter/pkg/test"
"sigs.k8s.io/karpenter/pkg/utils/resources"

"github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1"

Expand Down Expand Up @@ -225,7 +225,7 @@ var _ = Describe("StandaloneNodeClaim", func() {
env.EventuallyExpectNotFound(nodeClaim, node)

Eventually(func(g Gomega) {
g.Expect(lo.FromPtr(env.GetInstanceByID(instanceID).State.Name)).To(Equal("shutting-down"))
g.Expect(lo.FromPtr(env.GetInstanceByID(instanceID).State.Name)).To(BeElementOf("terminated", "shutting-down"))
}, time.Second*10).Should(Succeed())
})
It("should delete a NodeClaim from the node termination finalizer", func() {
Expand Down Expand Up @@ -264,7 +264,7 @@ var _ = Describe("StandaloneNodeClaim", func() {
env.EventuallyExpectNotFound(nodeClaim, node)

Eventually(func(g Gomega) {
g.Expect(lo.FromPtr(env.GetInstanceByID(instanceID).State.Name)).To(Equal("shutting-down"))
g.Expect(lo.FromPtr(env.GetInstanceByID(instanceID).State.Name)).To(BeElementOf("terminated", "shutting-down"))
}, time.Second*10).Should(Succeed())
})
It("should create a NodeClaim with custom labels passed through the userData", func() {
Expand Down
Loading