diff --git a/test/pkg/environment/common/expectations.go b/test/pkg/environment/common/expectations.go index 60f0c3c50f91..6069b0aee838 100644 --- a/test/pkg/environment/common/expectations.go +++ b/test/pkg/environment/common/expectations.go @@ -279,6 +279,21 @@ func (env *Environment) EventuallyExpectHealthy(pods ...*corev1.Pod) { env.EventuallyExpectHealthyWithTimeout(-1, pods...) } +func (env *Environment) EventuallyExpectTerminating(pods ...*corev1.Pod) { + GinkgoHelper() + env.EventuallyExpectTerminatingWithTimeout(-1, pods...) +} + +func (env *Environment) EventuallyExpectTerminatingWithTimeout(timeout time.Duration, pods ...*corev1.Pod) { + GinkgoHelper() + Eventually(func(g Gomega) { + for _, pod := range pods { + g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(pod), pod)).To(Succeed()) + g.Expect(pod.DeletionTimestamp.IsZero()).To(BeFalse()) + } + }).WithTimeout(timeout).Should(Succeed()) +} + func (env *Environment) EventuallyExpectHealthyWithTimeout(timeout time.Duration, pods ...*corev1.Pod) { GinkgoHelper() Eventually(func(g Gomega) { @@ -290,7 +305,17 @@ func (env *Environment) EventuallyExpectHealthyWithTimeout(timeout time.Duration ))) } }).WithTimeout(timeout).Should(Succeed()) +} +func (env *Environment) ConsistentlyExpectTerminatingPods(duration time.Duration, pods ...*corev1.Pod) { + GinkgoHelper() + By(fmt.Sprintf("expecting %d pods to be terminating for %s", len(pods), duration)) + Consistently(func(g Gomega) { + for _, pod := range pods { + g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(pod), pod)).To(Succeed()) + g.Expect(pod.DeletionTimestamp.IsZero()).To(BeFalse()) + } + }, duration.String()).Should(Succeed()) } func (env *Environment) ConsistentlyExpectHealthyPods(duration time.Duration, pods ...*corev1.Pod) { diff --git a/test/suites/termination/termination_grace_period_test.go b/test/suites/termination/termination_grace_period_test.go index 2cd7c1aec5e4..941297c35555 100644 --- a/test/suites/termination/termination_grace_period_test.go +++ b/test/suites/termination/termination_grace_period_test.go @@ -33,20 +33,19 @@ import ( var _ = Describe("TerminationGracePeriod", func() { BeforeEach(func() { nodePool.Spec.Template.Spec.TerminationGracePeriod = &metav1.Duration{Duration: time.Second * 30} - // Set the expireAfter value to get the node deleted - nodePool.Spec.Template.Spec.ExpireAfter = karpv1.MustParseNillableDuration("90s") }) - It("should delete pod that tolerates do-not-disrupt after termination grace period seconds", func() { + It("should delete pod with do-not-disrupt when it reaches its terminationGracePeriodSeconds", func() { pod := coretest.UnschedulablePod(coretest.PodOptions{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{ karpv1.DoNotDisruptAnnotationKey: "true", - }}}) + }}, TerminationGracePeriodSeconds: lo.ToPtr(int64(15))}) env.ExpectCreated(nodeClass, nodePool, pod) nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0] node := env.EventuallyExpectCreatedNodeCount("==", 1)[0] env.EventuallyExpectHealthy(pod) - // Check that pod remains healthy until termination grace period - env.ConsistentlyExpectHealthyPods(time.Second*30, pod) + + // Delete the nodeclaim to start the TerminationGracePeriod + env.ExpectDeleted(nodeClaim) // Eventually the node will be tainted Eventually(func(g Gomega) { @@ -55,19 +54,28 @@ var _ = Describe("TerminationGracePeriod", func() { return karpv1.IsDisruptingTaint(t) }) g.Expect(ok).To(BeTrue()) - }).Should(Succeed()) + }).WithTimeout(3 * time.Second).WithPolling(100 * time.Millisecond).Should(Succeed()) + + // Check that pod remains healthy until termination grace period + // subtract the polling time of the eventually above to reduce any races. + env.ConsistentlyExpectHealthyPods(time.Second*15-100*time.Millisecond, pod) // Both nodeClaim and node should be gone once terminationGracePeriod is reached - env.EventuallyExpectNotFound(nodeClaim, node) + env.EventuallyExpectNotFound(nodeClaim, node, pod) }) It("should delete pod that has a pre-stop hook after termination grace period seconds", func() { - pod := coretest.UnschedulablePod(coretest.PodOptions{PreStopSleep: lo.ToPtr(int64(300))}) + pod := coretest.UnschedulablePod(coretest.PodOptions{ + PreStopSleep: lo.ToPtr(int64(300)), + TerminationGracePeriodSeconds: lo.ToPtr(int64(15)), + Image: "alpine:3.20.2", + Command: []string{"/bin/sh", "-c", "sleep 30"}}) env.ExpectCreated(nodeClass, nodePool, pod) nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0] node := env.EventuallyExpectCreatedNodeCount("==", 1)[0] env.EventuallyExpectHealthy(pod) - // Check that pod remains healthy until termination grace period - env.ConsistentlyExpectHealthyPods(time.Second*30, pod) + + // Delete the nodeclaim to start the TerminationGracePeriod + env.ExpectDeleted(nodeClaim) // Eventually the node will be tainted Eventually(func(g Gomega) { @@ -76,8 +84,15 @@ var _ = Describe("TerminationGracePeriod", func() { return karpv1.IsDisruptingTaint(t) }) g.Expect(ok).To(BeTrue()) - }).Should(Succeed()) + }).WithTimeout(3 * time.Second).WithPolling(100 * time.Millisecond).Should(Succeed()) + + env.EventuallyExpectTerminating(pod) + + // Check that pod remains healthy until termination grace period + // subtract the polling time of the eventually above to reduce any races. + env.ConsistentlyExpectTerminatingPods(time.Second*15-100*time.Millisecond, pod) + // Both nodeClaim and node should be gone once terminationGracePeriod is reached - env.EventuallyExpectNotFound(nodeClaim, node) + env.EventuallyExpectNotFound(nodeClaim, node, pod) }) })