diff --git a/test/pkg/environment/common/expectations.go b/test/pkg/environment/common/expectations.go index 51751c4c824a..f58940c003b6 100644 --- a/test/pkg/environment/common/expectations.go +++ b/test/pkg/environment/common/expectations.go @@ -458,6 +458,27 @@ func (env *Environment) ConsistentlyExpectNodeCount(comparator string, count int return lo.ToSlicePtr(nodeList.Items) } +func (env *Environment) ConsistentlyExpectNoDisruptions(nodeCount int, duration string) { + GinkgoHelper() + Consistently(func(g Gomega) { + // Ensure we don't change our NodeClaims + nodeClaimList := &corev1beta1.NodeClaimList{} + g.Expect(env.Client.List(env, nodeClaimList, client.HasLabels{test.DiscoveryLabel})).To(Succeed()) + g.Expect(nodeClaimList.Items).To(HaveLen(nodeCount)) + + nodeList := &v1.NodeList{} + g.Expect(env.Client.List(env, nodeList, client.HasLabels{test.DiscoveryLabel})).To(Succeed()) + g.Expect(nodeList.Items).To(HaveLen(nodeCount)) + + for _, node := range nodeList.Items { + _, ok := lo.Find(node.Spec.Taints, func(t v1.Taint) bool { + return corev1beta1.IsDisruptingTaint(t) + }) + g.Expect(ok).To(BeFalse()) + } + }, duration).Should(Succeed()) +} + func (env *Environment) EventuallyExpectTaintedNodeCount(comparator string, count int) []*v1.Node { GinkgoHelper() By(fmt.Sprintf("waiting for tainted nodes to be %s to %d", comparator, count)) @@ -572,6 +593,7 @@ func (env *Environment) EventuallyExpectCreatedNodeClaimCount(comparator string, } func (env *Environment) EventuallyExpectNodeClaimsReady(nodeClaims ...*corev1beta1.NodeClaim) { + GinkgoHelper() Eventually(func(g Gomega) { for _, nc := range nodeClaims { temp := &corev1beta1.NodeClaim{} @@ -581,6 +603,36 @@ func (env *Environment) EventuallyExpectNodeClaimsReady(nodeClaims ...*corev1bet }).Should(Succeed()) } +func (env *Environment) EventuallyExpectExpired(nodeClaims ...*corev1beta1.NodeClaim) { + GinkgoHelper() + Eventually(func(g Gomega) { + for _, nc := range nodeClaims { + g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(nc), nc)).To(Succeed()) + g.Expect(nc.StatusConditions().GetCondition(corev1beta1.Expired).IsTrue()).To(BeTrue()) + } + }).Should(Succeed()) +} + +func (env *Environment) EventuallyExpectDrifted(nodeClaims ...*corev1beta1.NodeClaim) { + GinkgoHelper() + Eventually(func(g Gomega) { + for _, nc := range nodeClaims { + g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(nc), nc)).To(Succeed()) + g.Expect(nc.StatusConditions().GetCondition(corev1beta1.Drifted).IsTrue()).To(BeTrue()) + } + }).Should(Succeed()) +} + +func (env *Environment) EventuallyExpectEmpty(nodeClaims ...*corev1beta1.NodeClaim) { + GinkgoHelper() + Eventually(func(g Gomega) { + for _, nc := range nodeClaims { + g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(nc), nc)).To(Succeed()) + g.Expect(nc.StatusConditions().GetCondition(corev1beta1.Empty).IsTrue()).To(BeTrue()) + } + }).Should(Succeed()) +} + func (env *Environment) GetNode(nodeName string) v1.Node { GinkgoHelper() var node v1.Node diff --git a/test/suites/drift/suite_test.go b/test/suites/drift/suite_test.go index 3950df033e95..6479d4207b77 100644 --- a/test/suites/drift/suite_test.go +++ b/test/suites/drift/suite_test.go @@ -389,6 +389,54 @@ var _ = Describe("Drift", Label("AWS"), func() { // the node should be gone env.EventuallyExpectNotFound(nodes[0], nodes[1], nodes[2]) }) + It("should not allow drift if the budget is fully blocking", func() { + // We're going to define a budget that doesn't allow any drift to happen + nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{ + Nodes: "0", + }} + + dep.Spec.Template.Annotations = nil + env.ExpectCreated(nodeClass, nodePool, dep) + + nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0] + env.EventuallyExpectCreatedNodeCount("==", 1) + env.EventuallyExpectHealthyPodCount(selector, numPods) + + By("drifting the nodes") + // Drift the nodeclaims + nodePool.Spec.Template.Annotations = map[string]string{"test": "annotation"} + env.ExpectUpdated(nodePool) + + env.EventuallyExpectDrifted(nodeClaim) + env.ConsistentlyExpectNoDisruptions(1, "1m") + }) + It("should not allow drift if the budget is fully blocking during a scheduled time", func() { + // We're going to define a budget that doesn't allow any drift to happen + // This is going to be on a schedule that only lasts 30 minutes, whose window starts 15 minutes before + // the current time and extends 15 minutes past the current time + // Times need to be in UTC since the karpenter containers were built in UTC time + windowStart := time.Now().Add(-time.Minute * 15).UTC() + nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{ + Nodes: "0", + Schedule: lo.ToPtr(fmt.Sprintf("%d %d * * *", windowStart.Minute(), windowStart.Hour())), + Duration: &metav1.Duration{Duration: time.Minute * 30}, + }} + + dep.Spec.Template.Annotations = nil + env.ExpectCreated(nodeClass, nodePool, dep) + + nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0] + env.EventuallyExpectCreatedNodeCount("==", 1) + env.EventuallyExpectHealthyPodCount(selector, numPods) + + By("drifting the nodes") + // Drift the nodeclaims + nodePool.Spec.Template.Annotations = map[string]string{"test": "annotation"} + env.ExpectUpdated(nodePool) + + env.EventuallyExpectDrifted(nodeClaim) + env.ConsistentlyExpectNoDisruptions(1, "1m") + }) }) It("should disrupt nodes that have drifted due to AMIs", func() { // choose an old static image diff --git a/test/suites/expiration/suite_test.go b/test/suites/expiration/suite_test.go index 80f912fe27bf..fe8f1cc7c2b4 100644 --- a/test/suites/expiration/suite_test.go +++ b/test/suites/expiration/suite_test.go @@ -15,10 +15,12 @@ limitations under the License. package expiration_test import ( + "fmt" "testing" "time" "github.com/samber/lo" + appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -68,6 +70,28 @@ var _ = AfterEach(func() { env.Cleanup() }) var _ = AfterEach(func() { env.AfterEach() }) var _ = Describe("Expiration", func() { + var dep *appsv1.Deployment + var selector labels.Selector + var numPods int + BeforeEach(func() { + numPods = 1 + // Add pods with a do-not-disrupt annotation so that we can check node metadata before we disrupt + dep = coretest.Deployment(coretest.DeploymentOptions{ + Replicas: int32(numPods), + PodOptions: coretest.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "my-app", + }, + Annotations: map[string]string{ + corev1beta1.DoNotDisruptAnnotationKey: "true", + }, + }, + TerminationGracePeriodSeconds: lo.ToPtr[int64](0), + }, + }) + selector = labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) + }) Context("Budgets", func() { It("should respect budgets for empty expiration", func() { coretest.ReplaceRequirements(nodePool, @@ -82,9 +106,9 @@ var _ = Describe("Expiration", func() { }} nodePool.Spec.Disruption.ExpireAfter = corev1beta1.NillableDuration{} - var numPods int32 = 6 - dep := coretest.Deployment(coretest.DeploymentOptions{ - Replicas: numPods, + numPods = 6 + dep = coretest.Deployment(coretest.DeploymentOptions{ + Replicas: int32(numPods), PodOptions: coretest.PodOptions{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ @@ -100,12 +124,12 @@ var _ = Describe("Expiration", func() { }, }, }) - selector := labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) + selector = labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) env.ExpectCreated(nodeClass, nodePool, dep) env.EventuallyExpectCreatedNodeClaimCount("==", 3) env.EventuallyExpectCreatedNodeCount("==", 3) - env.EventuallyExpectHealthyPodCount(selector, int(numPods)) + env.EventuallyExpectHealthyPodCount(selector, numPods) env.Monitor.Reset() // Reset the monitor so that we can expect a single node to be spun up after expiration nodes := env.EventuallyExpectNodeCount("==", 3) @@ -167,9 +191,9 @@ var _ = Describe("Expiration", func() { }} // disable expiration so that we can enable it later when we want. nodePool.Spec.Disruption.ExpireAfter = corev1beta1.NillableDuration{} - var numPods int32 = 9 - dep := coretest.Deployment(coretest.DeploymentOptions{ - Replicas: numPods, + numPods = 9 + dep = coretest.Deployment(coretest.DeploymentOptions{ + Replicas: int32(numPods), PodOptions: coretest.PodOptions{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ @@ -185,12 +209,12 @@ var _ = Describe("Expiration", func() { }, }, }) - selector := labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) + selector = labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) env.ExpectCreated(nodeClass, nodePool, dep) env.EventuallyExpectCreatedNodeClaimCount("==", 3) env.EventuallyExpectCreatedNodeCount("==", 3) - env.EventuallyExpectHealthyPodCount(selector, int(numPods)) + env.EventuallyExpectHealthyPodCount(selector, numPods) env.Monitor.Reset() // Reset the monitor so that we can expect a single node to be spun up after expiration By("scaling down the deployment") @@ -287,9 +311,9 @@ var _ = Describe("Expiration", func() { Nodes: "50%", }} nodePool.Spec.Disruption.ExpireAfter = corev1beta1.NillableDuration{} - var numPods int32 = 3 - dep := coretest.Deployment(coretest.DeploymentOptions{ - Replicas: numPods, + numPods = 3 + dep = coretest.Deployment(coretest.DeploymentOptions{ + Replicas: int32(numPods), PodOptions: coretest.PodOptions{ ObjectMeta: metav1.ObjectMeta{ Annotations: map[string]string{ @@ -305,12 +329,12 @@ var _ = Describe("Expiration", func() { }, }, }) - selector := labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) + selector = labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) env.ExpectCreated(nodeClass, nodePool, dep) env.EventuallyExpectCreatedNodeClaimCount("==", 3) env.EventuallyExpectCreatedNodeCount("==", 3) - env.EventuallyExpectHealthyPodCount(selector, int(numPods)) + env.EventuallyExpectHealthyPodCount(selector, numPods) env.Monitor.Reset() // Reset the monitor so that we can expect a single node to be spun up after drift By("cordoning and adding finalizer to the nodes") @@ -370,26 +394,51 @@ var _ = Describe("Expiration", func() { // the node should be gone env.EventuallyExpectNotFound(nodes[0], nodes[1], nodes[2]) }) + It("should not allow expiration if the budget is fully blocking", func() { + // We're going to define a budget that doesn't allow any expirations to happen + nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{ + Nodes: "0", + }} + + dep.Spec.Template.Annotations = nil + env.ExpectCreated(nodeClass, nodePool, dep) + + nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0] + env.EventuallyExpectCreatedNodeCount("==", 1) + env.EventuallyExpectHealthyPodCount(selector, numPods) + + env.EventuallyExpectExpired(nodeClaim) + env.ConsistentlyExpectNoDisruptions(1, "1m") + }) + It("should not allow expiration if the budget is fully blocking during a scheduled time", func() { + // We're going to define a budget that doesn't allow any expirations to happen + // This is going to be on a schedule that only lasts 30 minutes, whose window starts 15 minutes before + // the current time and extends 15 minutes past the current time + // Times need to be in UTC since the karpenter containers were built in UTC time + windowStart := time.Now().Add(-time.Minute * 15).UTC() + nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{ + Nodes: "0", + Schedule: lo.ToPtr(fmt.Sprintf("%d %d * * *", windowStart.Minute(), windowStart.Hour())), + Duration: &metav1.Duration{Duration: time.Minute * 30}, + }} + + dep.Spec.Template.Annotations = nil + env.ExpectCreated(nodeClass, nodePool, dep) + + nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0] + env.EventuallyExpectCreatedNodeCount("==", 1) + env.EventuallyExpectHealthyPodCount(selector, numPods) + + env.EventuallyExpectExpired(nodeClaim) + env.ConsistentlyExpectNoDisruptions(1, "1m") + }) }) It("should expire the node after the expiration is reached", func() { - var numPods int32 = 1 - dep := coretest.Deployment(coretest.DeploymentOptions{ - Replicas: numPods, - PodOptions: coretest.PodOptions{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - corev1beta1.DoNotDisruptAnnotationKey: "true", - }, - Labels: map[string]string{"app": "large-app"}, - }, - }, - }) - selector := labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) env.ExpectCreated(nodeClass, nodePool, dep) nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0] node := env.EventuallyExpectCreatedNodeCount("==", 1)[0] - env.EventuallyExpectHealthyPodCount(selector, int(numPods)) + env.EventuallyExpectHealthyPodCount(selector, numPods) env.Monitor.Reset() // Reset the monitor so that we can expect a single node to be spun up after expiration // Expect that the NodeClaim will get an expired status condition @@ -425,7 +474,7 @@ var _ = Describe("Expiration", func() { env.EventuallyExpectCreatedNodeClaimCount("==", 1) env.EventuallyExpectCreatedNodeCount("==", 1) - env.EventuallyExpectHealthyPodCount(selector, int(numPods)) + env.EventuallyExpectHealthyPodCount(selector, numPods) }) It("should replace expired node with a single node and schedule all pods", func() { var numPods int32 = 5 diff --git a/test/suites/integration/emptiness_test.go b/test/suites/integration/emptiness_test.go index 424c969c719c..2425029a804d 100644 --- a/test/suites/integration/emptiness_test.go +++ b/test/suites/integration/emptiness_test.go @@ -15,9 +15,12 @@ limitations under the License. package integration_test import ( + "fmt" "time" "github.com/samber/lo" + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "knative.dev/pkg/ptr" @@ -31,8 +34,69 @@ import ( ) var _ = Describe("Emptiness", func() { - It("should terminate an empty node", func() { + var dep *appsv1.Deployment + var selector labels.Selector + var numPods int + BeforeEach(func() { nodePool.Spec.Disruption.ConsolidationPolicy = corev1beta1.ConsolidationPolicyWhenEmpty + nodePool.Spec.Disruption.ConsolidateAfter = &corev1beta1.NillableDuration{Duration: lo.ToPtr(time.Duration(0))} + + numPods = 1 + dep = test.Deployment(test.DeploymentOptions{ + Replicas: int32(numPods), + PodOptions: test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": "large-app"}, + }, + }, + }) + selector = labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) + }) + Context("Budgets", func() { + It("should not allow emptiness if the budget is fully blocking", func() { + // We're going to define a budget that doesn't allow any emptiness disruption to happen + nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{ + Nodes: "0", + }} + + env.ExpectCreated(nodeClass, nodePool, dep) + + nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0] + env.EventuallyExpectCreatedNodeCount("==", 1) + env.EventuallyExpectHealthyPodCount(selector, numPods) + + // Delete the deployment so there is nothing running on the node + env.ExpectDeleted(dep) + + env.EventuallyExpectEmpty(nodeClaim) + env.ConsistentlyExpectNoDisruptions(1, "1m") + }) + It("should not allow emptiness if the budget is fully blocking during a scheduled time", func() { + // We're going to define a budget that doesn't allow any emptiness disruption to happen + // This is going to be on a schedule that only lasts 30 minutes, whose window starts 15 minutes before + // the current time and extends 15 minutes past the current time + // Times need to be in UTC since the karpenter containers were built in UTC time + windowStart := time.Now().Add(-time.Minute * 15).UTC() + nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{ + Nodes: "0", + Schedule: lo.ToPtr(fmt.Sprintf("%d %d * * *", windowStart.Minute(), windowStart.Hour())), + Duration: &metav1.Duration{Duration: time.Minute * 30}, + }} + + env.ExpectCreated(nodeClass, nodePool, dep) + + nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0] + env.EventuallyExpectCreatedNodeCount("==", 1) + env.EventuallyExpectHealthyPodCount(selector, numPods) + + // Delete the deployment so there is nothing running on the node + env.ExpectDeleted(dep) + + env.EventuallyExpectEmpty(nodeClaim) + env.ConsistentlyExpectNoDisruptions(1, "1m") + }) + }) + It("should terminate an empty node", func() { nodePool.Spec.Disruption.ConsolidateAfter = &corev1beta1.NillableDuration{Duration: lo.ToPtr(time.Hour * 300)} const numPods = 1