Add E2E testing for fully blocking budgets with and without schedule

test/pkg/environment/common/expectations.go

@@ -458,6 +458,27 @@ func (env *Environment) ConsistentlyExpectNodeCount(comparator string, count int
 	return lo.ToSlicePtr(nodeList.Items)
 }
 
+func (env *Environment) ConsistentlyExpectNoDisruptions(nodeCount int, duration string) {
+	GinkgoHelper()
+	Consistently(func(g Gomega) {
+		// Ensure we don't change our NodeClaims
+		nodeClaimList := &corev1beta1.NodeClaimList{}
+		g.Expect(env.Client.List(env, nodeClaimList, client.HasLabels{test.DiscoveryLabel})).To(Succeed())
+		g.Expect(nodeClaimList.Items).To(HaveLen(nodeCount))
+
+		nodeList := &v1.NodeList{}
+		g.Expect(env.Client.List(env, nodeList, client.HasLabels{test.DiscoveryLabel})).To(Succeed())
+		g.Expect(nodeList.Items).To(HaveLen(nodeCount))
+
+		for _, node := range nodeList.Items {
+			_, ok := lo.Find(node.Spec.Taints, func(t v1.Taint) bool {
+				return corev1beta1.IsDisruptingTaint(t)
+			})
+			g.Expect(ok).To(BeFalse())
+		}
+	}, duration).Should(Succeed())
+}
+
 func (env *Environment) EventuallyExpectTaintedNodeCount(comparator string, count int) []*v1.Node {
 	GinkgoHelper()
 	By(fmt.Sprintf("waiting for tainted nodes to be %s to %d", comparator, count))
@@ -572,6 +593,7 @@ func (env *Environment) EventuallyExpectCreatedNodeClaimCount(comparator string,
 }
 
 func (env *Environment) EventuallyExpectNodeClaimsReady(nodeClaims ...*corev1beta1.NodeClaim) {
+	GinkgoHelper()
 	Eventually(func(g Gomega) {
 		for _, nc := range nodeClaims {
 			temp := &corev1beta1.NodeClaim{}
@@ -581,6 +603,36 @@ func (env *Environment) EventuallyExpectNodeClaimsReady(nodeClaims ...*corev1bet
 	}).Should(Succeed())
 }
 
+func (env *Environment) EventuallyExpectExpired(nodeClaims ...*corev1beta1.NodeClaim) {
+	GinkgoHelper()
+	Eventually(func(g Gomega) {
+		for _, nc := range nodeClaims {
+			g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(nc), nc)).To(Succeed())
+			g.Expect(nc.StatusConditions().GetCondition(corev1beta1.Expired).IsTrue()).To(BeTrue())
+		}
+	}).Should(Succeed())
+}
+
+func (env *Environment) EventuallyExpectDrifted(nodeClaims ...*corev1beta1.NodeClaim) {
+	GinkgoHelper()
+	Eventually(func(g Gomega) {
+		for _, nc := range nodeClaims {
+			g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(nc), nc)).To(Succeed())
+			g.Expect(nc.StatusConditions().GetCondition(corev1beta1.Drifted).IsTrue()).To(BeTrue())
+		}
+	}).Should(Succeed())
+}
+
+func (env *Environment) EventuallyExpectEmpty(nodeClaims ...*corev1beta1.NodeClaim) {
+	GinkgoHelper()
+	Eventually(func(g Gomega) {
+		for _, nc := range nodeClaims {
+			g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(nc), nc)).To(Succeed())
+			g.Expect(nc.StatusConditions().GetCondition(corev1beta1.Empty).IsTrue()).To(BeTrue())
+		}
+	}).Should(Succeed())
+}
+
 func (env *Environment) GetNode(nodeName string) v1.Node {
 	GinkgoHelper()
 	var node v1.Node

test/suites/drift/suite_test.go

@@ -389,6 +389,78 @@ var _ = Describe("Drift", Label("AWS"), func() {
 			// the node should be gone
 			env.EventuallyExpectNotFound(nodes[0], nodes[1], nodes[2])
 		})
+		It("should not allow drift if the budget is fully blocking", func() {
+			// We're going to define a budget that doesn't allow any drift to happen
+			nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{
+				Nodes: "0",
+			}}
+
+			var numPods int32 = 1
+			dep := coretest.Deployment(coretest.DeploymentOptions{
+				Replicas: numPods,
+				PodOptions: coretest.PodOptions{
+					ObjectMeta: metav1.ObjectMeta{
+						Annotations: map[string]string{
+							corev1beta1.DoNotDisruptAnnotationKey: "true",
+						},
+						Labels: map[string]string{"app": "large-app"},
+					},
+				},
+			})
+			selector := labels.SelectorFromSet(dep.Spec.Selector.MatchLabels)
+			env.ExpectCreated(nodeClass, nodePool, dep)
+
+			nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0]
+			env.EventuallyExpectCreatedNodeCount("==", 1)
+			env.EventuallyExpectHealthyPodCount(selector, int(numPods))
+
+			By("drifting the nodes")
+			// Drift the nodeclaims
+			nodePool.Spec.Template.Annotations = map[string]string{"test": "annotation"}
+			env.ExpectUpdated(nodePool)
+
+			env.EventuallyExpectDrifted(nodeClaim)
+			env.ConsistentlyExpectNoDisruptions(1, "1m")
+		})
+		It("should not allow drift if the budget is fully blocking during a scheduled time", func() {
+			// We're going to define a budget that doesn't allow any drift to happen
+			// This is going to be on a schedule that only lasts 30 minutes, whose window starts 15 minutes before
+			// the current time and extends 15 minutes past the current time
+			// Times need to be in UTC since the karpenter containers were built in UTC time
+			windowStart := time.Now().Add(-time.Minute * 15).UTC()
+			nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{
+				Nodes:    "0",
+				Schedule: lo.ToPtr(fmt.Sprintf("%d %d * * *", windowStart.Minute(), windowStart.Hour())),
+				Duration: &metav1.Duration{Duration: time.Minute * 30},
+			}}
+
+			var numPods int32 = 1
+			dep := coretest.Deployment(coretest.DeploymentOptions{
+				Replicas: numPods,
+				PodOptions: coretest.PodOptions{
+					ObjectMeta: metav1.ObjectMeta{
+						Annotations: map[string]string{
+							corev1beta1.DoNotDisruptAnnotationKey: "true",
+						},
+						Labels: map[string]string{"app": "large-app"},
+					},
+				},
+			})
+			selector := labels.SelectorFromSet(dep.Spec.Selector.MatchLabels)
+			env.ExpectCreated(nodeClass, nodePool, dep)
+
+			nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0]
+			env.EventuallyExpectCreatedNodeCount("==", 1)
+			env.EventuallyExpectHealthyPodCount(selector, int(numPods))
+
+			By("drifting the nodes")
+			// Drift the nodeclaims
+			nodePool.Spec.Template.Annotations = map[string]string{"test": "annotation"}
+			env.ExpectUpdated(nodePool)
+
+			env.EventuallyExpectDrifted(nodeClaim)
+			env.ConsistentlyExpectNoDisruptions(1, "1m")
+		})
 	})
 	It("should disrupt nodes that have drifted due to AMIs", func() {
 		// choose an old static image

test/suites/expiration/suite_test.go

@@ -15,6 +15,7 @@ limitations under the License.
 package expiration_test
 
 import (
+	"fmt"
 	"testing"
 	"time"
 
@@ -370,6 +371,68 @@ var _ = Describe("Expiration", func() {
 			// the node should be gone
 			env.EventuallyExpectNotFound(nodes[0], nodes[1], nodes[2])
 		})
+		It("should not allow expiration if the budget is fully blocking", func() {
+			// We're going to define a budget that doesn't allow any expirations to happen
+			nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{
+				Nodes: "0",
+			}}
+
+			var numPods int32 = 1
+			dep := coretest.Deployment(coretest.DeploymentOptions{
+				Replicas: numPods,
+				PodOptions: coretest.PodOptions{
+					ObjectMeta: metav1.ObjectMeta{
+						Annotations: map[string]string{
+							corev1beta1.DoNotDisruptAnnotationKey: "true",
+						},
+						Labels: map[string]string{"app": "large-app"},
+					},
+				},
+			})
+			selector := labels.SelectorFromSet(dep.Spec.Selector.MatchLabels)
+			env.ExpectCreated(nodeClass, nodePool, dep)
+
+			nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0]
+			env.EventuallyExpectCreatedNodeCount("==", 1)
+			env.EventuallyExpectHealthyPodCount(selector, int(numPods))
+
+			env.EventuallyExpectExpired(nodeClaim)
+			env.ConsistentlyExpectNoDisruptions(1, "1m")
+		})
+		It("should not allow expiration if the budget is fully blocking during a scheduled time", func() {
+			// We're going to define a budget that doesn't allow any expirations to happen
+			// This is going to be on a schedule that only lasts 30 minutes, whose window starts 15 minutes before
+			// the current time and extends 15 minutes past the current time
+			// Times need to be in UTC since the karpenter containers were built in UTC time
+			windowStart := time.Now().Add(-time.Minute * 15).UTC()
+			nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{
+				Nodes:    "0",
+				Schedule: lo.ToPtr(fmt.Sprintf("%d %d * * *", windowStart.Minute(), windowStart.Hour())),
+				Duration: &metav1.Duration{Duration: time.Minute * 30},
+			}}
+
+			var numPods int32 = 1
+			dep := coretest.Deployment(coretest.DeploymentOptions{
+				Replicas: numPods,
+				PodOptions: coretest.PodOptions{
+					ObjectMeta: metav1.ObjectMeta{
+						Annotations: map[string]string{
+							corev1beta1.DoNotDisruptAnnotationKey: "true",
+						},
+						Labels: map[string]string{"app": "large-app"},
+					},
+				},
+			})
+			selector := labels.SelectorFromSet(dep.Spec.Selector.MatchLabels)
+			env.ExpectCreated(nodeClass, nodePool, dep)
+
+			nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0]
+			env.EventuallyExpectCreatedNodeCount("==", 1)
+			env.EventuallyExpectHealthyPodCount(selector, int(numPods))
+
+			env.EventuallyExpectExpired(nodeClaim)
+			env.ConsistentlyExpectNoDisruptions(1, "1m")
+		})
 	})
 	It("should expire the node after the expiration is reached", func() {
 		var numPods int32 = 1

test/suites/integration/emptiness_test.go

@@ -15,9 +15,11 @@ limitations under the License.
 package integration_test
 
 import (
+	"fmt"
 	"time"
 
 	"github.com/samber/lo"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
 	"knative.dev/pkg/ptr"
 
@@ -31,8 +33,81 @@ import (
 )
 
 var _ = Describe("Emptiness", func() {
-	It("should terminate an empty node", func() {
+	BeforeEach(func() {
 		nodePool.Spec.Disruption.ConsolidationPolicy = corev1beta1.ConsolidationPolicyWhenEmpty
+		nodePool.Spec.Disruption.ConsolidateAfter = &corev1beta1.NillableDuration{Duration: lo.ToPtr(time.Duration(0))}
+	})
+	Context("Budgets", func() {
+		It("should not allow emptiness if the budget is fully blocking", func() {
+			// We're going to define a budget that doesn't allow any emptiness disruption to happen
+			nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{
+				Nodes: "0",
+			}}
+
+			var numPods int32 = 1
+			dep := test.Deployment(test.DeploymentOptions{
+				Replicas: numPods,
+				PodOptions: test.PodOptions{
+					ObjectMeta: metav1.ObjectMeta{
+						Annotations: map[string]string{
+							corev1beta1.DoNotDisruptAnnotationKey: "true",
+						},
+						Labels: map[string]string{"app": "large-app"},
+					},
+				},
+			})
+			selector := labels.SelectorFromSet(dep.Spec.Selector.MatchLabels)
+			env.ExpectCreated(nodeClass, nodePool, dep)
+
+			nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0]
+			env.EventuallyExpectCreatedNodeCount("==", 1)
+			env.EventuallyExpectHealthyPodCount(selector, int(numPods))
+
+			// Delete the deployment so there is nothing running on the node
+			env.ExpectDeleted(dep)
+
+			env.EventuallyExpectEmpty(nodeClaim)
+			env.ConsistentlyExpectNoDisruptions(1, "1m")
+		})
+		It("should not allow emptiness if the budget is fully blocking during a scheduled time", func() {
+			// We're going to define a budget that doesn't allow any emptiness disruption to happen
+			// This is going to be on a schedule that only lasts 30 minutes, whose window starts 15 minutes before
+			// the current time and extends 15 minutes past the current time
+			// Times need to be in UTC since the karpenter containers were built in UTC time
+			windowStart := time.Now().Add(-time.Minute * 15).UTC()
+			nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{
+				Nodes:    "0",
+				Schedule: lo.ToPtr(fmt.Sprintf("%d %d * * *", windowStart.Minute(), windowStart.Hour())),
+				Duration: &metav1.Duration{Duration: time.Minute * 30},
+			}}
+
+			var numPods int32 = 1
+			dep := test.Deployment(test.DeploymentOptions{
+				Replicas: numPods,
+				PodOptions: test.PodOptions{
+					ObjectMeta: metav1.ObjectMeta{
+						Annotations: map[string]string{
+							corev1beta1.DoNotDisruptAnnotationKey: "true",
+						},
+						Labels: map[string]string{"app": "large-app"},
+					},
+				},
+			})
+			selector := labels.SelectorFromSet(dep.Spec.Selector.MatchLabels)
+			env.ExpectCreated(nodeClass, nodePool, dep)
+
+			nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0]
+			env.EventuallyExpectCreatedNodeCount("==", 1)
+			env.EventuallyExpectHealthyPodCount(selector, int(numPods))
+
+			// Delete the deployment so there is nothing running on the node
+			env.ExpectDeleted(dep)
+
+			env.EventuallyExpectEmpty(nodeClaim)
+			env.ConsistentlyExpectNoDisruptions(1, "1m")
+		})
+	})
+	It("should terminate an empty node", func() {
 		nodePool.Spec.Disruption.ConsolidateAfter = &corev1beta1.NillableDuration{Duration: lo.ToPtr(time.Hour * 300)}
 
 		const numPods = 1