Skip to content

Commit

Permalink
Add E2E testing for fully blocking budgets with and without schedule
Browse files Browse the repository at this point in the history
  • Loading branch information
@jonathan-innis
jonathan-innis committed Jan 17, 2024
1 parent 95f812e commit ce979c9
Show file tree
Hide file tree
Showing 4 changed files with 244 additions and 31 deletions.
52 changes: 52 additions & 0 deletions test/pkg/environment/common/expectations.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -458,6 +458,27 @@ func (env *Environment) ConsistentlyExpectNodeCount(comparator string, count int
return lo.ToSlicePtr(nodeList.Items)
}

func (env *Environment) ConsistentlyExpectNoDisruptions(nodeCount int, duration string) {
GinkgoHelper()
Consistently(func(g Gomega) {
// Ensure we don't change our NodeClaims
nodeClaimList := &corev1beta1.NodeClaimList{}
g.Expect(env.Client.List(env, nodeClaimList, client.HasLabels{test.DiscoveryLabel})).To(Succeed())
g.Expect(nodeClaimList.Items).To(HaveLen(nodeCount))

nodeList := &v1.NodeList{}
g.Expect(env.Client.List(env, nodeList, client.HasLabels{test.DiscoveryLabel})).To(Succeed())
g.Expect(nodeList.Items).To(HaveLen(nodeCount))

for _, node := range nodeList.Items {
_, ok := lo.Find(node.Spec.Taints, func(t v1.Taint) bool {
return corev1beta1.IsDisruptingTaint(t)
})
g.Expect(ok).To(BeFalse())
}
}, duration).Should(Succeed())
}

func (env *Environment) EventuallyExpectTaintedNodeCount(comparator string, count int) []*v1.Node {
GinkgoHelper()
By(fmt.Sprintf("waiting for tainted nodes to be %s to %d", comparator, count))
Expand Down Expand Up @@ -572,6 +593,7 @@ func (env *Environment) EventuallyExpectCreatedNodeClaimCount(comparator string,
}

func (env *Environment) EventuallyExpectNodeClaimsReady(nodeClaims ...*corev1beta1.NodeClaim) {
GinkgoHelper()
Eventually(func(g Gomega) {
for _, nc := range nodeClaims {
temp := &corev1beta1.NodeClaim{}
Expand All @@ -581,6 +603,36 @@ func (env *Environment) EventuallyExpectNodeClaimsReady(nodeClaims ...*corev1bet
}).Should(Succeed())
}

func (env *Environment) EventuallyExpectExpired(nodeClaims ...*corev1beta1.NodeClaim) {
GinkgoHelper()
Eventually(func(g Gomega) {
for _, nc := range nodeClaims {
g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(nc), nc)).To(Succeed())
g.Expect(nc.StatusConditions().GetCondition(corev1beta1.Expired).IsTrue()).To(BeTrue())
}
}).Should(Succeed())
}

func (env *Environment) EventuallyExpectDrifted(nodeClaims ...*corev1beta1.NodeClaim) {
GinkgoHelper()
Eventually(func(g Gomega) {
for _, nc := range nodeClaims {
g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(nc), nc)).To(Succeed())
g.Expect(nc.StatusConditions().GetCondition(corev1beta1.Drifted).IsTrue()).To(BeTrue())
}
}).Should(Succeed())
}

func (env *Environment) EventuallyExpectEmpty(nodeClaims ...*corev1beta1.NodeClaim) {
GinkgoHelper()
Eventually(func(g Gomega) {
for _, nc := range nodeClaims {
g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(nc), nc)).To(Succeed())
g.Expect(nc.StatusConditions().GetCondition(corev1beta1.Empty).IsTrue()).To(BeTrue())
}
}).Should(Succeed())
}

func (env *Environment) GetNode(nodeName string) v1.Node {
GinkgoHelper()
var node v1.Node
Expand Down
48 changes: 48 additions & 0 deletions test/suites/drift/suite_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -389,6 +389,54 @@ var _ = Describe("Drift", Label("AWS"), func() {
// the node should be gone
env.EventuallyExpectNotFound(nodes[0], nodes[1], nodes[2])
})
It("should not allow drift if the budget is fully blocking", func() {
// We're going to define a budget that doesn't allow any drift to happen
nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{
Nodes: "0",
}}

dep.Spec.Template.Annotations = nil
env.ExpectCreated(nodeClass, nodePool, dep)

nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0]
env.EventuallyExpectCreatedNodeCount("==", 1)
env.EventuallyExpectHealthyPodCount(selector, numPods)

By("drifting the nodes")
// Drift the nodeclaims
nodePool.Spec.Template.Annotations = map[string]string{"test": "annotation"}
env.ExpectUpdated(nodePool)

env.EventuallyExpectDrifted(nodeClaim)
env.ConsistentlyExpectNoDisruptions(1, "1m")
})
It("should not allow drift if the budget is fully blocking during a scheduled time", func() {
// We're going to define a budget that doesn't allow any drift to happen
// This is going to be on a schedule that only lasts 30 minutes, whose window starts 15 minutes before
// the current time and extends 15 minutes past the current time
// Times need to be in UTC since the karpenter containers were built in UTC time
windowStart := time.Now().Add(-time.Minute * 15).UTC()
nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{
Nodes: "0",
Schedule: lo.ToPtr(fmt.Sprintf("%d %d * * *", windowStart.Minute(), windowStart.Hour())),
Duration: &metav1.Duration{Duration: time.Minute * 30},
}}

dep.Spec.Template.Annotations = nil
env.ExpectCreated(nodeClass, nodePool, dep)

nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0]
env.EventuallyExpectCreatedNodeCount("==", 1)
env.EventuallyExpectHealthyPodCount(selector, numPods)

By("drifting the nodes")
// Drift the nodeclaims
nodePool.Spec.Template.Annotations = map[string]string{"test": "annotation"}
env.ExpectUpdated(nodePool)

env.EventuallyExpectDrifted(nodeClaim)
env.ConsistentlyExpectNoDisruptions(1, "1m")
})
})
It("should disrupt nodes that have drifted due to AMIs", func() {
// choose an old static image
Expand Down
109 changes: 79 additions & 30 deletions test/suites/expiration/suite_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,12 @@ limitations under the License.
package expiration_test

import (
"fmt"
"testing"
"time"

"github.com/samber/lo"
appsv1 "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand Down Expand Up @@ -68,6 +70,28 @@ var _ = AfterEach(func() { env.Cleanup() })
var _ = AfterEach(func() { env.AfterEach() })

var _ = Describe("Expiration", func() {
var dep *appsv1.Deployment
var selector labels.Selector
var numPods int
BeforeEach(func() {
numPods = 1
// Add pods with a do-not-disrupt annotation so that we can check node metadata before we disrupt
dep = coretest.Deployment(coretest.DeploymentOptions{
Replicas: int32(numPods),
PodOptions: coretest.PodOptions{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
"app": "my-app",
},
Annotations: map[string]string{
corev1beta1.DoNotDisruptAnnotationKey: "true",
},
},
TerminationGracePeriodSeconds: lo.ToPtr[int64](0),
},
})
selector = labels.SelectorFromSet(dep.Spec.Selector.MatchLabels)
})
Context("Budgets", func() {
It("should respect budgets for empty expiration", func() {
coretest.ReplaceRequirements(nodePool,
Expand All @@ -82,9 +106,9 @@ var _ = Describe("Expiration", func() {
}}
nodePool.Spec.Disruption.ExpireAfter = corev1beta1.NillableDuration{}

var numPods int32 = 6
dep := coretest.Deployment(coretest.DeploymentOptions{
Replicas: numPods,
numPods = 6
dep = coretest.Deployment(coretest.DeploymentOptions{
Replicas: int32(numPods),
PodOptions: coretest.PodOptions{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
Expand All @@ -100,12 +124,12 @@ var _ = Describe("Expiration", func() {
},
},
})
selector := labels.SelectorFromSet(dep.Spec.Selector.MatchLabels)
selector = labels.SelectorFromSet(dep.Spec.Selector.MatchLabels)
env.ExpectCreated(nodeClass, nodePool, dep)

env.EventuallyExpectCreatedNodeClaimCount("==", 3)
env.EventuallyExpectCreatedNodeCount("==", 3)
env.EventuallyExpectHealthyPodCount(selector, int(numPods))
env.EventuallyExpectHealthyPodCount(selector, numPods)
env.Monitor.Reset() // Reset the monitor so that we can expect a single node to be spun up after expiration

nodes := env.EventuallyExpectNodeCount("==", 3)
Expand Down Expand Up @@ -167,9 +191,9 @@ var _ = Describe("Expiration", func() {
}}
// disable expiration so that we can enable it later when we want.
nodePool.Spec.Disruption.ExpireAfter = corev1beta1.NillableDuration{}
var numPods int32 = 9
dep := coretest.Deployment(coretest.DeploymentOptions{
Replicas: numPods,
numPods = 9
dep = coretest.Deployment(coretest.DeploymentOptions{
Replicas: int32(numPods),
PodOptions: coretest.PodOptions{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
Expand All @@ -185,12 +209,12 @@ var _ = Describe("Expiration", func() {
},
},
})
selector := labels.SelectorFromSet(dep.Spec.Selector.MatchLabels)
selector = labels.SelectorFromSet(dep.Spec.Selector.MatchLabels)
env.ExpectCreated(nodeClass, nodePool, dep)

env.EventuallyExpectCreatedNodeClaimCount("==", 3)
env.EventuallyExpectCreatedNodeCount("==", 3)
env.EventuallyExpectHealthyPodCount(selector, int(numPods))
env.EventuallyExpectHealthyPodCount(selector, numPods)
env.Monitor.Reset() // Reset the monitor so that we can expect a single node to be spun up after expiration

By("scaling down the deployment")
Expand Down Expand Up @@ -287,9 +311,9 @@ var _ = Describe("Expiration", func() {
Nodes: "50%",
}}
nodePool.Spec.Disruption.ExpireAfter = corev1beta1.NillableDuration{}
var numPods int32 = 3
dep := coretest.Deployment(coretest.DeploymentOptions{
Replicas: numPods,
numPods = 3
dep = coretest.Deployment(coretest.DeploymentOptions{
Replicas: int32(numPods),
PodOptions: coretest.PodOptions{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
Expand All @@ -305,12 +329,12 @@ var _ = Describe("Expiration", func() {
},
},
})
selector := labels.SelectorFromSet(dep.Spec.Selector.MatchLabels)
selector = labels.SelectorFromSet(dep.Spec.Selector.MatchLabels)
env.ExpectCreated(nodeClass, nodePool, dep)

env.EventuallyExpectCreatedNodeClaimCount("==", 3)
env.EventuallyExpectCreatedNodeCount("==", 3)
env.EventuallyExpectHealthyPodCount(selector, int(numPods))
env.EventuallyExpectHealthyPodCount(selector, numPods)
env.Monitor.Reset() // Reset the monitor so that we can expect a single node to be spun up after drift

By("cordoning and adding finalizer to the nodes")
Expand Down Expand Up @@ -370,26 +394,51 @@ var _ = Describe("Expiration", func() {
// the node should be gone
env.EventuallyExpectNotFound(nodes[0], nodes[1], nodes[2])
})
It("should not allow expiration if the budget is fully blocking", func() {
// We're going to define a budget that doesn't allow any expirations to happen
nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{
Nodes: "0",
}}

dep.Spec.Template.Annotations = nil
env.ExpectCreated(nodeClass, nodePool, dep)

nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0]
env.EventuallyExpectCreatedNodeCount("==", 1)
env.EventuallyExpectHealthyPodCount(selector, numPods)

env.EventuallyExpectExpired(nodeClaim)
env.ConsistentlyExpectNoDisruptions(1, "1m")
})
It("should not allow expiration if the budget is fully blocking during a scheduled time", func() {
// We're going to define a budget that doesn't allow any expirations to happen
// This is going to be on a schedule that only lasts 30 minutes, whose window starts 15 minutes before
// the current time and extends 15 minutes past the current time
// Times need to be in UTC since the karpenter containers were built in UTC time
windowStart := time.Now().Add(-time.Minute * 15).UTC()
nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{
Nodes: "0",
Schedule: lo.ToPtr(fmt.Sprintf("%d %d * * *", windowStart.Minute(), windowStart.Hour())),
Duration: &metav1.Duration{Duration: time.Minute * 30},
}}

dep.Spec.Template.Annotations = nil
env.ExpectCreated(nodeClass, nodePool, dep)

nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0]
env.EventuallyExpectCreatedNodeCount("==", 1)
env.EventuallyExpectHealthyPodCount(selector, numPods)

env.EventuallyExpectExpired(nodeClaim)
env.ConsistentlyExpectNoDisruptions(1, "1m")
})
})
It("should expire the node after the expiration is reached", func() {
var numPods int32 = 1
dep := coretest.Deployment(coretest.DeploymentOptions{
Replicas: numPods,
PodOptions: coretest.PodOptions{
ObjectMeta: metav1.ObjectMeta{
Annotations: map[string]string{
corev1beta1.DoNotDisruptAnnotationKey: "true",
},
Labels: map[string]string{"app": "large-app"},
},
},
})
selector := labels.SelectorFromSet(dep.Spec.Selector.MatchLabels)
env.ExpectCreated(nodeClass, nodePool, dep)

nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0]
node := env.EventuallyExpectCreatedNodeCount("==", 1)[0]
env.EventuallyExpectHealthyPodCount(selector, int(numPods))
env.EventuallyExpectHealthyPodCount(selector, numPods)
env.Monitor.Reset() // Reset the monitor so that we can expect a single node to be spun up after expiration

// Expect that the NodeClaim will get an expired status condition
Expand Down Expand Up @@ -425,7 +474,7 @@ var _ = Describe("Expiration", func() {

env.EventuallyExpectCreatedNodeClaimCount("==", 1)
env.EventuallyExpectCreatedNodeCount("==", 1)
env.EventuallyExpectHealthyPodCount(selector, int(numPods))
env.EventuallyExpectHealthyPodCount(selector, numPods)
})
It("should replace expired node with a single node and schedule all pods", func() {
var numPods int32 = 5
Expand Down
Loading

0 comments on commit ce979c9

Please sign in to comment.