test: Adding Provisioner Static Drift (aws#4305)

fffinkel · Jul 25, 2023 · 4cecea2 · 4cecea2
1 parent 4b0d545
commit 4cecea2
Show file tree

Hide file tree

Showing 3 changed files with 76 additions and 6 deletions.
diff --git a/charts/karpenter/templates/clusterrole-core.yaml b/charts/karpenter/templates/clusterrole-core.yaml
@@ -49,8 +49,11 @@ rules:
     verbs: [ "get", "list", "watch" ]
   # Write
   - apiGroups: ["karpenter.sh"]
-    resources: ["provisioners/status", "machines", "machines/status"]
+    resources: ["machines", "machines/status"]
     verbs: ["create", "delete", "update", "patch"]
+  - apiGroups: ["karpenter.sh"]
+    resources: ["provisioners", "provisioners/status"]
+    verbs: ["update", "patch"]
   - apiGroups: [""]
     resources: ["events"]
     verbs: ["create", "patch"]

diff --git a/test/suites/drift/suite_test.go b/test/suites/drift/suite_test.go
@@ -16,6 +16,7 @@ package drift
 
 import (
 	"fmt"
+	"sort"
 	"strings"
 	"testing"
 	"time"
@@ -269,4 +270,70 @@ var _ = Describe("Drift", Label("AWS"), func() {
 		env.ExpectUpdated(pod)
 		env.EventuallyExpectNotFound(pod, node)
 	})
+	Describe("Provisioner Drift", func() {
+		var pod *v1.Pod
+		var nodeTemplate *v1alpha1.AWSNodeTemplate
+		var provisioner *v1alpha5.Provisioner
+		BeforeEach(func() {
+			nodeTemplate = awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{
+				SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName},
+				SubnetSelector:        map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName},
+			}})
+			provisioner = test.Provisioner(test.ProvisionerOptions{ProviderRef: &v1alpha5.MachineTemplateRef{Name: nodeTemplate.Name}})
+			// Add a do-not-evict pod so that we can check node metadata before we deprovision
+			pod = test.Pod(test.PodOptions{
+				ObjectMeta: metav1.ObjectMeta{
+					Annotations: map[string]string{
+						v1alpha5.DoNotEvictPodAnnotationKey: "true",
+					},
+				},
+			})
+		})
+		DescribeTable("provisioner static drift", func(fieldName string, provisionerOption test.ProvisionerOptions) {
+			provisionerOption.ObjectMeta = provisioner.ObjectMeta
+			updatedProvisioner := test.Provisioner(
+				test.ProvisionerOptions{ProviderRef: &v1alpha5.MachineTemplateRef{Name: nodeTemplate.Name}},
+				provisionerOption,
+			)
+
+			env.ExpectCreated(pod, nodeTemplate, provisioner)
+			machine := env.EventuallyExpectCreatedMachineCount("==", 1)[0]
+			node := env.EventuallyExpectCreatedNodeCount("==", 1)[0]
+			env.EventuallyExpectHealthy(pod)
+
+			env.ExpectCreatedOrUpdated(updatedProvisioner)
+
+			By("validating the drifted status condition has propagated")
+			EventuallyWithOffset(1, func(g Gomega) {
+				g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(machine), machine)).To(Succeed())
+				g.Expect(machine.StatusConditions().GetCondition(v1alpha5.MachineDrifted)).ToNot(BeNil())
+				g.Expect(machine.StatusConditions().GetCondition(v1alpha5.MachineDrifted).IsTrue()).To(BeTrue())
+			}).Should(Succeed())
+
+			delete(pod.Annotations, v1alpha5.DoNotEvictPodAnnotationKey)
+			env.ExpectUpdated(pod)
+
+			// Nodes will need to have the start-up taint removed before the node can be considered as initialized
+			if fieldName == "Start-up Taint" {
+				nodes := env.EventuallyExpectCreatedNodeCount("==", 2)
+				sort.Slice(nodes, func(i int, j int) bool {
+					return nodes[i].CreationTimestamp.Before(&nodes[j].CreationTimestamp)
+				})
+				nodeTwo := nodes[1]
+				nodeTwo.Spec.Taints = []v1.Taint{}
+				env.ExpectCreatedOrUpdated(nodeTwo)
+			}
+
+			env.EventuallyExpectNotFound(pod, node)
+		},
+			Entry("Annotation Drift", "Annotation", test.ProvisionerOptions{Annotations: map[string]string{"keyAnnotationTest": "valueAnnotationTest"}}),
+			Entry("Labels Drift", "Labels", test.ProvisionerOptions{Labels: map[string]string{"keyLabelTest": "valueLabelTest"}}),
+			Entry("Taints Drift", "Taints", test.ProvisionerOptions{Taints: []v1.Taint{{Key: "example.com/another-taint-2", Effect: v1.TaintEffectPreferNoSchedule}}}),
+			Entry("KubeletConfiguration Drift", "KubeletConfiguration", test.ProvisionerOptions{Kubelet: &v1alpha5.KubeletConfiguration{
+				EvictionSoft:            map[string]string{"memory.available": "5%"},
+				EvictionSoftGracePeriod: map[string]metav1.Duration{"memory.available": {Duration: time.Minute}},
+			}}),
+			Entry("Start-up Taints Drift", "Start-up Taint", test.ProvisionerOptions{StartupTaints: []v1.Taint{{Key: "example.com/another-taint-2", Effect: v1.TaintEffectPreferNoSchedule}}}),
+		)
+	})
 })
diff --git a/website/content/en/preview/concepts/deprovisioning.md b/website/content/en/preview/concepts/deprovisioning.md
@@ -154,12 +154,12 @@ Read the [Drift Design](https://github.com/aws/karpenter-core/pull/366/files) fo
 
 |Provisioner Fields          | Static | Dynamic | Behavioral | Implemented |
 |----------------------------|  :---: |  :---:  |   :---:    |    :---:    |
-| Startup Taints             |    x   |         |            |             |
-| Taints                     |    x   |         |            |             |
-| Labels                     |    x   |         |            |             |
-| Annotations                |    x   |         |            |             |
+| Startup Taints             |    x   |         |            |      x      |
+| Taints                     |    x   |         |            |      x      |
+| Labels                     |    x   |         |            |      x      |
+| Annotations                |    x   |         |            |      x      |
 | Node Requirements          |        |    x    |            |             |
-| Kubelet Configuration      |    x   |         |            |             |
+| Kubelet Configuration      |    x   |         |            |      x      |
 | Weight                     |        |         |      x     |     NA      |
 | Limits                     |        |         |      x     |     NA      |
 | Consolidation              |        |         |      x     |     NA      |