docs: add karpenter startup taint custom userdata requirement (#6388)

aws · Jul 8, 2024 · ba0d623 · ba0d623
1 parent 2d9970e
commit ba0d623
Show file tree

Hide file tree

Showing 16 changed files with 36 additions and 16 deletions.
diff --git a/pkg/providers/amifamily/bootstrap/bootstrap.go b/pkg/providers/amifamily/bootstrap/bootstrap.go
@@ -78,12 +78,9 @@ func (o Options) kubeletExtraArgs() (args []string) {
 }
 
 func (o Options) nodeTaintArg() string {
-	if len(o.Taints) == 0 {
-		return ""
-	}
 	var taintStrings []string
 	for _, taint := range o.Taints {
-		taintStrings = append(taintStrings, fmt.Sprintf("%s=%s:%s", taint.Key, taint.Value, taint.Effect))
+		taintStrings = append(taintStrings, taint.ToString())
 	}
 	return fmt.Sprintf("--register-with-taints=%q", strings.Join(taintStrings, ","))
 }

diff --git a/pkg/providers/amifamily/bootstrap/bottlerocket.go b/pkg/providers/amifamily/bootstrap/bottlerocket.go
@@ -77,7 +77,8 @@ func (b Bottlerocket) Script() (string, error) {
 
 	s.Settings.Kubernetes.NodeTaints = map[string][]string{}
 	for _, taint := range b.Taints {
-		s.Settings.Kubernetes.NodeTaints[taint.Key] = append(s.Settings.Kubernetes.NodeTaints[taint.Key], fmt.Sprintf("%s:%s", taint.Value, taint.Effect))
+		s.Settings.Kubernetes.NodeTaints[taint.Key] = append(s.Settings.Kubernetes.NodeTaints[taint.Key], fmt.Sprintf("%s%s",
+			lo.Ternary(taint.Value == "", "", taint.Value+":"), taint.Effect))
 	}
 	script, err := s.MarshalTOML()
 	if err != nil {

diff --git a/pkg/providers/amifamily/bootstrap/nodeadm.go b/pkg/providers/amifamily/bootstrap/nodeadm.go
@@ -115,10 +115,8 @@ func (n Nodeadm) generateInlineKubeletConfiguration() (map[string]runtime.RawExt
 	if err != nil {
 		return nil, err
 	}
-	if len(n.Taints) != 0 {
-		kubeConfigMap["registerWithTaints"] = runtime.RawExtension{
-			Raw: lo.Must(json.Marshal(n.Taints)),
-		}
+	kubeConfigMap["registerWithTaints"] = runtime.RawExtension{
+		Raw: lo.Must(json.Marshal(n.Taints)),
 	}
 	return kubeConfigMap, nil
 }

diff --git a/pkg/providers/amifamily/resolver.go b/pkg/providers/amifamily/resolver.go
@@ -216,11 +216,21 @@ func (r Resolver) resolveLaunchTemplate(nodeClass *v1beta1.EC2NodeClass, nodeCla
 	if kubeletConfig.MaxPods == nil {
 		kubeletConfig.MaxPods = lo.ToPtr(int32(maxPods))
 	}
+	taints := lo.Flatten([][]core.Taint{
+		nodeClaim.Spec.Taints,
+		nodeClaim.Spec.StartupTaints,
+	})
+	if _, found := lo.Find(taints, func(t core.Taint) bool {
+		return t.MatchTaint(&core.Taint{Key: "karpenter.sh/unregistered", Effect: core.TaintEffectNoExecute})
+	}); !found {
+		taints = append(taints, core.Taint{Key: "karpenter.sh/unregistered", Effect: core.TaintEffectNoExecute})
+	}
+
 	resolved := &LaunchTemplate{
 		Options: options,
 		UserData: amiFamily.UserData(
 			r.defaultClusterDNS(options, kubeletConfig),
-			append(nodeClaim.Spec.Taints, nodeClaim.Spec.StartupTaints...),
+			taints,
 			options.Labels,
 			options.CABundle,
 			instanceTypes,

diff --git a/pkg/providers/launchtemplate/suite_test.go b/pkg/providers/launchtemplate/suite_test.go
@@ -1678,7 +1678,7 @@ var _ = Describe("LaunchTemplate Provider", func() {
 						Expect(ok).To(BeTrue())
 						taints := []v1.Taint{}
 						Expect(yaml.Unmarshal(taintsRaw.Raw, &taints)).To(Succeed())
-						Expect(len(taints)).To(Equal(2))
+						Expect(len(taints)).To(Equal(3))
 						Expect(taints).To(ContainElements(lo.Map(desiredTaints, func(t v1.Taint, _ int) interface{} {
 							return interface{}(t)
 						})))

diff --git a/pkg/providers/launchtemplate/testdata/al2023_mime_userdata_merged.golden b/pkg/providers/launchtemplate/testdata/al2023_mime_userdata_merged.golden
@@ -23,6 +23,9 @@ spec:
       clusterDNS:
       - 10.0.100.10
       maxPods: 110
+      registerWithTaints:
+      - effect: NoExecute
+        key: karpenter.sh/unregistered
     flags:
     - --node-labels="karpenter.sh/capacity-type=on-demand,%s=%s,testing/cluster=unspecified"
 

diff --git a/pkg/providers/launchtemplate/testdata/al2023_shell_userdata_merged.golden b/pkg/providers/launchtemplate/testdata/al2023_shell_userdata_merged.golden
@@ -23,6 +23,9 @@ spec:
       clusterDNS:
       - 10.0.100.10
       maxPods: 110
+      registerWithTaints:
+      - effect: NoExecute
+        key: karpenter.sh/unregistered
     flags:
     - --node-labels="karpenter.sh/capacity-type=on-demand,%s=%s,testing/cluster=unspecified"
 

diff --git a/pkg/providers/launchtemplate/testdata/al2023_userdata_unmerged.golden b/pkg/providers/launchtemplate/testdata/al2023_userdata_unmerged.golden
@@ -23,6 +23,9 @@ spec:
       clusterDNS:
       - 10.0.100.10
       maxPods: 110
+      registerWithTaints:
+      - effect: NoExecute
+        key: karpenter.sh/unregistered
     flags:
     - --node-labels="karpenter.sh/capacity-type=on-demand,%s=%s,testing/cluster=unspecified"
 

diff --git a/pkg/providers/launchtemplate/testdata/al2023_yaml_userdata_merged.golden b/pkg/providers/launchtemplate/testdata/al2023_yaml_userdata_merged.golden
@@ -23,6 +23,9 @@ spec:
       clusterDNS:
       - 10.0.100.10
       maxPods: 110
+      registerWithTaints:
+      - effect: NoExecute
+        key: karpenter.sh/unregistered
     flags:
     - --node-labels="karpenter.sh/capacity-type=on-demand,%s=%s,testing/cluster=unspecified"
 

diff --git a/pkg/providers/launchtemplate/testdata/al2_userdata_merged.golden b/pkg/providers/launchtemplate/testdata/al2_userdata_merged.golden
@@ -15,5 +15,5 @@ exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1
 /etc/eks/bootstrap.sh 'test-cluster' --apiserver-endpoint 'https://test-cluster' --b64-cluster-ca 'ca-bundle' \
 --dns-cluster-ip '10.0.100.10' \
 --use-max-pods false \
---kubelet-extra-args '--node-labels="karpenter.sh/capacity-type=on-demand,%s=%s,testing/cluster=unspecified" --max-pods=110'
+--kubelet-extra-args '--node-labels="karpenter.sh/capacity-type=on-demand,%s=%s,testing/cluster=unspecified" --register-with-taints="karpenter.sh/unregistered:NoExecute" --max-pods=110'
 --//--
diff --git a/pkg/providers/launchtemplate/testdata/al2_userdata_unmerged.golden b/pkg/providers/launchtemplate/testdata/al2_userdata_unmerged.golden
@@ -9,5 +9,5 @@ exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1
 /etc/eks/bootstrap.sh 'test-cluster' --apiserver-endpoint 'https://test-cluster' --b64-cluster-ca 'ca-bundle' \
 --dns-cluster-ip '10.0.100.10' \
 --use-max-pods false \
---kubelet-extra-args '--node-labels="karpenter.sh/capacity-type=on-demand,%s=%s,testing/cluster=unspecified" --max-pods=110'
+--kubelet-extra-args '--node-labels="karpenter.sh/capacity-type=on-demand,%s=%s,testing/cluster=unspecified" --register-with-taints="karpenter.sh/unregistered:NoExecute" --max-pods=110'
 --//--
diff --git a/pkg/providers/launchtemplate/testdata/br_userdata_merged.golden b/pkg/providers/launchtemplate/testdata/br_userdata_merged.golden
@@ -16,6 +16,7 @@ custom-node-label = 'custom'
 [settings.kubernetes.node-taints]
 baz = ['bin:NoExecute']
 foo = ['bar:NoExecute']
+'karpenter.sh/unregistered' = ['NoExecute']
 
 [settings.kubernetes.eviction-hard]
 'memory.available' = '12%%'

diff --git a/pkg/providers/launchtemplate/testdata/br_userdata_unmerged.golden b/pkg/providers/launchtemplate/testdata/br_userdata_unmerged.golden
@@ -14,3 +14,4 @@ max-pods = 110
 [settings.kubernetes.node-taints]
 baz = ['bin:NoExecute']
 foo = ['bar:NoExecute']
+'karpenter.sh/unregistered' = ['NoExecute']
diff --git a/pkg/providers/launchtemplate/testdata/windows_userdata_merged.golden b/pkg/providers/launchtemplate/testdata/windows_userdata_merged.golden
@@ -2,5 +2,5 @@
 Write-Host "Running custom user data script"
 Write-Host "Finished running custom user data script"
 [string]$EKSBootstrapScriptFile = "$env:ProgramFiles\Amazon\EKS\Start-EKSBootstrap.ps1"
-& $EKSBootstrapScriptFile -EKSClusterName 'test-cluster' -APIServerEndpoint 'https://test-cluster' -Base64ClusterCA 'ca-bundle' -KubeletExtraArgs '--node-labels="karpenter.sh/capacity-type=spot,%s=%s,testing/cluster=unspecified" --max-pods=110' -DNSClusterIP '10.0.100.10'
+& $EKSBootstrapScriptFile -EKSClusterName 'test-cluster' -APIServerEndpoint 'https://test-cluster' -Base64ClusterCA 'ca-bundle' -KubeletExtraArgs '--node-labels="karpenter.sh/capacity-type=spot,%s=%s,testing/cluster=unspecified" --register-with-taints="karpenter.sh/unregistered:NoExecute" --max-pods=110' -DNSClusterIP '10.0.100.10'
 </powershell>
diff --git a/pkg/providers/launchtemplate/testdata/windows_userdata_unmerged.golden b/pkg/providers/launchtemplate/testdata/windows_userdata_unmerged.golden
@@ -1,4 +1,4 @@
 <powershell>
 [string]$EKSBootstrapScriptFile = "$env:ProgramFiles\Amazon\EKS\Start-EKSBootstrap.ps1"
-& $EKSBootstrapScriptFile -EKSClusterName 'test-cluster' -APIServerEndpoint 'https://test-cluster' -Base64ClusterCA 'ca-bundle' -KubeletExtraArgs '--node-labels="karpenter.sh/capacity-type=spot,%s=%s,testing/cluster=unspecified" --max-pods=110' -DNSClusterIP '10.0.100.10'
+& $EKSBootstrapScriptFile -EKSClusterName 'test-cluster' -APIServerEndpoint 'https://test-cluster' -Base64ClusterCA 'ca-bundle' -KubeletExtraArgs '--node-labels="karpenter.sh/capacity-type=spot,%s=%s,testing/cluster=unspecified" --register-with-taints="karpenter.sh/unregistered:NoExecute" --max-pods=110' -DNSClusterIP '10.0.100.10'
 </powershell>
diff --git a/website/content/en/preview/concepts/nodeclasses.md b/website/content/en/preview/concepts/nodeclasses.md
@@ -270,7 +270,7 @@ Karpenter will automatically query for the appropriate [EKS optimized AMI](https
 
 ### Custom
 
-The `Custom` AMIFamily ships without any default userData to allow you to configure custom bootstrapping for control planes or images that don't support the default methods from the other families.
+The `Custom` AMIFamily ships without any default userData to allow you to configure custom bootstrapping for control planes or images that don't support the default methods from the other families. For this AMIFamily, kubelet must add the taint `karpenter.sh/unregistered:NoExecute` via the `--register-with-taints` flag ([flags](https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/#options)) or the KubeletConfiguration spec ([options](https://kubernetes.io/docs/reference/config-api/kubelet-config.v1beta1/#kubelet-config-k8s-io-v1beta1-KubeletConfiguration) and [docs](https://kubernetes.io/docs/tasks/administer-cluster/kubelet-config-file/)). Karpenter will fail to register nodes that do not have this taint.
 
 ## spec.subnetSelectorTerms