diff --git a/pkg/providers/amifamily/bootstrap/bootstrap.go b/pkg/providers/amifamily/bootstrap/bootstrap.go index b1b54d094e86..5387c4fa665f 100644 --- a/pkg/providers/amifamily/bootstrap/bootstrap.go +++ b/pkg/providers/amifamily/bootstrap/bootstrap.go @@ -78,12 +78,9 @@ func (o Options) kubeletExtraArgs() (args []string) { } func (o Options) nodeTaintArg() string { - if len(o.Taints) == 0 { - return "" - } var taintStrings []string for _, taint := range o.Taints { - taintStrings = append(taintStrings, fmt.Sprintf("%s=%s:%s", taint.Key, taint.Value, taint.Effect)) + taintStrings = append(taintStrings, taint.ToString()) } return fmt.Sprintf("--register-with-taints=%q", strings.Join(taintStrings, ",")) } diff --git a/pkg/providers/amifamily/bootstrap/bottlerocket.go b/pkg/providers/amifamily/bootstrap/bottlerocket.go index f7dbddd1b9e3..c8454342ba91 100644 --- a/pkg/providers/amifamily/bootstrap/bottlerocket.go +++ b/pkg/providers/amifamily/bootstrap/bottlerocket.go @@ -77,7 +77,8 @@ func (b Bottlerocket) Script() (string, error) { s.Settings.Kubernetes.NodeTaints = map[string][]string{} for _, taint := range b.Taints { - s.Settings.Kubernetes.NodeTaints[taint.Key] = append(s.Settings.Kubernetes.NodeTaints[taint.Key], fmt.Sprintf("%s:%s", taint.Value, taint.Effect)) + s.Settings.Kubernetes.NodeTaints[taint.Key] = append(s.Settings.Kubernetes.NodeTaints[taint.Key], fmt.Sprintf("%s%s", + lo.Ternary(taint.Value == "", "", taint.Value+":"), taint.Effect)) } script, err := s.MarshalTOML() if err != nil { diff --git a/pkg/providers/amifamily/bootstrap/nodeadm.go b/pkg/providers/amifamily/bootstrap/nodeadm.go index 8c82268199c6..dfd0b5bd4566 100644 --- a/pkg/providers/amifamily/bootstrap/nodeadm.go +++ b/pkg/providers/amifamily/bootstrap/nodeadm.go @@ -115,10 +115,8 @@ func (n Nodeadm) generateInlineKubeletConfiguration() (map[string]runtime.RawExt if err != nil { return nil, err } - if len(n.Taints) != 0 { - kubeConfigMap["registerWithTaints"] = runtime.RawExtension{ - Raw: lo.Must(json.Marshal(n.Taints)), - } + kubeConfigMap["registerWithTaints"] = runtime.RawExtension{ + Raw: lo.Must(json.Marshal(n.Taints)), } return kubeConfigMap, nil } diff --git a/pkg/providers/amifamily/resolver.go b/pkg/providers/amifamily/resolver.go index c257111900b1..836f469b69ef 100644 --- a/pkg/providers/amifamily/resolver.go +++ b/pkg/providers/amifamily/resolver.go @@ -216,11 +216,21 @@ func (r Resolver) resolveLaunchTemplate(nodeClass *v1beta1.EC2NodeClass, nodeCla if kubeletConfig.MaxPods == nil { kubeletConfig.MaxPods = lo.ToPtr(int32(maxPods)) } + taints := lo.Flatten([][]core.Taint{ + nodeClaim.Spec.Taints, + nodeClaim.Spec.StartupTaints, + }) + if _, found := lo.Find(taints, func(t core.Taint) bool { + return t.MatchTaint(&core.Taint{Key: "karpenter.sh/unregistered", Effect: core.TaintEffectNoExecute}) + }); !found { + taints = append(taints, core.Taint{Key: "karpenter.sh/unregistered", Effect: core.TaintEffectNoExecute}) + } + resolved := &LaunchTemplate{ Options: options, UserData: amiFamily.UserData( r.defaultClusterDNS(options, kubeletConfig), - append(nodeClaim.Spec.Taints, nodeClaim.Spec.StartupTaints...), + taints, options.Labels, options.CABundle, instanceTypes, diff --git a/pkg/providers/launchtemplate/suite_test.go b/pkg/providers/launchtemplate/suite_test.go index e08c21125e93..0cd5cdb12c27 100644 --- a/pkg/providers/launchtemplate/suite_test.go +++ b/pkg/providers/launchtemplate/suite_test.go @@ -1678,7 +1678,7 @@ var _ = Describe("LaunchTemplate Provider", func() { Expect(ok).To(BeTrue()) taints := []v1.Taint{} Expect(yaml.Unmarshal(taintsRaw.Raw, &taints)).To(Succeed()) - Expect(len(taints)).To(Equal(2)) + Expect(len(taints)).To(Equal(3)) Expect(taints).To(ContainElements(lo.Map(desiredTaints, func(t v1.Taint, _ int) interface{} { return interface{}(t) }))) diff --git a/pkg/providers/launchtemplate/testdata/al2023_mime_userdata_merged.golden b/pkg/providers/launchtemplate/testdata/al2023_mime_userdata_merged.golden index c65e73381035..21deb6e57e90 100644 --- a/pkg/providers/launchtemplate/testdata/al2023_mime_userdata_merged.golden +++ b/pkg/providers/launchtemplate/testdata/al2023_mime_userdata_merged.golden @@ -23,6 +23,9 @@ spec: clusterDNS: - 10.0.100.10 maxPods: 110 + registerWithTaints: + - effect: NoExecute + key: karpenter.sh/unregistered flags: - --node-labels="karpenter.sh/capacity-type=on-demand,%s=%s,testing/cluster=unspecified" diff --git a/pkg/providers/launchtemplate/testdata/al2023_shell_userdata_merged.golden b/pkg/providers/launchtemplate/testdata/al2023_shell_userdata_merged.golden index 6dd0bb7ac6dd..fae35eef55d9 100644 --- a/pkg/providers/launchtemplate/testdata/al2023_shell_userdata_merged.golden +++ b/pkg/providers/launchtemplate/testdata/al2023_shell_userdata_merged.golden @@ -23,6 +23,9 @@ spec: clusterDNS: - 10.0.100.10 maxPods: 110 + registerWithTaints: + - effect: NoExecute + key: karpenter.sh/unregistered flags: - --node-labels="karpenter.sh/capacity-type=on-demand,%s=%s,testing/cluster=unspecified" diff --git a/pkg/providers/launchtemplate/testdata/al2023_userdata_unmerged.golden b/pkg/providers/launchtemplate/testdata/al2023_userdata_unmerged.golden index 191107c780fb..cf18ed3508f4 100644 --- a/pkg/providers/launchtemplate/testdata/al2023_userdata_unmerged.golden +++ b/pkg/providers/launchtemplate/testdata/al2023_userdata_unmerged.golden @@ -23,6 +23,9 @@ spec: clusterDNS: - 10.0.100.10 maxPods: 110 + registerWithTaints: + - effect: NoExecute + key: karpenter.sh/unregistered flags: - --node-labels="karpenter.sh/capacity-type=on-demand,%s=%s,testing/cluster=unspecified" diff --git a/pkg/providers/launchtemplate/testdata/al2023_yaml_userdata_merged.golden b/pkg/providers/launchtemplate/testdata/al2023_yaml_userdata_merged.golden index 724f80b5b79d..128c1d991587 100644 --- a/pkg/providers/launchtemplate/testdata/al2023_yaml_userdata_merged.golden +++ b/pkg/providers/launchtemplate/testdata/al2023_yaml_userdata_merged.golden @@ -23,6 +23,9 @@ spec: clusterDNS: - 10.0.100.10 maxPods: 110 + registerWithTaints: + - effect: NoExecute + key: karpenter.sh/unregistered flags: - --node-labels="karpenter.sh/capacity-type=on-demand,%s=%s,testing/cluster=unspecified" diff --git a/pkg/providers/launchtemplate/testdata/al2_userdata_merged.golden b/pkg/providers/launchtemplate/testdata/al2_userdata_merged.golden index 3d1e9bc60623..22c8a37dfc03 100644 --- a/pkg/providers/launchtemplate/testdata/al2_userdata_merged.golden +++ b/pkg/providers/launchtemplate/testdata/al2_userdata_merged.golden @@ -15,5 +15,5 @@ exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 /etc/eks/bootstrap.sh 'test-cluster' --apiserver-endpoint 'https://test-cluster' --b64-cluster-ca 'ca-bundle' \ --dns-cluster-ip '10.0.100.10' \ --use-max-pods false \ ---kubelet-extra-args '--node-labels="karpenter.sh/capacity-type=on-demand,%s=%s,testing/cluster=unspecified" --max-pods=110' +--kubelet-extra-args '--node-labels="karpenter.sh/capacity-type=on-demand,%s=%s,testing/cluster=unspecified" --register-with-taints="karpenter.sh/unregistered:NoExecute" --max-pods=110' --//-- diff --git a/pkg/providers/launchtemplate/testdata/al2_userdata_unmerged.golden b/pkg/providers/launchtemplate/testdata/al2_userdata_unmerged.golden index 0e9c6773d191..39ecb867db43 100644 --- a/pkg/providers/launchtemplate/testdata/al2_userdata_unmerged.golden +++ b/pkg/providers/launchtemplate/testdata/al2_userdata_unmerged.golden @@ -9,5 +9,5 @@ exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 /etc/eks/bootstrap.sh 'test-cluster' --apiserver-endpoint 'https://test-cluster' --b64-cluster-ca 'ca-bundle' \ --dns-cluster-ip '10.0.100.10' \ --use-max-pods false \ ---kubelet-extra-args '--node-labels="karpenter.sh/capacity-type=on-demand,%s=%s,testing/cluster=unspecified" --max-pods=110' +--kubelet-extra-args '--node-labels="karpenter.sh/capacity-type=on-demand,%s=%s,testing/cluster=unspecified" --register-with-taints="karpenter.sh/unregistered:NoExecute" --max-pods=110' --//-- diff --git a/pkg/providers/launchtemplate/testdata/br_userdata_merged.golden b/pkg/providers/launchtemplate/testdata/br_userdata_merged.golden index a0f12195a192..900faa7faf75 100644 --- a/pkg/providers/launchtemplate/testdata/br_userdata_merged.golden +++ b/pkg/providers/launchtemplate/testdata/br_userdata_merged.golden @@ -16,6 +16,7 @@ custom-node-label = 'custom' [settings.kubernetes.node-taints] baz = ['bin:NoExecute'] foo = ['bar:NoExecute'] +'karpenter.sh/unregistered' = ['NoExecute'] [settings.kubernetes.eviction-hard] 'memory.available' = '12%%' diff --git a/pkg/providers/launchtemplate/testdata/br_userdata_unmerged.golden b/pkg/providers/launchtemplate/testdata/br_userdata_unmerged.golden index 67ad47bf0ebe..5d2d64d1e93e 100644 --- a/pkg/providers/launchtemplate/testdata/br_userdata_unmerged.golden +++ b/pkg/providers/launchtemplate/testdata/br_userdata_unmerged.golden @@ -14,3 +14,4 @@ max-pods = 110 [settings.kubernetes.node-taints] baz = ['bin:NoExecute'] foo = ['bar:NoExecute'] +'karpenter.sh/unregistered' = ['NoExecute'] diff --git a/pkg/providers/launchtemplate/testdata/windows_userdata_merged.golden b/pkg/providers/launchtemplate/testdata/windows_userdata_merged.golden index e1181fc5de59..d4c7b6285210 100644 --- a/pkg/providers/launchtemplate/testdata/windows_userdata_merged.golden +++ b/pkg/providers/launchtemplate/testdata/windows_userdata_merged.golden @@ -2,5 +2,5 @@ Write-Host "Running custom user data script" Write-Host "Finished running custom user data script" [string]$EKSBootstrapScriptFile = "$env:ProgramFiles\Amazon\EKS\Start-EKSBootstrap.ps1" -& $EKSBootstrapScriptFile -EKSClusterName 'test-cluster' -APIServerEndpoint 'https://test-cluster' -Base64ClusterCA 'ca-bundle' -KubeletExtraArgs '--node-labels="karpenter.sh/capacity-type=spot,%s=%s,testing/cluster=unspecified" --max-pods=110' -DNSClusterIP '10.0.100.10' +& $EKSBootstrapScriptFile -EKSClusterName 'test-cluster' -APIServerEndpoint 'https://test-cluster' -Base64ClusterCA 'ca-bundle' -KubeletExtraArgs '--node-labels="karpenter.sh/capacity-type=spot,%s=%s,testing/cluster=unspecified" --register-with-taints="karpenter.sh/unregistered:NoExecute" --max-pods=110' -DNSClusterIP '10.0.100.10' diff --git a/pkg/providers/launchtemplate/testdata/windows_userdata_unmerged.golden b/pkg/providers/launchtemplate/testdata/windows_userdata_unmerged.golden index 2c6f4b138f3b..a0b309c57f9c 100644 --- a/pkg/providers/launchtemplate/testdata/windows_userdata_unmerged.golden +++ b/pkg/providers/launchtemplate/testdata/windows_userdata_unmerged.golden @@ -1,4 +1,4 @@ [string]$EKSBootstrapScriptFile = "$env:ProgramFiles\Amazon\EKS\Start-EKSBootstrap.ps1" -& $EKSBootstrapScriptFile -EKSClusterName 'test-cluster' -APIServerEndpoint 'https://test-cluster' -Base64ClusterCA 'ca-bundle' -KubeletExtraArgs '--node-labels="karpenter.sh/capacity-type=spot,%s=%s,testing/cluster=unspecified" --max-pods=110' -DNSClusterIP '10.0.100.10' +& $EKSBootstrapScriptFile -EKSClusterName 'test-cluster' -APIServerEndpoint 'https://test-cluster' -Base64ClusterCA 'ca-bundle' -KubeletExtraArgs '--node-labels="karpenter.sh/capacity-type=spot,%s=%s,testing/cluster=unspecified" --register-with-taints="karpenter.sh/unregistered:NoExecute" --max-pods=110' -DNSClusterIP '10.0.100.10' diff --git a/website/content/en/preview/concepts/nodeclasses.md b/website/content/en/preview/concepts/nodeclasses.md index 37306468461d..975e65c32c87 100644 --- a/website/content/en/preview/concepts/nodeclasses.md +++ b/website/content/en/preview/concepts/nodeclasses.md @@ -270,7 +270,7 @@ Karpenter will automatically query for the appropriate [EKS optimized AMI](https ### Custom -The `Custom` AMIFamily ships without any default userData to allow you to configure custom bootstrapping for control planes or images that don't support the default methods from the other families. +The `Custom` AMIFamily ships without any default userData to allow you to configure custom bootstrapping for control planes or images that don't support the default methods from the other families. For this AMIFamily, kubelet must add the taint `karpenter.sh/unregistered:NoExecute` via the `--register-with-taints` flag ([flags](https://kubernetes.io/docs/reference/command-line-tools-reference/kubelet/#options)) or the KubeletConfiguration spec ([options](https://kubernetes.io/docs/reference/config-api/kubelet-config.v1beta1/#kubelet-config-k8s-io-v1beta1-KubeletConfiguration) and [docs](https://kubernetes.io/docs/tasks/administer-cluster/kubelet-config-file/)). Karpenter will fail to register nodes that do not have this taint. ## spec.subnetSelectorTerms