Merge branch 'main' into networkinterfaces-simplified

.github/actions/install-deps/action.yaml

@@ -3,7 +3,7 @@ description: 'Installs Go Downloads and installs Karpenter Dependencies'
 inputs:
   k8sVersion:
     description: Kubernetes version to use when installing the toolchain
-    default: "1.24.x"
+    default: "1.27.x"
 runs:
   using: "composite"
   steps:

.github/workflows/e2e-upgrade.yaml

@@ -59,6 +59,7 @@ jobs:
         with:
           name: "${{ github.workflow }} / e2e (Upgrade) / ${{ github.job }} (snapshot)"
           git_ref: ${{ inputs.to_git_ref }}
+      - uses: ./.github/actions/install-deps
       - uses: actions/checkout@v3
         with:
           ref: ${{ inputs.from_git_ref }}

.github/workflows/e2e.yaml

@@ -74,6 +74,7 @@ jobs:
         with:
           name: "${{ github.workflow }} / e2e (${{ inputs.suite }}) / ${{ github.job }} (snapshot)"
           git_ref: ${{ inputs.git_ref }}
+      - uses: ./.github/actions/install-deps
       - name: configure aws credentials
         uses: aws-actions/configure-aws-credentials@v2
         with:

charts/karpenter/templates/role.yaml

@@ -63,3 +63,24 @@ rules:
     resources: ["services"]
     resourceNames: ["kube-dns"]
     verbs: ["get"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ include "karpenter.fullname" . }}-lease
+  namespace: kube-node-lease
+  labels:
+    {{- include "karpenter.labels" . | nindent 4 }}
+  {{- with .Values.additionalAnnotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+rules:
+  # Read
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "list", "watch"]
+  # Write
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["delete"]

charts/karpenter/templates/rolebinding.yaml

@@ -33,6 +33,26 @@ roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: Role
   name: {{ include "karpenter.fullname" . }}-dns
+subjects:
+  - kind: ServiceAccount
+    name: {{ template "karpenter.serviceAccountName" . }}
+    namespace: {{ .Release.Namespace }}
+--- 
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ include "karpenter.fullname" . }}-lease
+  namespace: kube-node-lease
+  labels:
+    {{- include "karpenter.labels" . | nindent 4 }}
+  {{- with .Values.additionalAnnotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ include "karpenter.fullname" . }}-lease
 subjects:
   - kind: ServiceAccount
     name: {{ template "karpenter.serviceAccountName" . }}

charts/karpenter/values.yaml

@@ -170,6 +170,8 @@ settings:
     assumeRoleARN: ""
     # -- Duration of assumed credentials in minutes. Default value is 15 minutes. Not used unless aws.assumeRoleARN set.
     assumeRoleDuration: 15m
+    # -- Cluster CA bundle for TLS configuration of provisioned nodes. If not set, this is taken from the controller's TLS configuration for the API server.
+    clusterCABundle: ""
     # -- Cluster name.
     clusterName: ""
     # -- Cluster endpoint. If not set, will be discovered during startup (EKS only)

go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/PuerkitoBio/goquery v1.8.1
 	github.com/avast/retry-go v3.0.0+incompatible
 	github.com/aws/aws-sdk-go v1.44.328
-	github.com/aws/karpenter-core v0.30.0-rc.0.0.20230816121541-da2c2f5594a3
+	github.com/aws/karpenter-core v0.30.0-rc.0.0.20230822175121-c71cf73b5a52
 	github.com/go-playground/validator/v10 v10.15.1
 	github.com/imdario/mergo v0.3.16
 	github.com/mitchellh/hashstructure/v2 v2.0.2

go.sum

@@ -53,8 +53,8 @@ github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHS
 github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
 github.com/aws/aws-sdk-go v1.44.328 h1:WBwlf8ym9SDQ/GTIBO9eXyvwappKJyOetWJKl4mT7ZU=
 github.com/aws/aws-sdk-go v1.44.328/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
-github.com/aws/karpenter-core v0.30.0-rc.0.0.20230816121541-da2c2f5594a3 h1:sgiCpYhOZIOlIurhSBlsewuC/5QUPmFh6FyFFOnFGVw=
-github.com/aws/karpenter-core v0.30.0-rc.0.0.20230816121541-da2c2f5594a3/go.mod h1:lcICoIGQFejLdYTsD5lx+l24Zadnekacoc9urRxt+9s=
+github.com/aws/karpenter-core v0.30.0-rc.0.0.20230822175121-c71cf73b5a52 h1:vjppWOFi+YWrtNE3KhKka/2YLMet9obGiq/dsikxDJQ=
+github.com/aws/karpenter-core v0.30.0-rc.0.0.20230822175121-c71cf73b5a52/go.mod h1:AQl8m8OtgO2N8IlZlzAU6MTrJTJSbe6K4GwdRUNSJVc=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
 github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=

hack/docs/metrics_gen_docs.go

@@ -267,7 +267,7 @@ func getIdentMapping(identName string) (string, error) {
 		"metrics.Namespace": metrics.Namespace,
 		"Namespace":         metrics.Namespace,
 
-		"nodeSubsystem":           "nodes",
+		"NodeSubsystem":           "nodes",
 		"machineSubsystem":        "machines",
 		"nodeClaimSubsystem":      "nodeclaims",
 		"interruptionSubsystem":   "interruption",

pkg/apis/crds/karpenter.sh_machines.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.12.1
+    controller-gen.kubebuilder.io/version: v0.13.0
   name: machines.karpenter.sh
 spec:
   group: karpenter.sh

pkg/apis/crds/karpenter.sh_nodeclaims.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.12.1
+    controller-gen.kubebuilder.io/version: v0.13.0
   name: nodeclaims.karpenter.sh
 spec:
   group: karpenter.sh

pkg/apis/crds/karpenter.sh_nodepools.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.12.1
+    controller-gen.kubebuilder.io/version: v0.13.0
   name: nodepools.karpenter.sh
 spec:
   group: karpenter.sh

pkg/apis/crds/karpenter.sh_provisioners.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.12.1
+    controller-gen.kubebuilder.io/version: v0.13.0
   name: provisioners.karpenter.sh
 spec:
   group: karpenter.sh

pkg/apis/settings/settings.go

@@ -37,6 +37,7 @@ var ContextKey = settingsKeyType{}
 var defaultSettings = &Settings{
 	AssumeRoleARN:              "",
 	AssumeRoleDuration:         time.Minute * 15,
+	ClusterCABundle:            "",
 	ClusterName:                "",
 	ClusterEndpoint:            "",
 	DefaultInstanceProfile:     "",
@@ -53,7 +54,8 @@ var defaultSettings = &Settings{
 type Settings struct {
 	AssumeRoleARN              string
 	AssumeRoleDuration         time.Duration `validate:"min=15m"`
-	ClusterName                string        `validate:"required"`
+	ClusterCABundle            string
+	ClusterName                string `validate:"required"`
 	ClusterEndpoint            string
 	DefaultInstanceProfile     string
 	EnablePodENI               bool
@@ -76,6 +78,7 @@ func (*Settings) Inject(ctx context.Context, cm *v1.ConfigMap) (context.Context,
 	if err := configmap.Parse(cm.Data,
 		configmap.AsString("aws.assumeRoleARN", &s.AssumeRoleARN),
 		configmap.AsDuration("aws.assumeRoleDuration", &s.AssumeRoleDuration),
+		configmap.AsString("aws.clusterCABundle", &s.ClusterCABundle),
 		configmap.AsString("aws.clusterName", &s.ClusterName),
 		configmap.AsString("aws.clusterEndpoint", &s.ClusterEndpoint),
 		configmap.AsString("aws.defaultInstanceProfile", &s.DefaultInstanceProfile),

pkg/apis/settings/suite_test.go

@@ -48,6 +48,7 @@ var _ = Describe("Validation", func() {
 		s := settings.FromContext(ctx)
 		Expect(s.AssumeRoleARN).To(Equal(""))
 		Expect(s.AssumeRoleDuration).To(Equal(time.Duration(15) * time.Minute))
+		Expect(s.ClusterCABundle).To(Equal(""))
 		Expect(s.DefaultInstanceProfile).To(Equal(""))
 		Expect(s.EnablePodENI).To(BeFalse())
 		Expect(s.EnableENILimitedPodDensity).To(BeTrue())
@@ -61,6 +62,7 @@ var _ = Describe("Validation", func() {
 			Data: map[string]string{
 				"aws.assumeRoleARN":              "arn:aws:iam::111222333444:role/testrole",
 				"aws.assumeRoleDuration":         "27m",
+				"aws.clusterCABundle":            "ca-bundle",
 				"aws.clusterEndpoint":            "https://00000000000000000000000.gr7.us-west-2.eks.amazonaws.com",
 				"aws.clusterName":                "my-cluster",
 				"aws.defaultInstanceProfile":     "karpenter",
@@ -77,6 +79,7 @@ var _ = Describe("Validation", func() {
 		s := settings.FromContext(ctx)
 		Expect(s.AssumeRoleARN).To(Equal("arn:aws:iam::111222333444:role/testrole"))
 		Expect(s.AssumeRoleDuration).To(Equal(time.Duration(27) * time.Minute))
+		Expect(s.ClusterCABundle).To(Equal("ca-bundle"))
 		Expect(s.DefaultInstanceProfile).To(Equal("karpenter"))
 		Expect(s.EnablePodENI).To(BeTrue())
 		Expect(s.EnableENILimitedPodDensity).To(BeFalse())

pkg/operator/operator.go

@@ -136,7 +136,7 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont
 		amiResolver,
 		securityGroupProvider,
 		subnetProvider,
-		lo.Must(getCABundle(operator.GetConfig())),
+		lo.Must(getCABundle(ctx, operator.GetConfig())),
 		operator.Elected(),
 		kubeDNSIP,
 		clusterEndpoint,
@@ -207,11 +207,14 @@ func ResolveClusterEndpoint(ctx context.Context, eksAPI eksiface.EKSAPI) (string
 	return *out.Cluster.Endpoint, nil
 }
 
-func getCABundle(restConfig *rest.Config) (*string, error) {
+func getCABundle(ctx context.Context, restConfig *rest.Config) (*string, error) {
 	// Discover CA Bundle from the REST client. We could alternatively
 	// have used the simpler client-go InClusterConfig() method.
 	// However, that only works when Karpenter is running as a Pod
 	// within the same cluster it's managing.
+	if caBundle := settings.FromContext(ctx).ClusterCABundle; caBundle != "" {
+		return lo.ToPtr(caBundle), nil
+	}
 	transportConfig, err := restConfig.TransportConfig()
 	if err != nil {
 		return nil, fmt.Errorf("discovering caBundle, loading transport config, %w", err)

test/Dockerfile

@@ -22,7 +22,7 @@ RUN set -ex; \
     ./aws/install --bin-dir /aws-cli-bin; \
     /aws-cli-bin/aws --version
 
-FROM golang:1.20-alpine
+FROM golang:1.21-alpine
 
 # Install APK packages
 RUN echo "@community http://dl-cdn.alpinelinux.org/alpine/edge/community" >>/etc/apk/repositories

test/pkg/environment/common/expectations.go

@@ -43,6 +43,7 @@ import (
 	pscheduling "github.com/aws/karpenter-core/pkg/controllers/provisioning/scheduling"
 	"github.com/aws/karpenter-core/pkg/scheduling"
 	"github.com/aws/karpenter-core/pkg/test"
+	nodepoolutil "github.com/aws/karpenter-core/pkg/utils/nodepool"
 )
 
 func (env *Environment) ExpectCreatedWithOffset(offset int, objects ...client.Object) {
@@ -572,8 +573,8 @@ func (env *Environment) GetDaemonSetCount(prov *v1alpha5.Provisioner) int {
 
 	return lo.CountBy(daemonSetList.Items, func(d appsv1.DaemonSet) bool {
 		p := &v1.Pod{Spec: d.Spec.Template.Spec}
-		nodeTemplate := pscheduling.NewMachineTemplate(prov)
-		if err := nodeTemplate.Taints.Tolerates(p); err != nil {
+		nodeTemplate := pscheduling.NewNodeClaimTemplate(nodepoolutil.New(prov))
+		if err := scheduling.Taints(nodeTemplate.Spec.Taints).Tolerates(p); err != nil {
 			return false
 		}
 		if err := nodeTemplate.Requirements.Compatible(scheduling.NewPodRequirements(p)); err != nil {

test/suites/integration/lease_garbagecollection_test.go

@@ -0,0 +1,44 @@
+/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package integration_test
+
+import (
+	"time"
+
+	coordinationsv1 "k8s.io/api/coordination/v1"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/aws/karpenter-core/pkg/test"
+
+	. "github.com/onsi/ginkgo/v2"
+)
+
+var _ = Describe("Lease Garbage Collection", func() {
+	var badLease *coordinationsv1.Lease
+	BeforeEach(func() {
+		badLease = &coordinationsv1.Lease{
+			ObjectMeta: v1.ObjectMeta{
+				CreationTimestamp: v1.Time{Time: time.Now().Add(-time.Hour * 2)},
+				Name:              "new-lease",
+				Namespace:         "kube-node-lease",
+				Labels:            map[string]string{test.DiscoveryLabel: "unspecified"},
+			},
+		}
+	})
+	It("should delete node lease that does not contain an OwnerReference", func() {
+		env.ExpectCreated(badLease)
+		env.EventuallyExpectNotFound(badLease)
+	})
+})

website/content/en/docs/concepts/provisioners.md

@@ -436,7 +436,7 @@ This field points to the cloud provider-specific custom resource. Learn more abo
 
 ## spec.consolidation
 
-You can configure Karpenter to deprovision instances through your Provisioner in multiple ways. You can use `spec.TTLSecondsAfterEmpty`, `spec.ttlSecondsUntilExpired` or `spec.consolidation.enabled`. Read [Deprovisioning](../deprovisioning/) for more.
+You can configure Karpenter to deprovision instances through your Provisioner in multiple ways. You can use `spec.ttlSecondsAfterEmpty`, `spec.ttlSecondsUntilExpired` or `spec.consolidation.enabled`. Read [Deprovisioning](../deprovisioning/) for more.
 
 ## Example Use-Cases
 

website/content/en/preview/concepts/provisioners.md

@@ -436,7 +436,7 @@ This field points to the cloud provider-specific custom resource. Learn more abo
 
 ## spec.consolidation
 
-You can configure Karpenter to deprovision instances through your Provisioner in multiple ways. You can use `spec.TTLSecondsAfterEmpty`, `spec.ttlSecondsUntilExpired` or `spec.consolidation.enabled`. Read [Deprovisioning](../deprovisioning/) for more.
+You can configure Karpenter to deprovision instances through your Provisioner in multiple ways. You can use `spec.ttlSecondsAfterEmpty`, `spec.ttlSecondsUntilExpired` or `spec.consolidation.enabled`. Read [Deprovisioning](../deprovisioning/) for more.
 
 ## Example Use-Cases
 

website/content/en/preview/concepts/settings.md

@@ -49,6 +49,8 @@ data:
   aws.assumeRoleARN: arn:aws:iam::111222333444:role/examplerole
   # Duration of assumed credentials in minutes. Default value is 15 minutes. Not used unless aws.assumeRole set.
   aws.assumeRoleDuration: 15m
+  # Cluster CA bundle for nodes to use for TLS connections with the API server. If not set, this is taken from the controller's TLS configuration.
+  aws.clusterCABundle: "LS0tLS1..."
   # [REQUIRED] The kubernetes cluster name for resource discovery
   aws.clusterName: karpenter-cluster
   # [REQUIRED] The external kubernetes cluster endpoint for new nodes to connect with

website/content/en/preview/faq.md

@@ -175,6 +175,15 @@ To avoid new DaemonSets failing to schedule to existing Nodes, you should [set a
 
 The Karpenter maintainer team is also discussing a consolidation mechanism [in this Github issue](https://github.com/aws/karpenter/issues/3256) that would allow existing capacity to be rolled when a new DaemonSet is deployed without having to set [priority or preemption](https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/) on the pods.
 
+
+### Why aren’t my Topology Spread Constraints spreading pods across zones?
+
+Karpenter will provision nodes according to `topologySpreadConstraints`. However, the Kubernetes scheduler may schedule pods to nodes that do not fulfill zonal spread constraints if the `minDomains` field is not set. If Karpenter launches nodes that can handle more than the required number of pods, and the newly launched nodes initialize at different times, then the Kubernetes scheduler may place more than the desired number of pods on the first node that is Ready.
+
+The preferred solution is to use the [`minDomains` field in `topologySpreadConstraints`](https://kubernetes.io/docs/concepts/scheduling-eviction/topology-spread-constraints/#topologyspreadconstraints-field), which is enabled by default starting in Kubernetes 1.27.
+
+Before `minDomains` was available, another workaround has been to launch a lower [Priority](https://kubernetes.io/docs/concepts/scheduling-eviction/pod-priority-preemption/) pause container to each zone before launching the pods that you want to spread across the zones. The lower Priority on these pause pods would mean that they would be preempted when your desired pods are scheduled.
+
 ## Workloads
 
 ### How can someone deploying pods take advantage of Karpenter?

website/content/en/v0.26/concepts/provisioners.md

@@ -425,7 +425,7 @@ This field points to the cloud provider-specific custom resource. Learn more abo
 
 ## spec.consolidation
 
-You can configure Karpenter to deprovision instances through your Provisioner in multiple ways. You can use `spec.TTLSecondsAfterEmpty`, `spec.ttlSecondsUntilExpired` or `spec.consolidation.enabled`. Read [Deprovisioning](../deprovisioning/) for more.
+You can configure Karpenter to deprovision instances through your Provisioner in multiple ways. You can use `spec.ttlSecondsAfterEmpty`, `spec.ttlSecondsUntilExpired` or `spec.consolidation.enabled`. Read [Deprovisioning](../deprovisioning/) for more.
 
 ## Example Use-Cases
 

website/content/en/v0.27/concepts/provisioners.md

@@ -435,7 +435,7 @@ This field points to the cloud provider-specific custom resource. Learn more abo
 
 ## spec.consolidation
 
-You can configure Karpenter to deprovision instances through your Provisioner in multiple ways. You can use `spec.TTLSecondsAfterEmpty`, `spec.ttlSecondsUntilExpired` or `spec.consolidation.enabled`. Read [Deprovisioning](../deprovisioning/) for more.
+You can configure Karpenter to deprovision instances through your Provisioner in multiple ways. You can use `spec.ttlSecondsAfterEmpty`, `spec.ttlSecondsUntilExpired` or `spec.consolidation.enabled`. Read [Deprovisioning](../deprovisioning/) for more.
 
 ## Example Use-Cases
 

website/content/en/v0.28/concepts/provisioners.md

@@ -435,7 +435,7 @@ This field points to the cloud provider-specific custom resource. Learn more abo
 
 ## spec.consolidation
 
-You can configure Karpenter to deprovision instances through your Provisioner in multiple ways. You can use `spec.TTLSecondsAfterEmpty`, `spec.ttlSecondsUntilExpired` or `spec.consolidation.enabled`. Read [Deprovisioning](../deprovisioning/) for more.
+You can configure Karpenter to deprovision instances through your Provisioner in multiple ways. You can use `spec.ttlSecondsAfterEmpty`, `spec.ttlSecondsUntilExpired` or `spec.consolidation.enabled`. Read [Deprovisioning](../deprovisioning/) for more.
 
 ## Example Use-Cases
 

website/content/en/v0.29/concepts/provisioners.md

@@ -436,7 +436,7 @@ This field points to the cloud provider-specific custom resource. Learn more abo
 
 ## spec.consolidation
 
-You can configure Karpenter to deprovision instances through your Provisioner in multiple ways. You can use `spec.TTLSecondsAfterEmpty`, `spec.ttlSecondsUntilExpired` or `spec.consolidation.enabled`. Read [Deprovisioning](../deprovisioning/) for more.
+You can configure Karpenter to deprovision instances through your Provisioner in multiple ways. You can use `spec.ttlSecondsAfterEmpty`, `spec.ttlSecondsUntilExpired` or `spec.consolidation.enabled`. Read [Deprovisioning](../deprovisioning/) for more.
 
 ## Example Use-Cases
 

website/content/en/v0.30.0-rc/concepts/provisioners.md

@@ -436,7 +436,7 @@ This field points to the cloud provider-specific custom resource. Learn more abo
 
 ## spec.consolidation
 
-You can configure Karpenter to deprovision instances through your Provisioner in multiple ways. You can use `spec.TTLSecondsAfterEmpty`, `spec.ttlSecondsUntilExpired` or `spec.consolidation.enabled`. Read [Deprovisioning](../deprovisioning/) for more.
+You can configure Karpenter to deprovision instances through your Provisioner in multiple ways. You can use `spec.ttlSecondsAfterEmpty`, `spec.ttlSecondsUntilExpired` or `spec.consolidation.enabled`. Read [Deprovisioning](../deprovisioning/) for more.
 
 ## Example Use-Cases