BREAKING CHANGE: Drop Alpha Settings and Add Beta Defaults to Chart (#…

…5159)

.github/actions/e2e/create-cluster/action.yaml

@@ -215,4 +215,5 @@ runs:
       ecr_account_id: ${{ inputs.ecr_account_id }}
       ecr_region: ${{ inputs.ecr_region }}
       cluster_name: ${{ inputs.cluster_name }}
+      k8s_version: ${{ inputs.k8s_version }}
       git_ref: ${{ inputs.git_ref }}

.github/actions/e2e/install-karpenter/action.yaml

@@ -19,6 +19,9 @@ inputs:
   cluster_name:
     description: 'Name of the cluster to be launched by eksctl'
     required: true
+  k8s_version:
+    description: 'Version of Kubernetes to use for the launched cluster'
+    default: "1.28"
   git_ref:
     description: "The git commit, tag, or branch to check out. Requires a corresponding Karpenter snapshot release"
 runs:
@@ -45,12 +48,21 @@ runs:
     shell: bash
     run: |
       aws eks update-kubeconfig --name "${{ inputs.cluster_name }}"
-        helm upgrade --install karpenter oci://${{ inputs.ecr_account_id }}.dkr.ecr.${{ inputs.ecr_region }}.amazonaws.com/karpenter/snapshot/karpenter \
+      
+      # Parse minor version to determine whether to enable the webhooks
+      VERSION=${{ inputs.k8s_version }}
+      RELEASE_VERSION_MINOR="${VERSION#*.}"
+      WEBHOOK_ENABLED=false
+      if (( RELEASE_VRESION_MINOR < 25 )); then
+        WEBHOOK_ENABLED=true
+      fi
+      
+      helm upgrade --install karpenter oci://${{ inputs.ecr_account_id }}.dkr.ecr.${{ inputs.ecr_region }}.amazonaws.com/karpenter/snapshot/karpenter \
         -n karpenter \
         --version "v0-$(git rev-parse HEAD)" \
         --set serviceAccount.annotations."eks\.amazonaws\.com/role-arn"="arn:aws:iam::${{ inputs.account_id }}:role/karpenter-irsa-${{ inputs.cluster_name }}" \
+        --set webhook.enabled=${WEBHOOK_ENABLED} \
         --set settings.clusterName="${{ inputs.cluster_name }}" \
-        --set settings.aws.defaultInstanceProfile="KarpenterNodeInstanceProfile-${{ inputs.cluster_name }}" \
         --set settings.interruptionQueue="${{ inputs.cluster_name }}" \
         --set controller.resources.requests.cpu=3 \
         --set controller.resources.requests.memory=3Gi \

Makefile

@@ -13,18 +13,15 @@ AWS_ACCOUNT_ID ?= $(shell aws sts get-caller-identity --query Account --output t
 KARPENTER_IAM_ROLE_ARN ?= arn:aws:iam::${AWS_ACCOUNT_ID}:role/${CLUSTER_NAME}-karpenter
 HELM_OPTS ?= --set serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn=${KARPENTER_IAM_ROLE_ARN} \
       		--set settings.clusterName=${CLUSTER_NAME} \
-			--set settings.clusterEndpoint=${CLUSTER_ENDPOINT} \
-			--set settings.aws.defaultInstanceProfile=KarpenterNodeInstanceProfile-${CLUSTER_NAME} \
 			--set settings.interruptionQueue=${CLUSTER_NAME} \
-			--set settings.featureGates.drift=true \
 			--set controller.resources.requests.cpu=1 \
 			--set controller.resources.requests.memory=1Gi \
 			--set controller.resources.limits.cpu=1 \
 			--set controller.resources.limits.memory=1Gi \
 			--create-namespace
 
 # CR for local builds of Karpenter
-SYSTEM_NAMESPACE ?= karpenter
+KARPENTER_NAMESPACE ?= karpenter
 KARPENTER_VERSION ?= $(shell git tag --sort=committerdate | tail -1)
 KO_DOCKER_REPO ?= ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/dev
 GETTING_STARTED_SCRIPT_DIR = website/content/en/preview/getting-started/getting-started-with-karpenter/scripts
@@ -47,24 +44,15 @@ ci-test: test coverage ## Runs tests and submits coverage
 ci-non-test: verify licenses vulncheck ## Runs checks other than tests
 
 run: ## Run Karpenter controller binary against your local cluster
-	kubectl create configmap -n ${SYSTEM_NAMESPACE} karpenter-global-settings \
-		--from-literal=aws.defaultInstanceProfile=KarpenterNodeInstanceProfile-${CLUSTER_NAME} \
-		--dry-run=client -o yaml | kubectl apply -f -
-
-
-	SYSTEM_NAMESPACE=${SYSTEM_NAMESPACE} \
+	SYSTEM_NAMESPACE=${KARPENTER_NAMESPACE} \
 		KUBERNETES_MIN_VERSION="1.19.0-0" \
 		LEADER_ELECT=false \
 		DISABLE_WEBHOOK=true \
 		CLUSTER_NAME=${CLUSTER_NAME} \
-		CLUSTER_ENDPOINT=${CLUSTER_ENDPOINT} \
 		INTERRUPTION_QUEUE=${CLUSTER_NAME} \
 		FEATURE_GATES="Drift=true" \
 		go run ./cmd/controller/main.go
 
-clean-run: ## Clean resources deployed by the run target
-	kubectl delete configmap -n ${SYSTEM_NAMESPACE} karpenter-global-settings --ignore-not-found
-
 test: ## Run tests
 	go test -v ./pkg/$(shell echo $(TEST_SUITE) | tr A-Z a-z)/... \
 		-cover -coverprofile=coverage.out -outputdir=. -coverpkg=./... \
@@ -150,15 +138,15 @@ image: ## Build the Karpenter controller images using ko build
 	$(eval IMG_DIGEST=$(shell echo $(CONTROLLER_IMG) | cut -d "@" -f 2))
 
 apply: image ## Deploy the controller from the current state of your git repository into your ~/.kube/config cluster
-	helm upgrade --install karpenter charts/karpenter --namespace ${SYSTEM_NAMESPACE} \
+	helm upgrade --install karpenter charts/karpenter --namespace ${KARPENTER_NAMESPACE} \
 		$(HELM_OPTS) \
 		--set controller.image.repository=$(IMG_REPOSITORY) \
 		--set controller.image.tag=$(IMG_TAG) \
 		--set controller.image.digest=$(IMG_DIGEST)
 
 install:  ## Deploy the latest released version into your ~/.kube/config cluster
 	@echo Upgrading to ${KARPENTER_VERSION}
-	helm upgrade --install karpenter oci://public.ecr.aws/karpenter/karpenter --version ${KARPENTER_VERSION} --namespace ${SYSTEM_NAMESPACE} \
+	helm upgrade --install karpenter oci://public.ecr.aws/karpenter/karpenter --version ${KARPENTER_VERSION} --namespace ${KARPENTER_NAMESPACE} \
 		$(HELM_OPTS)
 
 delete: ## Delete the controller from your ~/.kube/config cluster

charts/karpenter/README.md

@@ -10,15 +10,14 @@ For full Karpenter documentation please checkout [https://karpenter.sh](https://
 
 ## Installing the Chart
 
-You can follow the detailed installation instruction in the [documentation](https://karpenter.sh/docs/getting-started/getting-started-with-karpenter/#install) which covers the Karpenter prerequisites and installation options. The outcome of these instructions should result in something like the following command.
+You can follow the detailed installation instruction in the [documentation](https://karpenter.sh/docs/getting-started/getting-started-with-karpenter) which covers the Karpenter prerequisites and installation options. The outcome of these instructions should result in something like the following command.
 
 ```bash
 helm upgrade --install --namespace karpenter --create-namespace \
   karpenter oci://public.ecr.aws/karpenter/karpenter \
   --version v0.32.2 \
   --set "serviceAccount.annotations.eks\.amazonaws\.com/role-arn=${KARPENTER_IAM_ROLE_ARN}" \
   --set settings.clusterName=${CLUSTER_NAME} \
-  --set settings.clusterEndpoint=${CLUSTER_ENDPOINT} \
   --set settings.interruptionQueue=${CLUSTER_NAME} \
   --wait
 ```
@@ -49,16 +48,16 @@ helm upgrade --install --namespace karpenter --create-namespace \
 | hostNetwork | bool | `false` | Bind the pod to the host network. This is required when using a custom CNI. |
 | imagePullPolicy | string | `"IfNotPresent"` | Image pull policy for Docker images. |
 | imagePullSecrets | list | `[]` | Image pull secrets for Docker images. |
-| logConfig | object | `{"enabled":true,"errorOutputPaths":["stderr"],"logEncoding":"json","logLevel":{"controller":"debug","global":"debug","webhook":"error"},"outputPaths":["stdout"]}` | Log configuration (Deprecated: Logging configuration will be dropped by v1, use logLevel instead) |
-| logConfig.enabled | bool | `true` | Whether to enable provisioning and mounting the log ConfigMap |
+| logConfig | object | `{"enabled":false,"errorOutputPaths":["stderr"],"logEncoding":"json","logLevel":{"controller":"info","global":"info","webhook":"error"},"outputPaths":["stdout"]}` | Log configuration (Deprecated: Logging configuration will be dropped by v1, use logLevel instead) |
+| logConfig.enabled | bool | `false` | Whether to enable provisioning and mounting the log ConfigMap |
 | logConfig.errorOutputPaths | list | `["stderr"]` | Log errorOutputPaths - defaults to stderr only |
 | logConfig.logEncoding | string | `"json"` | Log encoding - defaults to json - must be one of 'json', 'console' |
-| logConfig.logLevel | object | `{"controller":"debug","global":"debug","webhook":"error"}` | Component-based log configuration |
-| logConfig.logLevel.controller | string | `"debug"` | Controller log level, defaults to 'debug' |
-| logConfig.logLevel.global | string | `"debug"` | Global log level, defaults to 'debug' |
+| logConfig.logLevel | object | `{"controller":"info","global":"info","webhook":"error"}` | Component-based log configuration |
+| logConfig.logLevel.controller | string | `"info"` | Controller log level, defaults to 'info' |
+| logConfig.logLevel.global | string | `"info"` | Global log level, defaults to 'info' |
 | logConfig.logLevel.webhook | string | `"error"` | Error log level, defaults to 'error' |
 | logConfig.outputPaths | list | `["stdout"]` | Log outputPaths - defaults to stdout only |
-| logLevel | string | `"debug"` | Global log level |
+| logLevel | string | `"info"` | Global log level, defaults to 'info' |
 | nameOverride | string | `""` | Overrides the chart's name. |
 | nodeSelector | object | `{"kubernetes.io/os":"linux"}` | Node selectors to schedule the pod to nodes with labels. |
 | podAnnotations | object | `{}` | Additional annotations for the pod. |
@@ -74,17 +73,16 @@ helm upgrade --install --namespace karpenter --create-namespace \
 | serviceMonitor.additionalLabels | object | `{}` | Additional labels for the ServiceMonitor. |
 | serviceMonitor.enabled | bool | `false` | Specifies whether a ServiceMonitor should be created. |
 | serviceMonitor.endpointConfig | object | `{}` | Endpoint configuration for the ServiceMonitor. |
-| settings | object | `{"assumeRoleARN":"","assumeRoleDuration":"15m","aws":{},"batchIdleDuration":"1s","batchMaxDuration":"10s","clusterCABundle":"","clusterEndpoint":"","clusterName":"","featureGates":{"drift":false},"interruptionQueue":"","isolatedVPC":false,"reservedENIs":"0","vmMemoryOverheadPercent":0.075}` | Global Settings to configure Karpenter |
+| settings | object | `{"assumeRoleARN":"","assumeRoleDuration":"15m","batchIdleDuration":"1s","batchMaxDuration":"10s","clusterCABundle":"","clusterEndpoint":"","clusterName":"","featureGates":{"drift":true},"interruptionQueue":"","isolatedVPC":false,"reservedENIs":"0","vmMemoryOverheadPercent":0.075}` | Global Settings to configure Karpenter |
 | settings.assumeRoleARN | string | `""` | Role to assume for calling AWS services. |
 | settings.assumeRoleDuration | string | `"15m"` | Duration of assumed credentials in minutes. Default value is 15 minutes. Not used unless assumeRoleARN set. |
-| settings.aws | object | `{}` | AWS-specific settings (Deprecated: The AWS block inside of settings was flattened into settings) |
 | settings.batchIdleDuration | string | `"1s"` | The maximum amount of time with no new ending pods that if exceeded ends the current batching window. If pods arrive faster than this time, the batching window will be extended up to the maxDuration. If they arrive slower, the pods will be batched separately. |
 | settings.batchMaxDuration | string | `"10s"` | The maximum length of a batch window. The longer this is, the more pods we can consider for provisioning at one time which usually results in fewer but larger nodes. |
 | settings.clusterCABundle | string | `""` | Cluster CA bundle for TLS configuration of provisioned nodes. If not set, this is taken from the controller's TLS configuration for the API server. |
 | settings.clusterEndpoint | string | `""` | Cluster endpoint. If not set, will be discovered during startup (EKS only) |
 | settings.clusterName | string | `""` | Cluster name. |
-| settings.featureGates | object | `{"drift":false}` | Feature Gate configuration values. Feature Gates will follow the same graduation process and requirements as feature gates in Kubernetes. More information here https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/#feature-gates-for-alpha-or-beta-features |
-| settings.featureGates.drift | bool | `false` | drift is in ALPHA and is disabled by default. Setting drift to true enables the drift disruption method to watch for drift between currently deployed nodes and the desired state of nodes set in provisioners and node templates |
+| settings.featureGates | object | `{"drift":true}` | Feature Gate configuration values. Feature Gates will follow the same graduation process and requirements as feature gates in Kubernetes. More information here https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/#feature-gates-for-alpha-or-beta-features |
+| settings.featureGates.drift | bool | `true` | drift is in BETA and is enabled by default. Setting drift to false disables the drift disruption method to watch for drift between currently deployed nodes and the desired state of nodes set in provisioners and node templates |
 | settings.interruptionQueue | string | `""` | interruptionQueue is disabled if not specified. Enabling interruption handling may require additional permissions on the controller service account. Additional permissions are outlined in the docs. |
 | settings.isolatedVPC | bool | `false` | If true then assume we can't reach AWS services which don't have a VPC endpoint This also has the effect of disabling look-ups to the AWS pricing endpoint |
 | settings.reservedENIs | string | `"0"` | Reserved ENIs are not included in the calculations for max-pods or kube-reserved This is most often used in the VPC CNI custom networking setup https://docs.aws.amazon.com/eks/latest/userguide/cni-custom-network.html |
@@ -93,7 +91,7 @@ helm upgrade --install --namespace karpenter --create-namespace \
 | terminationGracePeriodSeconds | string | `nil` | Override the default termination grace period for the pod. |
 | tolerations | list | `[{"key":"CriticalAddonsOnly","operator":"Exists"}]` | Tolerations to allow the pod to be scheduled to nodes with taints. |
 | topologySpreadConstraints | list | `[{"maxSkew":1,"topologyKey":"topology.kubernetes.io/zone","whenUnsatisfiable":"ScheduleAnyway"}]` | Topology spread constraints to increase the controller resilience by distributing pods across the cluster zones. If an explicit label selector is not provided one will be created from the pod selector labels. |
-| webhook.enabled | bool | `true` | Whether to enable the webhooks and webhook permissions. |
+| webhook.enabled | bool | `false` | Whether to enable the webhooks and webhook permissions. |
 | webhook.metrics.port | int | `8001` | The container port to use for webhook metrics. |
 | webhook.port | int | `8443` | The container port to use for the webhook. |
 

charts/karpenter/README.md.gotmpl

@@ -9,15 +9,14 @@ For full Karpenter documentation please checkout [https://karpenter.sh](https://
 
 ## Installing the Chart
 
-You can follow the detailed installation instruction in the [documentation](https://karpenter.sh/docs/getting-started/getting-started-with-karpenter/#install) which covers the Karpenter prerequisites and installation options. The outcome of these instructions should result in something like the following command.
+You can follow the detailed installation instruction in the [documentation](https://karpenter.sh/docs/getting-started/getting-started-with-karpenter) which covers the Karpenter prerequisites and installation options. The outcome of these instructions should result in something like the following command.
 
 ```bash
 helm upgrade --install --namespace karpenter --create-namespace \
   karpenter oci://public.ecr.aws/karpenter/{{ template "chart.name" . }} \
   --version v{{ template "chart.version" . }} \
   --set "serviceAccount.annotations.eks\.amazonaws\.com/role-arn=${KARPENTER_IAM_ROLE_ARN}" \
   --set settings.clusterName=${CLUSTER_NAME} \
-  --set settings.clusterEndpoint=${CLUSTER_ENDPOINT} \
   --set settings.interruptionQueue=${CLUSTER_NAME} \
   --wait
 ```

charts/karpenter/templates/_helpers.tpl

@@ -146,11 +146,6 @@ Flatten the stdout logging outputs from args provided
 */}}
 {{- define "karpenter.outputPathsList" -}}
 {{ $paths := list -}}
-{{- range .Values.controller.outputPaths -}}
-    {{- if not (has (printf "%s" . | quote) $paths) -}}
-        {{- $paths = printf "%s" . | quote  | append $paths -}}
-    {{- end -}}
-{{- end -}}
 {{- range .Values.logConfig.outputPaths -}}
     {{- if not (has (printf "%s" . | quote) $paths) -}}
         {{- $paths = printf "%s" . | quote  | append $paths -}}
@@ -164,11 +159,6 @@ Flatten the stderr logging outputs from args provided
 */}}
 {{- define "karpenter.errorOutputPathsList" -}}
 {{ $paths := list -}}
-{{- range .Values.controller.errorOutputPaths -}}
-    {{- if not (has (printf "%s" . | quote) $paths) -}}
-        {{- $paths = printf "%s" . | quote  | append $paths -}}
-    {{- end -}}
-{{- end -}}
 {{- range .Values.logConfig.errorOutputPaths -}}
     {{- if not (has (printf "%s" . | quote) $paths) -}}
         {{- $paths = printf "%s" . | quote  | append $paths -}}

charts/karpenter/templates/aggregate-clusterrole.yaml

@@ -10,15 +10,9 @@ metadata:
     {{- toYaml . | nindent 4 }}
   {{- end }}
 rules:
-  - apiGroups: ["karpenter.sh"]
-    resources: ["provisioners", "provisioners/status", "machines", "machines/status"]
-    verbs: ["get", "list", "watch", "create", "delete", "patch"]
   - apiGroups: ["karpenter.sh"]
     resources: ["nodepools", "nodepools/status", "nodeclaims", "nodeclaims/status"]
     verbs: ["get", "list", "watch", "create", "delete", "patch"]
-  - apiGroups: ["karpenter.k8s.aws"]
-    resources: ["awsnodetemplates"]
-    verbs: ["get", "list", "watch", "create", "delete", "patch"]
   - apiGroups: ["karpenter.k8s.aws"]
     resources: ["ec2nodeclasses"]
     verbs: ["get", "list", "watch", "create", "delete", "patch"]

charts/karpenter/templates/clusterrole-core.yaml

@@ -29,9 +29,6 @@ metadata:
   {{- end }}
 rules:
   # Read
-  - apiGroups: ["karpenter.sh"]
-    resources: ["provisioners", "provisioners/status", "machines", "machines/status"]
-    verbs: ["get", "list", "watch"]
   - apiGroups: ["karpenter.sh"]
     resources: ["nodepools", "nodepools/status", "nodeclaims", "nodeclaims/status"]
     verbs: ["get", "list", "watch"]
@@ -53,12 +50,6 @@ rules:
     resources: ["poddisruptionbudgets"]
     verbs: ["get", "list", "watch"]
   # Write
-  - apiGroups: ["karpenter.sh"]
-    resources: ["machines", "machines/status"]
-    verbs: ["create", "delete", "update", "patch"]
-  - apiGroups: ["karpenter.sh"]
-    resources: ["provisioners", "provisioners/status"]
-    verbs: ["update", "patch"]
   - apiGroups: ["karpenter.sh"]
     resources: ["nodeclaims", "nodeclaims/status"]
     verbs: ["create", "delete", "update", "patch"]

charts/karpenter/templates/clusterrole.yaml

@@ -30,11 +30,11 @@ metadata:
 rules:
   # Read
   - apiGroups: ["karpenter.k8s.aws"]
-    resources: ["awsnodetemplates", "ec2nodeclasses"]
+    resources: ["ec2nodeclasses"]
     verbs: ["get", "list", "watch"]
   # Write
   - apiGroups: ["karpenter.k8s.aws"]
-    resources: ["awsnodetemplates", "awsnodetemplates/status", "ec2nodeclasses", "ec2nodeclasses/status"]
+    resources: ["ec2nodeclasses", "ec2nodeclasses/status"]
     verbs: ["patch", "update"]
 {{- if .Values.webhook.enabled }}
   - apiGroups: ["admissionregistration.k8s.io"]

charts/karpenter/templates/configmap-logging.yaml

@@ -24,7 +24,7 @@ data:
       },
       "outputPaths": [{{ include "karpenter.outputPathsList" . }}],
       "errorOutputPaths": [{{ include "karpenter.errorOutputPathsList" . }}],
-      "encoding": "{{ or .Values.logEncoding .Values.logConfig.logEncoding }}",
+      "encoding": "{{ .Values.logConfig.logEncoding }}",
       "encoderConfig": {
         "timeKey": "time",
         "levelKey": "level",
@@ -36,6 +36,6 @@ data:
         "timeEncoder": "iso8601"
       }
     }
-  loglevel.controller: {{ or .Values.controller.logLevel .Values.logConfig.logLevel.controller }}
-  loglevel.webhook: {{ or .Values.webhook.logLevel .Values.logConfig.logLevel.webhook }}
+  loglevel.controller: {{ .Values.logConfig.logLevel.controller }}
+  loglevel.webhook: {{ .Values.logConfig.logLevel.webhook }}
 {{- end }}