Fixes post merge

.github/actions/e2e/cleanup/action.yaml

@@ -39,10 +39,10 @@ runs:
         for name in $(aws iam list-instance-profiles --query "InstanceProfiles[*].{Name:InstanceProfileName}" --output text); do
           tags=$(aws iam list-instance-profile-tags --instance-profile-name $name --output json || true)
           if [[ $(echo $tags | jq -r '.Tags[] | select(.Key == "testing/cluster") | .Value') == "${{ inputs.cluster_name }}" ]]; then
+              echo "Deleting instance profile '$name'..."
               roleName=$(aws iam get-instance-profile --instance-profile-name $name --query "InstanceProfile.Roles[*].{Name:RoleName}" --output text)
               aws iam remove-role-from-instance-profile --instance-profile-name $name --role-name $roleName
               aws iam delete-instance-profile --instance-profile-name $name
-              break
           fi
         done
     - name: delete-cluster

.github/actions/e2e/create-cluster/action.yaml

@@ -113,6 +113,10 @@ runs:
           - key: CriticalAddonsOnly
             value: "true"
             effect: NoSchedule
+      cloudWatch:
+        clusterLogging:
+          enableTypes: ["*"]
+          logRetentionInDays: 30
       iam:
         serviceRolePermissionsBoundary: "arn:aws:iam::${{ inputs.account_id }}:policy/GithubActionsPermissionsBoundary"
         serviceAccounts:
@@ -152,8 +156,6 @@ runs:
       # We need to call these update iamserviceaccount commands again since the "eksctl upgrade cluster" action
       # doesn't handle updates to IAM serviceaccounts correctly when the roles assigned to them change
       eksctl update iamserviceaccount -f clusterconfig.yaml --approve
-      
-
   - name: tag oidc provider of the cluster
     if: always()
     shell: bash

.github/workflows/e2e-upgrade.yaml

@@ -152,7 +152,7 @@ jobs:
       - name: run the Upgrade test suite
         run: |
           aws eks update-kubeconfig --name ${{ env.CLUSTER_NAME }}
-          TEST_SUITE="Integration" make e2etests
+          CLUSTER_NAME=${{ env.CLUSTER_NAME }} INTERRUPTION_QUEUE=${{ env.CLUSTER_NAME }} CLUSTER_ENDPOINT="$(aws eks describe-cluster --name ${{ env.CLUSTER_NAME }} --query "cluster.endpoint" --output text)" TEST_SUITE="Integration" make e2etests
       - name: notify slack of success or failure
         uses: ./.github/actions/e2e/slack/notify
         if: (success() || failure()) && github.event_name != 'workflow_run' && github.event_name != 'conformance'

.github/workflows/e2e.yaml

@@ -131,7 +131,9 @@ jobs:
       - name: run the ${{ inputs.suite }} test suite
         run: |
           aws eks update-kubeconfig --name ${{ env.CLUSTER_NAME }}
-          TEST_SUITE="${{ inputs.suite }}" ENABLE_METRICS=${{ inputs.enable_metrics }} METRICS_REGION=${{ vars.TIMESTREAM_REGION }} GIT_REF="$(git rev-parse HEAD)" make e2etests
+          TEST_SUITE="${{ inputs.suite }}" ENABLE_METRICS=${{ inputs.enable_metrics }} METRICS_REGION=${{ vars.TIMESTREAM_REGION }} GIT_REF="$(git rev-parse HEAD)" \
+            CLUSTER_NAME="${{ env.CLUSTER_NAME }}" CLUSTER_ENDPOINT="$(aws eks describe-cluster --name ${{ env.CLUSTER_NAME }} --query "cluster.endpoint" --output text)" \
+            INTERRUPTION_QUEUE="${{ env.CLUSTER_NAME }}" make e2etests
       - name: notify slack of success or failure
         uses: ./.github/actions/e2e/slack/notify
         if: (success() || failure()) && github.event_name != 'workflow_run' && inputs.workflow_trigger != 'conformance'

ADOPTERS.md

@@ -38,6 +38,7 @@ If you are open to others contacting you about your use of Karpenter on Slack, a
 | PITS Global Data Recovery Services | Used to manage continuous integration and continuous delivery/deployment workflows. | N/A | [PITS Global Data Recovery Services](https://www.pitsdatarecovery.net/) |
 | PlanetScale | Leveraging Karpenter to dynamically deploy serverless MySQL workloads. | `@jtcunning` | [Homepage](https://www.planetscale.com/) |
 | QuestDB | Using Karpenter for the service nodes of the QuestBD Cloud (time-series database). | [questdb slack group](https://slack.questdb.io/) | [QuestDB](https://questdb.io/) |
+| Rapid7 | Using Karpenter across all of our Kubernetes infrastructure for efficient autoscaling, both in terms of speed and cost | `@arobinson`, `@Ross Kirk`, `@Ryan Williams` | [Homepage](https://www.rapid7.com/) |
 | Sendcloud | Using Karpenter to scale our k8s clusters for Europe’s #1 shipping automation platform  | N/A | [Homepage](https://www.sendcloud.com/) |
 | Sentra | Using Karpenter to scale our EKS clusters, running our platform and workflows while maximizing cost-efficiency with minimal operational overhead | `@Roei Jacobovich` | [Homepage](https://sentra.io/) |
 | Stone Pagamentos | Using Karpenter to do smart sizing of our clusters  | `@fabiano-amaral` | [Stone Pagamentos](https://www.stone.com.br/) |

Makefile

@@ -1,12 +1,8 @@
 export K8S_VERSION ?= 1.27.x
 CLUSTER_NAME ?= $(shell kubectl config view --minify -o jsonpath='{.clusters[].name}' | rev | cut -d"/" -f1 | rev | cut -d"." -f1)
 
-## Inject the app version into project.Version
-ifdef SNAPSHOT_TAG
-LDFLAGS ?= -ldflags=-X=github.com/aws/karpenter/pkg/utils/project.Version=$(SNAPSHOT_TAG)
-else
-LDFLAGS ?= -ldflags=-X=github.com/aws/karpenter/pkg/utils/project.Version=$(shell git describe --tags --always)
-endif
+## Inject the app version into operator.Version
+LDFLAGS ?= -ldflags=-X=github.com/aws/karpenter-core/pkg/operator.Version=$(shell git describe --tags --always)
 
 GOFLAGS ?= $(LDFLAGS)
 WITH_GOFLAGS = GOFLAGS="$(GOFLAGS)"
@@ -68,6 +64,7 @@ clean-run: ## Clean resources deployed by the run target
 
 test: ## Run tests
 	go test -v ./pkg/$(shell echo $(TEST_SUITE) | tr A-Z a-z)/... --ginkgo.focus="${FOCUS}" --ginkgo.vv
+	cd tools/karpenter-convert && go test -v ./pkg/... --ginkgo.focus="${FOCUS}" --ginkgo.vv
 
 battletest: ## Run randomized, racing, code-covered tests
 	go test -v ./pkg/... \

charts/karpenter-crd/templates/karpenter.k8s.aws_ec2nodeclasses.yaml

@@ -0,0 +1 @@
+../../../pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml

charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml

@@ -0,0 +1 @@
+../../../pkg/apis/crds/karpenter.sh_nodeclaims.yaml

charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml

@@ -0,0 +1 @@
+../../../pkg/apis/crds/karpenter.sh_nodepools.yaml

charts/karpenter/README.md

@@ -58,6 +58,9 @@ helm upgrade --install --namespace karpenter --create-namespace \
 | logConfig.errorOutputPaths | list | `["stderr"]` | Log errorOutputPaths - defaults to stderr only |
 | logConfig.logEncoding | string | `"console"` | Log encoding - defaults to console - must be one of 'json', 'console' |
 | logConfig.logLevel | object | `{"controller":"debug","global":"debug","webhook":"error"}` | Component-based log configuration |
+| logConfig.logLevel.controller | string | `"debug"` | Controller log level, defaults to 'debug' |
+| logConfig.logLevel.global | string | `"debug"` | Global log level, defaults to 'debug' |
+| logConfig.logLevel.webhook | string | `"error"` | Error log level, defaults to 'error' |
 | logConfig.outputPaths | list | `["stdout"]` | Log outputPaths - defaults to stdout only |
 | logEncoding | string | `"console"` | Global log encoding (Deprecated: Use logConfig.logEncoding instead) |
 | logLevel | string | `"debug"` | Global log level |

charts/karpenter/crds/karpenter.k8s.aws_ec2nodeclasses.yaml

@@ -0,0 +1 @@
+../../../pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml

charts/karpenter/crds/karpenter.sh_nodeclaims.yaml

@@ -0,0 +1 @@
+../../../pkg/apis/crds/karpenter.sh_nodeclaims.yaml

charts/karpenter/crds/karpenter.sh_nodepools.yaml

@@ -0,0 +1 @@
+../../../pkg/apis/crds/karpenter.sh_nodepools.yaml

charts/karpenter/templates/aggregate-clusterrole.yaml

@@ -13,6 +13,12 @@ rules:
   - apiGroups: ["karpenter.sh"]
     resources: ["provisioners", "provisioners/status", "machines", "machines/status"]
     verbs: ["get", "list", "watch", "create", "delete", "patch"]
+  - apiGroups: ["karpenter.sh"]
+    resources: ["nodepools", "nodepools/status", "nodeclaims", "nodeclaims/status"]
+    verbs: ["get", "list", "watch", "create", "delete", "patch"]
   - apiGroups: ["karpenter.k8s.aws"]
     resources: ["awsnodetemplates"]
     verbs: ["get", "list", "watch", "create", "delete", "patch"]
+  - apiGroups: ["karpenter.k8s.aws"]
+    resources: ["ec2nodeclasses"]
+    verbs: ["get", "list", "watch", "create", "delete", "patch"]

charts/karpenter/templates/clusterrole-core.yaml

@@ -32,6 +32,9 @@ rules:
   - apiGroups: ["karpenter.sh"]
     resources: ["provisioners", "provisioners/status", "machines", "machines/status"]
     verbs: ["get", "list", "watch"]
+  - apiGroups: ["karpenter.sh"]
+    resources: ["nodepools", "nodepools/status", "nodeclaims", "nodeclaims/status"]
+    verbs: ["get", "list", "watch"]
   - apiGroups: [""]
     resources: ["pods", "nodes", "persistentvolumes", "persistentvolumeclaims", "replicationcontrollers", "namespaces"]
     verbs: ["get", "list", "watch"]
@@ -56,6 +59,12 @@ rules:
   - apiGroups: ["karpenter.sh"]
     resources: ["provisioners", "provisioners/status"]
     verbs: ["update", "patch"]
+  - apiGroups: ["karpenter.sh"]
+    resources: ["nodeclaims", "nodeclaims/status"]
+    verbs: ["create", "delete", "update", "patch"]
+  - apiGroups: ["karpenter.sh"]
+    resources: ["nodepools", "nodepools/status"]
+    verbs: ["update", "patch"]
   - apiGroups: [""]
     resources: ["events"]
     verbs: ["create", "patch"]

charts/karpenter/templates/clusterrole.yaml

@@ -30,11 +30,11 @@ metadata:
 rules:
   # Read
   - apiGroups: ["karpenter.k8s.aws"]
-    resources: ["awsnodetemplates"]
+    resources: ["awsnodetemplates", "ec2nodeclasses"]
     verbs: ["get", "list", "watch"]
   # Write
   - apiGroups: ["karpenter.k8s.aws"]
-    resources: ["awsnodetemplates", "awsnodetemplates/status"]
+    resources: ["awsnodetemplates", "awsnodetemplates/status", "ec2nodeclasses", "ec2nodeclasses/status"]
     verbs: ["patch", "update"]
 {{- if .Values.webhook.enabled }}
   - apiGroups: ["admissionregistration.k8s.io"]
@@ -45,4 +45,4 @@ rules:
     resources: ["mutatingwebhookconfigurations"]
     verbs: ["update"]
     resourceNames: ["defaulting.webhook.karpenter.k8s.aws"]
-{{- end }}
+{{- end }}

charts/karpenter/templates/webhooks-core.yaml

@@ -24,12 +24,36 @@ webhooks:
           - karpenter.sh
         apiVersions:
           - v1alpha5
+        operations:
+          - CREATE
+          - UPDATE
         resources:
           - provisioners
           - provisioners/status
+        scope: '*'
+      - apiGroups:
+          - karpenter.sh
+        apiVersions:
+          - v1beta1
+        operations:
+          - CREATE
+          - UPDATE
+          - DELETE
+        resources:
+          - nodeclaims
+          - nodeclaims/status
+        scope: '*'
+      - apiGroups:
+          - karpenter.sh
+        apiVersions:
+          - v1beta1
         operations:
           - CREATE
           - UPDATE
+        resources:
+          - nodepools
+          - nodepools/status
+        scope: '*'
 ---
 apiVersion: admissionregistration.k8s.io/v1
 kind: ValidatingWebhookConfiguration

charts/karpenter/templates/webhooks.yaml

@@ -31,16 +31,28 @@ webhooks:
           - awsnodetemplates
           - awsnodetemplates/status
         scope: '*'
+      - apiGroups:
+          - karpenter.k8s.aws
+        apiVersions:
+          - v1beta1
+        operations:
+          - CREATE
+          - UPDATE
+        resources:
+          - ec2nodeclasses
+          - ec2nodeclasses/status
+        scope: '*'
       - apiGroups:
           - karpenter.sh
         apiVersions:
           - v1alpha5
-        resources:
-          - provisioners
-          - provisioners/status
         operations:
           - CREATE
           - UPDATE
+        resources:
+          - provisioners
+          - provisioners/status
+        scope: '*'
 ---
 apiVersion: admissionregistration.k8s.io/v1
 kind: ValidatingWebhookConfiguration
@@ -74,14 +86,26 @@ webhooks:
           - awsnodetemplates
           - awsnodetemplates/status
         scope: '*'
+      - apiGroups:
+          - karpenter.k8s.aws
+        apiVersions:
+          - v1beta1
+        operations:
+          - CREATE
+          - UPDATE
+        resources:
+          - ec2nodeclasses
+          - ec2nodeclasses/status
+        scope: '*'
       - apiGroups:
           - karpenter.sh
         apiVersions:
           - v1alpha5
-        resources:
-          - provisioners
-          - provisioners/status
         operations:
           - CREATE
           - UPDATE
+        resources:
+          - provisioners
+          - provisioners/status
+        scope: '*'
 {{- end }}

charts/karpenter/values.yaml

@@ -168,8 +168,11 @@ logConfig:
   logEncoding: console
   # -- Component-based log configuration
   logLevel:
+    # -- Global log level, defaults to 'debug'
     global: debug
+    # -- Controller log level, defaults to 'debug'
     controller: debug
+    # -- Error log level, defaults to 'error'
     webhook: error
 # -- Global Settings to configure Karpenter
 settings:

cmd/controller/main.go

@@ -64,6 +64,7 @@ func main() {
 			op.SubnetProvider,
 			op.SecurityGroupProvider,
 			op.InstanceProfileProvider,
+			op.InstanceProvider,
 			op.PricingProvider,
 			op.AMIProvider,
 			op.LicenseProvider,

designs/v1beta1-api.md

@@ -153,11 +153,11 @@ spec:
       owner: amazon
   subnetSelectorTerms:
     - tags:
-        compute.k8s.aws/discovery: cluster-name
+        karpenter.sh/discovery: cluster-name
     - id: subnet-1234
   securityGroupSelectorTerms:
     - tags:
-        compute.k8s.aws/discovery: cluster-name
+        karpenter.sh/discovery: cluster-name
     - name: default-security-group
   role: karpenter-node-role
   userData: |
@@ -253,7 +253,7 @@ spec:
   nodeClass:
     name: default
     kind: EC2NodeClass
-    apiVersion: compute.k8s.aws/v1beta1
+    apiVersion: karpenter.k8s.aws/v1beta1
   taints:
     - key: example.com/special-taint
       effect: NoSchedule

examples/v1beta1/100-cpu-limit.yaml

@@ -0,0 +1,49 @@
+# This example NodePool limits the amount of compute managed by
+# Karpenter for this NodePool. Karpenter will not provision compute that
+# takes the pool over a total of 100 (virtual or physical) CPU cores.
+---
+apiVersion: karpenter.sh/v1beta1
+kind: NodePool
+metadata:
+  name: limitcpu100
+  annotations:
+    kubernetes.io/description: "NodePool to restrict the number of cpus provisioned to 100"
+spec:
+  template:
+    spec:
+      requirements:
+        - key: kubernetes.io/arch
+          operator: In
+          values: ["amd64"]
+        - key: kubernetes.io/os
+          operator: In
+          values: ["linux"]
+        - key: karpenter.sh/capacity-type
+          operator: In
+          values: ["on-demand"]
+        - key: karpenter.k8s.aws/instance-category
+          operator: In
+          values: ["c", "m", "r"]
+        - key: karpenter.k8s.aws/instance-generation
+          operator: Gt
+          values: ["2"]
+      nodeClassRef:
+        name: default
+  limits:
+    cpu: 100
+---
+apiVersion: karpenter.k8s.aws/v1beta1
+kind: EC2NodeClass
+metadata:
+  name: default
+  annotations:
+    kubernetes.io/description: "General purpose EC2NodeClass for running Amazon Linux 2 nodes"
+spec:
+  amiFamily: AL2 # Amazon Linux 2
+  role: "KarpenterNodeRole-${CLUSTER_NAME}" # replace with your cluster name
+  subnetSelectorTerms:
+    - tags:
+        karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name
+  securityGroupSelectorTerms:
+    - tags:
+        karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name