Merge branch 'main' into networkinterfaces-simplified

.github/actions/e2e/cleanup/action.yaml

@@ -30,6 +30,32 @@ runs:
     - uses: ./.github/actions/e2e/install-eksctl
       with:
         eksctl_version: v0.147.0
+    - name: delete-instance-profiles
+      shell:
+      run: |
+        for name in $(aws iam list-instance-profiles --query "InstanceProfiles[*].{Name:InstanceProfileName}" --output text); do
+          tags=$(aws iam list-instance-profile-tags --instance-profile-name $name --output json || true)
+          if [[ $(echo $tags | jq -r '.Tags[] | select(.Key == "testing.karpenter.sh/cluster") | .Value') == "${{ inputs.cluster_name }}" ]]; then 
+              roleName=$(aws iam get-instance-profile --instance-profile-name $name --query "InstanceProfile.Roles[*].{Name:RoleName}" --output text)
+              aws iam remove-role-from-instance-profile --instance-profile-name $name --role-name $roleName
+              aws iam delete-instance-profile --instance-profile-name $name
+              break
+          fi
+        done
+    - name: delete-security-group
+      shell: bash
+      # For drift testing, we create a security group and need to clean it up here 
+      # to avoid leaks if the tests is not fully completed
+      run: | 
+        aws ec2 describe-security-groups \
+          --filters Name=group-name,Values=security-group-drift Name=tag:karpenter.sh/discovery,Values=${{ inputs.cluster_name }} \
+          --query "SecurityGroups[*].{ID:GroupId}" \
+          --output text |
+        xargs \
+          -n 1 \
+          -r \
+          aws ec2 delete-security-group \
+            --group-id
     - name: delete-cluster
       shell: bash
       run: |

.github/actions/e2e/create-cluster/action.yaml

@@ -125,6 +125,14 @@ runs:
         wellKnownPolicies:
           ebsCSIController: true
       EOF
+  - name: tag oidc provider of the cluster
+    if: always()
+    shell: bash
+    run: |
+      oidc_id=$(aws eks describe-cluster --name ${{ inputs.cluster_name }} --query "cluster.identity.oidc.issuer" --output text | cut -d '/' -f 3,4,5)
+      arn="arn:aws:iam::${{ inputs.account_id }}:oidc-provider/${oidc_id}"
+      aws iam tag-open-id-connect-provider --open-id-connect-provider-arn $arn \
+         --tags Key=testing.karpenter.sh/type,Value=e2e  Key=github.com/run-url,Value=https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
   - name: give KarpenterNodeRole permission to bootstrap
     shell: bash
     run: |

.github/actions/e2e/install-prometheus/action.yaml

@@ -43,12 +43,15 @@ runs:
           --set prometheus.prometheusSpec.remoteWrite[0].url=https://aps-workspaces.${{ inputs.region }}.amazonaws.com/workspaces/${{ inputs.workspace_id }}/api/v1/remote_write \
           --set prometheus.prometheusSpec.remoteWrite[0].sigv4.region=${{ inputs.region }} \
           --set prometheus.serviceAccount.annotations."eks\.amazonaws\.com/role-arn"="arn:aws:iam::${{ inputs.account_id }}:role/prometheus-irsa-${{ inputs.cluster_name }}" \
-          --set "kubelet.serviceMonitor.relabelings[0].targetLabel=clusterName" \
-          --set "kubelet.serviceMonitor.relabelings[0].replacement=${{ inputs.cluster_name }}" \
-          --set "kubelet.serviceMonitor.relabelings[1].targetLabel=gitRef" \
-          --set "kubelet.serviceMonitor.relabelings[1].replacement=$(git rev-parse HEAD)" \
-          --set "kubelet.serviceMonitor.relabelings[2].targetLabel=mostRecentTag" \
-          --set "kubelet.serviceMonitor.relabelings[2].replacement=$(git describe --abbrev=0 --tags)" \
-          --set "kubelet.serviceMonitor.relabelings[3].targetLabel=commitsAfterTag" \
-          --set "kubelet.serviceMonitor.relabelings[3].replacement=\"$(git describe --tags | cut -d '-' -f 2)\"" \
+          --set "kubelet.serviceMonitor.cAdvisorRelabelings[0].targetLabel=metrics_path" \
+          --set "kubelet.serviceMonitor.cAdvisorRelabelings[0].action=replace" \
+          --set "kubelet.serviceMonitor.cAdvisorRelabelings[0].sourceLabels[0]=__metrics_path__" \
+          --set "kubelet.serviceMonitor.cAdvisorRelabelings[1].targetLabel=clusterName" \
+          --set "kubelet.serviceMonitor.cAdvisorRelabelings[1].replacement=${{ inputs.cluster_name }}" \
+          --set "kubelet.serviceMonitor.cAdvisorRelabelings[2].targetLabel=gitRef" \
+          --set "kubelet.serviceMonitor.cAdvisorRelabelings[2].replacement=$(git rev-parse HEAD)" \
+          --set "kubelet.serviceMonitor.cAdvisorRelabelings[3].targetLabel=mostRecentTag" \
+          --set "kubelet.serviceMonitor.cAdvisorRelabelings[3].replacement=$(git describe --abbrev=0 --tags)" \
+          --set "kubelet.serviceMonitor.cAdvisorRelabelings[4].targetLabel=commitsAfterTag" \
+          --set "kubelet.serviceMonitor.cAdvisorRelabelings[4].replacement=\"$(git describe --tags | cut -d '-' -f 2)\"" \
           --wait

.github/actions/e2e/install-prometheus/values.yaml

@@ -28,6 +28,15 @@ alertmanager:
         operator: Exists
 kubelet:
   serviceMonitor:
+    # Enable CAdvisor metrics (this is enabled by default but explicitly specified for clarity)
+    # https://github.com/prometheus-community/helm-charts/blob/main/charts/kube-prometheus-stack/templates/exporters/kubelet/servicemonitor.yaml#L41
+    cAdvisor: true
+    # Disable probes scrape endpoint
+    # https://github.com/prometheus-community/helm-charts/blob/main/charts/kube-prometheus-stack/templates/exporters/kubelet/servicemonitor.yaml#L68
+    probes: false
+    # Disable resource scrape endpoint
+    # https://github.com/prometheus-community/helm-charts/blob/main/charts/kube-prometheus-stack/templates/exporters/kubelet/servicemonitor.yaml#L95
+    resource: false
     additionalLabels:
       scrape: enabled
 prometheus:

.github/pull-request-template.md

@@ -16,7 +16,7 @@ Fixes #N/A <!-- issue number -->
 **How was this change tested?**
 
 **Does this change impact docs?**
-- [ ] Yes, PR includes docs updates
+- [ ] Yes, PR includes docs updates <!-- docs must be added to /preview to be included in future version releases -->
 - [ ] Yes, issue opened: # <!-- issue number -->
 - [x] No
 

.github/workflows/sweeper.yaml

@@ -9,14 +9,18 @@ permissions:
 jobs:
   sweeper:
     if: github.repository == 'aws/karpenter' || github.event_name == 'workflow_dispatch'
+    strategy:
+      fail-fast: false
+      matrix:
+        region: [us-east-2, us-west-2, eu-west-1]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
       - name: configure aws credentials
         uses: aws-actions/configure-aws-credentials@v2
         with:
           role-to-assume: arn:aws:iam::${{ vars.ACCOUNT_ID }}:role/${{ vars.ROLE_NAME }}
-          aws-region: ${{ vars.AWS_REGION }}
+          aws-region: ${{ matrix.region }}
       - uses: actions/setup-go@v4
         with:
           go-version-file: test/hack/cleanup/go.mod

Makefile

@@ -31,7 +31,7 @@ HELM_OPTS ?= --set serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn=${K
 # CR for local builds of Karpenter
 SYSTEM_NAMESPACE ?= karpenter
 KARPENTER_VERSION ?= $(shell git tag --sort=committerdate | tail -1)
-KO_DOCKER_REPO ?= ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/karpenter
+KO_DOCKER_REPO ?= ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/dev
 GETTING_STARTED_SCRIPT_DIR = website/content/en/preview/getting-started/getting-started-with-karpenter/scripts
 
 # Common Directories
@@ -112,7 +112,7 @@ coverage:
 verify: tidy download ## Verify code. Includes dependencies, linting, formatting, etc
 	go generate ./...
 	hack/boilerplate.sh
-	cp $(KARPENTER_CORE_DIR)/pkg/apis/crds/* pkg/apis/crds
+	cp  $(KARPENTER_CORE_DIR)/pkg/apis/crds/* pkg/apis/crds
 	$(foreach dir,$(MOD_DIRS),cd $(dir) && golangci-lint run $(newline))
 	@git diff --quiet ||\
 		{ echo "New file modification detected in the Git working tree. Please check in before commit."; git --no-pager diff --name-only | uniq | awk '{print "  - " $$0}'; \
@@ -131,13 +131,13 @@ licenses: download ## Verifies dependency licenses
 setup: ## Sets up the IAM roles needed prior to deploying the karpenter-controller. This command only needs to be run once
 	CLUSTER_NAME=${CLUSTER_NAME} ./$(GETTING_STARTED_SCRIPT_DIR)/add-roles.sh $(KARPENTER_VERSION)
 
-build: ## Build the Karpenter controller images using ko build
-	$(eval CONTROLLER_IMG=$(shell $(WITH_GOFLAGS) KO_DOCKER_REPO="$(KO_DOCKER_REPO)" ko build -B github.com/aws/karpenter/cmd/controller))
+image: ## Build the Karpenter controller images using ko build
+	$(eval CONTROLLER_IMG=$(shell $(WITH_GOFLAGS) KO_DOCKER_REPO="$(KO_DOCKER_REPO)" ko build --bare github.com/aws/karpenter/cmd/controller))
 	$(eval IMG_REPOSITORY=$(shell echo $(CONTROLLER_IMG) | cut -d "@" -f 1 | cut -d ":" -f 1))
 	$(eval IMG_TAG=$(shell echo $(CONTROLLER_IMG) | cut -d "@" -f 1 | cut -d ":" -f 2 -s))
 	$(eval IMG_DIGEST=$(shell echo $(CONTROLLER_IMG) | cut -d "@" -f 2))
 
-apply: build ## Deploy the controller from the current state of your git repository into your ~/.kube/config cluster
+apply: image ## Deploy the controller from the current state of your git repository into your ~/.kube/config cluster
 	helm upgrade --install karpenter charts/karpenter --namespace ${SYSTEM_NAMESPACE} \
 		$(HELM_OPTS) \
 		--set controller.image.repository=$(IMG_REPOSITORY) \

charts/karpenter-crd/Chart.yaml

@@ -2,8 +2,8 @@ apiVersion: v2
 name: karpenter-crd
 description: A Helm chart for Karpenter Custom Resource Definitions (CRDs)
 type: application
-version: 0.29.2
-appVersion: 0.29.2
+version: 0.30.0-rc.0
+appVersion: 0.30.0-rc.0
 keywords:
   - cluster
   - node

charts/karpenter/Chart.yaml

@@ -2,8 +2,8 @@ apiVersion: v2
 name: karpenter
 description: A Helm chart for Karpenter, an open-source node provisioning project built for Kubernetes.
 type: application
-version: 0.29.2
-appVersion: 0.29.2
+version: 0.30.0-rc.0
+appVersion: 0.30.0-rc.0
 keywords:
   - cluster
   - node

charts/karpenter/README.md

@@ -2,7 +2,7 @@
 
 A Helm chart for Karpenter, an open-source node provisioning project built for Kubernetes.
 
-![Version: 0.29.2](https://img.shields.io/badge/Version-0.29.2-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.29.2](https://img.shields.io/badge/AppVersion-0.29.2-informational?style=flat-square)
+![Version: 0.30.0-rc.0](https://img.shields.io/badge/Version-0.30.0--rc.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.30.0-rc.0](https://img.shields.io/badge/AppVersion-0.30.0--rc.0-informational?style=flat-square)
 
 ## Documentation
 
@@ -15,7 +15,7 @@ You can follow the detailed installation instruction in the [documentation](http
 ```bash
 helm upgrade --install --namespace karpenter --create-namespace \
   karpenter oci://public.ecr.aws/karpenter/karpenter \
-  --version v0.29.2 \
+  --version v0.30.0-rc.0 \
   --set serviceAccount.annotations.eks\.amazonaws\.com/role-arn=${KARPENTER_IAM_ROLE_ARN} \
   --set settings.aws.clusterName=${CLUSTER_NAME} \
   --set settings.aws.clusterEndpoint=${CLUSTER_ENDPOINT} \
@@ -37,9 +37,9 @@ helm upgrade --install --namespace karpenter --create-namespace \
 | controller.errorOutputPaths | list | `["stderr"]` | Controller errorOutputPaths - default to stderr only |
 | controller.extraVolumeMounts | list | `[]` | Additional volumeMounts for the controller pod. |
 | controller.healthProbe.port | int | `8081` | The container port to use for http health probe. |
-| controller.image.digest | string | `"sha256:bac5ea470c09df21eb3742cf9448e9b806138ed5b6321d4e04697bbdf122eac6"` | SHA256 digest of the controller image. |
+| controller.image.digest | string | `"sha256:06f08a2c9b5125a2da57936a5ccfb54e5796677e247fb0bb960e70e79242a147"` | SHA256 digest of the controller image. |
 | controller.image.repository | string | `"public.ecr.aws/karpenter/controller"` | Repository path to the controller image. |
-| controller.image.tag | string | `"v0.29.2"` | Tag of the controller image. |
+| controller.image.tag | string | `"v0.30.0-rc.0"` | Tag of the controller image. |
 | controller.logEncoding | string | `""` | Controller log encoding, defaults to the global log encoding |
 | controller.logLevel | string | `""` | Controller log level, defaults to the global log level |
 | controller.metrics.port | int | `8000` | The container port to use for metrics. |

charts/karpenter/crds/karpenter.k8s.aws_awsnodetemplates.yaml

@@ -0,0 +1 @@
+../../../pkg/apis/crds/karpenter.k8s.aws_awsnodetemplates.yaml

charts/karpenter/crds/karpenter.sh_machines.yaml

@@ -0,0 +1 @@
+../../../pkg/apis/crds/karpenter.sh_machines.yaml

charts/karpenter/crds/karpenter.sh_provisioners.yaml

@@ -0,0 +1 @@
+../../../pkg/apis/crds/karpenter.sh_provisioners.yaml

charts/karpenter/values.yaml

@@ -99,9 +99,9 @@ controller:
     # -- Repository path to the controller image.
     repository: public.ecr.aws/karpenter/controller
     # -- Tag of the controller image.
-    tag: v0.29.2
+    tag: v0.30.0-rc.0
     # -- SHA256 digest of the controller image.
-    digest: sha256:bac5ea470c09df21eb3742cf9448e9b806138ed5b6321d4e04697bbdf122eac6
+    digest: sha256:06f08a2c9b5125a2da57936a5ccfb54e5796677e247fb0bb960e70e79242a147
   # -- SecurityContext for the controller container.
   securityContext: {}
   # -- Additional environment variables for the controller pod.
@@ -166,6 +166,10 @@ settings:
   batchIdleDuration: 1s
   # -- AWS-specific configuration values
   aws:
+    # -- Role to assume for calling AWS services.
+    assumeRoleARN: ""
+    # -- Duration of assumed credentials in minutes. Default value is 15 minutes. Not used unless aws.assumeRoleARN set.
+    assumeRoleDuration: 15m
     # -- Cluster name.
     clusterName: ""
     # -- Cluster endpoint. If not set, will be discovered during startup (EKS only)

go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/PuerkitoBio/goquery v1.8.1
 	github.com/avast/retry-go v3.0.0+incompatible
 	github.com/aws/aws-sdk-go v1.44.294
-	github.com/aws/karpenter-core v0.29.2-0.20230725171823-d34a21d5648d
+	github.com/aws/karpenter-core v0.30.0-rc.0
 	github.com/go-playground/validator/v10 v10.13.0
 	github.com/imdario/mergo v0.3.16
 	github.com/mitchellh/hashstructure/v2 v2.0.2

go.sum

@@ -53,8 +53,8 @@ github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHS
 github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
 github.com/aws/aws-sdk-go v1.44.294 h1:3x7GaEth+pDU9HwFcAU0awZlEix5CEdyIZvV08SlHa8=
 github.com/aws/aws-sdk-go v1.44.294/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
-github.com/aws/karpenter-core v0.29.2-0.20230725171823-d34a21d5648d h1:NRv2tSoPVyyYnLWco4w+OSXiEo434QLxNCq7ecxb8Eo=
-github.com/aws/karpenter-core v0.29.2-0.20230725171823-d34a21d5648d/go.mod h1:+C8X0N378fQ/+YmopvRHflj2JFrVP8sPs9xL7v4A6eM=
+github.com/aws/karpenter-core v0.30.0-rc.0 h1:Hhl+vZTOTdu8rrTNQkkOC1MPFmfBz0d6oS7d9thYfzU=
+github.com/aws/karpenter-core v0.30.0-rc.0/go.mod h1:90jYPLiwUxg0wFFE03Tqij4iUgvfYwAut8Hzxhm/3jY=
 github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=

hack/docs/metrics_gen_docs.go

@@ -79,6 +79,10 @@ description: >
 		"These metrics are available by default at `karpenter.karpenter.svc.cluster.local:8000/metrics` configurable via the `METRICS_PORT` environment variable documented [here](../settings)\n")
 	previousSubsystem := ""
 
+	// Ignore nodeClaimSubsystem metrics until NodeClaims are released
+	allMetrics = lo.Reject(allMetrics, func(m metricInfo, _ int) bool {
+		return m.subsystem == "nodeclaims"
+	})
 	for _, metric := range allMetrics {
 		// Controller Runtime naming is different in that they don't specify a namespace or subsystem
 		// Getting the metrics requires special parsing logic
@@ -265,6 +269,7 @@ func getIdentMapping(identName string) (string, error) {
 
 		"nodeSubsystem":           "nodes",
 		"machineSubsystem":        "machines",
+		"nodeClaimSubsystem":      "nodeclaims",
 		"interruptionSubsystem":   "interruption",
 		"nodeTemplateSubsystem":   "nodetemplate",
 		"deprovisioningSubsystem": "deprovisioning",

hack/release/common.sh

@@ -12,6 +12,7 @@ config(){
   CURRENT_MAJOR_VERSION="0"
   RELEASE_PLATFORM="--platform=linux/amd64,linux/arm64"
 
+  MAIN_GITHUB_ACCOUNT="aws"
   RELEASE_TYPE_STABLE="stable"
   RELEASE_TYPE_SNAPSHOT="snapshot"
 }

pkg/apis/apis.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/aws/karpenter-core/pkg/operator/scheme"
 	"github.com/aws/karpenter/pkg/apis/settings"
+	"github.com/aws/karpenter/pkg/apis/v1beta1"
 
 	"github.com/samber/lo"
 
@@ -36,6 +37,7 @@ var (
 	// Builder includes all types within the apis package
 	Builder = runtime.NewSchemeBuilder(
 		v1alpha1.SchemeBuilder.AddToScheme,
+		v1beta1.SchemeBuilder.AddToScheme,
 	)
 	// AddToScheme may be used to add all resources defined in the project to a Scheme
 	AddToScheme = Builder.AddToScheme
@@ -46,7 +48,12 @@ var (
 var (
 	//go:embed crds/karpenter.k8s.aws_awsnodetemplates.yaml
 	AWSNodeTemplateCRD []byte
-	CRDs               = append(apis.CRDs, lo.Must(functional.Unmarshal[v1.CustomResourceDefinition](AWSNodeTemplateCRD)))
+	//go:embed crds/compute.k8s.aws_nodeclasses.yaml
+	NodeClassCRD []byte
+	CRDs         = append(apis.CRDs,
+		lo.Must(functional.Unmarshal[v1.CustomResourceDefinition](AWSNodeTemplateCRD)),
+		lo.Must(functional.Unmarshal[v1.CustomResourceDefinition](NodeClassCRD)),
+	)
 )
 
 func init() {