Merge branch 'main' of https://github.com/aws/karpenter-provider-aws …

…into pageniate-instance-profile

.github/actions/e2e/cleanup/action.yaml

@@ -24,7 +24,7 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+    - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
       with:
         ref: ${{ inputs.git_ref }}
     - uses: ./.github/actions/e2e/install-eksctl
@@ -37,7 +37,7 @@ runs:
         CLUSTER_NAME: ${{ inputs.cluster_name }}
       run: |
         eksctl delete cluster --name "$CLUSTER_NAME" --timeout 60m --wait || true
-    - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
+    - uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1
       with:
         go-version-file: test/hack/resource/go.mod
         cache-dependency-path: test/hack/resource/go.sum

.github/actions/e2e/install-karpenter/action.yaml

@@ -30,7 +30,7 @@ inputs:
 runs:
   using: "composite"
   steps:
-  - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+  - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
     with:
       ref: ${{ inputs.git_ref }}
   - uses: ./.github/actions/e2e/install-helm

.github/actions/e2e/install-prometheus/action.yaml

@@ -27,7 +27,7 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+    - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
       with:
         ref: ${{ inputs.git_ref }}
     - uses: ./.github/actions/e2e/install-helm

.github/actions/e2e/install-prometheus/values.yaml

@@ -47,10 +47,10 @@ prometheus:
     resources:
       requests:
         cpu: 1
-        memory: 5Gi
+        memory: 15Gi
       limits:
         cpu: 1
-        memory: 5Gi
+        memory: 15Gi
     serviceMonitorSelector:
       matchLabels:
         scrape: enabled

.github/actions/e2e/setup-cluster/action.yaml

@@ -50,7 +50,7 @@ inputs:
 runs:
   using: "composite"
   steps:
-  - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+  - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
     with:
       ref: ${{ inputs.git_ref }}
   - uses: ./.github/actions/e2e/install-eksctl
@@ -78,6 +78,7 @@ runs:
         --capabilities CAPABILITY_NAMED_IAM \
         --parameter-overrides "ClusterName=$CLUSTER_NAME" \
         --tags "testing/type=e2e" "testing/cluster=$CLUSTER_NAME" "github.com/run-url=https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" "karpenter.sh/discovery=$CLUSTER_NAME"
+      aws iam attach-role-policy --policy-arn arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy --role-name KarpenterNodeRole-$CLUSTER_NAME
   - name: create or upgrade cluster
     shell: bash
     env:
@@ -174,6 +175,8 @@ runs:
         withOIDC: true
       addons:
       - name: amazon-cloudwatch-observability
+        # Pin addon version due to undiagnosed e2e failures after 1.6.0 release
+        version: '1.5.5-eksbuild.1'
         permissionsBoundary: "arn:aws:iam::$ACCOUNT_ID:policy/GithubActionsPermissionsBoundary"
       - name: vpc-cni
         permissionsBoundary: "arn:aws:iam::$ACCOUNT_ID:policy/GithubActionsPermissionsBoundary"
@@ -213,9 +216,22 @@ runs:
       fi
 
       # Adding taints after all necessary pods have scheduled to the manged node group nodes
-      # amazon-cloudwatch-observability pods do no not tolerate CriticalAddonsOnly=true:NoSchedule and 
+      # amazon-cloudwatch-observability pods do no not tolerate CriticalAddonsOnly=true:NoSchedule and
       # amazon-cloudwatch-observability addons does not allow to add tolerations to the addon pods as part of the advanced configuration
-      kubectl taint nodes CriticalAddonsOnly=true:NoSchedule --all
+      # Overwrite existing taints to ensure we don't fail here on upgrade
+      kubectl taint nodes CriticalAddonsOnly=true:NoSchedule --all --overwrite
+
+      # We delete DaemonSets that we don't care about because it causes inconsistencies in scheduling due to
+      # dcgm-exporter and neuron-monitor selecting on specific instance types
+      # See https://github.com/kubernetes-sigs/karpenter/issues/715 for more detail
+      kubectl delete daemonsets -n amazon-cloudwatch dcgm-exporter neuron-monitor --ignore-not-found
+
+      # We patch the priorityClass onto all DaemonSets to ensure that DaemonSets always schedule to nodes so we don't get scheduling inconsistencies
+      # See https://karpenter.sh/docs/faq/#when-deploying-an-additional-daemonset-to-my-cluster-why-does-karpenter-not-scale-up-my-nodes-to-support-the-extra-daemonset for more detail
+      # Additionally, we patch an everything toleration onto the daemonsets to prevent them from being included in drain operations.
+      for DAEMONSET in "cloudwatch-agent" "cloudwatch-agent-windows" "fluent-bit" "fluent-bit-windows"; do
+        kubectl patch daemonset -n amazon-cloudwatch $DAEMONSET -p '{"spec":{"template":{"spec":{"priorityClassName":"system-node-critical","tolerations": [{"operator": "Exists"}]}}}}' --type=merge
+      done
   - name: tag oidc provider of the cluster
     if: always()
     shell: bash

.github/actions/e2e/slack/notify/action.yaml

@@ -17,7 +17,7 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+    - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
       with:
         ref: ${{ inputs.git_ref }}
     - id: get-run-name

.github/actions/e2e/upgrade-crds/action.yaml

@@ -24,7 +24,7 @@ runs:
         role-to-assume: arn:aws:iam::${{ inputs.account_id }}:role/${{ inputs.role }}
         aws-region: ${{ inputs.region }}
         role-duration-seconds: 21600
-    - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
+    - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
       with:
         ref: ${{ inputs.git_ref }}
     - name: install-karpenter

.github/actions/install-deps/action.yaml

@@ -7,7 +7,7 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
+    - uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1
       id: setup-go
       with:
         go-version-file: go.mod

.github/workflows/e2e-matrix.yaml

@@ -95,7 +95,7 @@ jobs:
       statuses: write # ./.github/actions/commit-status/start
     uses: ./.github/workflows/e2e-upgrade.yaml
     with:
-      from_git_ref: 283e7b2a51ec73903a6d3f9362fc3009b898ef33
+      from_git_ref: 969530cc8ac4ee8a8c2efed9af823c44813b4ec2
       to_git_ref: ${{ inputs.git_ref }}
       region: ${{ inputs.region }}
       k8s_version: ${{ inputs.k8s_version }}

.github/workflows/e2e-soak-trigger.yaml

@@ -17,7 +17,7 @@ jobs:
       with:
        role-to-assume: arn:aws:iam::${{ vars.CI_ACCOUNT_ID }}:role/${{ vars.CI_ROLE_NAME }}
        aws-region: eu-north-1
-    - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
+    - uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1
       with:
         go-version-file: test/hack/soak/go.mod
         cache-dependency-path: test/hack/soak/go.sum

.github/workflows/e2e-upgrade.yaml

@@ -108,7 +108,7 @@ jobs:
           region: ${{ inputs.region }}
           cluster_name: ${{ steps.generate-cluster-name.outputs.CLUSTER_NAME }}
           k8s_version: ${{ inputs.k8s_version }}
-          eksctl_version: v0.169.0
+          eksctl_version: v0.175.0
           ip_family: IPv4 # Set the value to IPv6 if IPv6 suite, else IPv4
           git_ref: ${{ inputs.to_git_ref }}
           ecr_account_id: ${{ vars.SNAPSHOT_ACCOUNT_ID }}
@@ -136,7 +136,7 @@ jobs:
           suite: Upgrade
           git_ref: ${{ inputs.to_git_ref }}
       - name: add log retention policy
-        if: ${{ inputs.workflow_trigger != 'private_cluster' }}
+        if: always() && inputs.workflow_trigger != 'private_cluster'
         env:
           CLUSTER_NAME: ${{ steps.generate-cluster-name.outputs.CLUSTER_NAME }}
         run: |

.github/workflows/e2e.yaml

@@ -188,7 +188,7 @@ jobs:
           git_ref: ${{ inputs.git_ref }}
           workflow_trigger: ${{ inputs.workflow_trigger }}
       - name: add log retention policy
-        if: (success() || failure()) && inputs.workflow_trigger != 'private_cluster'
+        if:  always() && inputs.workflow_trigger != 'private_cluster'
         env:
           CLUSTER_NAME: ${{ steps.generate-cluster-name.outputs.CLUSTER_NAME }}
         run: |

.github/workflows/resource-count.yaml

@@ -20,7 +20,7 @@ jobs:
         with:
           role-to-assume: arn:aws:iam::${{ vars.CI_ACCOUNT_ID }}:role/${{ vars.CI_ROLE_NAME }}
           aws-region: ${{ matrix.region }}
-      - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
+      - uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1
         with:
           go-version-file: test/hack/resource/go.mod
           check-latest: true

.github/workflows/sweeper.yaml

@@ -20,7 +20,7 @@ jobs:
         with:
           role-to-assume: arn:aws:iam::${{ vars.CI_ACCOUNT_ID }}:role/${{ vars.CI_ROLE_NAME }}
           aws-region: ${{ matrix.region }}
-      - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
+      - uses: actions/setup-go@cdcb36043654635271a94b9a6d1392de5bb323a7 # v5.0.1
         with:
           go-version-file: test/hack/resource/go.mod
           check-latest: true

.golangci.yaml

@@ -1,16 +1,7 @@
 # See https://github.com/golangci/golangci-lint/blob/master/.golangci.example.yml
 run:
   tests: true
-
   timeout: 5m
-
-  skip-dirs:
-    - tools
-    - website
-    - hack
-    - charts
-    - designs
-
 linters:
   enable:
     - asciicheck
@@ -31,12 +22,13 @@ linters:
     - nilerr
   disable:
     - prealloc
-
 linters-settings:
   gocyclo:
     min-complexity: 11
   govet:
-    check-shadowing: true
+    enable-all: true
+    disable:
+      - fieldalignment
   revive:
     rules:
     - name: dot-imports
@@ -66,6 +58,12 @@ linters-settings:
 issues:
   fix: true
   exclude: ['declaration of "(err|ctx)" shadows declaration at']
+  exclude-dirs:
+    - tools
+    - website
+    - hack
+    - charts
+    - designs
   exclude-rules:
   - linters:
     - goheader

Makefile

@@ -128,7 +128,8 @@ image: ## Build the Karpenter controller images using ko build
 	$(eval IMG_TAG=$(shell echo $(CONTROLLER_IMG) | cut -d "@" -f 1 | cut -d ":" -f 2 -s))
 	$(eval IMG_DIGEST=$(shell echo $(CONTROLLER_IMG) | cut -d "@" -f 2))
 
-apply: image ## Deploy the controller from the current state of your git repository into your ~/.kube/config cluster
+apply: verify image ## Deploy the controller from the current state of your git repository into your ~/.kube/config cluster
+	kubectl apply -f ./pkg/apis/crds/
 	helm upgrade --install karpenter charts/karpenter --namespace ${KARPENTER_NAMESPACE} \
         $(HELM_OPTS) \
         --set logLevel=debug \

cmd/controller/main.go

@@ -31,14 +31,14 @@ import (
 
 func main() {
 	ctx, op := operator.NewOperator(coreoperator.NewOperator())
+
 	awsCloudProvider := cloudprovider.New(
 		op.InstanceTypesProvider,
 		op.InstanceProvider,
 		op.EventRecorder,
 		op.GetClient(),
 		op.AMIProvider,
 		op.SecurityGroupProvider,
-		op.SubnetProvider,
 	)
 	lo.Must0(op.AddHealthzCheck("cloud-provider", awsCloudProvider.LivenessProbe))
 	cloudProvider := metrics.Decorate(awsCloudProvider)

designs/v1beta1-api.md

@@ -346,6 +346,7 @@ status:
 8. `karpenter.k8s.aws/instance-cpu`
 9. `karpenter.k8s.aws/instance-cpu-manufacturer`
 10. `karpenter.k8s.aws/instance-memory`
+11. `karpenter.k8s.aws/instance-ebs-bandwidth`
 11. `karpenter.k8s.aws/instance-network-bandwidth`
 12. `karpenter.k8s.aws/instance-gpu-name`
 13. `karpenter.k8s.aws/instance-gpu-manufacturer`

go.mod

@@ -1,35 +1,36 @@
 module github.com/aws/karpenter-provider-aws
 
-go 1.22
+go 1.22.3
 
 require (
 	github.com/Pallinder/go-randomdata v1.2.0
-	github.com/PuerkitoBio/goquery v1.9.1
+	github.com/PuerkitoBio/goquery v1.9.2
 	github.com/avast/retry-go v3.0.0+incompatible
-	github.com/aws/aws-sdk-go v1.51.30
+	github.com/aws/aws-sdk-go v1.53.6
 	github.com/aws/karpenter-provider-aws/tools/kompat v0.0.0-20240410220356-6b868db24881
 	github.com/awslabs/amazon-eks-ami/nodeadm v0.0.0-20240229193347-cfab22a10647
-	github.com/awslabs/operatorpkg v0.0.0-20240502203521-a2115dcf4ac0
+	github.com/awslabs/operatorpkg v0.0.0-20240518001059-1e35978ba21b
 	github.com/go-logr/zapr v1.3.0
 	github.com/imdario/mergo v0.3.16
 	github.com/mitchellh/hashstructure/v2 v2.0.2
-	github.com/onsi/ginkgo/v2 v2.17.2
-	github.com/onsi/gomega v1.33.0
+	github.com/onsi/ginkgo/v2 v2.17.3
+	github.com/onsi/gomega v1.33.1
 	github.com/patrickmn/go-cache v2.1.0+incompatible
-	github.com/pelletier/go-toml/v2 v2.2.1
-	github.com/prometheus/client_golang v1.19.0
+	github.com/pelletier/go-toml/v2 v2.2.2
+	github.com/prometheus/client_golang v1.19.1
 	github.com/samber/lo v1.39.0
 	go.uber.org/multierr v1.11.0
 	go.uber.org/zap v1.27.0
 	golang.org/x/sync v0.7.0
-	k8s.io/api v0.29.3
-	k8s.io/apiextensions-apiserver v0.29.3
-	k8s.io/apimachinery v0.29.3
-	k8s.io/client-go v0.29.3
+	k8s.io/api v0.30.1
+	k8s.io/apiextensions-apiserver v0.30.1
+	k8s.io/apimachinery v0.30.1
+	k8s.io/client-go v0.30.1
+	k8s.io/klog/v2 v2.120.1
 	k8s.io/utils v0.0.0-20240102154912-e7106e64919e
 	knative.dev/pkg v0.0.0-20231010144348-ca8c009405dd
-	sigs.k8s.io/controller-runtime v0.17.3
-	sigs.k8s.io/karpenter v0.36.0
+	sigs.k8s.io/controller-runtime v0.18.2
+	sigs.k8s.io/karpenter v0.36.1-0.20240524020535-a30f67aaf181
 	sigs.k8s.io/yaml v1.4.0
 )
 
@@ -79,7 +80,7 @@ require (
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/prometheus/client_model v0.6.0 // indirect
+	github.com/prometheus/client_model v0.6.1 // indirect
 	github.com/prometheus/common v0.53.0 // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/prometheus/statsd_exporter v0.24.0 // indirect
@@ -94,7 +95,7 @@ require (
 	golang.org/x/oauth2 v0.18.0 // indirect
 	golang.org/x/sys v0.19.0 // indirect
 	golang.org/x/term v0.19.0 // indirect
-	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/text v0.15.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
 	golang.org/x/tools v0.20.0 // indirect
 	gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect
@@ -108,11 +109,10 @@ require (
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/cloud-provider v0.29.3 // indirect
-	k8s.io/component-base v0.29.3 // indirect
-	k8s.io/csi-translation-lib v0.29.3 // indirect
-	k8s.io/klog/v2 v2.120.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect
+	k8s.io/cloud-provider v0.30.1 // indirect
+	k8s.io/component-base v0.30.1 // indirect
+	k8s.io/csi-translation-lib v0.30.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
 )