diff --git a/.github/ISSUE_TEMPLATE/bug.yaml b/.github/ISSUE_TEMPLATE/bug.yaml index d1c4495e334f..9b02ca4f553d 100644 --- a/.github/ISSUE_TEMPLATE/bug.yaml +++ b/.github/ISSUE_TEMPLATE/bug.yaml @@ -4,12 +4,15 @@ labels: - bug - needs-triage body: +- type: markdown + attributes: + value: | + > [!IMPORTANT] + > If your issue is not specific to AWS, please cut a ticket in [kubernetes-sigs/karpenter](https://github.com/kubernetes-sigs/karpenter/issues/new/choose). - type: textarea attributes: label: Description value: | - ** READ BEFORE CONTINUING: If your issue is not specific to AWS, please cut a ticket in [kubernetes-sigs/karpenter](https://github.com/kubernetes-sigs/karpenter/issues/new/choose). - **Observed Behavior**: **Expected Behavior**: diff --git a/.github/ISSUE_TEMPLATE/feature.yaml b/.github/ISSUE_TEMPLATE/feature.yaml index 33ea8e0d71cc..af43d6a1a3d2 100644 --- a/.github/ISSUE_TEMPLATE/feature.yaml +++ b/.github/ISSUE_TEMPLATE/feature.yaml @@ -4,12 +4,15 @@ labels: - feature - needs-triage body: +- type: markdown + attributes: + value: | + > [!IMPORTANT] + > If your issue is not specific to AWS, please cut a ticket in [kubernetes-sigs/karpenter](https://github.com/kubernetes-sigs/karpenter/issues/new/choose). - type: textarea attributes: label: Description value: | - ** READ BEFORE CONTINUING: If your issue is not specific to AWS, please cut a ticket in [kubernetes-sigs/karpenter](https://github.com/kubernetes-sigs/karpenter/issues/new/choose). - **What problem are you trying to solve?** **How important is this feature to you?** diff --git a/ADOPTERS.md b/ADOPTERS.md index 89f8ac549a87..7ab9dd062e0b 100644 --- a/ADOPTERS.md +++ b/ADOPTERS.md @@ -28,6 +28,7 @@ If you are open to others contacting you about your use of Karpenter on Slack, a | Cordial | Using Karpenter to scale multiple EKS clusters quickly | `@dschaaff` | [Cordial](https://cordial.com) | | Dig Security | Protecting our customers data - Using Karpenter to manage production and development workloads on EKS, We are using only Spot Instances in production. | `@Shahar Danus` | [Dig Security](https://dig.security/) | | Docker | Using Karpenter to scale Docker Hub on our EKS clusters | N/A | [Docker](https://www.docker.com) | +| GlobalDots | Using Karpenter to scale Kubernetes clusters for a lot of our clients & for internal needs | `@vainkop` | [GlobalDots](https://globaldots.com) | | Grafana Labs | Using Karpenter as our Autoscaling tool on EKS | `@paulajulve`, `@logyball` | [Homepage](https://grafana.com/) & [Blog](https://grafana.com/blog/2023/11/09/how-grafana-labs-switched-to-karpenter-to-reduce-costs-and-complexities-in-amazon-eks/) | | H2O.ai | Dynamically scaling CPU and GPU nodes for AI workloads | `@Ophir Zahavi`, `@Asaf Oren` | [H2O.ai](https://h2o.ai/) | | idealo | Scaling multi-arch IPv6 clusters hosting web and event-driven applications | `@Heiko Rothe` | [Homepage](https://www.idealo.de) | @@ -49,6 +50,7 @@ If you are open to others contacting you about your use of Karpenter on Slack, a | The Scale Factory | Using Karpenter (controllers on EC2/Fargate) to efficiently scale K8s workloads and empowering our customer teams to do the same | `@marko` | [Homepage](https://www.scalefactory.com) | | Tyk Cloud | Scaling workloads for the Cloud Free plan | `@Artem Hluvchynskyi`, `@gowtham` | [Tyk Cloud](https://tyk.io/cloud/) | | Wehkamp | Using Karpenter to scale the EKS clusters for our e-commerce platforms | `@ChrisV` | [Wehkamp](https://www.wehkamp.nl) & [Wehkamp Techblog](https://medium.com/wehkamp-techblog)| +| Whoosh | Using Karpenter to scale the EKS clusters for many purposes | `@vainkop` | [Whoosh](https://whoosh.bike) | | Next Insurance | Using Karpenter to manage the nodes in all our EKS clusters, including dev and prod, on demand and spots | `@moshebs` | [Homepage](https://www.nextinsurance.com)| | Grover Group GmbH | We use Karpenter for efficient and cost effective scaling of our nodes in all of our EKS clusters | `@suraj2410` | [Homepage](https://www.grover.com/de-en) & [Engineering Techblog](https://engineering.grover.com)| | Logz.io | Using Karpenter in all of our EKS clusters for efficient and cost effective scaling of all our K8s workloads | `@pincher95`, `@Samplify` | [Homepage](https://logz.io/)| diff --git a/go.mod b/go.mod index e9dcb0627f19..c6a70d0b61f0 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.21 require ( github.com/Pallinder/go-randomdata v1.2.0 github.com/PuerkitoBio/goquery v1.8.1 - github.com/aws/aws-sdk-go v1.49.4 + github.com/aws/aws-sdk-go v1.49.13 github.com/aws/karpenter-provider-aws/tools/kompat v0.0.0-20231207011214-752356948623 github.com/go-logr/zapr v1.3.0 github.com/imdario/mergo v0.3.16 @@ -14,20 +14,20 @@ require ( github.com/onsi/gomega v1.30.0 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pelletier/go-toml/v2 v2.1.1 - github.com/prometheus/client_golang v1.17.0 + github.com/prometheus/client_golang v1.18.0 github.com/samber/lo v1.39.0 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.26.0 golang.org/x/sync v0.5.0 golang.org/x/time v0.5.0 - k8s.io/api v0.28.4 - k8s.io/apiextensions-apiserver v0.28.4 - k8s.io/apimachinery v0.28.4 - k8s.io/client-go v0.28.4 + k8s.io/api v0.29.0 + k8s.io/apiextensions-apiserver v0.29.0 + k8s.io/apimachinery v0.29.0 + k8s.io/client-go v0.29.0 k8s.io/utils v0.0.0-20230726121419-3b25d923346b knative.dev/pkg v0.0.0-20231010144348-ca8c009405dd sigs.k8s.io/controller-runtime v0.16.3 - sigs.k8s.io/karpenter v0.33.1-0.20231208060535-cc54b340f630 + sigs.k8s.io/karpenter v0.33.1-0.20231229170439-99f33e0a3e0c ) require ( @@ -46,10 +46,10 @@ require ( github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch v5.7.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.7.0 // indirect - github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-kit/log v0.2.1 // indirect github.com/go-logfmt/logfmt v0.6.0 // indirect - github.com/go-logr/logr v1.3.0 // indirect + github.com/go-logr/logr v1.4.1 // indirect github.com/go-openapi/jsonpointer v0.20.0 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.4 // indirect @@ -72,7 +72,7 @@ require ( github.com/kelseyhightower/envconfig v1.4.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect @@ -80,7 +80,7 @@ require ( github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/client_model v0.5.0 // indirect - github.com/prometheus/common v0.44.0 // indirect + github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/prometheus/statsd_exporter v0.24.0 // indirect github.com/rivo/uniseg v0.4.4 // indirect @@ -92,7 +92,7 @@ require ( golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect golang.org/x/net v0.17.0 // indirect golang.org/x/oauth2 v0.13.0 // indirect - golang.org/x/sys v0.14.0 // indirect + golang.org/x/sys v0.15.0 // indirect golang.org/x/term v0.13.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/tools v0.14.0 // indirect @@ -107,12 +107,12 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/cloud-provider v0.28.4 // indirect - k8s.io/component-base v0.28.4 // indirect - k8s.io/csi-translation-lib v0.28.4 // indirect + k8s.io/cloud-provider v0.29.0 // indirect + k8s.io/component-base v0.29.0 // indirect + k8s.io/csi-translation-lib v0.29.0 // indirect k8s.io/klog/v2 v2.110.1 // indirect k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) diff --git a/go.sum b/go.sum index cd1166ebaf52..647b9e87b6a2 100644 --- a/go.sum +++ b/go.sum @@ -55,8 +55,8 @@ github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= -github.com/aws/aws-sdk-go v1.49.4 h1:qiXsqEeLLhdLgUIyfr5ot+N/dGPWALmtM1SetRmbUlY= -github.com/aws/aws-sdk-go v1.49.4/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= +github.com/aws/aws-sdk-go v1.49.13 h1:f4mGztsgnx2dR9r8FQYa9YW/RsKb+N7bgef4UGrOW1Y= +github.com/aws/aws-sdk-go v1.49.13/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/aws/karpenter-provider-aws/tools/kompat v0.0.0-20231207011214-752356948623 h1:DQEFtmPyyMVHOyqva+DaWR6iAQG4h0KJbpSJAYlsnEo= github.com/aws/karpenter-provider-aws/tools/kompat v0.0.0-20231207011214-752356948623/go.mod h1:fpKKbSoh7nKrbAw8V44Ov1sgosfUvR1ZtyN9k44zHfY= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -96,8 +96,8 @@ github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/evanphx/json-patch/v5 v5.7.0 h1:nJqP7uwL84RJInrohHfW0Fx3awjbm8qZeFv0nW9SYGc= github.com/evanphx/json-patch/v5 v5.7.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= -github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= -github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -114,8 +114,9 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= -github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= @@ -254,8 +255,8 @@ github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= -github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4= @@ -296,8 +297,8 @@ github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqr github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= -github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= -github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= +github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= +github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -310,8 +311,8 @@ github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9 github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.35.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= -github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= -github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= +github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= +github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= @@ -531,11 +532,10 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= -golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= @@ -734,20 +734,20 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.28.4 h1:8ZBrLjwosLl/NYgv1P7EQLqoO8MGQApnbgH8tu3BMzY= -k8s.io/api v0.28.4/go.mod h1:axWTGrY88s/5YE+JSt4uUi6NMM+gur1en2REMR7IRj0= -k8s.io/apiextensions-apiserver v0.28.4 h1:AZpKY/7wQ8n+ZYDtNHbAJBb+N4AXXJvyZx6ww6yAJvU= -k8s.io/apiextensions-apiserver v0.28.4/go.mod h1:pgQIZ1U8eJSMQcENew/0ShUTlePcSGFq6dxSxf2mwPM= -k8s.io/apimachinery v0.28.4 h1:zOSJe1mc+GxuMnFzD4Z/U1wst50X28ZNsn5bhgIIao8= -k8s.io/apimachinery v0.28.4/go.mod h1:wI37ncBvfAoswfq626yPTe6Bz1c22L7uaJ8dho83mgg= -k8s.io/client-go v0.28.4 h1:Np5ocjlZcTrkyRJ3+T3PkXDpe4UpatQxj85+xjaD2wY= -k8s.io/client-go v0.28.4/go.mod h1:0VDZFpgoZfelyP5Wqu0/r/TRYcLYuJ2U1KEeoaPa1N4= -k8s.io/cloud-provider v0.28.4 h1:7obmeuJJ5CYTO9HANDqemf/d2v95U+F0t8aeH4jNOsQ= -k8s.io/cloud-provider v0.28.4/go.mod h1:xbhmGZ7wRHgXFP3SNsvdmFRO87KJIvirDYQA5ydMgGA= -k8s.io/component-base v0.28.4 h1:c/iQLWPdUgI90O+T9TeECg8o7N3YJTiuz2sKxILYcYo= -k8s.io/component-base v0.28.4/go.mod h1:m9hR0uvqXDybiGL2nf/3Lf0MerAfQXzkfWhUY58JUbU= -k8s.io/csi-translation-lib v0.28.4 h1:4TrU2zefZGU5HQCyPZvcPxkS6IowqZ/jBs2Qi/dPUpc= -k8s.io/csi-translation-lib v0.28.4/go.mod h1:oxwDdx0hyVqViINOUF7TGrVt51eqsOkQ0BTI+A9QcQs= +k8s.io/api v0.29.0 h1:NiCdQMY1QOp1H8lfRyeEf8eOwV6+0xA6XEE44ohDX2A= +k8s.io/api v0.29.0/go.mod h1:sdVmXoz2Bo/cb77Pxi71IPTSErEW32xa4aXwKH7gfBA= +k8s.io/apiextensions-apiserver v0.29.0 h1:0VuspFG7Hj+SxyF/Z/2T0uFbI5gb5LRgEyUVE3Q4lV0= +k8s.io/apiextensions-apiserver v0.29.0/go.mod h1:TKmpy3bTS0mr9pylH0nOt/QzQRrW7/h7yLdRForMZwc= +k8s.io/apimachinery v0.29.0 h1:+ACVktwyicPz0oc6MTMLwa2Pw3ouLAfAon1wPLtG48o= +k8s.io/apimachinery v0.29.0/go.mod h1:eVBxQ/cwiJxH58eK/jd/vAk4mrxmVlnpBH5J2GbMeis= +k8s.io/client-go v0.29.0 h1:KmlDtFcrdUzOYrBhXHgKw5ycWzc3ryPX5mQe0SkG3y8= +k8s.io/client-go v0.29.0/go.mod h1:yLkXH4HKMAywcrD82KMSmfYg2DlE8mepPR4JGSo5n38= +k8s.io/cloud-provider v0.29.0 h1:Qgk/jHsSKGRk/ltTlN6e7eaNuuamLROOzVBd0RPp94M= +k8s.io/cloud-provider v0.29.0/go.mod h1:gBCt7YYKFV4oUcJ/0xF9lS/9il4MxKunJ+ZKvh39WGo= +k8s.io/component-base v0.29.0 h1:T7rjd5wvLnPBV1vC4zWd/iWRbV8Mdxs+nGaoaFzGw3s= +k8s.io/component-base v0.29.0/go.mod h1:sADonFTQ9Zc9yFLghpDpmNXEdHyQmFIGbiuZbqAXQ1M= +k8s.io/csi-translation-lib v0.29.0 h1:we4X1yUlDikvm5Rv0dwMuPHNw6KwjwsQiAuOPWXha8M= +k8s.io/csi-translation-lib v0.29.0/go.mod h1:Cp6t3CNBSm1dXS17V8IImUjkqfIB6KCj8Fs8wf6uyTA= k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= @@ -763,9 +763,9 @@ sigs.k8s.io/controller-runtime v0.16.3 h1:2TuvuokmfXvDUamSx1SuAOO3eTyye+47mJCigw sigs.k8s.io/controller-runtime v0.16.3/go.mod h1:j7bialYoSn142nv9sCOJmQgDXQXxnroFU4VnX/brVJ0= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/karpenter v0.33.1-0.20231208060535-cc54b340f630 h1:l+zO3G5VV49YnSiX5RPSVGzWiNpidD1jiXz9stgcEhU= -sigs.k8s.io/karpenter v0.33.1-0.20231208060535-cc54b340f630/go.mod h1:J/nUafEcmZrz34hS0B8H51hqgoAd5LhGeS63kMauOjs= -sigs.k8s.io/structured-merge-diff/v4 v4.3.0 h1:UZbZAZfX0wV2zr7YZorDz6GXROfDFj6LvqCRm4VUVKk= -sigs.k8s.io/structured-merge-diff/v4 v4.3.0/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/karpenter v0.33.1-0.20231229170439-99f33e0a3e0c h1:TsWYFc2Ojl75MxcAEp6WXs/CN1YvqR0/ZfzwH4nNwaU= +sigs.k8s.io/karpenter v0.33.1-0.20231229170439-99f33e0a3e0c/go.mod h1:OAiiFe16SakE7GJNTuC2YTWZq0rwRUgqkunY1Y9TqYY= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/hack/code/vpc_limits_gen/main.go b/hack/code/vpc_limits_gen/main.go index 5baaa0460050..179bbeb578a4 100644 --- a/hack/code/vpc_limits_gen/main.go +++ b/hack/code/vpc_limits_gen/main.go @@ -62,5 +62,5 @@ func main() { out.WriteString(newRespData) defer out.Close() - fmt.Printf("Downloaded vpc/limits.go from \"%s\" to file \"%s\"\n", limitsURL.String(), out.Name) + fmt.Printf("Downloaded vpc/limits.go from \"%s\" to file \"%s\"\n", limitsURL.String(), out.Name()) } diff --git a/hack/codegen.sh b/hack/codegen.sh index 3baee053c78a..ec603b72ff7c 100755 --- a/hack/codegen.sh +++ b/hack/codegen.sh @@ -9,13 +9,10 @@ echo "codegen running ENABLE_GIT_PUSH: ${ENABLE_GIT_PUSH}" bandwidth() { GENERATED_FILE="pkg/providers/instancetype/zz_generated.bandwidth.go" - NO_UPDATE='' - SUBJECT="Bandwidth" go run hack/code/bandwidth_gen/main.go -- "${GENERATED_FILE}" - GIT_DIFF=$(git diff --stat "${GENERATED_FILE}") - checkForUpdates "${GIT_DIFF}" "${NO_UPDATE}" "${SUBJECT}" "${GENERATED_FILE}" + checkForUpdates "${GENERATED_FILE}" } pricing() { @@ -27,57 +24,58 @@ pricing() { for partition in "${PARTITIONS[@]}"; do GENERATED_FILE="pkg/providers/pricing/zz_generated.pricing_${partition//-/_}.go" - NO_UPDATE=" ${GENERATED_FILE} "$'| 4 ++--\n 1 file changed, 2 insertions(+), 2 deletions(-)' - SUBJECT="Pricing" go run hack/code/prices_gen/main.go --partition "$partition" --output "$GENERATED_FILE" - GIT_DIFF=$(git diff --stat "${GENERATED_FILE}") - checkForUpdates "${GIT_DIFF}" "${NO_UPDATE}" "${SUBJECT} beside timestamps since last update" "${GENERATED_FILE}" + IGNORE_PATTERN="// generated at" + checkForUpdates "${GENERATED_FILE}" "${IGNORE_PATTERN}" done } vpcLimits() { GENERATED_FILE="pkg/providers/instancetype/zz_generated.vpclimits.go" - NO_UPDATE='' - SUBJECT="VPC Limits" go run hack/code/vpc_limits_gen/main.go -- \ --url=https://raw.githubusercontent.com/aws/amazon-vpc-resource-controller-k8s/master/pkg/aws/vpc/limits.go \ --output="${GENERATED_FILE}" - GIT_DIFF=$(git diff --stat "${GENERATED_FILE}") - checkForUpdates "${GIT_DIFF}" "${NO_UPDATE}" "${SUBJECT}" "${GENERATED_FILE}" + checkForUpdates "${GENERATED_FILE}" } instanceTypeTestData() { GENERATED_FILE="pkg/fake/zz_generated.describe_instance_types.go" - NO_UPDATE='' - SUBJECT="Instance Type Test Data" go run hack/code/instancetype_testdata_gen/main.go --out-file ${GENERATED_FILE} \ --instance-types t3.large,m5.large,m5.xlarge,p3.8xlarge,g4dn.8xlarge,c6g.large,inf1.2xlarge,inf1.6xlarge,trn1.2xlarge,m5.metal,dl1.24xlarge,m6idn.32xlarge,t4g.small,t4g.xlarge,t4g.medium - GIT_DIFF=$(git diff --stat "${GENERATED_FILE}") - checkForUpdates "${GIT_DIFF}" "${NO_UPDATE}" "${SUBJECT}" "${GENERATED_FILE}" + checkForUpdates "${GENERATED_FILE}" } +# checkForUpdates is a helper function that takes in a file and an optional ignore pattern +# to determine if there is a diff between the previous iteration of the file and the newly generated data +# If it fines a difference between the new and the old file and the ENABLE_GIT_PUSH environment variable is set, +# it will push the updated file with an automatic commit to the "codegen" branch +# USAGE: +# checkForUpdates "pkg/providers/pricing/zz_generated.pricing_aws.go" "// generated at" checkForUpdates() { - GIT_DIFF=$1 - NO_UPDATE=$2 - SUBJECT=$3 - GENERATED_FILE=$4 - - echo "Checking git diff for updates. ${GIT_DIFF}, ${NO_UPDATE}" - if [[ "${GIT_DIFF}" == "${NO_UPDATE}" ]]; then - noUpdates "${SUBJECT}" - git checkout "${GENERATED_FILE}" + GENERATED_FILE=$1 + IGNORE_PATTERN=${2:-""} + + if [[ -z "$IGNORE_PATTERN" ]]; then + GIT_DIFF=$(git diff --stat --ignore-blank-lines "${GENERATED_FILE}") else - echo "true" >/tmp/codegen-updates - git add "${GENERATED_FILE}" + GIT_DIFF=$(git diff --stat --ignore-blank-lines --ignore-matching-lines="${IGNORE_PATTERN}" "${GENERATED_FILE}") + fi + + echo "Checking git diff for updates..." + if [[ -n "${GIT_DIFF}" ]]; then + echo "$GIT_DIFF" if [[ $ENABLE_GIT_PUSH == true ]]; then - gitCommitAndPush "${SUBJECT}" + gitCommitAndPush "${GENERATED_FILE}" fi + else + noUpdates "${GENERATED_FILE}" + git checkout "${GENERATED_FILE}" fi } @@ -87,15 +85,16 @@ gitOpenAndPullBranch() { } gitCommitAndPush() { - UPDATE_SUBJECT=$1 - git commit -m "CodeGen updates from AWS API for ${UPDATE_SUBJECT}" + GENERATED_FILE=$1 + git add "${GENERATED_FILE}" + git commit -m "CodeGen updates from AWS API for ${GENERATED_FILE}" # Force push the branch since we might have left the branch around from the last codegen git push --set-upstream origin codegen --force } noUpdates() { - UPDATE_SUBJECT=$1 - echo "No updates from AWS API for ${UPDATE_SUBJECT}" + GENERATED_FILE=$1 + echo "No updates from AWS API for ${GENERATED_FILE}" } if [[ $ENABLE_GIT_PUSH == true ]]; then diff --git a/pkg/apis/crds/karpenter.sh_nodepools.yaml b/pkg/apis/crds/karpenter.sh_nodepools.yaml index 4fd3720548f2..bcea499e74c0 100644 --- a/pkg/apis/crds/karpenter.sh_nodepools.yaml +++ b/pkg/apis/crds/karpenter.sh_nodepools.yaml @@ -59,11 +59,11 @@ spec: type: string nodes: default: 10% - description: 'Nodes dictates how many NodeClaims owned by this NodePool can be terminating at once. It must be set. This only considers NodeClaims with the karpenter.sh/disruption taint. We can''t use an intstr.IntOrString since kubebuilder doesn''t support pattern checking for int nodes for IntOrString nodes. Ref: https://github.com/kubernetes-sigs/controller-tools/blob/55efe4be40394a288216dab63156b0a64fb82929/pkg/crd/markers/validation.go#L379-L388' + description: 'Nodes dictates the maximum number of NodeClaims owned by this NodePool that can be terminating at once. This is calculated by counting nodes that have a deletion timestamp set, or are actively being deleted by Karpenter. This field is required when specifying a budget. This cannot be of type intstr.IntOrString since kubebuilder doesn''t support pattern checking for int nodes for IntOrString nodes. Ref: https://github.com/kubernetes-sigs/controller-tools/blob/55efe4be40394a288216dab63156b0a64fb82929/pkg/crd/markers/validation.go#L379-L388' pattern: ^((100|[0-9]{1,2})%|[0-9]+)$ type: string schedule: - description: Schedule specifies when a budget begins being active, using the upstream cronjob syntax. If omitted, the budget is always active. Currently timezones are not supported. This is required if Duration is set. + description: Schedule specifies when a budget begins being active, following the upstream cronjob syntax. If omitted, the budget is always active. Timezones are not supported. This field is required if Duration is set. pattern: ^(@(annually|yearly|monthly|weekly|daily|midnight|hourly))|((.+)\s(.+)\s(.+)\s(.+)\s(.+))$ type: string required: diff --git a/pkg/controllers/interruption/controller.go b/pkg/controllers/interruption/controller.go index f1e9ba9b5596..e2f3cfc4798e 100644 --- a/pkg/controllers/interruption/controller.go +++ b/pkg/controllers/interruption/controller.go @@ -20,6 +20,7 @@ import ( "time" sqsapi "github.com/aws/aws-sdk-go/service/sqs" + "github.com/prometheus/client_golang/prometheus" "github.com/samber/lo" "go.uber.org/multierr" v1 "k8s.io/api/core/v1" @@ -29,6 +30,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/karpenter/pkg/metrics" "sigs.k8s.io/karpenter/pkg/apis/v1beta1" "sigs.k8s.io/karpenter/pkg/utils/pretty" @@ -42,7 +44,6 @@ import ( "sigs.k8s.io/karpenter/pkg/events" corecontroller "sigs.k8s.io/karpenter/pkg/operator/controller" - nodeclaimutil "sigs.k8s.io/karpenter/pkg/utils/nodeclaim" ) type Action string @@ -208,12 +209,15 @@ func (c *Controller) deleteNodeClaim(ctx context.Context, nodeClaim *v1beta1.Nod if !nodeClaim.DeletionTimestamp.IsZero() { return nil } - if err := nodeclaimutil.Delete(ctx, c.kubeClient, nodeClaim); err != nil { + if err := c.kubeClient.Delete(ctx, nodeClaim); err != nil { return client.IgnoreNotFound(fmt.Errorf("deleting the node on interruption message, %w", err)) } logging.FromContext(ctx).Infof("initiating delete from interruption message") c.recorder.Publish(interruptionevents.TerminatingOnInterruption(node, nodeClaim)...) - nodeclaimutil.TerminatedCounter(nodeClaim, terminationReasonLabel).Inc() + metrics.NodeClaimsTerminatedCounter.With(prometheus.Labels{ + metrics.ReasonLabel: terminationReasonLabel, + metrics.NodePoolLabel: nodeClaim.Labels[v1beta1.NodePoolLabelKey], + }).Inc() return nil } @@ -245,8 +249,8 @@ func (c *Controller) notifyForMessage(msg messages.Message, nodeClaim *v1beta1.N // NodeClaim .status.providerID and the NodeClaim func (c *Controller) makeNodeClaimInstanceIDMap(ctx context.Context) (map[string]*v1beta1.NodeClaim, error) { m := map[string]*v1beta1.NodeClaim{} - nodeClaimList, err := nodeclaimutil.List(ctx, c.kubeClient) - if err != nil { + nodeClaimList := &v1beta1.NodeClaimList{} + if err := c.kubeClient.List(ctx, nodeClaimList); err != nil { return nil, err } for i := range nodeClaimList.Items { diff --git a/pkg/controllers/nodeclaim/garbagecollection/controller.go b/pkg/controllers/nodeclaim/garbagecollection/controller.go index 6955d23a0392..6e06163bb28b 100644 --- a/pkg/controllers/nodeclaim/garbagecollection/controller.go +++ b/pkg/controllers/nodeclaim/garbagecollection/controller.go @@ -32,7 +32,6 @@ import ( "sigs.k8s.io/karpenter/pkg/apis/v1beta1" corecloudprovider "sigs.k8s.io/karpenter/pkg/cloudprovider" "sigs.k8s.io/karpenter/pkg/operator/controller" - nodeclaimutil "sigs.k8s.io/karpenter/pkg/utils/nodeclaim" "github.com/aws/karpenter-provider-aws/pkg/cloudprovider" ) @@ -66,12 +65,12 @@ func (c *Controller) Reconcile(ctx context.Context, _ reconcile.Request) (reconc managedRetrieved := lo.Filter(retrieved, func(nc *v1beta1.NodeClaim, _ int) bool { return nc.Annotations[v1beta1.ManagedByAnnotationKey] != "" && nc.DeletionTimestamp.IsZero() }) - nodeClaimList, err := nodeclaimutil.List(ctx, c.kubeClient) - if err != nil { + nodeClaimList := &v1beta1.NodeClaimList{} + if err = c.kubeClient.List(ctx, nodeClaimList); err != nil { return reconcile.Result{}, err } nodeList := &v1.NodeList{} - if err := c.kubeClient.List(ctx, nodeList); err != nil { + if err = c.kubeClient.List(ctx, nodeList); err != nil { return reconcile.Result{}, err } resolvedProviderIDs := sets.New[string](lo.FilterMap(nodeClaimList.Items, func(n v1beta1.NodeClaim, _ int) (string, bool) { diff --git a/test/pkg/environment/common/expectations.go b/test/pkg/environment/common/expectations.go index edbeb2f5688e..51751c4c824a 100644 --- a/test/pkg/environment/common/expectations.go +++ b/test/pkg/environment/common/expectations.go @@ -38,6 +38,7 @@ import ( "knative.dev/pkg/logging" "knative.dev/pkg/ptr" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" pscheduling "sigs.k8s.io/karpenter/pkg/controllers/provisioning/scheduling" @@ -79,7 +80,7 @@ func (env *Environment) ExpectUpdated(objects ...client.Object) { current := o.DeepCopyObject().(client.Object) g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(current), current)).To(Succeed()) if current.GetResourceVersion() != o.GetResourceVersion() { - logging.FromContext(env).Infof("detected an update to an object with an outdated resource version, did you get the latest version of the object before patching?") + logging.FromContext(env).Infof("detected an update to an object (%s) with an outdated resource version, did you get the latest version of the object before patching?", lo.Must(apiutil.GVKForObject(o, env.Client.Scheme()))) } o.SetResourceVersion(current.GetResourceVersion()) g.Expect(env.Client.Update(env.Context, o)).To(Succeed()) @@ -670,6 +671,20 @@ func (env *Environment) ExpectDaemonSetEnvironmentVariableUpdated(obj client.Obj Expect(env.Client.Patch(env.Context, ds, patch)).To(Succeed()) } +func (env *Environment) ExpectHealthyPodsForNode(nodeName string) []*v1.Pod { + GinkgoHelper() + podList := &v1.PodList{} + Expect(env.Client.List(env, podList, client.MatchingFields{"spec.nodeName": nodeName}, client.HasLabels{test.DiscoveryLabel})).To(Succeed()) + + // Return the healthy pods + return lo.Filter(lo.ToSlicePtr(podList.Items), func(p *v1.Pod, _ int) bool { + _, found := lo.Find(p.Status.Conditions, func(cond v1.PodCondition) bool { + return cond.Type == v1.PodReady && cond.Status == v1.ConditionTrue + }) + return found + }) +} + func (env *Environment) ExpectCABundle() string { // Discover CA Bundle from the REST client. We could alternatively // have used the simpler client-go InClusterConfig() method. diff --git a/test/pkg/environment/common/setup.go b/test/pkg/environment/common/setup.go index 3f1ad4809289..dda838310832 100644 --- a/test/pkg/environment/common/setup.go +++ b/test/pkg/environment/common/setup.go @@ -25,6 +25,7 @@ import ( policyv1 "k8s.io/api/policy/v1" schedulingv1 "k8s.io/api/scheduling/v1" storagev1 "k8s.io/api/storage/v1" + "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/util/workqueue" "sigs.k8s.io/controller-runtime/pkg/client" @@ -41,6 +42,8 @@ import ( . "github.com/onsi/gomega" ) +const TestingFinalizer = "testing/finalizer" + var ( CleanableObjects = []client.Object{ &v1.Pod{}, @@ -122,6 +125,7 @@ func (env *Environment) CleanupObjects(cleanableObjects ...client.Object) { // are deleting so that we avoid getting client-side throttled workqueue.ParallelizeUntil(env, 50, len(metaList.Items), func(i int) { defer GinkgoRecover() + g.Expect(env.ExpectTestingFinalizerRemoved(&metaList.Items[i])).To(Succeed()) g.Expect(client.IgnoreNotFound(env.Client.Delete(env, &metaList.Items[i], client.PropagationPolicy(metav1.DeletePropagationForeground)))).To(Succeed()) }) // If the deletes eventually succeed, we should have no elements here at the end of the test @@ -132,3 +136,21 @@ func (env *Environment) CleanupObjects(cleanableObjects ...client.Object) { } wg.Wait() } + +func (env *Environment) ExpectTestingFinalizerRemoved(obj client.Object) error { + metaObj := &metav1.PartialObjectMetadata{} + metaObj.SetGroupVersionKind(lo.Must(apiutil.GVKForObject(obj, env.Client.Scheme()))) + if err := env.Client.Get(env, client.ObjectKeyFromObject(obj), metaObj); err != nil { + return client.IgnoreNotFound(err) + } + + deepCopy := metaObj.DeepCopy() + metaObj.Finalizers = lo.Reject(metaObj.Finalizers, func(finalizer string, _ int) bool { + return finalizer == TestingFinalizer + }) + + if !equality.Semantic.DeepEqual(metaObj, deepCopy) { + return client.IgnoreNotFound(env.Client.Patch(env, metaObj, client.MergeFrom(deepCopy))) + } + return nil +} diff --git a/test/suites/drift/suite_test.go b/test/suites/drift/suite_test.go index 57b23a587f97..5b6b534b0f94 100644 --- a/test/suites/drift/suite_test.go +++ b/test/suites/drift/suite_test.go @@ -41,6 +41,7 @@ import ( "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" "github.com/aws/karpenter-provider-aws/pkg/test" "github.com/aws/karpenter-provider-aws/test/pkg/environment/aws" + "github.com/aws/karpenter-provider-aws/test/pkg/environment/common" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -96,6 +97,299 @@ var _ = Describe("Drift", Label("AWS"), func() { env.ExpectSettingsOverridden(v1.EnvVar{Name: "FEATURE_GATES", Value: "Drift=true"}) }) + Context("Budgets", func() { + It("should respect budgets for empty drift", func() { + nodePool = coretest.ReplaceRequirements(nodePool, + v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceSize, + Operator: v1.NodeSelectorOpIn, + Values: []string{"2xlarge"}, + }, + ) + // We're expecting to create 3 nodes, so we'll expect to see 2 nodes deleting at one time. + nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{ + Nodes: "50%", + }} + var numPods int32 = 6 + dep = coretest.Deployment(coretest.DeploymentOptions{ + Replicas: numPods, + PodOptions: coretest.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + corev1beta1.DoNotDisruptAnnotationKey: "true", + }, + Labels: map[string]string{"app": "large-app"}, + }, + // Each 2xlarge has 8 cpu, so each node should fit 2 pods. + ResourceRequirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("3"), + }, + }, + }, + }) + selector = labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) + env.ExpectCreated(nodeClass, nodePool, dep) + + env.EventuallyExpectCreatedNodeClaimCount("==", 3) + env.EventuallyExpectCreatedNodeCount("==", 3) + env.EventuallyExpectHealthyPodCount(selector, int(numPods)) + env.Monitor.Reset() // Reset the monitor so that we can expect a single node to be spun up after expiration + + // List nodes so that we get any updated information on the nodes. If we don't + // we have the potential to over-write any changes Karpenter makes to the nodes. + nodes := env.EventuallyExpectNodeCount("==", 3) + // Add a finalizer to each node so that we can stop termination disruptions + By("adding finalizers to the nodes to prevent termination") + for _, node := range nodes { + node.Finalizers = append(node.Finalizers, common.TestingFinalizer) + env.ExpectUpdated(node) + } + + By("making the nodes empty") + // Delete the deployment to make all nodes empty. + env.ExpectDeleted(dep) + + // Drift the nodeclaims + By("drift the nodeclaims") + nodePool.Spec.Template.Annotations = map[string]string{"test": "annotation"} + env.ExpectUpdated(nodePool) + + // Expect that the NodeClaims will all be marked drifted + Eventually(func(g Gomega) { + nodeClaimList := &corev1beta1.NodeClaimList{} + err := env.Client.List(env.Context, nodeClaimList) + g.Expect(err).To(Succeed()) + lo.ForEach(nodeClaimList.Items, func(nc corev1beta1.NodeClaim, _ int) { + g.Expect(nc.StatusConditions().GetCondition(corev1beta1.Drifted).IsTrue()).To(BeTrue()) + }) + }).Should(Succeed()) + + nodes = env.EventuallyExpectTaintedNodeCount("==", 2) + + // Remove the finalizer from each node so that we can terminate + for _, node := range nodes { + Expect(env.ExpectTestingFinalizerRemoved(node)).To(Succeed()) + } + + // After the deletion timestamp is set and all pods are drained + // the node should be gone + env.EventuallyExpectNotFound(nodes[0], nodes[1]) + + nodes = env.EventuallyExpectTaintedNodeCount("==", 1) + Expect(env.ExpectTestingFinalizerRemoved(nodes[0])).To(Succeed()) + env.EventuallyExpectNotFound(nodes[0]) + }) + It("should respect budgets for non-empty delete drift", func() { + nodePool = coretest.ReplaceRequirements(nodePool, + v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceSize, + Operator: v1.NodeSelectorOpIn, + Values: []string{"2xlarge"}, + }, + ) + // We're expecting to create 3 nodes, so we'll expect to see at most 2 nodes deleting at one time. + nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{ + Nodes: "50%", + }} + var numPods int32 = 9 + dep = coretest.Deployment(coretest.DeploymentOptions{ + Replicas: numPods, + PodOptions: coretest.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + corev1beta1.DoNotDisruptAnnotationKey: "true", + }, + Labels: map[string]string{"app": "large-app"}, + }, + // Each 2xlarge has 8 cpu, so each node should fit no more than 3 pods. + ResourceRequirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("2100m"), + }, + }, + }, + }) + selector = labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) + env.ExpectCreated(nodeClass, nodePool, dep) + + env.EventuallyExpectCreatedNodeClaimCount("==", 3) + env.EventuallyExpectCreatedNodeCount("==", 3) + env.EventuallyExpectHealthyPodCount(selector, int(numPods)) + env.Monitor.Reset() // Reset the monitor so that we can expect a single node to be spun up after drift + + By("scaling down the deployment") + // Update the deployment to a third of the replicas. + dep.Spec.Replicas = lo.ToPtr[int32](3) + env.ExpectUpdated(dep) + + By("spreading the pods to each of the nodes") + env.EventuallyExpectHealthyPodCount(selector, 3) + var nodes []*v1.Node + // Delete pods from the deployment until each node has one pod. + nodePods := []*v1.Pod{} + for { + nodes = env.EventuallyExpectNodeCount("==", 3) + node, found := lo.Find(nodes, func(n *v1.Node) bool { + nodePods = env.ExpectHealthyPodsForNode(n.Name) + return len(nodePods) > 1 + }) + if !found { + break + } + // Set the nodes to unschedulable so that the pods won't reschedule. + node.Spec.Unschedulable = true + env.ExpectUpdated(node) + for _, pod := range nodePods[1:] { + env.ExpectDeleted(pod) + } + Eventually(func(g Gomega) { + g.Expect(len(env.ExpectHealthyPodsForNode(node.Name))).To(Equal(1)) + }).WithTimeout(5 * time.Second).Should(Succeed()) + } + env.EventuallyExpectHealthyPodCount(selector, 3) + + By("cordoning and adding finalizer to the nodes") + nodes = env.EventuallyExpectNodeCount("==", 3) + // Add a finalizer to each node so that we can stop termination disruptions + for _, node := range nodes { + node.Finalizers = append(node.Finalizers, common.TestingFinalizer) + // Set nodes as unschedulable so that pod nomination doesn't delay disruption for the second disruption action + node.Spec.Unschedulable = true + env.ExpectUpdated(node) + } + + By("drifting the nodes") + // Drift the nodeclaims + nodePool.Spec.Template.Annotations = map[string]string{"test": "annotation"} + env.ExpectUpdated(nodePool) + + // Expect that the NodeClaims will all be marked drifted + Eventually(func(g Gomega) { + nodeClaimList := &corev1beta1.NodeClaimList{} + err := env.Client.List(env.Context, nodeClaimList) + g.Expect(err).To(Succeed()) + lo.ForEach(nodeClaimList.Items, func(nc corev1beta1.NodeClaim, _ int) { + g.Expect(nc.StatusConditions().GetCondition(corev1beta1.Drifted).IsTrue()).To(BeTrue()) + }) + }).Should(Succeed()) + + By("enabling disruption by removing the do not disrupt annotation") + pods := env.EventuallyExpectHealthyPodCount(selector, 3) + // Remove the do-not-disrupt annotation so that the nodes are now disruptable + for _, pod := range pods { + delete(pod.Annotations, corev1beta1.DoNotDisruptAnnotationKey) + env.ExpectUpdated(pod) + } + + // List nodes so that we get any updated information on the nodes. If we don't + // we have the potential to over-write any changes Karpenter makes to the nodes. + nodes = env.EventuallyExpectNodeCount("==", 3) + + // Mark one node as schedulable so the other two nodes can schedule to this node and delete. + nodes[0].Spec.Unschedulable = false + env.ExpectUpdated(nodes[0]) + nodes = env.EventuallyExpectTaintedNodeCount("==", 2) + + By("removing the finalizer from the nodes") + Expect(env.ExpectTestingFinalizerRemoved(nodes[0])).To(Succeed()) + Expect(env.ExpectTestingFinalizerRemoved(nodes[1])).To(Succeed()) + + // After the deletion timestamp is set and all pods are drained + // the node should be gone + env.EventuallyExpectNotFound(nodes[0], nodes[1]) + }) + It("should respect budgets for non-empty replace drift", func() { + nodePool = coretest.ReplaceRequirements(nodePool, + v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceSize, + Operator: v1.NodeSelectorOpIn, + Values: []string{"2xlarge"}, + }, + ) + // We're expecting to create 3 nodes, so we'll expect to see at most 2 nodes deleting at one time. + nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{ + Nodes: "50%", + }} + var numPods int32 = 3 + dep = coretest.Deployment(coretest.DeploymentOptions{ + Replicas: numPods, + PodOptions: coretest.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + corev1beta1.DoNotDisruptAnnotationKey: "true", + }, + Labels: map[string]string{"app": "large-app"}, + }, + // Each 2xlarge has 8 cpu, so each node should fit no more than 3 pods. + ResourceRequirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("5"), + }, + }, + }, + }) + selector = labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) + env.ExpectCreated(nodeClass, nodePool, dep) + + env.EventuallyExpectCreatedNodeClaimCount("==", 3) + env.EventuallyExpectCreatedNodeCount("==", 3) + env.EventuallyExpectHealthyPodCount(selector, int(numPods)) + env.Monitor.Reset() // Reset the monitor so that we can expect a single node to be spun up after drift + + By("cordoning and adding finalizer to the nodes") + nodes := env.EventuallyExpectNodeCount("==", 3) + // Add a finalizer to each node so that we can stop termination disruptions + for _, node := range nodes { + node.Finalizers = append(node.Finalizers, common.TestingFinalizer) + // Set nodes as unschedulable so that pod nomination doesn't delay disruption for the second disruption action + env.ExpectUpdated(node) + } + + By("drifting the nodes") + // Drift the nodeclaims + nodePool.Spec.Template.Annotations = map[string]string{"test": "annotation"} + env.ExpectUpdated(nodePool) + + // Expect that the NodeClaims will all be marked drifted + Eventually(func(g Gomega) { + nodeClaimList := &corev1beta1.NodeClaimList{} + err := env.Client.List(env.Context, nodeClaimList) + g.Expect(err).To(Succeed()) + lo.ForEach(nodeClaimList.Items, func(nc corev1beta1.NodeClaim, _ int) { + g.Expect(nc.StatusConditions().GetCondition(corev1beta1.Drifted).IsTrue()).To(BeTrue()) + }) + }).Should(Succeed()) + + By("enabling disruption by removing the do not disrupt annotation") + pods := env.EventuallyExpectHealthyPodCount(selector, 3) + // Remove the do-not-disrupt annotation so that the nodes are now disruptable + for _, pod := range pods { + delete(pod.Annotations, corev1beta1.DoNotDisruptAnnotationKey) + env.ExpectUpdated(pod) + } + + nodes = env.EventuallyExpectNodeCount("==", 3) + // Expect two nodes tainted, and 2 nodes created + tainted := env.EventuallyExpectTaintedNodeCount("==", 2) + env.EventuallyExpectCreatedNodeCount("==", 2) + + Expect(env.ExpectTestingFinalizerRemoved(tainted[0])).To(Succeed()) + Expect(env.ExpectTestingFinalizerRemoved(tainted[1])).To(Succeed()) + + env.EventuallyExpectNotFound(tainted[0], tainted[1]) + + // Expect one node tainted and a one more new node created. + tainted = env.EventuallyExpectTaintedNodeCount("==", 1) + env.EventuallyExpectCreatedNodeCount("==", 3) + + Expect(env.ExpectTestingFinalizerRemoved(tainted[0])).To(Succeed()) + + // After the deletion timestamp is set and all pods are drained + // the node should be gone + env.EventuallyExpectNotFound(nodes[0], nodes[1], nodes[2]) + }) + }) It("should disrupt nodes that have drifted due to AMIs", func() { // choose an old static image parameter, err := env.SSMAPI.GetParameter(&ssm.GetParameterInput{ diff --git a/test/suites/expiration/suite_test.go b/test/suites/expiration/suite_test.go index 4d0557529e00..1305b8efb96a 100644 --- a/test/suites/expiration/suite_test.go +++ b/test/suites/expiration/suite_test.go @@ -20,6 +20,7 @@ import ( "github.com/samber/lo" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" @@ -33,6 +34,7 @@ import ( "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" "github.com/aws/karpenter-provider-aws/test/pkg/environment/aws" + "github.com/aws/karpenter-provider-aws/test/pkg/environment/common" coretest "sigs.k8s.io/karpenter/pkg/test" @@ -66,6 +68,309 @@ var _ = AfterEach(func() { env.Cleanup() }) var _ = AfterEach(func() { env.AfterEach() }) var _ = Describe("Expiration", func() { + Context("Budgets", func() { + It("should respect budgets for empty expiration", func() { + coretest.ReplaceRequirements(nodePool, + v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceSize, + Operator: v1.NodeSelectorOpIn, + Values: []string{"2xlarge"}, + }, + ) + nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{ + Nodes: "50%", + }} + nodePool.Spec.Disruption.ExpireAfter = corev1beta1.NillableDuration{} + + var numPods int32 = 6 + dep := coretest.Deployment(coretest.DeploymentOptions{ + Replicas: numPods, + PodOptions: coretest.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + corev1beta1.DoNotDisruptAnnotationKey: "true", + }, + Labels: map[string]string{"app": "large-app"}, + }, + // Each 2xlarge has 8 cpu, so each node should fit 2 pods. + ResourceRequirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("3"), + }, + }, + }, + }) + selector := labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) + env.ExpectCreated(nodeClass, nodePool, dep) + + env.EventuallyExpectCreatedNodeClaimCount("==", 3) + env.EventuallyExpectCreatedNodeCount("==", 3) + env.EventuallyExpectHealthyPodCount(selector, int(numPods)) + env.Monitor.Reset() // Reset the monitor so that we can expect a single node to be spun up after expiration + + nodes := env.EventuallyExpectNodeCount("==", 3) + By("adding finalizers to the nodes to prevent termination") + // Add a finalizer to each node so that we can stop termination disruptions + for _, node := range nodes { + node.Finalizers = append(node.Finalizers, common.TestingFinalizer) + env.ExpectUpdated(node) + } + + By("making the nodes empty") + // Delete the deployment to make all nodes empty. + env.ExpectDeleted(dep) + + By("enabling expiration") + nodePool.Spec.Disruption.ExpireAfter = corev1beta1.NillableDuration{Duration: lo.ToPtr(time.Second * 30)} + env.ExpectUpdated(nodePool) + + // Expect that the NodeClaims will all be marked expired + Eventually(func(g Gomega) { + nodeClaimList := &corev1beta1.NodeClaimList{} + err := env.Client.List(env.Context, nodeClaimList) + g.Expect(err).To(Succeed()) + lo.ForEach(nodeClaimList.Items, func(nc corev1beta1.NodeClaim, _ int) { + g.Expect(nc.StatusConditions().GetCondition(corev1beta1.Expired).IsTrue()).To(BeTrue()) + }) + }).Should(Succeed()) + + // Expect that two nodes are tainted. + nodes = env.EventuallyExpectTaintedNodeCount("==", 2) + + // Remove finalizers + for _, node := range nodes { + Expect(env.ExpectTestingFinalizerRemoved(node)).To(Succeed()) + } + + // After the deletion timestamp is set and all pods are drained + // the node should be gone + env.EventuallyExpectNotFound(nodes[0], nodes[1]) + + // Expect that only one node is tainted, even considering the new node that was just created. + nodes = env.EventuallyExpectTaintedNodeCount("==", 1) + + // Expect the finalizers to be removed and deleted. + Expect(env.ExpectTestingFinalizerRemoved(nodes[0])).To(Succeed()) + env.EventuallyExpectNotFound(nodes[0]) + }) + It("should respect budgets for non-empty delete expiration", func() { + nodePool = coretest.ReplaceRequirements(nodePool, + v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceSize, + Operator: v1.NodeSelectorOpIn, + Values: []string{"2xlarge"}, + }, + ) + // We're expecting to create 3 nodes, so we'll expect to see at most 2 nodes deleting at one time. + nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{ + Nodes: "50%", + }} + // disable expiration so that we can enable it later when we want. + nodePool.Spec.Disruption.ExpireAfter = corev1beta1.NillableDuration{} + var numPods int32 = 9 + dep := coretest.Deployment(coretest.DeploymentOptions{ + Replicas: numPods, + PodOptions: coretest.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + corev1beta1.DoNotDisruptAnnotationKey: "true", + }, + Labels: map[string]string{"app": "large-app"}, + }, + // Each 2xlarge has 8 cpu, so each node should fit no more than 3 pods. + ResourceRequirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("2100m"), + }, + }, + }, + }) + selector := labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) + env.ExpectCreated(nodeClass, nodePool, dep) + + env.EventuallyExpectCreatedNodeClaimCount("==", 3) + env.EventuallyExpectCreatedNodeCount("==", 3) + env.EventuallyExpectHealthyPodCount(selector, int(numPods)) + env.Monitor.Reset() // Reset the monitor so that we can expect a single node to be spun up after expiration + + By("scaling down the deployment") + // Update the deployment to a third of the replicas. + dep.Spec.Replicas = lo.ToPtr[int32](3) + env.ExpectUpdated(dep) + + By("spreading the pods to each of the nodes") + env.EventuallyExpectHealthyPodCount(selector, 3) + var nodes []*v1.Node + // Delete pods from the deployment until each node has one pod. + nodePods := []*v1.Pod{} + for { + nodes = env.EventuallyExpectNodeCount("==", 3) + node, found := lo.Find(nodes, func(n *v1.Node) bool { + nodePods = env.ExpectHealthyPodsForNode(n.Name) + return len(nodePods) > 1 + }) + if !found { + break + } + // Set the nodes to unschedulable so that the pods won't reschedule. + node.Spec.Unschedulable = true + env.ExpectUpdated(node) + for _, pod := range nodePods[1:] { + env.ExpectDeleted(pod) + } + Eventually(func(g Gomega) { + g.Expect(len(env.ExpectHealthyPodsForNode(node.Name))).To(Equal(1)) + }).WithTimeout(5 * time.Second).Should(Succeed()) + } + env.EventuallyExpectHealthyPodCount(selector, 3) + + By("cordoning and adding finalizer to the nodes") + nodes = env.EventuallyExpectNodeCount("==", 3) + // Add a finalizer to each node so that we can stop termination disruptions + for _, node := range nodes { + node.Finalizers = append(node.Finalizers, common.TestingFinalizer) + // Set nodes as unschedulable so that pod nomination doesn't delay disruption for the second disruption action + node.Spec.Unschedulable = true + env.ExpectUpdated(node) + } + + By("expiring the nodes") + // expire the nodeclaims + nodePool.Spec.Disruption.ExpireAfter = corev1beta1.NillableDuration{Duration: lo.ToPtr(time.Second * 30)} + env.ExpectUpdated(nodePool) + + // Expect that the NodeClaims will all be marked expired + Eventually(func(g Gomega) { + nodeClaimList := &corev1beta1.NodeClaimList{} + err := env.Client.List(env.Context, nodeClaimList) + g.Expect(err).To(Succeed()) + lo.ForEach(nodeClaimList.Items, func(nc corev1beta1.NodeClaim, _ int) { + g.Expect(nc.StatusConditions().GetCondition(corev1beta1.Expired).IsTrue()).To(BeTrue()) + }) + }).Should(Succeed()) + + By("enabling disruption by removing the do not disrupt annotation") + pods := env.EventuallyExpectHealthyPodCount(selector, 3) + // Remove the do-not-disrupt annotation so that the nodes are now disruptable + for _, pod := range pods { + delete(pod.Annotations, corev1beta1.DoNotDisruptAnnotationKey) + env.ExpectUpdated(pod) + } + + // List nodes so that we get any updated information on the nodes. If we don't + // we have the potential to over-write any changes Karpenter makes to the nodes. + nodes = env.EventuallyExpectNodeCount("==", 3) + + // Mark one node as schedulable so the other two nodes can schedule to this node and delete. + nodes[0].Spec.Unschedulable = false + env.ExpectUpdated(nodes[0]) + nodes = env.EventuallyExpectTaintedNodeCount("==", 2) + + By("removing the finalizer from the nodes") + Expect(env.ExpectTestingFinalizerRemoved(nodes[0])).To(Succeed()) + Expect(env.ExpectTestingFinalizerRemoved(nodes[1])).To(Succeed()) + + // After the deletion timestamp is set and all pods are drained + // the node should be gone + env.EventuallyExpectNotFound(nodes[0], nodes[1]) + }) + It("should respect budgets for non-empty replace expiration", func() { + nodePool = coretest.ReplaceRequirements(nodePool, + v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceSize, + Operator: v1.NodeSelectorOpIn, + Values: []string{"2xlarge"}, + }, + ) + // We're expecting to create 3 nodes, so we'll expect to see at most 2 nodes deleting at one time. + nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{ + Nodes: "50%", + }} + nodePool.Spec.Disruption.ExpireAfter = corev1beta1.NillableDuration{} + var numPods int32 = 3 + dep := coretest.Deployment(coretest.DeploymentOptions{ + Replicas: numPods, + PodOptions: coretest.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + corev1beta1.DoNotDisruptAnnotationKey: "true", + }, + Labels: map[string]string{"app": "large-app"}, + }, + // Each 2xlarge has 8 cpu, so each node should fit no more than 3 pods. + ResourceRequirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("5"), + }, + }, + }, + }) + selector := labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) + env.ExpectCreated(nodeClass, nodePool, dep) + + env.EventuallyExpectCreatedNodeClaimCount("==", 3) + env.EventuallyExpectCreatedNodeCount("==", 3) + env.EventuallyExpectHealthyPodCount(selector, int(numPods)) + env.Monitor.Reset() // Reset the monitor so that we can expect a single node to be spun up after drift + + By("cordoning and adding finalizer to the nodes") + nodes := env.EventuallyExpectNodeCount("==", 3) + // Add a finalizer to each node so that we can stop termination disruptions + for _, node := range nodes { + node.Finalizers = append(node.Finalizers, common.TestingFinalizer) + // Set nodes as unschedulable so that pod nomination doesn't delay disruption for the second disruption action + env.ExpectUpdated(node) + } + + By("expiring the nodes") + // Expire the nodeclaims + nodePool.Spec.Disruption.ExpireAfter = corev1beta1.NillableDuration{Duration: lo.ToPtr(time.Second * 30)} + env.ExpectUpdated(nodePool) + + // Expect that the NodeClaims will all be marked expired + Eventually(func(g Gomega) { + nodeClaimList := &corev1beta1.NodeClaimList{} + err := env.Client.List(env.Context, nodeClaimList) + g.Expect(err).To(Succeed()) + lo.ForEach(nodeClaimList.Items, func(nc corev1beta1.NodeClaim, _ int) { + g.Expect(nc.StatusConditions().GetCondition(corev1beta1.Expired).IsTrue()).To(BeTrue()) + }) + }).Should(Succeed()) + + By("enabling disruption by removing the do not disrupt annotation") + pods := env.EventuallyExpectHealthyPodCount(selector, 3) + // Remove the do-not-disrupt annotation so that the nodes are now disruptable + for _, pod := range pods { + delete(pod.Annotations, corev1beta1.DoNotDisruptAnnotationKey) + env.ExpectUpdated(pod) + } + + nodes = env.EventuallyExpectNodeCount("==", 3) + // Expect two nodes tainted and two nodes created + tainted := env.EventuallyExpectTaintedNodeCount("==", 2) + env.EventuallyExpectCreatedNodeCount("==", 2) + + Expect(env.ExpectTestingFinalizerRemoved(tainted[0])).To(Succeed()) + Expect(env.ExpectTestingFinalizerRemoved(tainted[1])).To(Succeed()) + + env.EventuallyExpectNotFound(tainted[0], tainted[1]) + + tainted = env.EventuallyExpectTaintedNodeCount("==", 1) + env.EventuallyExpectCreatedNodeCount("==", 3) + + // Set the expireAfter to "Never" to make sure new node isn't deleted + // This is CRITICAL since it prevents nodes that are immediately spun up from immediately being expired and + // racing at the end of the E2E test, leaking node resources into subsequent tests + nodePool.Spec.Disruption.ExpireAfter.Duration = nil + env.ExpectUpdated(nodePool) + + Expect(env.ExpectTestingFinalizerRemoved(tainted[0])).To(Succeed()) + + // After the deletion timestamp is set and all pods are drained + // the node should be gone + env.EventuallyExpectNotFound(nodes[0], nodes[1], nodes[2]) + }) + }) It("should expire the node after the expiration is reached", func() { var numPods int32 = 1 dep := coretest.Deployment(coretest.DeploymentOptions{ diff --git a/website/content/en/docs/concepts/disruption.md b/website/content/en/docs/concepts/disruption.md index 5e4fa193fc78..65d15696981e 100644 --- a/website/content/en/docs/concepts/disruption.md +++ b/website/content/en/docs/concepts/disruption.md @@ -168,7 +168,9 @@ When Karpenter detects one of these events will occur to your nodes, it automati For Spot interruptions, the NodePool will start a new node as soon as it sees the Spot interruption warning. Spot interruptions have a __2 minute notice__ before Amazon EC2 reclaims the instance. Karpenter's average node startup time means that, generally, there is sufficient time for the new node to become ready and to move the pods to the new node before the NodeClaim is reclaimed. {{% alert title="Note" color="primary" %}} -Karpenter publishes Kubernetes events to the node for all events listed above in addition to __Spot Rebalance Recommendations__. Karpenter does not currently support taint, drain, and terminate logic for Spot Rebalance Recommendations. +Karpenter publishes Kubernetes events to the node for all events listed above in addition to [__Spot Rebalance Recommendations__](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/rebalance-recommendations.html). Karpenter does not currently support taint, drain, and terminate logic for Spot Rebalance Recommendations. + +If you require handling for Spot Rebalance Recommendations, you can use the [AWS Node Termination Handler (NTH)](https://github.com/aws/aws-node-termination-handler) alongside Karpenter; however, note that the AWS Node Termination Handler cordons and drains nodes on rebalance recommendations, potentially causing more node churn in the cluster than with interruptions alone. Further information can be found in the [Troubleshooting Guide]({{< ref "../troubleshooting#aws-node-termination-handler-nth-interactions" >}}). {{% /alert %}} Karpenter enables this feature by watching an SQS queue which receives critical events from AWS services which may affect your nodes. Karpenter requires that an SQS queue be provisioned and EventBridge rules and targets be added that forward interruption events from AWS services to the SQS queue. Karpenter provides details for provisioning this infrastructure in the [CloudFormation template in the Getting Started Guide](../../getting-started/getting-started-with-karpenter/#create-the-karpenter-infrastructure-and-iam-roles). diff --git a/website/content/en/docs/concepts/nodeclasses.md b/website/content/en/docs/concepts/nodeclasses.md index 5079073cd3d1..21415e1b6a5d 100644 --- a/website/content/en/docs/concepts/nodeclasses.md +++ b/website/content/en/docs/concepts/nodeclasses.md @@ -66,7 +66,7 @@ spec: # Must specify one of "role" or "instanceProfile" for Karpenter to launch nodes instanceProfile: "KarpenterNodeInstanceProfile-${CLUSTER_NAME}" - # Optional, discovers amis to override the amiFamily's default + # Optional, discovers amis to override the amiFamily's default amis # Each term in the array of amiSelectorTerms is ORed together # Within a single term, all conditions are ANDed amiSelectorTerms: @@ -394,7 +394,9 @@ spec: ## spec.amiSelectorTerms -AMI Selector Terms are used to configure custom AMIs for Karpenter to use, where the AMIs are discovered through ids, owners, name, and [tags](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html). This selection logic is modeled as terms, where each term contains multiple conditions that must all be satisfied for the selector to match. Effectively, all requirements within a single term are ANDed together. It's possible that you may want to select on two different AMIs that have unrelated requirements. In this case, you can specify multiple terms which will be ORed together to form your selection logic. The example below shows how this selection logic is fulfilled. +AMI Selector Terms are used to configure custom AMIs for Karpenter to use, where the AMIs are discovered through ids, owners, name, and [tags](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html). **When you specify `amiSelectorTerms`, you fully override the default AMIs that are selected on by your EC2NodeClass [`amiFamily`]({{< ref "#specamifamily" >}}).** + +This selection logic is modeled as terms, where each term contains multiple conditions that must all be satisfied for the selector to match. Effectively, all requirements within a single term are ANDed together. It's possible that you may want to select on two different AMIs that have unrelated requirements. In this case, you can specify multiple terms which will be ORed together to form your selection logic. The example below shows how this selection logic is fulfilled. ```yaml amiSelectorTerms: @@ -488,7 +490,7 @@ spec: ## spec.instanceProfile -`InstanceProfile` is an optional field and is neccesary to tell Karpenter which identity nodes from this `EC2NodeClass` should assume. You must specify one of `role` or `instanceProfile` when creating a Karpenter `EC2NodeClasss`. If you use the `instanceProfile` field instead of `role`, Karpenter will not manage the InstanceProfile on your behalf. +`InstanceProfile` is an optional field and is necessary to tell Karpenter which identity nodes from this `EC2NodeClass` should assume. You must specify one of `role` or `instanceProfile` when creating a Karpenter `EC2NodeClasss`. If you use the `instanceProfile` field instead of `role`, Karpenter will not manage the InstanceProfile on your behalf. {{% alert title="Note" color="primary" %}} diff --git a/website/content/en/docs/reference/cloudformation.md b/website/content/en/docs/reference/cloudformation.md index ee2224feb8f2..d469ceb2dbac 100644 --- a/website/content/en/docs/reference/cloudformation.md +++ b/website/content/en/docs/reference/cloudformation.md @@ -113,7 +113,7 @@ Someone wanting to add Karpenter to an existing cluster, instead of using `cloud The AllowScopedEC2InstanceActions statement ID (Sid) identifies a set of EC2 resources that are allowed to be accessed with [RunInstances](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_RunInstances.html) and [CreateFleet](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html) actions. -For `RunInstances` and `CreateFleet` actions, the Karpenter controller can read (but not create) `image`, `snapshot`, `spot-instances-request`, `security-group`, `subnet` and `launch-template` EC2 resources, scoped for the particular AWS partition and region. +For `RunInstances` and `CreateFleet` actions, the Karpenter controller can read (but not create) `image`, `snapshot`, `security-group`, `subnet` and `launch-template` EC2 resources, scoped for the particular AWS partition and region. ```json { @@ -122,7 +122,6 @@ For `RunInstances` and `CreateFleet` actions, the Karpenter controller can read "Resource": [ "arn:${AWS::Partition}:ec2:${AWS::Region}::image/*", "arn:${AWS::Partition}:ec2:${AWS::Region}::snapshot/*", - "arn:${AWS::Partition}:ec2:${AWS::Region}:*:spot-instances-request/*", "arn:${AWS::Partition}:ec2:${AWS::Region}:*:security-group/*", "arn:${AWS::Partition}:ec2:${AWS::Region}:*:subnet/*", "arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*" @@ -138,7 +137,7 @@ For `RunInstances` and `CreateFleet` actions, the Karpenter controller can read The AllowScopedEC2InstanceActionsWithTags Sid allows the [RunInstances](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_RunInstances.html), [CreateFleet](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html), and [CreateLaunchTemplate](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateLaunchTemplate.html) -actions requested by the Karpenter controller to create all `fleet`, `instance`, `volume`, `network-interface`, or `launch-template` EC2 resources (for the partition and region), and requires that the `kubernetes.io/cluster/${ClusterName}` tag be set to `owned` and a `karpenter.sh/nodepool` tag be set to any value. This ensures that Karpenter is only allowed to create instances for a single EKS cluster. +actions requested by the Karpenter controller to create all `fleet`, `instance`, `volume`, `network-interface`, `launch-template` or `spot-instances-request` EC2 resources (for the partition and region), and requires that the `kubernetes.io/cluster/${ClusterName}` tag be set to `owned` and a `karpenter.sh/nodepool` tag be set to any value. This ensures that Karpenter is only allowed to create instances for a single EKS cluster. ```json { @@ -149,7 +148,8 @@ actions requested by the Karpenter controller to create all `fleet`, `instance`, "arn:${AWS::Partition}:ec2:${AWS::Region}:*:instance/*", "arn:${AWS::Partition}:ec2:${AWS::Region}:*:volume/*", "arn:${AWS::Partition}:ec2:${AWS::Region}:*:network-interface/*", - "arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*" + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*", + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:spot-instances-request/*" ], "Action": [ "ec2:RunInstances", @@ -170,7 +170,7 @@ actions requested by the Karpenter controller to create all `fleet`, `instance`, #### AllowScopedResourceCreationTagging The AllowScopedResourceCreationTagging Sid allows EC2 [CreateTags](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateTags.html) -actions on `fleet`, `instance`, `volume`, `network-interface`, and `launch-template` resources, While making `RunInstance`, `CreateFleet`, or `CreateLaunchTemplate` calls. Additionally, this ensures that resources can't be tagged arbitrarily by Karpenter after they are created. +actions on `fleet`, `instance`, `volume`, `network-interface`, `launch-template` and `spot-instances-request` resources, While making `RunInstance`, `CreateFleet`, or `CreateLaunchTemplate` calls. Additionally, this ensures that resources can't be tagged arbitrarily by Karpenter after they are created. ```json { @@ -181,7 +181,8 @@ actions on `fleet`, `instance`, `volume`, `network-interface`, and `launch-templ "arn:${AWS::Partition}:ec2:${AWS::Region}:*:instance/*", "arn:${AWS::Partition}:ec2:${AWS::Region}:*:volume/*", "arn:${AWS::Partition}:ec2:${AWS::Region}:*:network-interface/*", - "arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*" + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*", + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:spot-instances-request/*" ], "Action": "ec2:CreateTags", "Condition": { @@ -202,7 +203,7 @@ actions on `fleet`, `instance`, `volume`, `network-interface`, and `launch-templ #### AllowScopedResourceTagging -The AllowScopedResourceTagging Sid allows EC2 [CreateTags](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateTags.html) actions on all instances created by Karpenter after their creation. It enforces that Karpenter is only able to update the tags on cluster instances it is operating on through the `karpenter.sh/cluster/${ClusterName}`" and `karpenter.sh/nodepool` tags. +The AllowScopedResourceTagging Sid allows EC2 [CreateTags](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateTags.html) actions on all instances created by Karpenter after their creation. It enforces that Karpenter is only able to update the tags on cluster instances it is operating on through the `kubernetes.io/cluster/${ClusterName}`" and `karpenter.sh/nodepool` tags. ```json { "Sid": "AllowScopedResourceTagging", @@ -211,7 +212,7 @@ The AllowScopedResourceTagging Sid allows EC2 [CreateTags](https://docs.aws.amaz "Action": "ec2:CreateTags", "Condition": { "StringEquals": { - "aws:ResourceTag/karpenter.sh/cluster/${ClusterName}": "owned" + "aws:ResourceTag/kubernetes.io/cluster/${ClusterName}": "owned" }, "StringLike": { "aws:ResourceTag/karpenter.sh/nodepool": "*" diff --git a/website/content/en/docs/troubleshooting.md b/website/content/en/docs/troubleshooting.md index cbb5b07d1574..41e23bc964f7 100644 --- a/website/content/en/docs/troubleshooting.md +++ b/website/content/en/docs/troubleshooting.md @@ -637,6 +637,22 @@ This typically occurs when the node has not been considered fully initialized fo This error indicates that the `vpc.amazonaws.com/pod-eni` resource was never reported on the node. If you've enabled Pod ENI for Karpenter nodes via the `aws.enablePodENI` setting, you will need to make the corresponding change to the VPC CNI to enable [security groups for pods](https://docs.aws.amazon.com/eks/latest/userguide/security-groups-for-pods.html) which will cause the resource to be registered. +### AWS Node Termination Handler (NTH) interactions +Karpenter [doesn't currently support draining and terminating on spot rebalance recommendations]({{< ref "concepts/disruption#interruption" >}}). Users who want support for both drain and terminate on spot interruption as well as drain and termination on spot rebalance recommendations may install Node Termination Handler (NTH) on their clusters to support this behavior. + +These two components do not share information between each other, meaning if you have drain and terminate functionality enabled on NTH, NTH may remove a node for a spot rebalance recommendation. Karpenter will replace the node to fulfill the pod capacity that was being fulfilled by the old node; however, Karpenter won't be aware of the reason that that node was terminated. This means that Karpenter may launch the same instance type that was just deprovisioned, causing a spot rebalance recommendation to be sent again. This can result in very short-lived instances where NTH continually removes nodes and Karpeneter re-launches the same instance type over and over again. + +Karpenter doesn't recommend reacting to spot rebalance recommendations when running Karpenter with spot nodes; however, if you absolutely require this functionality, note that the above scenario is possible. +Spot instances are time limited and, therefore, interruptible. When a signal is sent by AWS, it triggers actions from NTH and Karpenter, where the former signals a shutdown and the later provisions, creating a recursive situation. +This can be mitigated by either completely removing NTH or by setting the following values: + +* enableSpotInterruptionDraining: If false, do not drain nodes when the spot interruption termination notice is received. Only used in IMDS mode. +enableSpotInterruptionDraining: false + +* enableRebalanceDrainin: If true, drain nodes when the rebalance recommendation notice is received. Only used in IMDS mode. +enableRebalanceDraining: false + + ## Pricing ### Stale pricing data on isolated subnet diff --git a/website/content/en/docs/upgrading/upgrade-guide.md b/website/content/en/docs/upgrading/upgrade-guide.md index 2d243e72c76c..254d81705721 100644 --- a/website/content/en/docs/upgrading/upgrade-guide.md +++ b/website/content/en/docs/upgrading/upgrade-guide.md @@ -36,7 +36,7 @@ kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0 ### Upgrading to v0.33.0+ * Karpenter now tags `spot-instances-request` with the same tags that it tags instances, volumes, and primary ENIs. This means that you will now need to add `ec2:CreateTags` permission for `spot-instances-request`. You can also further scope your controller policy for the `ec2:RunInstances` action to require that it launches the `spot-instances-request` with these specific tags. You can view an example of scoping these actions in the [Getting Started Guide's default CloudFormation controller policy](https://github.com/aws/karpenter/blob/v0.33.0/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml#L61). -* We now recommend that you set the installation namespace for your Karpenter controllers to `kube-system` to denote Karpenter as a critical cluster component. This ensures that requests from the Karpenter controllers are treated with higher priority by assigning them to a different [PriorityLevelConfiguration](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/#prioritylevelconfiguration) than generic requests from other namespaces. For more details on API Priority and Fairness, read the [Kubernetes API Priorirty and Fairness Conceptual Docs](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/). Note: Changing the namespace for your Karpenter release will cause the service account namespace to change. If you are using IRSA for authentication with AWS, you will need to change scoping set in the controller's trust policy from `karpenter:karpenter` to `kube-system:karpenter`. +* We now recommend that you set the installation namespace for your Karpenter controllers to `kube-system` to denote Karpenter as a critical cluster component. This ensures that requests from the Karpenter controllers are treated with higher priority by assigning them to a different [PriorityLevelConfiguration](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/#prioritylevelconfiguration) than generic requests from other namespaces. For more details on API Priority and Fairness, read the [Kubernetes API Priority and Fairness Conceptual Docs](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/). Note: Changing the namespace for your Karpenter release will cause the service account namespace to change. If you are using IRSA for authentication with AWS, you will need to change scoping set in the controller's trust policy from `karpenter:karpenter` to `kube-system:karpenter`. * `v0.33.x` disables mutating and validating webhooks by default in favor of using [Common Expression Language for CRD validation](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#validation). The Common Expression Language Validation Feature [is enabled by default on EKS 1.25](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#validation-rules). If you are using Kubernetes version >= 1.25, no further action is required. If you are using a Kubernetes version below 1.25, you now need to set `DISABLE_WEBHOOK=false` in your container environment variables or `--set webhook.enabled=true` if using Helm. View the [Webhook Support Deprecated in Favor of CEL Section of the v1beta1 Migration Guide]({{}}). * `v0.33.x` drops support for passing settings through the `karpenter-global-settings` ConfigMap. You should pass settings through the container environment variables in the Karpenter deployment manifest. View the [Global Settings Section of the v1beta1 Migration Guide]({{}}) for more details. * `v0.33.x` enables `Drift=true` by default in the `FEATURE_GATES`. If you previously didn't enable the feature gate, Karpenter will now check if there is a difference between the desired state of your nodes declared in your NodePool and the actual state of your nodes. View the [Drift Section of Disruption Conceptual Docs]({{}}) for more details. diff --git a/website/content/en/docs/upgrading/v1beta1-migration.md b/website/content/en/docs/upgrading/v1beta1-migration.md index 8ad11065c4f4..a98306c0d60e 100644 --- a/website/content/en/docs/upgrading/v1beta1-migration.md +++ b/website/content/en/docs/upgrading/v1beta1-migration.md @@ -752,6 +752,34 @@ Karpenter v1beta1 introduces changes to some common labels, annotations, and sta | MachineExpired | Expired | | MachineDrifted | Drifted | +### IAM Controller Permissions + +v1beta1 introduces changes to the IAM permissions assigned to the Karpenter controller policy used when deploying Karpenter to your cluster with [IRSA](https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/setting-up-enable-IAM.html) or [EKS Pod Identity](https://docs.aws.amazon.com/eks/latest/userguide/pod-identities.html). + +You can see a full example of the v1beta1 required controller permissions by viewing the [v1beta1 Controller Policy](https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.4/website/content/en/preview/upgrading/v1beta1-controller-policy.json). + +Additionally, read more detail about the full set of permissions assigned to the Karpenter controller policy in the [CloudFormation Reference Guide]({{< ref "../reference/cloudformation" >}}). + +#### Updating Tag-Based Permissions + +Since Karpenter v1beta1 introduces changes to custom resource, label, and annotation naming, this also changes the tag keys that Karpenter uses to tag instances, volumes launch templates, and any other resources that Karpenter deploys and manages. + +By default, when using the [Karpenter Getting Started Guide]({{< ref "../getting-started/getting-started-with-karpenter" >}}) to setup Karpenter on your cluster, you will deploy an IAM policy that scopes Karpenter's permissions based on tag keys and values using [ABAC](https://aws.amazon.com/identity/attribute-based-access-control/). Any part of the Karpenter alpha controller policy which previously referenced `aws:RequestTag:karpenter.sh/provisioner-name` or `aws:ResourceTag:karpenter.sh/provisioner-name` is now updated in v1beta1 to be `aws:RequestTag:karpenter.sh/nodepool` and `aws:ResourceTag:karpenter.sh/nodepool`. + +{{% alert title="Note" color="warning" %}} +While migrating between alpha and beta, you will need to maintain the old tag permissions as well as the new permissions. You can remove the old tag key permissions from the controller policy when you have fully migrated to beta resources. This process for maintaining both sets of permissions while migrating is also mentioned in greater detail in the [Upgrade Procedure]({{< ref "#upgrade-procedure" >}}) shown above. +{{% /alert %}} + +#### Updating IAM Instance Profile Permissions + +Additionally, starting in v1beta1, Karpenter removes the need for you to manage your own instance profiles used to launch EC2 instances, allowing you to only specify the role that you want assigned to your instances in the `spec.role` of the `EC2NodeClass`. When you do this, Karpenter will generate and manage an instance profile on your behalf. + +To enable this functionality, you need to add `iam:` permissions that give Karpenter permission to generate and managed instance profiles. These permissions include `iam:CreateInstanceProfile`, `iam:TagInstanceProfile`, `iam:AddRoleToInstanceProfile`, `iam:RemoveRoleFromInstanceProfile`, `iam:DeleteInstanceProfile`, and `iam:GetInstanceProfile`. Each of these permissions is scoped down to only operate on instance profiles generated from a single Karpenter instance on a single Karpenter cluster using [ABAC](https://aws.amazon.com/identity/attribute-based-access-control/). + +{{% alert title="Note" color="warning" %}} +These `iam:` permissions are not required if you do not intend to use the `spec.role` field to enable the managed instance profile feature. Instead, you can use the `spec.instanceProfile` field to tell Karpenter to use an unmanaged instance profile explicitly. Note that this means that you have to manually provision and manage the instance profile yourself as you did in alpha. +{{% /alert %}} + ### Metrics The following table shows v1alpha5 metrics and the v1beta1 version of each metric. All metrics on this table will exist simultaneously, while both v1alpha5 and v1beta1 are supported within the same version. diff --git a/website/content/en/preview/concepts/disruption.md b/website/content/en/preview/concepts/disruption.md index c59ea1279b15..27f76bd63c12 100644 --- a/website/content/en/preview/concepts/disruption.md +++ b/website/content/en/preview/concepts/disruption.md @@ -172,7 +172,9 @@ When Karpenter detects one of these events will occur to your nodes, it automati For Spot interruptions, the NodePool will start a new node as soon as it sees the Spot interruption warning. Spot interruptions have a __2 minute notice__ before Amazon EC2 reclaims the instance. Karpenter's average node startup time means that, generally, there is sufficient time for the new node to become ready and to move the pods to the new node before the NodeClaim is reclaimed. {{% alert title="Note" color="primary" %}} -Karpenter publishes Kubernetes events to the node for all events listed above in addition to __Spot Rebalance Recommendations__. Karpenter does not currently support taint, drain, and terminate logic for Spot Rebalance Recommendations. +Karpenter publishes Kubernetes events to the node for all events listed above in addition to [__Spot Rebalance Recommendations__](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/rebalance-recommendations.html). Karpenter does not currently support taint, drain, and terminate logic for Spot Rebalance Recommendations. + +If you require handling for Spot Rebalance Recommendations, you can use the [AWS Node Termination Handler (NTH)](https://github.com/aws/aws-node-termination-handler) alongside Karpenter; however, note that the AWS Node Termination Handler cordons and drains nodes on rebalance recommendations, potentially causing more node churn in the cluster than with interruptions alone. Further information can be found in the [Troubleshooting Guide]({{< ref "../troubleshooting#aws-node-termination-handler-nth-interactions" >}}). {{% /alert %}} Karpenter enables this feature by watching an SQS queue which receives critical events from AWS services which may affect your nodes. Karpenter requires that an SQS queue be provisioned and EventBridge rules and targets be added that forward interruption events from AWS services to the SQS queue. Karpenter provides details for provisioning this infrastructure in the [CloudFormation template in the Getting Started Guide](../../getting-started/getting-started-with-karpenter/#create-the-karpenter-infrastructure-and-iam-roles). diff --git a/website/content/en/preview/concepts/nodeclasses.md b/website/content/en/preview/concepts/nodeclasses.md index 3ad9e5ced259..c8fa5537fba3 100644 --- a/website/content/en/preview/concepts/nodeclasses.md +++ b/website/content/en/preview/concepts/nodeclasses.md @@ -66,7 +66,7 @@ spec: # Must specify one of "role" or "instanceProfile" for Karpenter to launch nodes instanceProfile: "KarpenterNodeInstanceProfile-${CLUSTER_NAME}" - # Optional, discovers amis to override the amiFamily's default + # Optional, discovers amis to override the amiFamily's default amis # Each term in the array of amiSelectorTerms is ORed together # Within a single term, all conditions are ANDed amiSelectorTerms: @@ -396,7 +396,9 @@ spec: ## spec.amiSelectorTerms -AMI Selector Terms are used to configure custom AMIs for Karpenter to use, where the AMIs are discovered through ids, owners, name, and [tags](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html). This selection logic is modeled as terms, where each term contains multiple conditions that must all be satisfied for the selector to match. Effectively, all requirements within a single term are ANDed together. It's possible that you may want to select on two different AMIs that have unrelated requirements. In this case, you can specify multiple terms which will be ORed together to form your selection logic. The example below shows how this selection logic is fulfilled. +AMI Selector Terms are used to configure custom AMIs for Karpenter to use, where the AMIs are discovered through ids, owners, name, and [tags](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html). **When you specify `amiSelectorTerms`, you fully override the default AMIs that are selected on by your EC2NodeClass [`amiFamily`]({{< ref "#specamifamily" >}}).** + +This selection logic is modeled as terms, where each term contains multiple conditions that must all be satisfied for the selector to match. Effectively, all requirements within a single term are ANDed together. It's possible that you may want to select on two different AMIs that have unrelated requirements. In this case, you can specify multiple terms which will be ORed together to form your selection logic. The example below shows how this selection logic is fulfilled. ```yaml amiSelectorTerms: @@ -490,7 +492,7 @@ spec: ## spec.instanceProfile -`InstanceProfile` is an optional field and is neccesary to tell Karpenter which identity nodes from this `EC2NodeClass` should assume. You must specify one of `role` or `instanceProfile` when creating a Karpenter `EC2NodeClasss`. If you use the `instanceProfile` field instead of `role`, Karpenter will not manage the InstanceProfile on your behalf. +`InstanceProfile` is an optional field and is necessary to tell Karpenter which identity nodes from this `EC2NodeClass` should assume. You must specify one of `role` or `instanceProfile` when creating a Karpenter `EC2NodeClasss`. If you use the `instanceProfile` field instead of `role`, Karpenter will not manage the InstanceProfile on your behalf. {{% alert title="Note" color="primary" %}} diff --git a/website/content/en/preview/reference/cloudformation.md b/website/content/en/preview/reference/cloudformation.md index dcc1e853fb01..5dc95af3e482 100644 --- a/website/content/en/preview/reference/cloudformation.md +++ b/website/content/en/preview/reference/cloudformation.md @@ -36,7 +36,7 @@ That name would then be appended to any name below where `${ClusterName}` is inc * Partition: Any time an ARN is used, it includes the [partition name](https://docs.aws.amazon.com/whitepapers/latest/aws-fault-isolation-boundaries/partitions.html) to identify where the object is found. In most cases, that partition name is `aws`. However, it could also be `aws-cn` (for China Regions) or `aws-us-gov` (for AWS GovCloud US Regions). -## Node Authorization +## Node Authorization The following sections of the `cloudformation.yaml` file set up IAM permissions for Kubernetes nodes created by Karpenter. In particular, this involves setting up a node role that can be attached and passed to instance profiles that Karpenter generates at runtime: @@ -79,7 +79,7 @@ The role created here includes several AWS managed policies, which are designed If you were to use a node role from an existing cluster, you could skip this provisioning step and pass this node role to any EC2NodeClasses that you create. Additionally, you would ensure that the [Controller Policy]({{< relref "#controllerpolicy" >}}) has `iam:PassRole` permission to the role attached to the generated instance profiles. -## Controller Authorization +## Controller Authorization This section sets the AWS permissions for the Karpenter Controller. When used in the Getting Started guide, `eksctl` uses these permissions to create a service account (karpenter) that is combined with the KarpenterControllerPolicy. @@ -113,7 +113,7 @@ Someone wanting to add Karpenter to an existing cluster, instead of using `cloud The AllowScopedEC2InstanceActions statement ID (Sid) identifies a set of EC2 resources that are allowed to be accessed with [RunInstances](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_RunInstances.html) and [CreateFleet](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html) actions. -For `RunInstances` and `CreateFleet` actions, the Karpenter controller can read (but not create) `image`, `snapshot`, `spot-instances-request`, `security-group`, `subnet` and `launch-template` EC2 resources, scoped for the particular AWS partition and region. +For `RunInstances` and `CreateFleet` actions, the Karpenter controller can read (but not create) `image`, `snapshot`, `security-group`, `subnet` and `launch-template` EC2 resources, scoped for the particular AWS partition and region. ```json { @@ -122,7 +122,6 @@ For `RunInstances` and `CreateFleet` actions, the Karpenter controller can read "Resource": [ "arn:${AWS::Partition}:ec2:${AWS::Region}::image/*", "arn:${AWS::Partition}:ec2:${AWS::Region}::snapshot/*", - "arn:${AWS::Partition}:ec2:${AWS::Region}:*:spot-instances-request/*", "arn:${AWS::Partition}:ec2:${AWS::Region}:*:security-group/*", "arn:${AWS::Partition}:ec2:${AWS::Region}:*:subnet/*", "arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*" @@ -136,9 +135,9 @@ For `RunInstances` and `CreateFleet` actions, the Karpenter controller can read #### AllowScopedEC2InstanceActionsWithTags -The AllowScopedEC2InstanceActionsWithTags Sid allows the +The AllowScopedEC2InstanceActionsWithTags Sid allows the [RunInstances](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_RunInstances.html), [CreateFleet](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html), and [CreateLaunchTemplate](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateLaunchTemplate.html) -actions requested by the Karpenter controller to create all `fleet`, `instance`, `volume`, `network-interface`, or `launch-template` EC2 resources (for the partition and region), and requires that the `kubernetes.io/cluster/${ClusterName}` tag be set to `owned` and a `karpenter.sh/nodepool` tag be set to any value. This ensures that Karpenter is only allowed to create instances for a single EKS cluster. +actions requested by the Karpenter controller to create all `fleet`, `instance`, `volume`, `network-interface`, `launch-template` or `spot-instances-request` EC2 resources (for the partition and region), and requires that the `kubernetes.io/cluster/${ClusterName}` tag be set to `owned` and a `karpenter.sh/nodepool` tag be set to any value. This ensures that Karpenter is only allowed to create instances for a single EKS cluster. ```json { @@ -149,7 +148,8 @@ actions requested by the Karpenter controller to create all `fleet`, `instance`, "arn:${AWS::Partition}:ec2:${AWS::Region}:*:instance/*", "arn:${AWS::Partition}:ec2:${AWS::Region}:*:volume/*", "arn:${AWS::Partition}:ec2:${AWS::Region}:*:network-interface/*", - "arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*" + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*", + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:spot-instances-request/*" ], "Action": [ "ec2:RunInstances", @@ -170,7 +170,7 @@ actions requested by the Karpenter controller to create all `fleet`, `instance`, #### AllowScopedResourceCreationTagging The AllowScopedResourceCreationTagging Sid allows EC2 [CreateTags](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateTags.html) -actions on `fleet`, `instance`, `volume`, `network-interface`, and `launch-template` resources, While making `RunInstance`, `CreateFleet`, or `CreateLaunchTemplate` calls. Additionally, this ensures that resources can't be tagged arbitrarily by Karpenter after they are created. +actions on `fleet`, `instance`, `volume`, `network-interface`, `launch-template` and `spot-instances-request` resources, While making `RunInstance`, `CreateFleet`, or `CreateLaunchTemplate` calls. Additionally, this ensures that resources can't be tagged arbitrarily by Karpenter after they are created. ```json { @@ -181,7 +181,8 @@ actions on `fleet`, `instance`, `volume`, `network-interface`, and `launch-templ "arn:${AWS::Partition}:ec2:${AWS::Region}:*:instance/*", "arn:${AWS::Partition}:ec2:${AWS::Region}:*:volume/*", "arn:${AWS::Partition}:ec2:${AWS::Region}:*:network-interface/*", - "arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*" + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*", + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:spot-instances-request/*" ], "Action": "ec2:CreateTags", "Condition": { @@ -202,7 +203,7 @@ actions on `fleet`, `instance`, `volume`, `network-interface`, and `launch-templ #### AllowScopedResourceTagging -The AllowScopedResourceTagging Sid allows EC2 [CreateTags](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateTags.html) actions on all instances created by Karpenter after their creation. It enforces that Karpenter is only able to update the tags on cluster instances it is operating on through the `karpenter.sh/cluster/${ClusterName}`" and `karpenter.sh/nodepool` tags. +The AllowScopedResourceTagging Sid allows EC2 [CreateTags](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateTags.html) actions on all instances created by Karpenter after their creation. It enforces that Karpenter is only able to update the tags on cluster instances it is operating on through the `kubernetes.io/cluster/${ClusterName}`" and `karpenter.sh/nodepool` tags. ```json { "Sid": "AllowScopedResourceTagging", @@ -211,7 +212,7 @@ The AllowScopedResourceTagging Sid allows EC2 [CreateTags](https://docs.aws.amaz "Action": "ec2:CreateTags", "Condition": { "StringEquals": { - "aws:ResourceTag/karpenter.sh/cluster/${ClusterName}": "owned" + "aws:ResourceTag/kubernetes.io/cluster/${ClusterName}": "owned" }, "StringLike": { "aws:ResourceTag/karpenter.sh/nodepool": "*" @@ -404,7 +405,7 @@ Also, `karpenter.k8s.aws/ec2nodeclass` must be set to some value. This ensures t #### AllowScopedInstanceProfileActions -The AllowScopedInstanceProfileActions Sid gives the Karpenter controller permission to perform [`iam:AddRoleToInstanceProfile`](https://docs.aws.amazon.com/IAM/latest/APIReference/API_AddRoleToInstanceProfile.html), [`iam:RemoveRoleFromInstanceProfile`](https://docs.aws.amazon.com/IAM/latest/APIReference/API_RemoveRoleFromInstanceProfile.html), and [`iam:DeleteInstanceProfile`](https://docs.aws.amazon.com/IAM/latest/APIReference/API_DeleteInstanceProfile.html) actions, +The AllowScopedInstanceProfileActions Sid gives the Karpenter controller permission to perform [`iam:AddRoleToInstanceProfile`](https://docs.aws.amazon.com/IAM/latest/APIReference/API_AddRoleToInstanceProfile.html), [`iam:RemoveRoleFromInstanceProfile`](https://docs.aws.amazon.com/IAM/latest/APIReference/API_RemoveRoleFromInstanceProfile.html), and [`iam:DeleteInstanceProfile`](https://docs.aws.amazon.com/IAM/latest/APIReference/API_DeleteInstanceProfile.html) actions, provided that the request is made to a cluster with `kubernetes.io/cluster/${ClusterName` set to owned and is made in the current region. Also, `karpenter.k8s.aws/ec2nodeclass` must be set to some value. This permission is further enforced by the `iam:PassRole` permission. If Karpenter attempts to add a role to an instance profile that it doesn't have `iam:PassRole` permission on, that call will fail. Therefore, if you configure Karpenter to use a new role through the `EC2NodeClass`, ensure that you also specify that role within your `iam:PassRole` permission. @@ -459,7 +460,7 @@ The AllowAPIServerEndpointDiscovery Sid allows the Karpenter controller to get t } ``` -## Interruption Handling +## Interruption Handling Settings in this section allow the Karpenter controller to stand-up an interruption queue to receive notification messages from other AWS services about the health and status of instances. For example, this interruption queue allows Karpenter to be aware of spot instance interruptions that are sent 2 minutes before spot instances are reclaimed by EC2. Adding this queue allows Karpenter to be proactive in migrating workloads to new nodes. See the [Interruption]({{< relref "../concepts/disruption#interruption" >}}) section of the Disruption page for details. diff --git a/website/content/en/preview/troubleshooting.md b/website/content/en/preview/troubleshooting.md index cbb5b07d1574..c33f04264e3b 100644 --- a/website/content/en/preview/troubleshooting.md +++ b/website/content/en/preview/troubleshooting.md @@ -637,6 +637,21 @@ This typically occurs when the node has not been considered fully initialized fo This error indicates that the `vpc.amazonaws.com/pod-eni` resource was never reported on the node. If you've enabled Pod ENI for Karpenter nodes via the `aws.enablePodENI` setting, you will need to make the corresponding change to the VPC CNI to enable [security groups for pods](https://docs.aws.amazon.com/eks/latest/userguide/security-groups-for-pods.html) which will cause the resource to be registered. +### AWS Node Termination Handler (NTH) interactions +Karpenter [doesn't currently support draining and terminating on spot rebalance recommendations]({{< ref "concepts/disruption#interruption" >}}). Users who want support for both drain and terminate on spot interruption as well as drain and termination on spot rebalance recommendations may install Node Termination Handler (NTH) on their clusters to support this behavior. + +These two components do not share information between each other, meaning if you have drain and terminate functionality enabled on NTH, NTH may remove a node for a spot rebalance recommendation. Karpenter will replace the node to fulfill the pod capacity that was being fulfilled by the old node; however, Karpenter won't be aware of the reason that that node was terminated. This means that Karpenter may launch the same instance type that was just deprovisioned, causing a spot rebalance recommendation to be sent again. This can result in very short-lived instances where NTH continually removes nodes and Karpeneter re-launches the same instance type over and over again. + +Karpenter doesn't recommend reacting to spot rebalance recommendations when running Karpenter with spot nodes; however, if you absolutely require this functionality, note that the above scenario is possible. +Spot instances are time limited and, therefore, interruptible. When a signal is sent by AWS, it triggers actions from NTH and Karpenter, where the former signals a shutdown and the later provisions, creating a recursive situation. +This can be mitigated by either completely removing NTH or by setting the following values: + +* enableSpotInterruptionDraining: If false, do not drain nodes when the spot interruption termination notice is received. Only used in IMDS mode. +enableSpotInterruptionDraining: false + +* enableRebalanceDrainin: If true, drain nodes when the rebalance recommendation notice is received. Only used in IMDS mode. +enableRebalanceDraining: false + ## Pricing ### Stale pricing data on isolated subnet diff --git a/website/content/en/preview/upgrading/upgrade-guide.md b/website/content/en/preview/upgrading/upgrade-guide.md index 06e33f4e8b76..14f5e7db0dc4 100644 --- a/website/content/en/preview/upgrading/upgrade-guide.md +++ b/website/content/en/preview/upgrading/upgrade-guide.md @@ -45,7 +45,7 @@ kubectl apply -f https://raw.githubusercontent.com/aws/karpenter{{< githubRelRef ### Upgrading to v0.33.0+ * Karpenter now tags `spot-instances-request` with the same tags that it tags instances, volumes, and primary ENIs. This means that you will now need to add `ec2:CreateTags` permission for `spot-instances-request`. You can also further scope your controller policy for the `ec2:RunInstances` action to require that it launches the `spot-instances-request` with these specific tags. You can view an example of scoping these actions in the [Getting Started Guide's default CloudFormation controller policy](https://github.com/aws/karpenter/blob/v0.33.0/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml#L61). -* We now recommend that you set the installation namespace for your Karpenter controllers to `kube-system` to denote Karpenter as a critical cluster component. This ensures that requests from the Karpenter controllers are treated with higher priority by assigning them to a different [PriorityLevelConfiguration](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/#prioritylevelconfiguration) than generic requests from other namespaces. For more details on API Priority and Fairness, read the [Kubernetes API Priorirty and Fairness Conceptual Docs](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/). Note: Changing the namespace for your Karpenter release will cause the service account namespace to change. If you are using IRSA for authentication with AWS, you will need to change scoping set in the controller's trust policy from `karpenter:karpenter` to `kube-system:karpenter`. +* We now recommend that you set the installation namespace for your Karpenter controllers to `kube-system` to denote Karpenter as a critical cluster component. This ensures that requests from the Karpenter controllers are treated with higher priority by assigning them to a different [PriorityLevelConfiguration](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/#prioritylevelconfiguration) than generic requests from other namespaces. For more details on API Priority and Fairness, read the [Kubernetes API Priority and Fairness Conceptual Docs](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/). Note: Changing the namespace for your Karpenter release will cause the service account namespace to change. If you are using IRSA for authentication with AWS, you will need to change scoping set in the controller's trust policy from `karpenter:karpenter` to `kube-system:karpenter`. * `v0.33.x` disables mutating and validating webhooks by default in favor of using [Common Expression Language for CRD validation](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#validation). The Common Expression Language Validation Feature [is enabled by default on EKS 1.25](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#validation-rules). If you are using Kubernetes version >= 1.25, no further action is required. If you are using a Kubernetes version below 1.25, you now need to set `DISABLE_WEBHOOK=false` in your container environment variables or `--set webhook.enabled=true` if using Helm. View the [Webhook Support Deprecated in Favor of CEL Section of the v1beta1 Migration Guide]({{}}). * `v0.33.x` drops support for passing settings through the `karpenter-global-settings` ConfigMap. You should pass settings through the container environment variables in the Karpenter deployment manifest. View the [Global Settings Section of the v1beta1 Migration Guide]({{}}) for more details. * `v0.33.x` enables `Drift=true` by default in the `FEATURE_GATES`. If you previously didn't enable the feature gate, Karpenter will now check if there is a difference between the desired state of your nodes declared in your NodePool and the actual state of your nodes. View the [Drift Section of Disruption Conceptual Docs]({{}}) for more details. diff --git a/website/content/en/preview/upgrading/v1beta1-migration.md b/website/content/en/preview/upgrading/v1beta1-migration.md index 8ad11065c4f4..a98306c0d60e 100644 --- a/website/content/en/preview/upgrading/v1beta1-migration.md +++ b/website/content/en/preview/upgrading/v1beta1-migration.md @@ -752,6 +752,34 @@ Karpenter v1beta1 introduces changes to some common labels, annotations, and sta | MachineExpired | Expired | | MachineDrifted | Drifted | +### IAM Controller Permissions + +v1beta1 introduces changes to the IAM permissions assigned to the Karpenter controller policy used when deploying Karpenter to your cluster with [IRSA](https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/setting-up-enable-IAM.html) or [EKS Pod Identity](https://docs.aws.amazon.com/eks/latest/userguide/pod-identities.html). + +You can see a full example of the v1beta1 required controller permissions by viewing the [v1beta1 Controller Policy](https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.4/website/content/en/preview/upgrading/v1beta1-controller-policy.json). + +Additionally, read more detail about the full set of permissions assigned to the Karpenter controller policy in the [CloudFormation Reference Guide]({{< ref "../reference/cloudformation" >}}). + +#### Updating Tag-Based Permissions + +Since Karpenter v1beta1 introduces changes to custom resource, label, and annotation naming, this also changes the tag keys that Karpenter uses to tag instances, volumes launch templates, and any other resources that Karpenter deploys and manages. + +By default, when using the [Karpenter Getting Started Guide]({{< ref "../getting-started/getting-started-with-karpenter" >}}) to setup Karpenter on your cluster, you will deploy an IAM policy that scopes Karpenter's permissions based on tag keys and values using [ABAC](https://aws.amazon.com/identity/attribute-based-access-control/). Any part of the Karpenter alpha controller policy which previously referenced `aws:RequestTag:karpenter.sh/provisioner-name` or `aws:ResourceTag:karpenter.sh/provisioner-name` is now updated in v1beta1 to be `aws:RequestTag:karpenter.sh/nodepool` and `aws:ResourceTag:karpenter.sh/nodepool`. + +{{% alert title="Note" color="warning" %}} +While migrating between alpha and beta, you will need to maintain the old tag permissions as well as the new permissions. You can remove the old tag key permissions from the controller policy when you have fully migrated to beta resources. This process for maintaining both sets of permissions while migrating is also mentioned in greater detail in the [Upgrade Procedure]({{< ref "#upgrade-procedure" >}}) shown above. +{{% /alert %}} + +#### Updating IAM Instance Profile Permissions + +Additionally, starting in v1beta1, Karpenter removes the need for you to manage your own instance profiles used to launch EC2 instances, allowing you to only specify the role that you want assigned to your instances in the `spec.role` of the `EC2NodeClass`. When you do this, Karpenter will generate and manage an instance profile on your behalf. + +To enable this functionality, you need to add `iam:` permissions that give Karpenter permission to generate and managed instance profiles. These permissions include `iam:CreateInstanceProfile`, `iam:TagInstanceProfile`, `iam:AddRoleToInstanceProfile`, `iam:RemoveRoleFromInstanceProfile`, `iam:DeleteInstanceProfile`, and `iam:GetInstanceProfile`. Each of these permissions is scoped down to only operate on instance profiles generated from a single Karpenter instance on a single Karpenter cluster using [ABAC](https://aws.amazon.com/identity/attribute-based-access-control/). + +{{% alert title="Note" color="warning" %}} +These `iam:` permissions are not required if you do not intend to use the `spec.role` field to enable the managed instance profile feature. Instead, you can use the `spec.instanceProfile` field to tell Karpenter to use an unmanaged instance profile explicitly. Note that this means that you have to manually provision and manage the instance profile yourself as you did in alpha. +{{% /alert %}} + ### Metrics The following table shows v1alpha5 metrics and the v1beta1 version of each metric. All metrics on this table will exist simultaneously, while both v1alpha5 and v1beta1 are supported within the same version. diff --git a/website/content/en/v0.31/concepts/_index.md b/website/content/en/v0.31/concepts/_index.md index 74ec1368b573..fec50b86b6d7 100755 --- a/website/content/en/v0.31/concepts/_index.md +++ b/website/content/en/v0.31/concepts/_index.md @@ -126,7 +126,9 @@ If interruption-handling is enabled for the controller, Karpenter will watch for When Karpenter detects one of these events will occur to your nodes, it automatically cordons, drains, and terminates the node(s) ahead of the interruption event to give the maximum amount of time for workload cleanup prior to compute disruption. This enables scenarios where the `terminationGracePeriod` for your workloads may be long or cleanup for your workloads is critical, and you want enough time to be able to gracefully clean-up your pods. {{% alert title="Note" color="warning" %}} -Karpenter publishes Kubernetes events to the node for all events listed above in addition to __Spot Rebalance Recommendations__. Karpenter does not currently support cordon, drain, and terminate logic for Spot Rebalance Recommendations. +Karpenter publishes Kubernetes events to the node for all events listed above in addition to [__Spot Rebalance Recommendations__](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/rebalance-recommendations.html). Karpenter does not currently support taint, drain, and terminate logic for Spot Rebalance Recommendations. + +If you require handling for Spot Rebalance Recommendations, you can use the [AWS Node Termination Handler (NTH)](https://github.com/aws/aws-node-termination-handler) alongside Karpenter; however, note that the AWS Node Termination Handler cordons and drains nodes on rebalance recommendations, potentially causing more node churn in the cluster than with interruptions alone. Further information can be found in the [Troubleshooting Guide]({{< ref "../troubleshooting#aws-node-termination-handler-nth-interactions" >}}). {{% /alert %}} ### Kubernetes cluster autoscaler diff --git a/website/content/en/v0.31/troubleshooting.md b/website/content/en/v0.31/troubleshooting.md index a87ba78b23e5..eff5bd1ec6e6 100644 --- a/website/content/en/v0.31/troubleshooting.md +++ b/website/content/en/v0.31/troubleshooting.md @@ -637,6 +637,21 @@ This typically occurs when the node has not been considered fully initialized fo This error indicates that the `vpc.amazonaws.com/pod-eni` resource was never reported on the node. If you've enabled Pod ENI for Karpenter nodes via the `aws.enablePodENI` setting, you will need to make the corresponding change to the VPC CNI to enable [security groups for pods](https://docs.aws.amazon.com/eks/latest/userguide/security-groups-for-pods.html) which will cause the resource to be registered. +### AWS Node Termination Handler (NTH) interactions +Karpenter [doesn't currently support draining and terminating on spot rebalance recommendations]({{< ref "concepts/deprovisioning#interruption" >}}). Users who want support for both drain and terminate on spot interruption as well as drain and termination on spot rebalance recommendations may install Node Termination Handler (NTH) on their clusters to support this behavior. + +These two components do not share information between each other, meaning if you have drain and terminate functionality enabled on NTH, NTH may remove a node for a spot rebalance recommendation. Karpenter will replace the node to fulfill the pod capacity that was being fulfilled by the old node; however, Karpenter won't be aware of the reason that that node was terminated. This means that Karpenter may launch the same instance type that was just deprovisioned, causing a spot rebalance recommendation to be sent again. This can result in very short-lived instances where NTH continually removes nodes and Karpeneter re-launches the same instance type over and over again. + +Karpenter doesn't recommend reacting to spot rebalance recommendations when running Karpenter with spot nodes; however, if you absolutely require this functionality, note that the above scenario is possible. +Spot instances are time limited and, therefore, interruptible. When a signal is sent by AWS, it triggers actions from NTH and Karpenter, where the former signals a shutdown and the later provisions, creating a recursive situation. +This can be mitigated by either completely removing NTH or by setting the following values: + +* enableSpotInterruptionDraining: If false, do not drain nodes when the spot interruption termination notice is received. Only used in IMDS mode. +enableSpotInterruptionDraining: false + +* enableRebalanceDrainin: If true, drain nodes when the rebalance recommendation notice is received. Only used in IMDS mode. +enableRebalanceDraining: false + ## Pricing ### Stale pricing data on isolated subnet diff --git a/website/content/en/v0.32/concepts/disruption.md b/website/content/en/v0.32/concepts/disruption.md index 5e4fa193fc78..65d15696981e 100644 --- a/website/content/en/v0.32/concepts/disruption.md +++ b/website/content/en/v0.32/concepts/disruption.md @@ -168,7 +168,9 @@ When Karpenter detects one of these events will occur to your nodes, it automati For Spot interruptions, the NodePool will start a new node as soon as it sees the Spot interruption warning. Spot interruptions have a __2 minute notice__ before Amazon EC2 reclaims the instance. Karpenter's average node startup time means that, generally, there is sufficient time for the new node to become ready and to move the pods to the new node before the NodeClaim is reclaimed. {{% alert title="Note" color="primary" %}} -Karpenter publishes Kubernetes events to the node for all events listed above in addition to __Spot Rebalance Recommendations__. Karpenter does not currently support taint, drain, and terminate logic for Spot Rebalance Recommendations. +Karpenter publishes Kubernetes events to the node for all events listed above in addition to [__Spot Rebalance Recommendations__](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/rebalance-recommendations.html). Karpenter does not currently support taint, drain, and terminate logic for Spot Rebalance Recommendations. + +If you require handling for Spot Rebalance Recommendations, you can use the [AWS Node Termination Handler (NTH)](https://github.com/aws/aws-node-termination-handler) alongside Karpenter; however, note that the AWS Node Termination Handler cordons and drains nodes on rebalance recommendations, potentially causing more node churn in the cluster than with interruptions alone. Further information can be found in the [Troubleshooting Guide]({{< ref "../troubleshooting#aws-node-termination-handler-nth-interactions" >}}). {{% /alert %}} Karpenter enables this feature by watching an SQS queue which receives critical events from AWS services which may affect your nodes. Karpenter requires that an SQS queue be provisioned and EventBridge rules and targets be added that forward interruption events from AWS services to the SQS queue. Karpenter provides details for provisioning this infrastructure in the [CloudFormation template in the Getting Started Guide](../../getting-started/getting-started-with-karpenter/#create-the-karpenter-infrastructure-and-iam-roles). diff --git a/website/content/en/v0.32/concepts/nodeclasses.md b/website/content/en/v0.32/concepts/nodeclasses.md index 5079073cd3d1..21415e1b6a5d 100644 --- a/website/content/en/v0.32/concepts/nodeclasses.md +++ b/website/content/en/v0.32/concepts/nodeclasses.md @@ -66,7 +66,7 @@ spec: # Must specify one of "role" or "instanceProfile" for Karpenter to launch nodes instanceProfile: "KarpenterNodeInstanceProfile-${CLUSTER_NAME}" - # Optional, discovers amis to override the amiFamily's default + # Optional, discovers amis to override the amiFamily's default amis # Each term in the array of amiSelectorTerms is ORed together # Within a single term, all conditions are ANDed amiSelectorTerms: @@ -394,7 +394,9 @@ spec: ## spec.amiSelectorTerms -AMI Selector Terms are used to configure custom AMIs for Karpenter to use, where the AMIs are discovered through ids, owners, name, and [tags](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html). This selection logic is modeled as terms, where each term contains multiple conditions that must all be satisfied for the selector to match. Effectively, all requirements within a single term are ANDed together. It's possible that you may want to select on two different AMIs that have unrelated requirements. In this case, you can specify multiple terms which will be ORed together to form your selection logic. The example below shows how this selection logic is fulfilled. +AMI Selector Terms are used to configure custom AMIs for Karpenter to use, where the AMIs are discovered through ids, owners, name, and [tags](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html). **When you specify `amiSelectorTerms`, you fully override the default AMIs that are selected on by your EC2NodeClass [`amiFamily`]({{< ref "#specamifamily" >}}).** + +This selection logic is modeled as terms, where each term contains multiple conditions that must all be satisfied for the selector to match. Effectively, all requirements within a single term are ANDed together. It's possible that you may want to select on two different AMIs that have unrelated requirements. In this case, you can specify multiple terms which will be ORed together to form your selection logic. The example below shows how this selection logic is fulfilled. ```yaml amiSelectorTerms: @@ -488,7 +490,7 @@ spec: ## spec.instanceProfile -`InstanceProfile` is an optional field and is neccesary to tell Karpenter which identity nodes from this `EC2NodeClass` should assume. You must specify one of `role` or `instanceProfile` when creating a Karpenter `EC2NodeClasss`. If you use the `instanceProfile` field instead of `role`, Karpenter will not manage the InstanceProfile on your behalf. +`InstanceProfile` is an optional field and is necessary to tell Karpenter which identity nodes from this `EC2NodeClass` should assume. You must specify one of `role` or `instanceProfile` when creating a Karpenter `EC2NodeClasss`. If you use the `instanceProfile` field instead of `role`, Karpenter will not manage the InstanceProfile on your behalf. {{% alert title="Note" color="primary" %}} diff --git a/website/content/en/v0.32/reference/cloudformation.md b/website/content/en/v0.32/reference/cloudformation.md index e6ee0760e120..0519f916225d 100644 --- a/website/content/en/v0.32/reference/cloudformation.md +++ b/website/content/en/v0.32/reference/cloudformation.md @@ -202,7 +202,7 @@ actions on `fleet`, `instance`, `volume`, `network-interface`, and `launch-templ #### AllowScopedResourceTagging -The AllowScopedResourceTagging Sid allows EC2 [CreateTags](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateTags.html) actions on all instances created by Karpenter after their creation. It enforces that Karpenter is only able to update the tags on cluster instances it is operating on through the `karpenter.sh/cluster/${ClusterName}`" and `karpenter.sh/nodepool` tags. +The AllowScopedResourceTagging Sid allows EC2 [CreateTags](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateTags.html) actions on all instances created by Karpenter after their creation. It enforces that Karpenter is only able to update the tags on cluster instances it is operating on through the `kubernetes.io/cluster/${ClusterName}`" and `karpenter.sh/nodepool` tags. ```json { "Sid": "AllowScopedResourceTagging", @@ -211,7 +211,7 @@ The AllowScopedResourceTagging Sid allows EC2 [CreateTags](https://docs.aws.amaz "Action": "ec2:CreateTags", "Condition": { "StringEquals": { - "aws:ResourceTag/karpenter.sh/cluster/${ClusterName}": "owned" + "aws:ResourceTag/kubernetes.io/cluster/${ClusterName}": "owned" }, "StringLike": { "aws:ResourceTag/karpenter.sh/nodepool": "*" diff --git a/website/content/en/v0.32/troubleshooting.md b/website/content/en/v0.32/troubleshooting.md index b380423054a0..db6a5c867350 100644 --- a/website/content/en/v0.32/troubleshooting.md +++ b/website/content/en/v0.32/troubleshooting.md @@ -637,6 +637,22 @@ This typically occurs when the node has not been considered fully initialized fo This error indicates that the `vpc.amazonaws.com/pod-eni` resource was never reported on the node. If you've enabled Pod ENI for Karpenter nodes via the `aws.enablePodENI` setting, you will need to make the corresponding change to the VPC CNI to enable [security groups for pods](https://docs.aws.amazon.com/eks/latest/userguide/security-groups-for-pods.html) which will cause the resource to be registered. +### AWS Node Termination Handler (NTH) interactions +Karpenter [doesn't currently support draining and terminating on spot rebalance recommendations]({{< ref "concepts/disruption#interruption" >}}). Users who want support for both drain and terminate on spot interruption as well as drain and termination on spot rebalance recommendations may install Node Termination Handler (NTH) on their clusters to support this behavior. + +These two components do not share information between each other, meaning if you have drain and terminate functionality enabled on NTH, NTH may remove a node for a spot rebalance recommendation. Karpenter will replace the node to fulfill the pod capacity that was being fulfilled by the old node; however, Karpenter won't be aware of the reason that that node was terminated. This means that Karpenter may launch the same instance type that was just deprovisioned, causing a spot rebalance recommendation to be sent again. This can result in very short-lived instances where NTH continually removes nodes and Karpeneter re-launches the same instance type over and over again. + +Karpenter doesn't recommend reacting to spot rebalance recommendations when running Karpenter with spot nodes; however, if you absolutely require this functionality, note that the above scenario is possible. +Spot instances are time limited and, therefore, interruptible. When a signal is sent by AWS, it triggers actions from NTH and Karpenter, where the former signals a shutdown and the later provisions, creating a recursive situation. +This can be mitigated by either completely removing NTH or by setting the following values: + +* enableSpotInterruptionDraining: If false, do not drain nodes when the spot interruption termination notice is received. Only used in IMDS mode. +enableSpotInterruptionDraining: false + +* enableRebalanceDrainin: If true, drain nodes when the rebalance recommendation notice is received. Only used in IMDS mode. +enableRebalanceDraining: false + + ## Pricing ### Stale pricing data on isolated subnet diff --git a/website/content/en/v0.32/upgrading/v1beta1-migration.md b/website/content/en/v0.32/upgrading/v1beta1-migration.md index 5c1b126e003f..18cff11f2059 100644 --- a/website/content/en/v0.32/upgrading/v1beta1-migration.md +++ b/website/content/en/v0.32/upgrading/v1beta1-migration.md @@ -752,6 +752,34 @@ Karpenter v1beta1 introduces changes to some common labels, annotations, and sta | MachineExpired | Expired | | MachineDrifted | Drifted | +### IAM Controller Permissions + +v1beta1 introduces changes to the IAM permissions assigned to the Karpenter controller policy used when deploying Karpenter to your cluster with [IRSA](https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/setting-up-enable-IAM.html) or [EKS Pod Identity](https://docs.aws.amazon.com/eks/latest/userguide/pod-identities.html). + +You can see a full example of the v1beta1 required controller permissions by viewing the [v1beta1 Controller Policy](https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.4/website/content/en/preview/upgrading/v1beta1-controller-policy.json). + +Additionally, read more detail about the full set of permissions assigned to the Karpenter controller policy in the [CloudFormation Reference Guide]({{< ref "../reference/cloudformation" >}}). + +#### Updating Tag-Based Permissions + +Since Karpenter v1beta1 introduces changes to custom resource, label, and annotation naming, this also changes the tag keys that Karpenter uses to tag instances, volumes launch templates, and any other resources that Karpenter deploys and manages. + +By default, when using the [Karpenter Getting Started Guide]({{< ref "../getting-started/getting-started-with-karpenter" >}}) to setup Karpenter on your cluster, you will deploy an IAM policy that scopes Karpenter's permissions based on tag keys and values using [ABAC](https://aws.amazon.com/identity/attribute-based-access-control/). Any part of the Karpenter alpha controller policy which previously referenced `aws:RequestTag:karpenter.sh/provisioner-name` or `aws:ResourceTag:karpenter.sh/provisioner-name` is now updated in v1beta1 to be `aws:RequestTag:karpenter.sh/nodepool` and `aws:ResourceTag:karpenter.sh/nodepool`. + +{{% alert title="Note" color="warning" %}} +While migrating between alpha and beta, you will need to maintain the old tag permissions as well as the new permissions. You can remove the old tag key permissions from the controller policy when you have fully migrated to beta resources. This process for maintaining both sets of permissions while migrating is also mentioned in greater detail in the [Upgrade Procedure]({{< ref "#upgrade-procedure" >}}) shown above. +{{% /alert %}} + +#### Updating IAM Instance Profile Permissions + +Additionally, starting in v1beta1, Karpenter removes the need for you to manage your own instance profiles used to launch EC2 instances, allowing you to only specify the role that you want assigned to your instances in the `spec.role` of the `EC2NodeClass`. When you do this, Karpenter will generate and manage an instance profile on your behalf. + +To enable this functionality, you need to add `iam:` permissions that give Karpenter permission to generate and managed instance profiles. These permissions include `iam:CreateInstanceProfile`, `iam:TagInstanceProfile`, `iam:AddRoleToInstanceProfile`, `iam:RemoveRoleFromInstanceProfile`, `iam:DeleteInstanceProfile`, and `iam:GetInstanceProfile`. Each of these permissions is scoped down to only operate on instance profiles generated from a single Karpenter instance on a single Karpenter cluster using [ABAC](https://aws.amazon.com/identity/attribute-based-access-control/). + +{{% alert title="Note" color="warning" %}} +These `iam:` permissions are not required if you do not intend to use the `spec.role` field to enable the managed instance profile feature. Instead, you can use the `spec.instanceProfile` field to tell Karpenter to use an unmanaged instance profile explicitly. Note that this means that you have to manually provision and manage the instance profile yourself as you did in alpha. +{{% /alert %}} + ### Metrics The following table shows v1alpha5 metrics and the v1beta1 version of each metric. All metrics on this table will exist simultaneously, while both v1alpha5 and v1beta1 are supported within the same version. diff --git a/website/content/en/v0.33/concepts/disruption.md b/website/content/en/v0.33/concepts/disruption.md index 5e4fa193fc78..65d15696981e 100644 --- a/website/content/en/v0.33/concepts/disruption.md +++ b/website/content/en/v0.33/concepts/disruption.md @@ -168,7 +168,9 @@ When Karpenter detects one of these events will occur to your nodes, it automati For Spot interruptions, the NodePool will start a new node as soon as it sees the Spot interruption warning. Spot interruptions have a __2 minute notice__ before Amazon EC2 reclaims the instance. Karpenter's average node startup time means that, generally, there is sufficient time for the new node to become ready and to move the pods to the new node before the NodeClaim is reclaimed. {{% alert title="Note" color="primary" %}} -Karpenter publishes Kubernetes events to the node for all events listed above in addition to __Spot Rebalance Recommendations__. Karpenter does not currently support taint, drain, and terminate logic for Spot Rebalance Recommendations. +Karpenter publishes Kubernetes events to the node for all events listed above in addition to [__Spot Rebalance Recommendations__](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/rebalance-recommendations.html). Karpenter does not currently support taint, drain, and terminate logic for Spot Rebalance Recommendations. + +If you require handling for Spot Rebalance Recommendations, you can use the [AWS Node Termination Handler (NTH)](https://github.com/aws/aws-node-termination-handler) alongside Karpenter; however, note that the AWS Node Termination Handler cordons and drains nodes on rebalance recommendations, potentially causing more node churn in the cluster than with interruptions alone. Further information can be found in the [Troubleshooting Guide]({{< ref "../troubleshooting#aws-node-termination-handler-nth-interactions" >}}). {{% /alert %}} Karpenter enables this feature by watching an SQS queue which receives critical events from AWS services which may affect your nodes. Karpenter requires that an SQS queue be provisioned and EventBridge rules and targets be added that forward interruption events from AWS services to the SQS queue. Karpenter provides details for provisioning this infrastructure in the [CloudFormation template in the Getting Started Guide](../../getting-started/getting-started-with-karpenter/#create-the-karpenter-infrastructure-and-iam-roles). diff --git a/website/content/en/v0.33/concepts/nodeclasses.md b/website/content/en/v0.33/concepts/nodeclasses.md index 5079073cd3d1..21415e1b6a5d 100644 --- a/website/content/en/v0.33/concepts/nodeclasses.md +++ b/website/content/en/v0.33/concepts/nodeclasses.md @@ -66,7 +66,7 @@ spec: # Must specify one of "role" or "instanceProfile" for Karpenter to launch nodes instanceProfile: "KarpenterNodeInstanceProfile-${CLUSTER_NAME}" - # Optional, discovers amis to override the amiFamily's default + # Optional, discovers amis to override the amiFamily's default amis # Each term in the array of amiSelectorTerms is ORed together # Within a single term, all conditions are ANDed amiSelectorTerms: @@ -394,7 +394,9 @@ spec: ## spec.amiSelectorTerms -AMI Selector Terms are used to configure custom AMIs for Karpenter to use, where the AMIs are discovered through ids, owners, name, and [tags](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html). This selection logic is modeled as terms, where each term contains multiple conditions that must all be satisfied for the selector to match. Effectively, all requirements within a single term are ANDed together. It's possible that you may want to select on two different AMIs that have unrelated requirements. In this case, you can specify multiple terms which will be ORed together to form your selection logic. The example below shows how this selection logic is fulfilled. +AMI Selector Terms are used to configure custom AMIs for Karpenter to use, where the AMIs are discovered through ids, owners, name, and [tags](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html). **When you specify `amiSelectorTerms`, you fully override the default AMIs that are selected on by your EC2NodeClass [`amiFamily`]({{< ref "#specamifamily" >}}).** + +This selection logic is modeled as terms, where each term contains multiple conditions that must all be satisfied for the selector to match. Effectively, all requirements within a single term are ANDed together. It's possible that you may want to select on two different AMIs that have unrelated requirements. In this case, you can specify multiple terms which will be ORed together to form your selection logic. The example below shows how this selection logic is fulfilled. ```yaml amiSelectorTerms: @@ -488,7 +490,7 @@ spec: ## spec.instanceProfile -`InstanceProfile` is an optional field and is neccesary to tell Karpenter which identity nodes from this `EC2NodeClass` should assume. You must specify one of `role` or `instanceProfile` when creating a Karpenter `EC2NodeClasss`. If you use the `instanceProfile` field instead of `role`, Karpenter will not manage the InstanceProfile on your behalf. +`InstanceProfile` is an optional field and is necessary to tell Karpenter which identity nodes from this `EC2NodeClass` should assume. You must specify one of `role` or `instanceProfile` when creating a Karpenter `EC2NodeClasss`. If you use the `instanceProfile` field instead of `role`, Karpenter will not manage the InstanceProfile on your behalf. {{% alert title="Note" color="primary" %}} diff --git a/website/content/en/v0.33/reference/cloudformation.md b/website/content/en/v0.33/reference/cloudformation.md index ee2224feb8f2..d469ceb2dbac 100644 --- a/website/content/en/v0.33/reference/cloudformation.md +++ b/website/content/en/v0.33/reference/cloudformation.md @@ -113,7 +113,7 @@ Someone wanting to add Karpenter to an existing cluster, instead of using `cloud The AllowScopedEC2InstanceActions statement ID (Sid) identifies a set of EC2 resources that are allowed to be accessed with [RunInstances](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_RunInstances.html) and [CreateFleet](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html) actions. -For `RunInstances` and `CreateFleet` actions, the Karpenter controller can read (but not create) `image`, `snapshot`, `spot-instances-request`, `security-group`, `subnet` and `launch-template` EC2 resources, scoped for the particular AWS partition and region. +For `RunInstances` and `CreateFleet` actions, the Karpenter controller can read (but not create) `image`, `snapshot`, `security-group`, `subnet` and `launch-template` EC2 resources, scoped for the particular AWS partition and region. ```json { @@ -122,7 +122,6 @@ For `RunInstances` and `CreateFleet` actions, the Karpenter controller can read "Resource": [ "arn:${AWS::Partition}:ec2:${AWS::Region}::image/*", "arn:${AWS::Partition}:ec2:${AWS::Region}::snapshot/*", - "arn:${AWS::Partition}:ec2:${AWS::Region}:*:spot-instances-request/*", "arn:${AWS::Partition}:ec2:${AWS::Region}:*:security-group/*", "arn:${AWS::Partition}:ec2:${AWS::Region}:*:subnet/*", "arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*" @@ -138,7 +137,7 @@ For `RunInstances` and `CreateFleet` actions, the Karpenter controller can read The AllowScopedEC2InstanceActionsWithTags Sid allows the [RunInstances](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_RunInstances.html), [CreateFleet](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html), and [CreateLaunchTemplate](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateLaunchTemplate.html) -actions requested by the Karpenter controller to create all `fleet`, `instance`, `volume`, `network-interface`, or `launch-template` EC2 resources (for the partition and region), and requires that the `kubernetes.io/cluster/${ClusterName}` tag be set to `owned` and a `karpenter.sh/nodepool` tag be set to any value. This ensures that Karpenter is only allowed to create instances for a single EKS cluster. +actions requested by the Karpenter controller to create all `fleet`, `instance`, `volume`, `network-interface`, `launch-template` or `spot-instances-request` EC2 resources (for the partition and region), and requires that the `kubernetes.io/cluster/${ClusterName}` tag be set to `owned` and a `karpenter.sh/nodepool` tag be set to any value. This ensures that Karpenter is only allowed to create instances for a single EKS cluster. ```json { @@ -149,7 +148,8 @@ actions requested by the Karpenter controller to create all `fleet`, `instance`, "arn:${AWS::Partition}:ec2:${AWS::Region}:*:instance/*", "arn:${AWS::Partition}:ec2:${AWS::Region}:*:volume/*", "arn:${AWS::Partition}:ec2:${AWS::Region}:*:network-interface/*", - "arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*" + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*", + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:spot-instances-request/*" ], "Action": [ "ec2:RunInstances", @@ -170,7 +170,7 @@ actions requested by the Karpenter controller to create all `fleet`, `instance`, #### AllowScopedResourceCreationTagging The AllowScopedResourceCreationTagging Sid allows EC2 [CreateTags](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateTags.html) -actions on `fleet`, `instance`, `volume`, `network-interface`, and `launch-template` resources, While making `RunInstance`, `CreateFleet`, or `CreateLaunchTemplate` calls. Additionally, this ensures that resources can't be tagged arbitrarily by Karpenter after they are created. +actions on `fleet`, `instance`, `volume`, `network-interface`, `launch-template` and `spot-instances-request` resources, While making `RunInstance`, `CreateFleet`, or `CreateLaunchTemplate` calls. Additionally, this ensures that resources can't be tagged arbitrarily by Karpenter after they are created. ```json { @@ -181,7 +181,8 @@ actions on `fleet`, `instance`, `volume`, `network-interface`, and `launch-templ "arn:${AWS::Partition}:ec2:${AWS::Region}:*:instance/*", "arn:${AWS::Partition}:ec2:${AWS::Region}:*:volume/*", "arn:${AWS::Partition}:ec2:${AWS::Region}:*:network-interface/*", - "arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*" + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*", + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:spot-instances-request/*" ], "Action": "ec2:CreateTags", "Condition": { @@ -202,7 +203,7 @@ actions on `fleet`, `instance`, `volume`, `network-interface`, and `launch-templ #### AllowScopedResourceTagging -The AllowScopedResourceTagging Sid allows EC2 [CreateTags](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateTags.html) actions on all instances created by Karpenter after their creation. It enforces that Karpenter is only able to update the tags on cluster instances it is operating on through the `karpenter.sh/cluster/${ClusterName}`" and `karpenter.sh/nodepool` tags. +The AllowScopedResourceTagging Sid allows EC2 [CreateTags](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateTags.html) actions on all instances created by Karpenter after their creation. It enforces that Karpenter is only able to update the tags on cluster instances it is operating on through the `kubernetes.io/cluster/${ClusterName}`" and `karpenter.sh/nodepool` tags. ```json { "Sid": "AllowScopedResourceTagging", @@ -211,7 +212,7 @@ The AllowScopedResourceTagging Sid allows EC2 [CreateTags](https://docs.aws.amaz "Action": "ec2:CreateTags", "Condition": { "StringEquals": { - "aws:ResourceTag/karpenter.sh/cluster/${ClusterName}": "owned" + "aws:ResourceTag/kubernetes.io/cluster/${ClusterName}": "owned" }, "StringLike": { "aws:ResourceTag/karpenter.sh/nodepool": "*" diff --git a/website/content/en/v0.33/troubleshooting.md b/website/content/en/v0.33/troubleshooting.md index cbb5b07d1574..c33f04264e3b 100644 --- a/website/content/en/v0.33/troubleshooting.md +++ b/website/content/en/v0.33/troubleshooting.md @@ -637,6 +637,21 @@ This typically occurs when the node has not been considered fully initialized fo This error indicates that the `vpc.amazonaws.com/pod-eni` resource was never reported on the node. If you've enabled Pod ENI for Karpenter nodes via the `aws.enablePodENI` setting, you will need to make the corresponding change to the VPC CNI to enable [security groups for pods](https://docs.aws.amazon.com/eks/latest/userguide/security-groups-for-pods.html) which will cause the resource to be registered. +### AWS Node Termination Handler (NTH) interactions +Karpenter [doesn't currently support draining and terminating on spot rebalance recommendations]({{< ref "concepts/disruption#interruption" >}}). Users who want support for both drain and terminate on spot interruption as well as drain and termination on spot rebalance recommendations may install Node Termination Handler (NTH) on their clusters to support this behavior. + +These two components do not share information between each other, meaning if you have drain and terminate functionality enabled on NTH, NTH may remove a node for a spot rebalance recommendation. Karpenter will replace the node to fulfill the pod capacity that was being fulfilled by the old node; however, Karpenter won't be aware of the reason that that node was terminated. This means that Karpenter may launch the same instance type that was just deprovisioned, causing a spot rebalance recommendation to be sent again. This can result in very short-lived instances where NTH continually removes nodes and Karpeneter re-launches the same instance type over and over again. + +Karpenter doesn't recommend reacting to spot rebalance recommendations when running Karpenter with spot nodes; however, if you absolutely require this functionality, note that the above scenario is possible. +Spot instances are time limited and, therefore, interruptible. When a signal is sent by AWS, it triggers actions from NTH and Karpenter, where the former signals a shutdown and the later provisions, creating a recursive situation. +This can be mitigated by either completely removing NTH or by setting the following values: + +* enableSpotInterruptionDraining: If false, do not drain nodes when the spot interruption termination notice is received. Only used in IMDS mode. +enableSpotInterruptionDraining: false + +* enableRebalanceDrainin: If true, drain nodes when the rebalance recommendation notice is received. Only used in IMDS mode. +enableRebalanceDraining: false + ## Pricing ### Stale pricing data on isolated subnet diff --git a/website/content/en/v0.33/upgrading/upgrade-guide.md b/website/content/en/v0.33/upgrading/upgrade-guide.md index 2d243e72c76c..254d81705721 100644 --- a/website/content/en/v0.33/upgrading/upgrade-guide.md +++ b/website/content/en/v0.33/upgrading/upgrade-guide.md @@ -36,7 +36,7 @@ kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0 ### Upgrading to v0.33.0+ * Karpenter now tags `spot-instances-request` with the same tags that it tags instances, volumes, and primary ENIs. This means that you will now need to add `ec2:CreateTags` permission for `spot-instances-request`. You can also further scope your controller policy for the `ec2:RunInstances` action to require that it launches the `spot-instances-request` with these specific tags. You can view an example of scoping these actions in the [Getting Started Guide's default CloudFormation controller policy](https://github.com/aws/karpenter/blob/v0.33.0/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml#L61). -* We now recommend that you set the installation namespace for your Karpenter controllers to `kube-system` to denote Karpenter as a critical cluster component. This ensures that requests from the Karpenter controllers are treated with higher priority by assigning them to a different [PriorityLevelConfiguration](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/#prioritylevelconfiguration) than generic requests from other namespaces. For more details on API Priority and Fairness, read the [Kubernetes API Priorirty and Fairness Conceptual Docs](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/). Note: Changing the namespace for your Karpenter release will cause the service account namespace to change. If you are using IRSA for authentication with AWS, you will need to change scoping set in the controller's trust policy from `karpenter:karpenter` to `kube-system:karpenter`. +* We now recommend that you set the installation namespace for your Karpenter controllers to `kube-system` to denote Karpenter as a critical cluster component. This ensures that requests from the Karpenter controllers are treated with higher priority by assigning them to a different [PriorityLevelConfiguration](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/#prioritylevelconfiguration) than generic requests from other namespaces. For more details on API Priority and Fairness, read the [Kubernetes API Priority and Fairness Conceptual Docs](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/). Note: Changing the namespace for your Karpenter release will cause the service account namespace to change. If you are using IRSA for authentication with AWS, you will need to change scoping set in the controller's trust policy from `karpenter:karpenter` to `kube-system:karpenter`. * `v0.33.x` disables mutating and validating webhooks by default in favor of using [Common Expression Language for CRD validation](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#validation). The Common Expression Language Validation Feature [is enabled by default on EKS 1.25](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#validation-rules). If you are using Kubernetes version >= 1.25, no further action is required. If you are using a Kubernetes version below 1.25, you now need to set `DISABLE_WEBHOOK=false` in your container environment variables or `--set webhook.enabled=true` if using Helm. View the [Webhook Support Deprecated in Favor of CEL Section of the v1beta1 Migration Guide]({{}}). * `v0.33.x` drops support for passing settings through the `karpenter-global-settings` ConfigMap. You should pass settings through the container environment variables in the Karpenter deployment manifest. View the [Global Settings Section of the v1beta1 Migration Guide]({{}}) for more details. * `v0.33.x` enables `Drift=true` by default in the `FEATURE_GATES`. If you previously didn't enable the feature gate, Karpenter will now check if there is a difference between the desired state of your nodes declared in your NodePool and the actual state of your nodes. View the [Drift Section of Disruption Conceptual Docs]({{}}) for more details. diff --git a/website/content/en/v0.33/upgrading/v1beta1-migration.md b/website/content/en/v0.33/upgrading/v1beta1-migration.md index 8ad11065c4f4..a98306c0d60e 100644 --- a/website/content/en/v0.33/upgrading/v1beta1-migration.md +++ b/website/content/en/v0.33/upgrading/v1beta1-migration.md @@ -752,6 +752,34 @@ Karpenter v1beta1 introduces changes to some common labels, annotations, and sta | MachineExpired | Expired | | MachineDrifted | Drifted | +### IAM Controller Permissions + +v1beta1 introduces changes to the IAM permissions assigned to the Karpenter controller policy used when deploying Karpenter to your cluster with [IRSA](https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/setting-up-enable-IAM.html) or [EKS Pod Identity](https://docs.aws.amazon.com/eks/latest/userguide/pod-identities.html). + +You can see a full example of the v1beta1 required controller permissions by viewing the [v1beta1 Controller Policy](https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.4/website/content/en/preview/upgrading/v1beta1-controller-policy.json). + +Additionally, read more detail about the full set of permissions assigned to the Karpenter controller policy in the [CloudFormation Reference Guide]({{< ref "../reference/cloudformation" >}}). + +#### Updating Tag-Based Permissions + +Since Karpenter v1beta1 introduces changes to custom resource, label, and annotation naming, this also changes the tag keys that Karpenter uses to tag instances, volumes launch templates, and any other resources that Karpenter deploys and manages. + +By default, when using the [Karpenter Getting Started Guide]({{< ref "../getting-started/getting-started-with-karpenter" >}}) to setup Karpenter on your cluster, you will deploy an IAM policy that scopes Karpenter's permissions based on tag keys and values using [ABAC](https://aws.amazon.com/identity/attribute-based-access-control/). Any part of the Karpenter alpha controller policy which previously referenced `aws:RequestTag:karpenter.sh/provisioner-name` or `aws:ResourceTag:karpenter.sh/provisioner-name` is now updated in v1beta1 to be `aws:RequestTag:karpenter.sh/nodepool` and `aws:ResourceTag:karpenter.sh/nodepool`. + +{{% alert title="Note" color="warning" %}} +While migrating between alpha and beta, you will need to maintain the old tag permissions as well as the new permissions. You can remove the old tag key permissions from the controller policy when you have fully migrated to beta resources. This process for maintaining both sets of permissions while migrating is also mentioned in greater detail in the [Upgrade Procedure]({{< ref "#upgrade-procedure" >}}) shown above. +{{% /alert %}} + +#### Updating IAM Instance Profile Permissions + +Additionally, starting in v1beta1, Karpenter removes the need for you to manage your own instance profiles used to launch EC2 instances, allowing you to only specify the role that you want assigned to your instances in the `spec.role` of the `EC2NodeClass`. When you do this, Karpenter will generate and manage an instance profile on your behalf. + +To enable this functionality, you need to add `iam:` permissions that give Karpenter permission to generate and managed instance profiles. These permissions include `iam:CreateInstanceProfile`, `iam:TagInstanceProfile`, `iam:AddRoleToInstanceProfile`, `iam:RemoveRoleFromInstanceProfile`, `iam:DeleteInstanceProfile`, and `iam:GetInstanceProfile`. Each of these permissions is scoped down to only operate on instance profiles generated from a single Karpenter instance on a single Karpenter cluster using [ABAC](https://aws.amazon.com/identity/attribute-based-access-control/). + +{{% alert title="Note" color="warning" %}} +These `iam:` permissions are not required if you do not intend to use the `spec.role` field to enable the managed instance profile feature. Instead, you can use the `spec.instanceProfile` field to tell Karpenter to use an unmanaged instance profile explicitly. Note that this means that you have to manually provision and manage the instance profile yourself as you did in alpha. +{{% /alert %}} + ### Metrics The following table shows v1alpha5 metrics and the v1beta1 version of each metric. All metrics on this table will exist simultaneously, while both v1alpha5 and v1beta1 are supported within the same version. diff --git a/website/static/_redirects b/website/static/_redirects index 12e744ecc390..3a0e4abc37c7 100644 --- a/website/static/_redirects +++ b/website/static/_redirects @@ -16,6 +16,36 @@ /preview/concepts/settings /preview/reference/settings /preview/concepts/threat-model /preview/reference/threat-model +# Redirect all docs documentation to the new routes +/docs/concepts/provisioners /docs/concepts/nodepools +/docs/concepts/node-templates /docs/concepts/nodeclasses +/docs/concepts/deprovisioning /docs/concepts/disruption +/docs/upgrade-guide /docs/upgrading/upgrade-guide +/docs/concepts/instance-types /docs/reference/instance-types +/docs/concepts/metrics /docs/reference/metrics +/docs/concepts/settings /docs/reference/settings +/docs/concepts/threat-model /docs/reference/threat-model + +# Redirect all v0.30 documentation from the new routes to the old routes +/v0.30/concepts/nodepools /v0.30/concepts/provisioners +/v0.30/concepts/nodeclasses /v0.30/concepts/node-templates +/v0.30/concepts/disruption /v0.30/concepts/deprovisioning +/v0.30/upgrading/upgrade-guide /v0.30/upgrade-guide +/v0.30/reference/instance-types /v0.30/concepts/instance-types +/v0.30/reference/metrics /v0.30/concepts/metrics +/v0.30/reference/settings /v0.30/concepts/settings +/v0.30/reference/threat-model /v0.30/concepts/threat-model + +# Redirect all v0.31 documentation from the new routes to the old routes +/v0.31/concepts/nodepools /v0.31/concepts/provisioners +/v0.31/concepts/nodeclasses /v0.31/concepts/node-templates +/v0.31/concepts/disruption /v0.31/concepts/deprovisioning +/v0.31/upgrading/upgrade-guide /v0.31/upgrade-guide +/v0.31/reference/instance-types /v0.31/concepts/instance-types +/v0.31/reference/metrics /v0.31/concepts/metrics +/v0.31/reference/settings /v0.31/concepts/settings +/v0.31/reference/threat-model /v0.31/concepts/threat-model + # Redirect all v0.32 documentation to the new routes /v0.32/concepts/provisioners /v0.32/concepts/nodepools /v0.32/concepts/node-templates /v0.32/concepts/nodeclasses @@ -26,12 +56,22 @@ /v0.32/concepts/settings /v0.32/reference/settings /v0.32/concepts/threat-model /v0.32/reference/threat-model -# Redirect all docs documentation to the new routes -/docs/concepts/provisioners /docs/concepts/nodepools -/docs/concepts/node-templates /docs/concepts/nodeclasses -/docs/concepts/deprovisioning /docs/concepts/disruption -/docs/upgrade-guide /docs/upgrading/upgrade-guide -/docs/concepts/instance-types /docs/reference/instance-types -/docs/concepts/metrics /docs/reference/metrics -/docs/concepts/settings /docs/reference/settings -/docs/concepts/threat-model /docs/reference/threat-model \ No newline at end of file +# Redirect all v0.33 documentation to the new routes +/v0.33/concepts/provisioners /v0.33/concepts/nodepools +/v0.33/concepts/node-templates /v0.33/concepts/nodeclasses +/v0.33/concepts/deprovisioning /v0.33/concepts/disruption +/v0.33/upgrade-guide /v0.33/upgrading/upgrade-guide +/v0.33/concepts/instance-types /v0.33/reference/instance-types +/v0.33/concepts/metrics /v0.33/reference/metrics +/v0.33/concepts/settings /v0.33/reference/settings +/v0.33/concepts/threat-model /v0.33/reference/threat-model + +# Redirect all v0.34 documentation to the new routes +/v0.34/concepts/provisioners /v0.34/concepts/nodepools +/v0.34/concepts/node-templates /v0.34/concepts/nodeclasses +/v0.34/concepts/deprovisioning /v0.34/concepts/disruption +/v0.34/upgrade-guide /v0.34/upgrading/upgrade-guide +/v0.34/concepts/instance-types /v0.34/reference/instance-types +/v0.34/concepts/metrics /v0.34/reference/metrics +/v0.34/concepts/settings /v0.34/reference/settings +/v0.34/concepts/threat-model /v0.34/reference/threat-model \ No newline at end of file