diff --git a/.github/actions/e2e/cleanup/action.yaml b/.github/actions/e2e/cleanup/action.yaml index 567a251d71b3..df1ff2f04229 100644 --- a/.github/actions/e2e/cleanup/action.yaml +++ b/.github/actions/e2e/cleanup/action.yaml @@ -39,10 +39,10 @@ runs: for name in $(aws iam list-instance-profiles --query "InstanceProfiles[*].{Name:InstanceProfileName}" --output text); do tags=$(aws iam list-instance-profile-tags --instance-profile-name $name --output json || true) if [[ $(echo $tags | jq -r '.Tags[] | select(.Key == "testing/cluster") | .Value') == "${{ inputs.cluster_name }}" ]]; then + echo "Deleting instance profile '$name'..." roleName=$(aws iam get-instance-profile --instance-profile-name $name --query "InstanceProfile.Roles[*].{Name:RoleName}" --output text) aws iam remove-role-from-instance-profile --instance-profile-name $name --role-name $roleName aws iam delete-instance-profile --instance-profile-name $name - break fi done - name: delete-cluster diff --git a/.github/actions/e2e/create-cluster/action.yaml b/.github/actions/e2e/create-cluster/action.yaml index 7c4ba3a42ee1..eb81c8abeb15 100644 --- a/.github/actions/e2e/create-cluster/action.yaml +++ b/.github/actions/e2e/create-cluster/action.yaml @@ -113,6 +113,10 @@ runs: - key: CriticalAddonsOnly value: "true" effect: NoSchedule + cloudWatch: + clusterLogging: + enableTypes: ["*"] + logRetentionInDays: 30 iam: serviceRolePermissionsBoundary: "arn:aws:iam::${{ inputs.account_id }}:policy/GithubActionsPermissionsBoundary" serviceAccounts: @@ -152,8 +156,6 @@ runs: # We need to call these update iamserviceaccount commands again since the "eksctl upgrade cluster" action # doesn't handle updates to IAM serviceaccounts correctly when the roles assigned to them change eksctl update iamserviceaccount -f clusterconfig.yaml --approve - - - name: tag oidc provider of the cluster if: always() shell: bash diff --git a/.github/workflows/e2e-upgrade.yaml b/.github/workflows/e2e-upgrade.yaml index d73890ff6c05..a7f1dff781ac 100644 --- a/.github/workflows/e2e-upgrade.yaml +++ b/.github/workflows/e2e-upgrade.yaml @@ -152,7 +152,7 @@ jobs: - name: run the Upgrade test suite run: | aws eks update-kubeconfig --name ${{ env.CLUSTER_NAME }} - TEST_SUITE="Integration" make e2etests + CLUSTER_NAME=${{ env.CLUSTER_NAME }} INTERRUPTION_QUEUE=${{ env.CLUSTER_NAME }} CLUSTER_ENDPOINT="$(aws eks describe-cluster --name ${{ env.CLUSTER_NAME }} --query "cluster.endpoint" --output text)" TEST_SUITE="Integration" make e2etests - name: notify slack of success or failure uses: ./.github/actions/e2e/slack/notify if: (success() || failure()) && github.event_name != 'workflow_run' && github.event_name != 'conformance' diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index f8d838ed17f4..6f36ed52974a 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -131,7 +131,9 @@ jobs: - name: run the ${{ inputs.suite }} test suite run: | aws eks update-kubeconfig --name ${{ env.CLUSTER_NAME }} - TEST_SUITE="${{ inputs.suite }}" ENABLE_METRICS=${{ inputs.enable_metrics }} METRICS_REGION=${{ vars.TIMESTREAM_REGION }} GIT_REF="$(git rev-parse HEAD)" make e2etests + TEST_SUITE="${{ inputs.suite }}" ENABLE_METRICS=${{ inputs.enable_metrics }} METRICS_REGION=${{ vars.TIMESTREAM_REGION }} GIT_REF="$(git rev-parse HEAD)" \ + CLUSTER_NAME="${{ env.CLUSTER_NAME }}" CLUSTER_ENDPOINT="$(aws eks describe-cluster --name ${{ env.CLUSTER_NAME }} --query "cluster.endpoint" --output text)" \ + INTERRUPTION_QUEUE="${{ env.CLUSTER_NAME }}" make e2etests - name: notify slack of success or failure uses: ./.github/actions/e2e/slack/notify if: (success() || failure()) && github.event_name != 'workflow_run' && inputs.workflow_trigger != 'conformance' diff --git a/ADOPTERS.md b/ADOPTERS.md index 999630ea2ba6..e1893a8fb798 100644 --- a/ADOPTERS.md +++ b/ADOPTERS.md @@ -38,6 +38,7 @@ If you are open to others contacting you about your use of Karpenter on Slack, a | PITS Global Data Recovery Services | Used to manage continuous integration and continuous delivery/deployment workflows. | N/A | [PITS Global Data Recovery Services](https://www.pitsdatarecovery.net/) | | PlanetScale | Leveraging Karpenter to dynamically deploy serverless MySQL workloads. | `@jtcunning` | [Homepage](https://www.planetscale.com/) | | QuestDB | Using Karpenter for the service nodes of the QuestBD Cloud (time-series database). | [questdb slack group](https://slack.questdb.io/) | [QuestDB](https://questdb.io/) | +| Rapid7 | Using Karpenter across all of our Kubernetes infrastructure for efficient autoscaling, both in terms of speed and cost | `@arobinson`, `@Ross Kirk`, `@Ryan Williams` | [Homepage](https://www.rapid7.com/) | | Sendcloud | Using Karpenter to scale our k8s clusters for Europe’s #1 shipping automation platform | N/A | [Homepage](https://www.sendcloud.com/) | | Sentra | Using Karpenter to scale our EKS clusters, running our platform and workflows while maximizing cost-efficiency with minimal operational overhead | `@Roei Jacobovich` | [Homepage](https://sentra.io/) | | Stone Pagamentos | Using Karpenter to do smart sizing of our clusters | `@fabiano-amaral` | [Stone Pagamentos](https://www.stone.com.br/) | diff --git a/Makefile b/Makefile index a0013374c069..260a1a70f974 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,8 @@ export K8S_VERSION ?= 1.27.x CLUSTER_NAME ?= $(shell kubectl config view --minify -o jsonpath='{.clusters[].name}' | rev | cut -d"/" -f1 | rev | cut -d"." -f1) -## Inject the app version into project.Version -ifdef SNAPSHOT_TAG -LDFLAGS ?= -ldflags=-X=github.com/aws/karpenter/pkg/utils/project.Version=$(SNAPSHOT_TAG) -else -LDFLAGS ?= -ldflags=-X=github.com/aws/karpenter/pkg/utils/project.Version=$(shell git describe --tags --always) -endif +## Inject the app version into operator.Version +LDFLAGS ?= -ldflags=-X=github.com/aws/karpenter-core/pkg/operator.Version=$(shell git describe --tags --always) GOFLAGS ?= $(LDFLAGS) WITH_GOFLAGS = GOFLAGS="$(GOFLAGS)" @@ -68,6 +64,7 @@ clean-run: ## Clean resources deployed by the run target test: ## Run tests go test -v ./pkg/$(shell echo $(TEST_SUITE) | tr A-Z a-z)/... --ginkgo.focus="${FOCUS}" --ginkgo.vv + cd tools/karpenter-convert && go test -v ./pkg/... --ginkgo.focus="${FOCUS}" --ginkgo.vv battletest: ## Run randomized, racing, code-covered tests go test -v ./pkg/... \ diff --git a/charts/karpenter-crd/templates/karpenter.k8s.aws_ec2nodeclasses.yaml b/charts/karpenter-crd/templates/karpenter.k8s.aws_ec2nodeclasses.yaml new file mode 120000 index 000000000000..3bb741dfaf65 --- /dev/null +++ b/charts/karpenter-crd/templates/karpenter.k8s.aws_ec2nodeclasses.yaml @@ -0,0 +1 @@ +../../../pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml \ No newline at end of file diff --git a/charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml b/charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml new file mode 120000 index 000000000000..3f572b57547e --- /dev/null +++ b/charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml @@ -0,0 +1 @@ +../../../pkg/apis/crds/karpenter.sh_nodeclaims.yaml \ No newline at end of file diff --git a/charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml b/charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml new file mode 120000 index 000000000000..36d2d1dd918a --- /dev/null +++ b/charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml @@ -0,0 +1 @@ +../../../pkg/apis/crds/karpenter.sh_nodepools.yaml \ No newline at end of file diff --git a/charts/karpenter/README.md b/charts/karpenter/README.md index b43423909809..9b4eee5cb7b0 100644 --- a/charts/karpenter/README.md +++ b/charts/karpenter/README.md @@ -58,6 +58,9 @@ helm upgrade --install --namespace karpenter --create-namespace \ | logConfig.errorOutputPaths | list | `["stderr"]` | Log errorOutputPaths - defaults to stderr only | | logConfig.logEncoding | string | `"console"` | Log encoding - defaults to console - must be one of 'json', 'console' | | logConfig.logLevel | object | `{"controller":"debug","global":"debug","webhook":"error"}` | Component-based log configuration | +| logConfig.logLevel.controller | string | `"debug"` | Controller log level, defaults to 'debug' | +| logConfig.logLevel.global | string | `"debug"` | Global log level, defaults to 'debug' | +| logConfig.logLevel.webhook | string | `"error"` | Error log level, defaults to 'error' | | logConfig.outputPaths | list | `["stdout"]` | Log outputPaths - defaults to stdout only | | logEncoding | string | `"console"` | Global log encoding (Deprecated: Use logConfig.logEncoding instead) | | logLevel | string | `"debug"` | Global log level | diff --git a/charts/karpenter/crds/karpenter.k8s.aws_ec2nodeclasses.yaml b/charts/karpenter/crds/karpenter.k8s.aws_ec2nodeclasses.yaml new file mode 120000 index 000000000000..3bb741dfaf65 --- /dev/null +++ b/charts/karpenter/crds/karpenter.k8s.aws_ec2nodeclasses.yaml @@ -0,0 +1 @@ +../../../pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml \ No newline at end of file diff --git a/charts/karpenter/crds/karpenter.sh_nodeclaims.yaml b/charts/karpenter/crds/karpenter.sh_nodeclaims.yaml new file mode 120000 index 000000000000..3f572b57547e --- /dev/null +++ b/charts/karpenter/crds/karpenter.sh_nodeclaims.yaml @@ -0,0 +1 @@ +../../../pkg/apis/crds/karpenter.sh_nodeclaims.yaml \ No newline at end of file diff --git a/charts/karpenter/crds/karpenter.sh_nodepools.yaml b/charts/karpenter/crds/karpenter.sh_nodepools.yaml new file mode 120000 index 000000000000..36d2d1dd918a --- /dev/null +++ b/charts/karpenter/crds/karpenter.sh_nodepools.yaml @@ -0,0 +1 @@ +../../../pkg/apis/crds/karpenter.sh_nodepools.yaml \ No newline at end of file diff --git a/charts/karpenter/templates/aggregate-clusterrole.yaml b/charts/karpenter/templates/aggregate-clusterrole.yaml index 155cccea565a..b46cfd8b2013 100644 --- a/charts/karpenter/templates/aggregate-clusterrole.yaml +++ b/charts/karpenter/templates/aggregate-clusterrole.yaml @@ -13,6 +13,12 @@ rules: - apiGroups: ["karpenter.sh"] resources: ["provisioners", "provisioners/status", "machines", "machines/status"] verbs: ["get", "list", "watch", "create", "delete", "patch"] + - apiGroups: ["karpenter.sh"] + resources: ["nodepools", "nodepools/status", "nodeclaims", "nodeclaims/status"] + verbs: ["get", "list", "watch", "create", "delete", "patch"] - apiGroups: ["karpenter.k8s.aws"] resources: ["awsnodetemplates"] verbs: ["get", "list", "watch", "create", "delete", "patch"] + - apiGroups: ["karpenter.k8s.aws"] + resources: ["ec2nodeclasses"] + verbs: ["get", "list", "watch", "create", "delete", "patch"] diff --git a/charts/karpenter/templates/clusterrole-core.yaml b/charts/karpenter/templates/clusterrole-core.yaml index 2b28bfb826e9..76752a65b039 100644 --- a/charts/karpenter/templates/clusterrole-core.yaml +++ b/charts/karpenter/templates/clusterrole-core.yaml @@ -32,6 +32,9 @@ rules: - apiGroups: ["karpenter.sh"] resources: ["provisioners", "provisioners/status", "machines", "machines/status"] verbs: ["get", "list", "watch"] + - apiGroups: ["karpenter.sh"] + resources: ["nodepools", "nodepools/status", "nodeclaims", "nodeclaims/status"] + verbs: ["get", "list", "watch"] - apiGroups: [""] resources: ["pods", "nodes", "persistentvolumes", "persistentvolumeclaims", "replicationcontrollers", "namespaces"] verbs: ["get", "list", "watch"] @@ -56,6 +59,12 @@ rules: - apiGroups: ["karpenter.sh"] resources: ["provisioners", "provisioners/status"] verbs: ["update", "patch"] + - apiGroups: ["karpenter.sh"] + resources: ["nodeclaims", "nodeclaims/status"] + verbs: ["create", "delete", "update", "patch"] + - apiGroups: ["karpenter.sh"] + resources: ["nodepools", "nodepools/status"] + verbs: ["update", "patch"] - apiGroups: [""] resources: ["events"] verbs: ["create", "patch"] diff --git a/charts/karpenter/templates/clusterrole.yaml b/charts/karpenter/templates/clusterrole.yaml index 66ba10b0a393..8580931421a0 100644 --- a/charts/karpenter/templates/clusterrole.yaml +++ b/charts/karpenter/templates/clusterrole.yaml @@ -30,11 +30,11 @@ metadata: rules: # Read - apiGroups: ["karpenter.k8s.aws"] - resources: ["awsnodetemplates"] + resources: ["awsnodetemplates", "ec2nodeclasses"] verbs: ["get", "list", "watch"] # Write - apiGroups: ["karpenter.k8s.aws"] - resources: ["awsnodetemplates", "awsnodetemplates/status"] + resources: ["awsnodetemplates", "awsnodetemplates/status", "ec2nodeclasses", "ec2nodeclasses/status"] verbs: ["patch", "update"] {{- if .Values.webhook.enabled }} - apiGroups: ["admissionregistration.k8s.io"] @@ -45,4 +45,4 @@ rules: resources: ["mutatingwebhookconfigurations"] verbs: ["update"] resourceNames: ["defaulting.webhook.karpenter.k8s.aws"] -{{- end }} \ No newline at end of file +{{- end }} diff --git a/charts/karpenter/templates/webhooks-core.yaml b/charts/karpenter/templates/webhooks-core.yaml index 08e9ab8fdfce..55fa5cfe57ec 100644 --- a/charts/karpenter/templates/webhooks-core.yaml +++ b/charts/karpenter/templates/webhooks-core.yaml @@ -24,12 +24,36 @@ webhooks: - karpenter.sh apiVersions: - v1alpha5 + operations: + - CREATE + - UPDATE resources: - provisioners - provisioners/status + scope: '*' + - apiGroups: + - karpenter.sh + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + - DELETE + resources: + - nodeclaims + - nodeclaims/status + scope: '*' + - apiGroups: + - karpenter.sh + apiVersions: + - v1beta1 operations: - CREATE - UPDATE + resources: + - nodepools + - nodepools/status + scope: '*' --- apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration diff --git a/charts/karpenter/templates/webhooks.yaml b/charts/karpenter/templates/webhooks.yaml index fb92da161387..e52fc722aab1 100644 --- a/charts/karpenter/templates/webhooks.yaml +++ b/charts/karpenter/templates/webhooks.yaml @@ -31,16 +31,28 @@ webhooks: - awsnodetemplates - awsnodetemplates/status scope: '*' + - apiGroups: + - karpenter.k8s.aws + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - ec2nodeclasses + - ec2nodeclasses/status + scope: '*' - apiGroups: - karpenter.sh apiVersions: - v1alpha5 - resources: - - provisioners - - provisioners/status operations: - CREATE - UPDATE + resources: + - provisioners + - provisioners/status + scope: '*' --- apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration @@ -74,14 +86,26 @@ webhooks: - awsnodetemplates - awsnodetemplates/status scope: '*' + - apiGroups: + - karpenter.k8s.aws + apiVersions: + - v1beta1 + operations: + - CREATE + - UPDATE + resources: + - ec2nodeclasses + - ec2nodeclasses/status + scope: '*' - apiGroups: - karpenter.sh apiVersions: - v1alpha5 - resources: - - provisioners - - provisioners/status operations: - CREATE - UPDATE + resources: + - provisioners + - provisioners/status + scope: '*' {{- end }} \ No newline at end of file diff --git a/charts/karpenter/values.yaml b/charts/karpenter/values.yaml index 309235699eac..d99f16c5261f 100644 --- a/charts/karpenter/values.yaml +++ b/charts/karpenter/values.yaml @@ -168,8 +168,11 @@ logConfig: logEncoding: console # -- Component-based log configuration logLevel: + # -- Global log level, defaults to 'debug' global: debug + # -- Controller log level, defaults to 'debug' controller: debug + # -- Error log level, defaults to 'error' webhook: error # -- Global Settings to configure Karpenter settings: diff --git a/cmd/controller/main.go b/cmd/controller/main.go index 9e900bb31825..a23c1b0b4f1e 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -64,6 +64,7 @@ func main() { op.SubnetProvider, op.SecurityGroupProvider, op.InstanceProfileProvider, + op.InstanceProvider, op.PricingProvider, op.AMIProvider, op.LicenseProvider, diff --git a/designs/v1beta1-api.md b/designs/v1beta1-api.md index 706e19394da5..13faad7ece38 100644 --- a/designs/v1beta1-api.md +++ b/designs/v1beta1-api.md @@ -153,11 +153,11 @@ spec: owner: amazon subnetSelectorTerms: - tags: - compute.k8s.aws/discovery: cluster-name + karpenter.sh/discovery: cluster-name - id: subnet-1234 securityGroupSelectorTerms: - tags: - compute.k8s.aws/discovery: cluster-name + karpenter.sh/discovery: cluster-name - name: default-security-group role: karpenter-node-role userData: | @@ -253,7 +253,7 @@ spec: nodeClass: name: default kind: EC2NodeClass - apiVersion: compute.k8s.aws/v1beta1 + apiVersion: karpenter.k8s.aws/v1beta1 taints: - key: example.com/special-taint effect: NoSchedule diff --git a/examples/v1beta1/100-cpu-limit.yaml b/examples/v1beta1/100-cpu-limit.yaml new file mode 100644 index 000000000000..b0fb8151366f --- /dev/null +++ b/examples/v1beta1/100-cpu-limit.yaml @@ -0,0 +1,49 @@ +# This example NodePool limits the amount of compute managed by +# Karpenter for this NodePool. Karpenter will not provision compute that +# takes the pool over a total of 100 (virtual or physical) CPU cores. +--- +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: limitcpu100 + annotations: + kubernetes.io/description: "NodePool to restrict the number of cpus provisioned to 100" +spec: + template: + spec: + requirements: + - key: kubernetes.io/arch + operator: In + values: ["amd64"] + - key: kubernetes.io/os + operator: In + values: ["linux"] + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand"] + - key: karpenter.k8s.aws/instance-category + operator: In + values: ["c", "m", "r"] + - key: karpenter.k8s.aws/instance-generation + operator: Gt + values: ["2"] + nodeClassRef: + name: default + limits: + cpu: 100 +--- +apiVersion: karpenter.k8s.aws/v1beta1 +kind: EC2NodeClass +metadata: + name: default + annotations: + kubernetes.io/description: "General purpose EC2NodeClass for running Amazon Linux 2 nodes" +spec: + amiFamily: AL2 # Amazon Linux 2 + role: "KarpenterNodeRole-${CLUSTER_NAME}" # replace with your cluster name + subnetSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + securityGroupSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name \ No newline at end of file diff --git a/examples/v1beta1/al2-custom-ami.yaml b/examples/v1beta1/al2-custom-ami.yaml new file mode 100644 index 000000000000..c1b808598b65 --- /dev/null +++ b/examples/v1beta1/al2-custom-ami.yaml @@ -0,0 +1,61 @@ +# This example NodePool will provision instances using a custom EKS-Optimized AMI that belongs to the +# AL2 AMIFamily. If your AMIs are built off https://github.com/awslabs/amazon-eks-ami and can be bootstrapped +# by Karpenter, this may be a good fit for you. +--- +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: default + annotations: + kubernetes.io/description: "General purpose NodePool for generic workloads" +spec: + template: + spec: + requirements: + - key: kubernetes.io/arch + operator: In + values: ["amd64"] + - key: kubernetes.io/os + operator: In + values: ["linux"] + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand"] + - key: karpenter.k8s.aws/instance-category + operator: In + values: ["c", "m", "r"] + - key: karpenter.k8s.aws/instance-generation + operator: Gt + values: ["2"] + nodeClassRef: + name: al2 +--- +apiVersion: karpenter.k8s.aws/v1beta1 +kind: EC2NodeClass +metadata: + name: al2 + annotations: + kubernetes.io/description: "EC2NodeClass for running Amazon Linux 2 nodes with a custom AMI" +spec: + amiFamily: AL2 # Amazon Linux 2 + role: "KarpenterNodeRole-${CLUSTER_NAME}" # replace with your cluster name + subnetSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + securityGroupSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + amiSelectorTerms: + - id: ami-123 + - id: ami-456 + userData: | + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="EXAMPLEBOUNDARY" + + --EXAMPLEBOUNDARY + Content-Type: text/x-shellscript; charset="us-ascii" + + #!/bin/bash + echo "Running a custom user data script" + + --EXAMPLEBOUNDARY-- diff --git a/examples/v1beta1/al2-custom-userdata.yaml b/examples/v1beta1/al2-custom-userdata.yaml new file mode 100644 index 000000000000..e9b5873884a2 --- /dev/null +++ b/examples/v1beta1/al2-custom-userdata.yaml @@ -0,0 +1,58 @@ +# This example NodePool will provision instances using the AL2 EKS-Optimized AMI. +# The UserData defined in spec.UserData needs to be in the MIME-multipart format, +# and will be prepended to a Karpenter managed section that will bootstrap the kubelet. +--- +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: default + annotations: + kubernetes.io/description: "General purpose NodePool for generic workloads" +spec: + template: + spec: + requirements: + - key: kubernetes.io/arch + operator: In + values: ["amd64"] + - key: kubernetes.io/os + operator: In + values: ["linux"] + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand"] + - key: karpenter.k8s.aws/instance-category + operator: In + values: ["c", "m", "r"] + - key: karpenter.k8s.aws/instance-generation + operator: Gt + values: ["2"] + nodeClassRef: + name: al2 +--- +apiVersion: karpenter.k8s.aws/v1beta1 +kind: EC2NodeClass +metadata: + name: al2 + annotations: + kubernetes.io/description: "EC2NodeClass for running Amazon Linux 2 nodes with custom user data" +spec: + amiFamily: AL2 # Amazon Linux 2 + role: "KarpenterNodeRole-${CLUSTER_NAME}" # replace with your cluster name + subnetSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + securityGroupSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + userData: | + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="BOUNDARY" + + --BOUNDARY + Content-Type: text/x-shellscript; charset="us-ascii" + + #!/bin/bash + echo "Running a custom user data script" + + --BOUNDARY-- diff --git a/examples/v1beta1/al2-kubelet-log-query.yaml b/examples/v1beta1/al2-kubelet-log-query.yaml new file mode 100644 index 000000000000..013469976c5b --- /dev/null +++ b/examples/v1beta1/al2-kubelet-log-query.yaml @@ -0,0 +1,69 @@ +# This example NodePool will provision instances using the AL2 EKS-Optimized AMI +# and will be prepended to a Karpenter managed section that will bootstrap the kubelet. +--- +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: default + annotations: + kubernetes.io/description: "General purpose NodePool for generic workloads" +spec: + template: + spec: + requirements: + - key: kubernetes.io/arch + operator: In + values: ["amd64"] + - key: kubernetes.io/os + operator: In + values: ["linux"] + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand"] + - key: karpenter.k8s.aws/instance-category + operator: In + values: ["c", "m", "r"] + - key: karpenter.k8s.aws/instance-generation + operator: Gt + values: ["2"] + nodeClassRef: + name: al2 +--- +apiVersion: karpenter.k8s.aws/v1beta1 +kind: EC2NodeClass +metadata: + name: al2 + annotations: + kubernetes.io/description: "EC2NodeClass for running Amazon Linux 2 nodes with user data that enables the NodeLogQuery feature gate" +spec: + amiFamily: AL2 # Amazon Linux 2 + role: "KarpenterNodeRole-${CLUSTER_NAME}" # replace with your cluster name + subnetSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + securityGroupSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + userData: | + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="BOUNDARY" + + --BOUNDARY + Content-Type: text/x-shellscript; charset="us-ascii" + + #!/bin/bash + + set -e + + # Add additional KUBELET_EXTRA_ARGS to the service + # Requires Kubernetes 1.27 (alpha feature) + cat << EOF > /etc/systemd/system/kubelet.service.d/90-kubelet-extra-args.conf + [Service] + Environment="KUBELET_EXTRA_ARGS=--feature-gates=NodeLogQuery=true $KUBELET_EXTRA_ARGS" + EOF + + # Enable log handler and log query to the kubelet configuration + echo "$(jq '.enableSystemLogHandler=true' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json + echo "$(jq '.enableSystemLogQuery=true' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json + + --BOUNDARY-- \ No newline at end of file diff --git a/examples/v1beta1/bottlerocket.yaml b/examples/v1beta1/bottlerocket.yaml new file mode 100644 index 000000000000..0c2574a4e35b --- /dev/null +++ b/examples/v1beta1/bottlerocket.yaml @@ -0,0 +1,58 @@ +# This example NodePool will provision instances +# running Bottlerocket OS +--- +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: default + annotations: + kubernetes.io/description: "General purpose NodePool for generic workloads" +spec: + template: + spec: + requirements: + - key: kubernetes.io/arch + operator: In + values: ["amd64"] + - key: kubernetes.io/os + operator: In + values: ["linux"] + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand"] + - key: karpenter.k8s.aws/instance-category + operator: In + values: ["c", "m", "r"] + - key: karpenter.k8s.aws/instance-generation + operator: Gt + values: ["2"] + nodeClassRef: + name: bottlerocket +--- +apiVersion: karpenter.k8s.aws/v1beta1 +kind: EC2NodeClass +metadata: + name: bottlerocket + annotations: + kubernetes.io/description: "EC2NodeClass for running Bottlerocket nodes" +spec: + amiFamily: Bottlerocket + role: "KarpenterNodeRole-${CLUSTER_NAME}" # replace with your cluster name + subnetSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + securityGroupSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeType: gp3 + volumeSize: 4Gi + deleteOnTermination: true + # Bottlerocket data volume + - deviceName: /dev/xvdb + ebs: + volumeType: gp3 + volumeSize: 20Gi # replace with your required disk size + deleteOnTermination: true diff --git a/examples/v1beta1/br-custom-userdata.yaml b/examples/v1beta1/br-custom-userdata.yaml new file mode 100644 index 000000000000..fa18decf4c3b --- /dev/null +++ b/examples/v1beta1/br-custom-userdata.yaml @@ -0,0 +1,50 @@ +# This example NodePool will provision instances +# running Bottlerocket OS and the user data settings specified in +# this EC2NodeClass will be merged into Karpenter defaults. +--- +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: default + annotations: + kubernetes.io/description: "General purpose NodePool for generic workloads" +spec: + template: + spec: + requirements: + - key: kubernetes.io/arch + operator: In + values: ["amd64"] + - key: kubernetes.io/os + operator: In + values: ["linux"] + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand"] + - key: karpenter.k8s.aws/instance-category + operator: In + values: ["c", "m", "r"] + - key: karpenter.k8s.aws/instance-generation + operator: Gt + values: ["2"] + nodeClassRef: + name: bottlerocket +--- +apiVersion: karpenter.k8s.aws/v1beta1 +kind: EC2NodeClass +metadata: + name: bottlerocket + annotations: + kubernetes.io/description: "EC2NodeClass for running Bottlerocket nodes with custom user data" +spec: + amiFamily: Bottlerocket + role: "KarpenterNodeRole-${CLUSTER_NAME}" # replace with your cluster name + subnetSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + securityGroupSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + userData: | + [settings.kubernetes] + kube-api-qps = 30 diff --git a/examples/v1beta1/custom-family.yaml b/examples/v1beta1/custom-family.yaml new file mode 100644 index 000000000000..daad19810935 --- /dev/null +++ b/examples/v1beta1/custom-family.yaml @@ -0,0 +1,63 @@ +# This example NodePool provisions instances using an AMI that belongs to a custom AMIFamily +# Keep in mind, that you're in charge of bootstrapping your worker nodes. +--- +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: default + annotations: + kubernetes.io/description: "General purpose NodePool for generic workloads" +spec: + template: + spec: + requirements: + - key: kubernetes.io/arch + operator: In + values: ["amd64"] + - key: kubernetes.io/os + operator: In + values: ["linux"] + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand"] + - key: karpenter.k8s.aws/instance-category + operator: In + values: ["c", "m", "r"] + - key: karpenter.k8s.aws/instance-generation + operator: Gt + values: ["2"] + nodeClassRef: + name: custom-family +--- +apiVersion: karpenter.k8s.aws/v1beta1 +kind: EC2NodeClass +metadata: + name: custom-family + annotations: + kubernetes.io/description: "EC2NodeClass for running Custom AMIFamily with custom user data that doesn't conform to the other AMIFamilies" +spec: + amiFamily: Custom + role: "KarpenterNodeRole-${CLUSTER_NAME}" # replace with your cluster name + subnetSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + securityGroupSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + amiSelectorTerms: + - id: ami-123 + - id: ami-456 + userData: | + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="BOUNDARY" + + --BOUNDARY + Content-Type: text/x-shellscript; charset="us-ascii" + + #!/bin/bash + echo "Running my custom set-up" + + # Have the kubelet label the node + /etc/eks/bootstrap.sh my-cluster --kubelet-extra-args='--node-labels=foo=bar' + + --BOUNDARY diff --git a/examples/v1beta1/general-purpose.yaml b/examples/v1beta1/general-purpose.yaml new file mode 100644 index 000000000000..43e32d8c436d --- /dev/null +++ b/examples/v1beta1/general-purpose.yaml @@ -0,0 +1,45 @@ +# This example NodePool will provision general purpose instances +--- +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: general-purpose + annotations: + kubernetes.io/description: "General purpose NodePool for generic workloads" +spec: + template: + spec: + requirements: + - key: kubernetes.io/arch + operator: In + values: ["amd64"] + - key: kubernetes.io/os + operator: In + values: ["linux"] + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand"] + - key: karpenter.k8s.aws/instance-category + operator: In + values: ["c", "m", "r"] + - key: karpenter.k8s.aws/instance-generation + operator: Gt + values: ["2"] + nodeClassRef: + name: default +--- +apiVersion: karpenter.k8s.aws/v1beta1 +kind: EC2NodeClass +metadata: + name: default + annotations: + kubernetes.io/description: "General purpose EC2NodeClass for running Amazon Linux 2 nodes" +spec: + amiFamily: AL2 # Amazon Linux 2 + role: "KarpenterNodeRole-${CLUSTER_NAME}" # replace with your cluster name + subnetSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + securityGroupSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name diff --git a/examples/v1beta1/large-instances.yaml b/examples/v1beta1/large-instances.yaml new file mode 100644 index 000000000000..da6fac3ba4ba --- /dev/null +++ b/examples/v1beta1/large-instances.yaml @@ -0,0 +1,37 @@ +# This example NodePool will avoid small instance types in the cluster +--- +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: large-instances + annotations: + kubernetes.io/description: "NodePool for provisioning larger instances using Gt/Lt requirements" +spec: + template: + spec: + requirements: + # exclude instances with < 4 cores and < 8GiB memory (8192 mebibytes) + - key: "karpenter.k8s.aws/instance-cpu" + operator: Gt + values: ["3"] + - key: "karpenter.k8s.aws/instance-memory" + operator: Gt + values: ["8191"] + nodeClassRef: + name: default +--- +apiVersion: karpenter.k8s.aws/v1beta1 +kind: EC2NodeClass +metadata: + name: default + annotations: + kubernetes.io/description: "General purpose EC2NodeClass for running Amazon Linux 2 nodes" +spec: + amiFamily: AL2 # Amazon Linux 2 + role: "KarpenterNodeRole-${CLUSTER_NAME}" # replace with your cluster name + subnetSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + securityGroupSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name diff --git a/examples/v1beta1/multiple-arch.yaml b/examples/v1beta1/multiple-arch.yaml new file mode 100644 index 000000000000..91529c628908 --- /dev/null +++ b/examples/v1beta1/multiple-arch.yaml @@ -0,0 +1,75 @@ +# This example allows you to create arm64 AND amd64 workloads +# Karpenter will choose the NodePool that suits the workload requirements and +# find an AMI automatically that matches the architecture requirements +--- +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: amd64 + annotations: + kubernetes.io/description: "NodePool for amd64 workloads" +spec: + template: + spec: + requirements: + - key: kubernetes.io/arch + operator: In + values: ["amd64"] + - key: kubernetes.io/os + operator: In + values: ["linux"] + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand"] + - key: karpenter.k8s.aws/instance-category + operator: In + values: ["c", "m", "r"] + - key: karpenter.k8s.aws/instance-generation + operator: Gt + values: ["2"] + nodeClassRef: + name: default +--- +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: arm64 + annotations: + kubernetes.io/description: "NodePool for arm64 workloads" +spec: + template: + spec: + requirements: + - key: kubernetes.io/arch + operator: In + values: ["arm64"] + - key: kubernetes.io/os + operator: In + values: ["linux"] + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand"] + - key: karpenter.k8s.aws/instance-category + operator: In + values: ["c", "a"] + - key: karpenter.k8s.aws/instance-generation + operator: Gt + values: ["2"] + nodeClassRef: + name: default +--- +apiVersion: karpenter.k8s.aws/v1beta1 +kind: EC2NodeClass +metadata: + name: default + annotations: + kubernetes.io/description: "General purpose EC2NodeClass for running Amazon Linux 2 nodes" +spec: + amiFamily: AL2 # Amazon Linux 2 + role: "KarpenterNodeRole-${CLUSTER_NAME}" # replace with your cluster name + subnetSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + securityGroupSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name \ No newline at end of file diff --git a/examples/v1beta1/multiple-ebs.yaml b/examples/v1beta1/multiple-ebs.yaml new file mode 100644 index 000000000000..5400e110bdcc --- /dev/null +++ b/examples/v1beta1/multiple-ebs.yaml @@ -0,0 +1,62 @@ +# This example NodePool will provision instances +# with multiple EBS volumes attached +--- +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: default + annotations: + kubernetes.io/description: "General purpose NodePool for generic workloads" +spec: + template: + spec: + requirements: + - key: kubernetes.io/arch + operator: In + values: ["amd64"] + - key: kubernetes.io/os + operator: In + values: ["linux"] + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand"] + - key: karpenter.k8s.aws/instance-category + operator: In + values: ["c", "m", "r"] + - key: karpenter.k8s.aws/instance-generation + operator: Gt + values: ["2"] + nodeClassRef: + name: multiple-ebs +--- +apiVersion: karpenter.k8s.aws/v1beta1 +kind: EC2NodeClass +metadata: + name: multiple-ebs + annotations: + kubernetes.io/description: "EC2NodeClass to provision multiple EBS volumes to attach to new nodes" +spec: + amiFamily: AL2 # Amazon Linux 2 + role: "KarpenterNodeRole-${CLUSTER_NAME}" # replace with your cluster name + subnetSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + securityGroupSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeType: gp3 + volumeSize: 20Gi + deleteOnTermination: true + - deviceName: /dev/xvdb + ebs: + volumeType: gp3 + volumeSize: 100Gi + deleteOnTermination: true + - deviceName: /dev/xvdc + ebs: + volumeType: gp3 + volumeSize: 2000Gi + deleteOnTermination: true diff --git a/examples/v1beta1/node-ttls.yaml b/examples/v1beta1/node-ttls.yaml new file mode 100644 index 000000000000..84c00f34f716 --- /dev/null +++ b/examples/v1beta1/node-ttls.yaml @@ -0,0 +1,51 @@ +# This example NodePool will provision instances +# that are replaced every 7 days and drain after 1 minute +# with no workloads +--- +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: default + annotations: + kubernetes.io/description: "General purpose NodePool for generic workloads" +spec: + template: + spec: + requirements: + - key: kubernetes.io/arch + operator: In + values: ["amd64"] + - key: kubernetes.io/os + operator: In + values: ["linux"] + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand"] + - key: karpenter.k8s.aws/instance-category + operator: In + values: ["c", "m", "r"] + - key: karpenter.k8s.aws/instance-generation + operator: Gt + values: ["2"] + nodeClassRef: + name: default + disruption: + consolidationPolicy: WhenEmpty + consolidateAfter: 60s # scale down nodes after 60 seconds without workloads (excluding daemons) + expireAfter: 168h # expire nodes after 7 days = 7 * 24h +--- +apiVersion: karpenter.k8s.aws/v1beta1 +kind: EC2NodeClass +metadata: + name: default + annotations: + kubernetes.io/description: "General purpose EC2NodeClass for running Amazon Linux 2 nodes" +spec: + amiFamily: AL2 # Amazon Linux 2 + role: "KarpenterNodeRole-${CLUSTER_NAME}" # replace with your cluster name + subnetSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + securityGroupSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name diff --git a/examples/v1beta1/spot.yaml b/examples/v1beta1/spot.yaml new file mode 100644 index 000000000000..7c8903748658 --- /dev/null +++ b/examples/v1beta1/spot.yaml @@ -0,0 +1,46 @@ +# This example will use spot instance type for all +# provisioned instances +--- +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: spot + annotations: + kubernetes.io/description: "NodePool for provisioning spot capacity" +spec: + template: + spec: + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["spot"] + - key: kubernetes.io/arch + operator: In + values: ["amd64"] + - key: kubernetes.io/os + operator: In + values: ["linux"] + - key: karpenter.k8s.aws/instance-category + operator: In + values: ["c", "m", "r"] + - key: karpenter.k8s.aws/instance-generation + operator: Gt + values: ["2"] + nodeClassRef: + name: default +--- +apiVersion: karpenter.k8s.aws/v1beta1 +kind: EC2NodeClass +metadata: + name: default + annotations: + kubernetes.io/description: "General purpose EC2NodeClass for running Amazon Linux 2 nodes" +spec: + amiFamily: AL2 # Amazon Linux 2 + role: "KarpenterNodeRole-${CLUSTER_NAME}" # replace with your cluster name + subnetSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + securityGroupSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name diff --git a/examples/v1beta1/ubuntu-kubelet-log-query.yaml b/examples/v1beta1/ubuntu-kubelet-log-query.yaml new file mode 100644 index 000000000000..9c6a4c27570c --- /dev/null +++ b/examples/v1beta1/ubuntu-kubelet-log-query.yaml @@ -0,0 +1,69 @@ +# This example NodePool provisions instances using the Ubuntu EKS AMI +# and will be prepended to a Karpenter managed section that will bootstrap the kubelet. +--- +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: ubuntu + annotations: + kubernetes.io/description: "General purpose NodePool for generic workloads" +spec: + template: + spec: + requirements: + - key: kubernetes.io/arch + operator: In + values: ["amd64"] + - key: kubernetes.io/os + operator: In + values: ["linux"] + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand"] + - key: karpenter.k8s.aws/instance-category + operator: In + values: ["c", "m", "r"] + - key: karpenter.k8s.aws/instance-generation + operator: Gt + values: ["2"] + nodeClassRef: + name: ubuntu +--- +apiVersion: karpenter.k8s.aws/v1beta1 +kind: EC2NodeClass +metadata: + name: ubuntu + annotations: + kubernetes.io/description: "EC2NodeClass for running Ubuntu nodes with user data that enables the NodeLogQuery feature gate" +spec: + amiFamily: Ubuntu + role: "KarpenterNodeRole-${CLUSTER_NAME}" # replace with your cluster name + subnetSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + securityGroupSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + userData: | + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="BOUNDARY" + + --BOUNDARY + Content-Type: text/x-shellscript; charset="us-ascii" + + #!/bin/bash + # There is currently a bug with log query and kubelet running inside a snap environment + # https://github.com/kubernetes/kubernetes/issues/120618 + # This example is provided for reference on how to change Ubuntu settings in user data + + set -e + + # This requires Kubernetes 1.27 or above (alpha feature) + # This modifies the configuration of the /etc/eks/bootstrap.sh script because /etc/kubernetes/kubelet/kubelet-config.json + # doesn't exist before bootstrap.sh is run + + sed -i 's/args="$KUBELET_EXTRA_ARGS"/args="--feature-gates=NodeLogQuery=true $KUBELET_EXTRA_ARGS"/g' /etc/eks/bootstrap.sh + sed -i '/# writes kubeReserved and evictionHard/a echo "$(jq .enableSystemLogHandler=true $KUBELET_CONFIG)" > $KUBELET_CONFIG' /etc/eks/bootstrap.sh + sed -i '/# writes kubeReserved and evictionHard/a echo "$(jq .enableSystemLogQuery=true $KUBELET_CONFIG)" > $KUBELET_CONFIG' /etc/eks/bootstrap.sh + + --BOUNDARY-- \ No newline at end of file diff --git a/examples/v1beta1/windows-custom-userdata.yaml b/examples/v1beta1/windows-custom-userdata.yaml new file mode 100644 index 000000000000..412519503eba --- /dev/null +++ b/examples/v1beta1/windows-custom-userdata.yaml @@ -0,0 +1,54 @@ +# This example NodePool provisions instances using the Windows 2022 EKS-Optimized AMI. +# The UserData defined in spec.UserData should be PowerShell commands +# and they will be prepended to a Karpenter managed section that will bootstrap the kubelet. +# This example also applies to the Windows 2019 EKS-Optimized AMI. +--- +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: windows2022 + annotations: + kubernetes.io/description: "General purpose NodePool for Windows workloads" +spec: + template: + spec: + requirements: + - key: kubernetes.io/os + operator: In + values: ["windows"] + - key: kubernetes.io/arch + operator: In + values: ["amd64"] + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand"] + - key: karpenter.k8s.aws/instance-category + operator: In + values: ["c", "m", "r"] + - key: karpenter.k8s.aws/instance-generation + operator: Gt + values: ["2"] + nodeClassRef: + name: windows2022 +--- +apiVersion: karpenter.k8s.aws/v1beta1 +kind: EC2NodeClass +metadata: + name: windows2022 + annotations: + kubernetes.io/description: "Nodes running Windows Server 2022" +spec: + amiFamily: Windows2022 + role: "KarpenterNodeRole-${CLUSTER_NAME}" # replace with your cluster name + subnetSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + securityGroupSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + metadataOptions: + httpProtocolIPv6: disabled + httpTokens: required + userData: | + New-Item -Path 'C:\temp\' -ItemType Directory + New-Item -Path 'C:\temp\sample.txt' -ItemType File \ No newline at end of file diff --git a/examples/v1beta1/windows2019.yaml b/examples/v1beta1/windows2019.yaml new file mode 100644 index 000000000000..8b3791ac8c22 --- /dev/null +++ b/examples/v1beta1/windows2019.yaml @@ -0,0 +1,48 @@ +# This example NodePool will provision instances running Windows Server 2019 +--- +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: windows2019 + annotations: + kubernetes.io/description: "General purpose NodePool for Windows workloads" +spec: + template: + spec: + requirements: + - key: kubernetes.io/os + operator: In + values: ["windows"] + - key: kubernetes.io/arch + operator: In + values: ["amd64"] + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand"] + - key: karpenter.k8s.aws/instance-category + operator: In + values: ["c", "m", "r"] + - key: karpenter.k8s.aws/instance-generation + operator: Gt + values: ["2"] + nodeClassRef: + name: windows2019 +--- +apiVersion: karpenter.k8s.aws/v1beta1 +kind: EC2NodeClass +metadata: + name: windows2019 + annotations: + kubernetes.io/description: "Nodes running Windows Server 2019" +spec: + amiFamily: Windows2019 + role: "KarpenterNodeRole-${CLUSTER_NAME}" # replace with your cluster name + subnetSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + securityGroupSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + metadataOptions: + httpProtocolIPv6: disabled + httpTokens: required \ No newline at end of file diff --git a/examples/v1beta1/windows2022.yaml b/examples/v1beta1/windows2022.yaml new file mode 100644 index 000000000000..899512fbc805 --- /dev/null +++ b/examples/v1beta1/windows2022.yaml @@ -0,0 +1,48 @@ +# This example NodePool will provision instances running Windows Server 2022 +--- +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: windows2022 + annotations: + kubernetes.io/description: "General purpose NodePool for Windows workloads" +spec: + template: + spec: + requirements: + - key: kubernetes.io/os + operator: In + values: ["windows"] + - key: kubernetes.io/arch + operator: In + values: ["amd64"] + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand"] + - key: karpenter.k8s.aws/instance-category + operator: In + values: ["c", "m", "r"] + - key: karpenter.k8s.aws/instance-generation + operator: Gt + values: ["2"] + nodeClassRef: + name: windows2022 +--- +apiVersion: karpenter.k8s.aws/v1beta1 +kind: EC2NodeClass +metadata: + name: windows2022 + annotations: + kubernetes.io/description: "Nodes running Windows Server 2022" +spec: + amiFamily: Windows2022 + role: "KarpenterNodeRole-${CLUSTER_NAME}" # replace with your cluster name + subnetSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + securityGroupSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + metadataOptions: + httpProtocolIPv6: disabled + httpTokens: required \ No newline at end of file diff --git a/go.mod b/go.mod index 0f9678513e97..46e05c333eb7 100644 --- a/go.mod +++ b/go.mod @@ -6,9 +6,9 @@ require ( github.com/Pallinder/go-randomdata v1.2.0 github.com/PuerkitoBio/goquery v1.8.1 github.com/avast/retry-go v3.0.0+incompatible - github.com/aws/aws-sdk-go v1.45.24 - github.com/aws/karpenter-core v0.31.1-0.20231005232319-8e6687ea72b6 - github.com/aws/karpenter/tools/kompat v0.0.0-20230915222222-abfbf5fa3644 + github.com/aws/aws-sdk-go v1.46.0 + github.com/aws/karpenter-core v0.31.1-0.20231019191151-73c0fd546f75 + github.com/aws/karpenter/tools/kompat v0.0.0-20231010173459-62c25a3ea85c github.com/imdario/mergo v0.3.16 github.com/mitchellh/hashstructure/v2 v2.0.2 github.com/onsi/ginkgo/v2 v2.13.0 @@ -21,20 +21,20 @@ require ( go.uber.org/zap v1.26.0 golang.org/x/sync v0.4.0 golang.org/x/time v0.3.0 - k8s.io/api v0.26.6 - k8s.io/apiextensions-apiserver v0.26.6 - k8s.io/apimachinery v0.26.6 - k8s.io/client-go v0.26.6 - k8s.io/utils v0.0.0-20230209194617-a36077c30491 - knative.dev/pkg v0.0.0-20230712131115-7051d301e7f4 - sigs.k8s.io/controller-runtime v0.14.6 + k8s.io/api v0.28.3 + k8s.io/apiextensions-apiserver v0.28.3 + k8s.io/apimachinery v0.28.3 + k8s.io/client-go v0.28.3 + k8s.io/utils v0.0.0-20230726121419-3b25d923346b + knative.dev/pkg v0.0.0-20231010144348-ca8c009405dd + sigs.k8s.io/controller-runtime v0.16.3 ) require ( contrib.go.opencensus.io/exporter/ocagent v0.7.1-0.20200907061046-05415f1de66d // indirect - contrib.go.opencensus.io/exporter/prometheus v0.4.0 // indirect + contrib.go.opencensus.io/exporter/prometheus v0.4.2 // indirect github.com/Masterminds/semver/v3 v3.2.1 // indirect - github.com/andybalholm/cascadia v1.3.1 // indirect + github.com/andybalholm/cascadia v1.3.2 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect github.com/blendle/zapdriver v1.3.1 // indirect @@ -42,35 +42,36 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/deckarep/golang-set v1.8.0 // indirect - github.com/emicklei/go-restful/v3 v3.9.0 // indirect - github.com/evanphx/json-patch v4.12.0+incompatible // indirect - github.com/evanphx/json-patch/v5 v5.6.0 // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/evanphx/json-patch v5.7.0+incompatible // indirect + github.com/evanphx/json-patch/v5 v5.7.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-kit/log v0.2.1 // indirect - github.com/go-logfmt/logfmt v0.5.1 // indirect + github.com/go-logfmt/logfmt v0.6.0 // indirect github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/zapr v1.2.4 // indirect - github.com/go-openapi/jsonpointer v0.19.6 // indirect - github.com/go-openapi/jsonreference v0.20.1 // indirect - github.com/go-openapi/swag v0.22.3 // indirect + github.com/go-openapi/jsonpointer v0.20.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.22.4 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect - github.com/gobuffalo/flect v0.2.4 // indirect + github.com/gobuffalo/flect v1.0.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect - github.com/google/gnostic v0.5.7-v3refs // indirect - github.com/google/go-cmp v0.5.9 // indirect + github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect - github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect - github.com/google/uuid v1.3.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3 // indirect - github.com/hashicorp/golang-lru v0.5.4 // indirect + github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 // indirect + github.com/google/uuid v1.3.1 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0 // indirect + github.com/hashicorp/golang-lru v1.0.2 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kelseyhightower/envconfig v1.4.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect - github.com/mattn/go-runewidth v0.0.9 // indirect + github.com/mattn/go-runewidth v0.0.15 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -78,36 +79,39 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect + github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.44.0 // indirect - github.com/prometheus/procfs v0.11.1 // indirect - github.com/prometheus/statsd_exporter v0.21.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect + github.com/prometheus/statsd_exporter v0.24.0 // indirect + github.com/rivo/uniseg v0.4.4 // indirect + github.com/spf13/cobra v1.7.0 // indirect github.com/spf13/pflag v1.0.5 // indirect go.opencensus.io v0.24.0 // indirect - go.uber.org/atomic v1.9.0 // indirect - go.uber.org/automaxprocs v1.4.0 // indirect - golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 // indirect - golang.org/x/net v0.14.0 // indirect - golang.org/x/oauth2 v0.8.0 // indirect - golang.org/x/sys v0.12.0 // indirect - golang.org/x/term v0.11.0 // indirect + go.uber.org/automaxprocs v1.5.3 // indirect + golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect + golang.org/x/net v0.17.0 // indirect + golang.org/x/oauth2 v0.13.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/term v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect - golang.org/x/tools v0.12.0 // indirect - gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect - google.golang.org/api v0.124.0 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect - google.golang.org/grpc v1.55.0 // indirect + golang.org/x/tools v0.14.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect + google.golang.org/api v0.146.0 // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/genproto v0.0.0-20231009173412-8bfb1ae86b6c // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20231009173412-8bfb1ae86b6c // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231009173412-8bfb1ae86b6c // indirect + google.golang.org/grpc v1.58.2 // indirect google.golang.org/protobuf v1.31.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/cloud-provider v0.26.6 // indirect - k8s.io/component-base v0.26.6 // indirect - k8s.io/csi-translation-lib v0.26.6 // indirect + k8s.io/cloud-provider v0.28.2 // indirect + k8s.io/component-base v0.28.3 // indirect + k8s.io/csi-translation-lib v0.28.2 // indirect k8s.io/klog/v2 v2.100.1 // indirect - k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect + k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) diff --git a/go.sum b/go.sum index 4c78e41984d9..e3db90e43d87 100644 --- a/go.sum +++ b/go.sum @@ -32,8 +32,8 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= contrib.go.opencensus.io/exporter/ocagent v0.7.1-0.20200907061046-05415f1de66d h1:LblfooH1lKOpp1hIhukktmSAxFkqMPFk9KR6iZ0MJNI= contrib.go.opencensus.io/exporter/ocagent v0.7.1-0.20200907061046-05415f1de66d/go.mod h1:IshRmMJBhDfFj5Y67nVhMYTTIze91RUeT73ipWKs/GY= -contrib.go.opencensus.io/exporter/prometheus v0.4.0 h1:0QfIkj9z/iVZgK31D9H9ohjjIDApI2GOPScCKwxedbs= -contrib.go.opencensus.io/exporter/prometheus v0.4.0/go.mod h1:o7cosnyfuPVK0tB8q0QmaQNhGnptITnPQB+z1+qeFB0= +contrib.go.opencensus.io/exporter/prometheus v0.4.2 h1:sqfsYl5GIY/L570iT+l93ehxaWJs2/OwXtiWwew3oAg= +contrib.go.opencensus.io/exporter/prometheus v0.4.2/go.mod h1:dvEHbiKmgvbr5pjaF9fpw1KeYcjrnC1J8B+JKjsZyRQ= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= @@ -48,17 +48,19 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= -github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= +github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= +github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= +github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= -github.com/aws/aws-sdk-go v1.45.24 h1:TZx/CizkmCQn8Rtsb11iLYutEQVGK5PK9wAhwouELBo= -github.com/aws/aws-sdk-go v1.45.24/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= -github.com/aws/karpenter-core v0.31.1-0.20231005232319-8e6687ea72b6 h1:2V3E9h5GUK7hB8setPdaUKcL/ubuDNoG0RAGQtvwfuE= -github.com/aws/karpenter-core v0.31.1-0.20231005232319-8e6687ea72b6/go.mod h1:4wXCSTj97gOWkWeB4D6LjWQMoqldrI8fo4tUOAhYTDs= -github.com/aws/karpenter/tools/kompat v0.0.0-20230915222222-abfbf5fa3644 h1:M1fxGlOfvSqFYI01HL2zzvomy8e7LiTHk77KDuChWZQ= -github.com/aws/karpenter/tools/kompat v0.0.0-20230915222222-abfbf5fa3644/go.mod h1:l/TIBsaCx/IrOr0Xvlj/cHLOf05QzuQKEZ1hx2XWmfU= +github.com/aws/aws-sdk-go v1.46.0 h1:Igh7W8P+sA6mXJ9yhreOSweefLapcqekhxQlY1llxcM= +github.com/aws/aws-sdk-go v1.46.0/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/karpenter-core v0.31.1-0.20231019191151-73c0fd546f75 h1:YGsJA1sH7xW749onvhczydUZksdMm3PB/AJr8qcZllA= +github.com/aws/karpenter-core v0.31.1-0.20231019191151-73c0fd546f75/go.mod h1:rb3kp/3cj38tACF6udfpmIvKoQMwirSVoHNlrd66LyE= +github.com/aws/karpenter/tools/kompat v0.0.0-20231010173459-62c25a3ea85c h1:oXWwIttmjYLbBKhLazG21aQvpJ3NOOr8IXhCJ/p6e/M= +github.com/aws/karpenter/tools/kompat v0.0.0-20231010173459-62c25a3ea85c/go.mod h1:l/TIBsaCx/IrOr0Xvlj/cHLOf05QzuQKEZ1hx2XWmfU= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= @@ -72,6 +74,7 @@ github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g= github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= @@ -79,24 +82,23 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deckarep/golang-set v1.8.0 h1:sk9/l/KqpunDwP7pSjUg0keiOOLEnOBHzykLrsPppp4= github.com/deckarep/golang-set v1.8.0/go.mod h1:5nI87KwE7wgsBU1F4GKAw2Qod7p5kyS383rP6+o6qqo= -github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= -github.com/emicklei/go-restful/v3 v3.9.0 h1:XwGDlfxEnQZzuopoqxwSEllNcCOM9DhhFyhFIIGKwxE= -github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= -github.com/evanphx/json-patch v4.12.0+incompatible h1:4onqiflcdA9EOZ4RxV643DvftH5pOlLGNtQ5lPWQu84= -github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= -github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= -github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= +github.com/evanphx/json-patch v5.7.0+incompatible h1:vgGkfT/9f8zE6tvSCe74nfpAVDQ2tG6yudJd8LBksgI= +github.com/evanphx/json-patch v5.7.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch/v5 v5.7.0 h1:nJqP7uwL84RJInrohHfW0Fx3awjbm8qZeFv0nW9SYGc= +github.com/evanphx/json-patch/v5 v5.7.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -106,29 +108,33 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= +github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= +github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= github.com/go-logr/zapr v1.2.4/go.mod h1:FyHWQIzQORZ0QVE1BtVHv3cKtNLuXsbNLtpuhNapBOA= -github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= -github.com/go-openapi/jsonreference v0.20.1 h1:FBLnyygC4/IZZr893oiomc9XaghoveYTrLC1F86HID8= -github.com/go-openapi/jsonreference v0.20.1/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= -github.com/go-openapi/swag v0.22.3 h1:yMBqmnQ0gyZvEb/+KzuWZOXgllrXT4SADYbvDaXHv/g= +github.com/go-openapi/jsonpointer v0.20.0 h1:ESKJdU9ASRfaPNOPRx12IUyA1vn3R9GiE3KYD14BXdQ= +github.com/go-openapi/jsonpointer v0.20.0/go.mod h1:6PGzBjjIIumbLYysB73Klnms1mwnU4G3YHOECG3CedA= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= +github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= -github.com/gobuffalo/flect v0.2.4 h1:BSYA8+T60cdyq+vynaSUjqSVI9mDEg9ZfQUXKmfjo4I= -github.com/gobuffalo/flect v0.2.4/go.mod h1:1ZyCLIbg0YD7sDkzvFdPoOydPtD8y9JQnrOROolUcM8= +github.com/gobuffalo/flect v1.0.2 h1:eqjPGSo2WmjgY2XlpGwo2NXgL3RucAKo4k4qQMNA5sA= +github.com/gobuffalo/flect v1.0.2/go.mod h1:A5msMlrHtLqh9umBSnvabjsMrCcCpAyzglnDvkbYKHs= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -167,8 +173,8 @@ github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/gnostic v0.5.7-v3refs h1:FhTMOKj2VhjpouxvWJAV1TL304uMlb9zcDqkl6cEI54= -github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -179,8 +185,10 @@ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -193,26 +201,26 @@ github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= -github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 h1:pUa4ghanp6q4IJHwE9RwLgmVFfReJN+KbQ8ExNEUUoQ= +github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/grpc-ecosystem/grpc-gateway v1.14.6/go.mod h1:zdiPV4Yse/1gnckTHtghG4GkDEdKCRJduHpTxT3/jcw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3 h1:lLT7ZLSzGLI08vc9cpd+tYmNWjdKDqyr/2L+f6U12Fk= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU0GSB0f8Nhgmxx0VIRUvaC0w= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0 h1:RtRsiaGvWxcwd8y3BiRZxsylPT8hLWZ5SPcfI+3IDNk= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0/go.mod h1:TzP6duP4Py2pHLVPPQp42aoYI92+PCrVotyR5e8Vqlk= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= -github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= +github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= -github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= -github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= @@ -237,7 +245,6 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -247,8 +254,9 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= +github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= @@ -283,44 +291,59 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= +github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= +github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM= -github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.28.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/common v0.35.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= +github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= -github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= -github.com/prometheus/statsd_exporter v0.21.0 h1:hA05Q5RFeIjgwKIYEdFd59xu5Wwaznf33yKI+pyX6T8= -github.com/prometheus/statsd_exporter v0.21.0/go.mod h1:rbT83sZq2V+p73lHhPZfMc3MLCHmSHelCh9hSGYNLTQ= +github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/prometheus/statsd_exporter v0.22.7/go.mod h1:N/TevpjkIh9ccs6nuzY3jQn9dFqnUakOjnEuMPJJJnI= +github.com/prometheus/statsd_exporter v0.24.0 h1:aZmN6CzS2H1Non1JKZdjkQlAkDtGoQBYIESk2SlU1OI= +github.com/prometheus/statsd_exporter v0.24.0/go.mod h1:+dQiRTqn9DnPmN5mI5Xond+k8nuRKzdgh1omxh9OgFY= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= +github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/samber/lo v1.38.1 h1:j2XEAqXKb09Am4ebOg31SpvzUTTs6EN3VfgeLUhPdXM= github.com/samber/lo v1.38.1/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= @@ -328,7 +351,6 @@ github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpE github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -336,6 +358,7 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stvp/go-udp-testing v0.0.0-20201019212854-469649b16807/go.mod h1:7jxmlfBCDBXRzr0eAQJ48XC1hBu1np4CS5+cHEYfwpc= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -352,13 +375,11 @@ go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= -go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/automaxprocs v1.4.0 h1:CpDZl6aOlLhReez+8S3eEotD7Jx0Os++lemPlMULQP0= -go.uber.org/automaxprocs v1.4.0/go.mod h1:/mTEdr7LvHhs0v7mjdxDreTz1OG5zdZGqgOnhWiR/+Q= +go.uber.org/automaxprocs v1.5.3 h1:kWazyxZUrS3Gs4qUpbwo5kEIMGe/DAvi5Z4tl2NW4j8= +go.uber.org/automaxprocs v1.5.3/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= -go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= +go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= +go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= @@ -384,8 +405,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 h1:3MTrJm4PyNL9NBqvYDSj3DHl46qQakyfqfWo4jgfaEM= -golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -408,8 +429,9 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= -golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= +golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -445,19 +467,24 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= -golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8= -golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY= +golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -469,7 +496,9 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -511,19 +540,24 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= -golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= +golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -531,8 +565,10 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -584,14 +620,15 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.12.0 h1:YW6HUoUmYBpwSgyaGaZq1fHjrBjX1rlpZ54T6mu2kss= -golang.org/x/tools v0.12.0/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= +golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY= -gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY= +gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= +gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -609,16 +646,16 @@ google.golang.org/api v0.25.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.124.0 h1:dP6Ef1VgOGqQ8eiv4GiY8RhmeyqzovcXBYPDUYG8Syo= -google.golang.org/api v0.124.0/go.mod h1:xu2HQurE5gi/3t1aFCvhPD781p0a3p11sdunTJ2BlP4= +google.golang.org/api v0.146.0 h1:9aBYT4vQXt9dhCuLNfwfd3zpwu8atg0yPkjBymwSrOM= +google.golang.org/api v0.146.0/go.mod h1:OARJqIfoYjXJj4C1AiBSXYZt03qsoz8FQYU6fBEfrHM= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= -google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -650,9 +687,12 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= +google.golang.org/genproto v0.0.0-20231009173412-8bfb1ae86b6c h1:ml3TAUoIIzQUtX88s/icpXCFW9lV5VwsuIuS1htNjKY= +google.golang.org/genproto v0.0.0-20231009173412-8bfb1ae86b6c/go.mod h1:MugzuwC+GYOxyF0XUGQvsT97bOgWCV7MM1XMc5FZv8E= +google.golang.org/genproto/googleapis/api v0.0.0-20231009173412-8bfb1ae86b6c h1:0RtEmmHjemvUXloH7+RuBSIw7n+GEHMOMY1CkGYnWq4= +google.golang.org/genproto/googleapis/api v0.0.0-20231009173412-8bfb1ae86b6c/go.mod h1:Wth13BrWMRN/G+guBLupKa6fslcWZv14R0ZKDRkNfY8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231009173412-8bfb1ae86b6c h1:jHkCUWkseRf+W+edG5hMzr/Uh1xkDREY4caybAq4dpY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231009173412-8bfb1ae86b6c/go.mod h1:4cYg8o5yUbm77w8ZX00LhMVNl/YVBFJRYWDc0uYWMs0= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -666,8 +706,8 @@ google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3Iji google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag= -google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= +google.golang.org/grpc v1.58.2 h1:SXUpjxeVF3FKrTYQI4f4KvbGD5u2xccdYdurwowix5I= +google.golang.org/grpc v1.58.2/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -680,6 +720,8 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= @@ -701,7 +743,6 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -711,36 +752,36 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.26.6 h1:RZsJGP5p/qdWuFVqj/JFyt+6ttfgL+8/K8gtyi7riuo= -k8s.io/api v0.26.6/go.mod h1:Z+i6M3de4+LJiXtIiWSz/yLpnG+YjxAkeW6cgZqoxn4= -k8s.io/apiextensions-apiserver v0.26.6 h1:BrrWb5gQlWuwvqGJs1xMV1Qtr+xQS6ri6A1QBT4rnz8= -k8s.io/apiextensions-apiserver v0.26.6/go.mod h1:T6zbudRhmwN0sxg9lD51co/3Ah3JuCduz0nbtxyRXrk= -k8s.io/apimachinery v0.26.6 h1:OT04J9US8G+AqfqvcJZZ8s3WUQkWbc3t6ePPWieDN6I= -k8s.io/apimachinery v0.26.6/go.mod h1:qYzLkrQ9lhrZRh0jNKo2cfvf/R1/kQONnSiyB7NUJU0= -k8s.io/client-go v0.26.6 h1:CtC0wOxkAwjYyG2URGzdEKo0nLILopSDYn5AmzOkdi4= -k8s.io/client-go v0.26.6/go.mod h1:HDjbQGY7XzFYFUWOPAfAsIYhvFXyc9l6Ne0pO0bOQ7o= -k8s.io/cloud-provider v0.26.6 h1:byNR1IYs4ykPAqreq7icYmoGiy7ViupWtT5cz7W1pfQ= -k8s.io/cloud-provider v0.26.6/go.mod h1:zJd8Em72WezikROPVJiq+xZ44vUfYDR+OuIQE7CqCro= -k8s.io/component-base v0.26.6 h1:/Tm16Z8l/ruLFcw1XbFKTRSuxD6gQULQxxYgmar8PI0= -k8s.io/component-base v0.26.6/go.mod h1:fsv8CPnT5gumGxRbiQvK1j8IGvqSNwqZaJS5XTlLM1s= -k8s.io/csi-translation-lib v0.26.6 h1:lIgfnC0rJvrxA6yzJtQBQ8vt3lDOkBsJkddrZUim8Gs= -k8s.io/csi-translation-lib v0.26.6/go.mod h1:sBIA1rnrhXq6Mske+l4xmHAmkoviJIb7qcTU/O8/SJo= +k8s.io/api v0.28.3 h1:Gj1HtbSdB4P08C8rs9AR94MfSGpRhJgsS+GF9V26xMM= +k8s.io/api v0.28.3/go.mod h1:MRCV/jr1dW87/qJnZ57U5Pak65LGmQVkKTzf3AtKFHc= +k8s.io/apiextensions-apiserver v0.28.3 h1:Od7DEnhXHnHPZG+W9I97/fSQkVpVPQx2diy+2EtmY08= +k8s.io/apiextensions-apiserver v0.28.3/go.mod h1:NE1XJZ4On0hS11aWWJUTNkmVB03j9LM7gJSisbRt8Lc= +k8s.io/apimachinery v0.28.3 h1:B1wYx8txOaCQG0HmYF6nbpU8dg6HvA06x5tEffvOe7A= +k8s.io/apimachinery v0.28.3/go.mod h1:uQTKmIqs+rAYaq+DFaoD2X7pcjLOqbQX2AOiO0nIpb8= +k8s.io/client-go v0.28.3 h1:2OqNb72ZuTZPKCl+4gTKvqao0AMOl9f3o2ijbAj3LI4= +k8s.io/client-go v0.28.3/go.mod h1:LTykbBp9gsA7SwqirlCXBWtK0guzfhpoW4qSm7i9dxo= +k8s.io/cloud-provider v0.28.2 h1:9qsYm86hm4bnPgZbl9LE29Zfgjuq3NZR2dgtPioJ40s= +k8s.io/cloud-provider v0.28.2/go.mod h1:40fqf6MtgYho5Eu4gkyLgh5abxU/QKTMTIwBxt4ILyU= +k8s.io/component-base v0.28.3 h1:rDy68eHKxq/80RiMb2Ld/tbH8uAE75JdCqJyi6lXMzI= +k8s.io/component-base v0.28.3/go.mod h1:fDJ6vpVNSk6cRo5wmDa6eKIG7UlIQkaFmZN2fYgIUD8= +k8s.io/csi-translation-lib v0.28.2 h1:63MIOXUn5bet2Mw7G+A7zFmLzQ/vzBrjvNYIlXYh/n0= +k8s.io/csi-translation-lib v0.28.2/go.mod h1:14Lusc0J0vnlRNXA/T7GlZcou4XFTRHC071jsz+SHvQ= k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f h1:2kWPakN3i/k81b0gvD5C5FJ2kxm1WrQFanWchyKuqGg= -k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f/go.mod h1:byini6yhqGC14c3ebc/QwanvYwhuMWF6yz2F8uwW8eg= -k8s.io/utils v0.0.0-20230209194617-a36077c30491 h1:r0BAOLElQnnFhE/ApUsg3iHdVYYPBjNSSOMowRZxxsY= -k8s.io/utils v0.0.0-20230209194617-a36077c30491/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -knative.dev/pkg v0.0.0-20230712131115-7051d301e7f4 h1:oO/BQJpVCFTSTMHF/S6u+nPtIvbHDTsvbPZvdCZAFjs= -knative.dev/pkg v0.0.0-20230712131115-7051d301e7f4/go.mod h1:eXobTqst4aI7CNa6W7sG73VhEsHGWPSrkefeMTb++a0= +k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= +k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +knative.dev/pkg v0.0.0-20231010144348-ca8c009405dd h1:KJXBX9dOmRTUWduHg1gnWtPGIEl+GMh8UHdrBEZgOXE= +knative.dev/pkg v0.0.0-20231010144348-ca8c009405dd/go.mod h1:36cYnaOVHkzmhgybmYX6zDaTl3PakFeJQJl7wi6/RLE= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/controller-runtime v0.14.6 h1:oxstGVvXGNnMvY7TAESYk+lzr6S3V5VFxQ6d92KcwQA= -sigs.k8s.io/controller-runtime v0.14.6/go.mod h1:WqIdsAY6JBsjfc/CqO0CORmNtoCtE4S6qbPc9s68h+0= +sigs.k8s.io/controller-runtime v0.16.3 h1:2TuvuokmfXvDUamSx1SuAOO3eTyye+47mJCigwG62c4= +sigs.k8s.io/controller-runtime v0.16.3/go.mod h1:j7bialYoSn142nv9sCOJmQgDXQXxnroFU4VnX/brVJ0= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE= -sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= +sigs.k8s.io/structured-merge-diff/v4 v4.3.0 h1:UZbZAZfX0wV2zr7YZorDz6GXROfDFj6LvqCRm4VUVKk= +sigs.k8s.io/structured-merge-diff/v4 v4.3.0/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/hack/code/bandwidth_gen.go b/hack/code/bandwidth_gen/main.go similarity index 100% rename from hack/code/bandwidth_gen.go rename to hack/code/bandwidth_gen/main.go diff --git a/hack/code/instancetype_testdata_gen.go b/hack/code/instancetype_testdata_gen/main.go similarity index 100% rename from hack/code/instancetype_testdata_gen.go rename to hack/code/instancetype_testdata_gen/main.go diff --git a/hack/code/prices_gen.go b/hack/code/prices_gen/main.go similarity index 98% rename from hack/code/prices_gen.go rename to hack/code/prices_gen/main.go index e6dfe86cb0bb..c2bac3781adc 100644 --- a/hack/code/prices_gen.go +++ b/hack/code/prices_gen/main.go @@ -34,7 +34,7 @@ import ( "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "github.com/aws/karpenter/pkg/apis/settings" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/providers/pricing" "github.com/aws/karpenter/pkg/test" ) @@ -93,7 +93,7 @@ func main() { os.Setenv("AWS_SDK_LOAD_CONFIG", "true") os.Setenv("AWS_REGION", region) ctx := context.Background() - ctx = settings.ToContext(ctx, test.Settings()) + ctx = options.ToContext(ctx, test.Options()) sess := session.Must(session.NewSession()) ec2 := ec22.New(sess) src := &bytes.Buffer{} diff --git a/hack/code/vpc_limits_gen.go b/hack/code/vpc_limits_gen/main.go similarity index 97% rename from hack/code/vpc_limits_gen.go rename to hack/code/vpc_limits_gen/main.go index 773777fc567c..5baaa0460050 100644 --- a/hack/code/vpc_limits_gen.go +++ b/hack/code/vpc_limits_gen/main.go @@ -26,13 +26,13 @@ import ( "time" ) -type options struct { +type Options struct { sourceOutput string urlInput string } func main() { - opts := options{} + opts := Options{} flag.StringVar(&opts.urlInput, "url", "https://raw.githubusercontent.com/aws/amazon-vpc-resource-controller-k8s/master/pkg/aws/vpc/limits.go", "url of the raw vpc/limits.go file in the github.com/aws/amazon-vpc-resource-controller-k8s repo") flag.StringVar(&opts.sourceOutput, "output", "pkg/providers/instancetype/zz_generated.vpclimits.go", "output location for the generated go source file") diff --git a/hack/codegen.sh b/hack/codegen.sh index 966c70c7cf5f..2781722d3bf2 100755 --- a/hack/codegen.sh +++ b/hack/codegen.sh @@ -12,7 +12,7 @@ bandwidth() { NO_UPDATE='' SUBJECT="Bandwidth" - go run hack/code/bandwidth_gen.go -- "${GENERATED_FILE}" + go run hack/code/bandwidth_gen/main.go -- "${GENERATED_FILE}" GIT_DIFF=$(git diff --stat "${GENERATED_FILE}") checkForUpdates "${GIT_DIFF}" "${NO_UPDATE}" "${SUBJECT}" "${GENERATED_FILE}" @@ -30,7 +30,7 @@ pricing() { NO_UPDATE=" ${GENERATED_FILE} "$'| 4 ++--\n 1 file changed, 2 insertions(+), 2 deletions(-)' SUBJECT="Pricing" - go run hack/code/prices_gen.go --partition "$partition" --output "$GENERATED_FILE" + go run hack/code/prices_gen/main.go --partition "$partition" --output "$GENERATED_FILE" GIT_DIFF=$(git diff --stat "${GENERATED_FILE}") checkForUpdates "${GIT_DIFF}" "${NO_UPDATE}" "${SUBJECT} beside timestamps since last update" "${GENERATED_FILE}" @@ -42,7 +42,7 @@ vpcLimits() { NO_UPDATE='' SUBJECT="VPC Limits" - go run hack/code/vpc_limits_gen.go -- \ + go run hack/code/vpc_limits_gen/main.go -- \ --url=https://raw.githubusercontent.com/aws/amazon-vpc-resource-controller-k8s/master/pkg/aws/vpc/limits.go \ --output="${GENERATED_FILE}" @@ -55,7 +55,7 @@ instanceTypeTestData() { NO_UPDATE='' SUBJECT="Instance Type Test Data" - go run hack/code/instancetype_testdata_gen.go --out-file ${GENERATED_FILE} \ + go run hack/code/instancetype_testdata_gen/main.go --out-file ${GENERATED_FILE} \ --instance-types t3.large,m5.large,m5.xlarge,p3.8xlarge,g4dn.8xlarge,c6g.large,inf1.2xlarge,inf1.6xlarge,trn1.2xlarge,m5.metal,dl1.24xlarge,m6idn.32xlarge,t4g.small,t4g.xlarge,t4g.medium GIT_DIFF=$(git diff --stat "${GENERATED_FILE}") diff --git a/hack/docgen.sh b/hack/docgen.sh index fe34554d9b96..58655b38b2c5 100755 --- a/hack/docgen.sh +++ b/hack/docgen.sh @@ -3,7 +3,7 @@ set -euo pipefail compatibilitymatrix() { go run hack/docs/version_compatibility.go hack/docs/compatibility-karpenter.yaml "$(git describe --exact-match --tags || echo "no tag")" - go run hack/docs/compatibilitymetrix_gen_docs.go website/content/en/preview/upgrade-guide.md hack/docs/compatibility-karpenter.yaml 6 + go run hack/docs/compatibilitymetrix_gen_docs.go website/content/en/preview/upgrading/compatibility.md hack/docs/compatibility-karpenter.yaml 6 } diff --git a/hack/docs/configuration_gen_docs.go b/hack/docs/configuration_gen_docs.go index 65556ace64a4..828369d0e405 100644 --- a/hack/docs/configuration_gen_docs.go +++ b/hack/docs/configuration_gen_docs.go @@ -21,7 +21,8 @@ import ( "os" "strings" - "github.com/aws/karpenter-core/pkg/operator/options" + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" + "github.com/aws/karpenter/pkg/operator/options" ) func main() { @@ -48,17 +49,20 @@ func main() { topDoc := fmt.Sprintf("%s%s\n\n", startDocSections[0], genStart) bottomDoc := fmt.Sprintf("\n%s%s", genEnd, endDocSections[1]) - opts := options.New() + fs := &coreoptions.FlagSet { + FlagSet: flag.NewFlagSet("karpenter", flag.ContinueOnError), + } + (&coreoptions.Options{}).AddFlags(fs) + (&options.Options{}).AddFlags(fs) envVarsBlock := "| Environment Variable | CLI Flag | Description |\n" envVarsBlock += "|--|--|--|\n" - opts.VisitAll(func(f *flag.Flag) { + fs.VisitAll(func(f *flag.Flag) { if f.DefValue == "" { envVarsBlock += fmt.Sprintf("| %s | %s | %s|\n", strings.ReplaceAll(strings.ToUpper(f.Name), "-", "_"), "\\-\\-"+f.Name, f.Usage) } else { envVarsBlock += fmt.Sprintf("| %s | %s | %s (default = %s)|\n", strings.ReplaceAll(strings.ToUpper(f.Name), "-", "_"), "\\-\\-"+f.Name, f.Usage, f.DefValue) } - }) log.Println("writing output to", outputFileName) diff --git a/hack/docs/instancetypes_gen_docs.go b/hack/docs/instancetypes_gen_docs.go index dcbbc9f1eb83..b87f3cb435ce 100644 --- a/hack/docs/instancetypes_gen_docs.go +++ b/hack/docs/instancetypes_gen_docs.go @@ -35,14 +35,14 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/manager" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" coreoperator "github.com/aws/karpenter-core/pkg/operator" + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" coretest "github.com/aws/karpenter-core/pkg/test" nodepoolutil "github.com/aws/karpenter-core/pkg/utils/nodepool" - "github.com/aws/karpenter/pkg/apis/settings" awscloudprovider "github.com/aws/karpenter/pkg/cloudprovider" "github.com/aws/karpenter/pkg/operator" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/test" "github.com/aws/karpenter-core/pkg/cloudprovider" @@ -63,10 +63,20 @@ func (m *FakeManager) GetConfig() *rest.Config { return &rest.Config{} } +func (m *FakeManager) GetFieldIndexer() client.FieldIndexer { + return &FakeFieldIndexer{} +} + func (m *FakeManager) Elected() <-chan struct{} { return make(chan struct{}, 1) } +type FakeFieldIndexer struct{} + +func (f *FakeFieldIndexer) IndexField(_ context.Context, _ client.Object, _ string, _ client.IndexerFunc) error { + return nil +} + func main() { flag.Parse() if flag.NArg() != 1 { @@ -77,8 +87,8 @@ func main() { lo.Must0(os.Setenv("AWS_SDK_LOAD_CONFIG", "true")) lo.Must0(os.Setenv("AWS_REGION", "us-east-1")) - ctx := coresettings.ToContext(context.Background(), coretest.Settings()) - ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ + ctx := coreoptions.ToContext(context.Background(), coretest.Options()) + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ ClusterName: lo.ToPtr("docs-gen"), ClusterEndpoint: lo.ToPtr("https://docs-gen.aws"), IsolatedVPC: lo.ToPtr(true), // disable pricing lookup diff --git a/hack/docs/metrics_gen_docs.go b/hack/docs/metrics_gen_docs.go index af8642a197ed..07deb83c94f1 100644 --- a/hack/docs/metrics_gen_docs.go +++ b/hack/docs/metrics_gen_docs.go @@ -79,10 +79,6 @@ description: > "These metrics are available by default at `karpenter.karpenter.svc.cluster.local:8000/metrics` configurable via the `METRICS_PORT` environment variable documented [here](../settings)\n") previousSubsystem := "" - // Ignore nodeClaimSubsystem and nodePoolSubsystem metrics until NodeClaims are released - allMetrics = lo.Reject(allMetrics, func(m metricInfo, _ int) bool { - return m.subsystem == "nodeclaims" || m.subsystem == "nodepools" - }) for _, metric := range allMetrics { // Controller Runtime naming is different in that they don't specify a namespace or subsystem // Getting the metrics requires special parsing logic @@ -275,6 +271,7 @@ func getIdentMapping(identName string) (string, error) { "interruptionSubsystem": "interruption", "nodeTemplateSubsystem": "nodetemplate", "deprovisioningSubsystem": "deprovisioning", + "disruptionSubsystem": "disruption", "consistencySubsystem": "consistency", "batcherSubsystem": "cloudprovider_batcher", "cloudProviderSubsystem": "cloudprovider", diff --git a/hack/release/release.sh b/hack/release/release.sh index 7410d22324d8..2775e026fb4c 100755 --- a/hack/release/release.sh +++ b/hack/release/release.sh @@ -12,5 +12,9 @@ SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) source "${SCRIPT_DIR}/common.sh" config -release "$GIT_TAG" +# Don't release with a dirty commit! +if [[ "$(git status --porcelain)" != "" ]]; then + exit 1 +fi +release "$GIT_TAG" diff --git a/hack/release/snapshot.sh b/hack/release/snapshot.sh index 5597c071be4d..bb792ba4c396 100755 --- a/hack/release/snapshot.sh +++ b/hack/release/snapshot.sh @@ -8,4 +8,9 @@ SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) source "${SCRIPT_DIR}/common.sh" config +# Don't release with a dirty commit! +if [[ "$(git status --porcelain)" != "" ]]; then + exit 1 +fi + snapshot "$HEAD_HASH" diff --git a/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml b/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml index 60be2238f1a0..717659ce402d 100644 --- a/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml +++ b/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml @@ -77,9 +77,21 @@ spec: description: Tags is a map of key/value tags used to select subnets Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') type: object + maxItems: 30 type: array + x-kubernetes-validations: + - message: expected at least one, got none, ['tags', 'id', 'name'] + rule: self.all(x, has(x.tags) || has(x.id) || has(x.name)) + - message: '''id'' is mutually exclusive, cannot be set with a combination + of other fields in amiSelectorTerms' + rule: '!self.all(x, has(x.id) && (has(x.tags) || has(x.name)) || + has(x.owner))' blockDeviceMappings: description: BlockDeviceMappings to be applied to provisioned nodes. items: @@ -133,29 +145,47 @@ spec: format: int64 type: integer volumeSize: + allOf: + - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + - pattern: ^((?:[1-9][0-9]{0,3}|[1-4][0-9]{4}|[5][0-8][0-9]{3}|59000)Gi|(?:[1-9][0-9]{0,3}|[1-5][0-9]{4}|[6][0-3][0-9]{3}|64000)G|([1-9]||[1-5][0-7]|58)Ti|([1-9]||[1-5][0-9]|6[0-3]|64)T)$ anyOf: - type: integer - type: string - description: "VolumeSize in GiBs. You must specify either - a snapshot ID or a volume size. The following are the - supported volumes sizes for each volume type: \n * gp2 - and gp3: 1-16,384 \n * io1 and io2: 4-16,384 \n * st1 - and sc1: 125-16,384 \n * standard: 1-1,024" - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + description: "VolumeSize in `Gi`, `G`, `Ti`, or `T`. You + must specify either a snapshot ID or a volume size. The + following are the supported volumes sizes for each volume + type: \n * gp2 and gp3: 1-16,384 \n * io1 and io2: 4-16,384 + \n * st1 and sc1: 125-16,384 \n * standard: 1-1,024" x-kubernetes-int-or-string: true volumeType: description: VolumeType of the block device. For more information, see Amazon EBS volume types (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html) in the Amazon Elastic Compute Cloud User Guide. + enum: + - standard + - io1 + - io2 + - gp2 + - sc1 + - st1 + - gp3 type: string type: object + x-kubernetes-validations: + - message: snapshotID or volumeSize must be defined + rule: has(self.snapshotID) || has(self.volumeSize) rootVolume: description: RootVolume is a flag indicating if this device is mounted as kubelet root dir. You can configure at most one root volume in BlockDeviceMappings. type: boolean type: object + maxItems: 50 type: array + x-kubernetes-validations: + - message: must have only one blockDeviceMappings with rootVolume + rule: self.filter(x, has(x.rootVolume)?x.rootVolume==true:false).size() + <= 1 context: description: Context is a Reserved field in EC2 APIs https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html type: string @@ -213,6 +243,9 @@ spec: but this parameter is not specified, the default state is \"enabled\". \n If you specify a value of \"disabled\", instance metadata will not be accessible on the node." + enum: + - enabled + - disabled type: string httpProtocolIPv6: default: disabled @@ -220,6 +253,9 @@ spec: for the instance metadata service on provisioned nodes. If metadata options is non-nil, but this parameter is not specified, the default state is "disabled". + enum: + - enabled + - disabled type: string httpPutResponseHopLimit: default: 2 @@ -229,6 +265,8 @@ spec: values are integers from 1 to 64. If metadata options is non-nil, but this parameter is not specified, the default value is 2. format: int64 + maximum: 64 + minimum: 1 type: integer httpTokens: default: required @@ -245,6 +283,9 @@ spec: retrieval requests. In this state, retrieving the IAM role credentials always returns the version 2.0 credentials; the version 1.0 credentials are not available." + enum: + - required + - optional type: string type: object placementGroupSelectorTerms: @@ -268,6 +309,9 @@ spec: collection and drift handling is implemented for the old instance profiles on an update. type: string + x-kubernetes-validations: + - message: immutable field changed + rule: self == oldSelf securityGroupSelectorTerms: description: SecurityGroupSelectorTerms is a list of or security group selector terms. The terms are ORed. @@ -290,9 +334,25 @@ spec: description: Tags is a map of key/value tags used to select subnets Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') type: object + maxItems: 30 type: array + x-kubernetes-validations: + - message: securityGroupSelectorTerms cannot be empty + rule: self.size() != 0 + - message: expected at least one, got none, ['tags', 'id', 'name'] + rule: self.all(x, has(x.tags) || has(x.id) || has(x.name)) + - message: '''id'' is mutually exclusive, cannot be set with a combination + of other fields in securityGroupSelectorTerms' + rule: '!self.all(x, has(x.id) && (has(x.tags) || has(x.name)))' + - message: '''name'' is mutually exclusive, cannot be set with a combination + of other fields in securityGroupSelectorTerms' + rule: '!self.all(x, has(x.name) && (has(x.tags) || has(x.id)))' subnetSelectorTerms: description: SubnetSelectorTerms is a list of or subnet selector terms. The terms are ORed. @@ -311,15 +371,39 @@ spec: description: Tags is a map of key/value tags used to select subnets Specifying '*' for a value selects all values for a given tag key. + maxProperties: 20 type: object + x-kubernetes-validations: + - message: empty tag keys or values aren't supported + rule: self.all(k, k != '' && self[k] != '') type: object + maxItems: 30 type: array + x-kubernetes-validations: + - message: subnetSelectorTerms cannot be empty + rule: self.size() != 0 + - message: expected at least one, got none, ['tags', 'id'] + rule: self.all(x, has(x.tags) || has(x.id)) + - message: '''id'' is mutually exclusive, cannot be set with a combination + of other fields in subnetSelectorTerms' + rule: '!self.all(x, has(x.id) && has(x.tags))' tags: additionalProperties: type: string description: Tags to be applied on ec2 resources like instances and launch templates. type: object + x-kubernetes-validations: + - message: empty tag keys aren't supported + rule: self.all(k, k != '') + - message: tag contains a restricted tag matching kubernetes.io/cluster/ + rule: self.all(k, !k.startsWith('kubernetes.io/cluster') ) + - message: tag contains a restricted tag matching karpenter.sh/provisioner-name + rule: self.all(k, k != 'karpenter.sh/provisioner-name') + - message: tag contains a restricted tag matching karpenter.sh/nodepool + rule: self.all(k, k != 'karpenter.sh/nodepool') + - message: tag contains a restricted tag matching karpenter.sh/managed-by + rule: self.all(k, k !='karpenter.sh/managed-by') userData: description: UserData to be applied to the provisioned nodes. It must be in the appropriate format based on the AMIFamily in use. Karpenter @@ -332,6 +416,10 @@ spec: - securityGroupSelectorTerms - subnetSelectorTerms type: object + x-kubernetes-validations: + - message: amiSelectorTerms is required when amiFamily == 'Custom' + rule: 'self.amiFamily == ''Custom'' ? self.amiSelectorTerms.size() != + 0 : true' status: description: EC2NodeClassStatus contains the resolved state of the EC2NodeClass properties: diff --git a/pkg/apis/crds/karpenter.sh_nodeclaims.yaml b/pkg/apis/crds/karpenter.sh_nodeclaims.yaml index de7af73fb226..3ceff377c01d 100644 --- a/pkg/apis/crds/karpenter.sh_nodeclaims.yaml +++ b/pkg/apis/crds/karpenter.sh_nodeclaims.yaml @@ -9,351 +9,316 @@ spec: group: karpenter.sh names: categories: - - karpenter + - karpenter kind: NodeClaim listKind: NodeClaimList plural: nodeclaims singular: nodeclaim scope: Cluster versions: - - additionalPrinterColumns: - - jsonPath: .metadata.labels.node\.kubernetes\.io/instance-type - name: Type - type: string - - jsonPath: .metadata.labels.topology\.kubernetes\.io/zone - name: Zone - type: string - - jsonPath: .status.nodeName - name: Node - type: string - - jsonPath: .status.conditions[?(@.type=="Ready")].status - name: Ready - type: string - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - - jsonPath: .metadata.labels.karpenter\.sh/capacity-type - name: Capacity - priority: 1 - type: string - - jsonPath: .metadata.labels.karpenter\.sh/nodepool - name: NodePool - priority: 1 - type: string - - jsonPath: .spec.nodeClassRef.name - name: NodeClass - priority: 1 - type: string - name: v1beta1 - schema: - openAPIV3Schema: - description: NodeClaim is the Schema for the NodeClaims API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: NodeClaimSpec describes the desired state of the NodeClaim - properties: - kubelet: - description: Kubelet defines args to be used when configuring kubelet - on provisioned nodes. They are a subset of the upstream types, recognizing - not all options may be supported. Wherever possible, the types and - names should reflect the upstream kubelet types. - properties: - clusterDNS: - description: clusterDNS is a list of IP addresses for the cluster - DNS server. Note that not all providers may use all addresses. - items: - type: string - type: array - cpuCFSQuota: - description: CPUCFSQuota enables CPU CFS quota enforcement for - containers that specify CPU limits. - type: boolean - evictionHard: - additionalProperties: - type: string - description: EvictionHard is the map of signal names to quantities - that define hard eviction thresholds - type: object - evictionMaxPodGracePeriod: - description: EvictionMaxPodGracePeriod is the maximum allowed - grace period (in seconds) to use when terminating pods in response - to soft eviction thresholds being met. - format: int32 - type: integer - evictionSoft: - additionalProperties: - type: string - description: EvictionSoft is the map of signal names to quantities - that define soft eviction thresholds - type: object - evictionSoftGracePeriod: - additionalProperties: - type: string - description: EvictionSoftGracePeriod is the map of signal names - to quantities that define grace periods for each eviction signal - type: object - imageGCHighThresholdPercent: - description: ImageGCHighThresholdPercent is the percent of disk - usage after which image garbage collection is always run. The - percent is calculated by dividing this field value by 100, so - this field must be between 0 and 100, inclusive. When specified, - the value must be greater than ImageGCLowThresholdPercent. - format: int32 - maximum: 100 - minimum: 0 - type: integer - imageGCLowThresholdPercent: - description: ImageGCLowThresholdPercent is the percent of disk - usage before which image garbage collection is never run. Lowest - disk usage to garbage collect to. The percent is calculated - by dividing this field value by 100, so the field value must - be between 0 and 100, inclusive. When specified, the value must - be less than imageGCHighThresholdPercent - format: int32 - maximum: 100 - minimum: 0 - type: integer - kubeReserved: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: KubeReserved contains resources reserved for Kubernetes - system components. - type: object - maxPods: - description: MaxPods is an override for the maximum number of - pods that can run on a worker node instance. - format: int32 - minimum: 0 - type: integer - podsPerCore: - description: PodsPerCore is an override for the number of pods - that can run on a worker node instance based on the number of - cpu cores. This value cannot exceed MaxPods, so, if MaxPods - is a lower value, that value will be used. - format: int32 - minimum: 0 - type: integer - systemReserved: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: SystemReserved contains resources reserved for OS - system daemons and kernel memory. - type: object - type: object - nodeClassRef: - description: NodeClassRef is a reference to an object that defines - provider specific configuration - properties: - apiVersion: - description: API version of the referent - type: string - kind: - description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' - type: string - name: - description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' - type: string - required: - - name - type: object - requirements: - description: Requirements are layered with GetLabels and applied to - every node. - items: - description: A node selector requirement is a selector that contains - values, a key, and an operator that relates the key and values. + - additionalPrinterColumns: + - jsonPath: .metadata.labels.node\.kubernetes\.io/instance-type + name: Type + type: string + - jsonPath: .metadata.labels.topology\.kubernetes\.io/zone + name: Zone + type: string + - jsonPath: .status.nodeName + name: Node + type: string + - jsonPath: .status.conditions[?(@.type=="Ready")].status + name: Ready + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .metadata.labels.karpenter\.sh/capacity-type + name: Capacity + priority: 1 + type: string + - jsonPath: .metadata.labels.karpenter\.sh/nodepool + name: NodePool + priority: 1 + type: string + - jsonPath: .spec.nodeClassRef.name + name: NodeClass + priority: 1 + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: NodeClaim is the Schema for the NodeClaims API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: NodeClaimSpec describes the desired state of the NodeClaim + properties: + kubelet: + description: Kubelet defines args to be used when configuring kubelet on provisioned nodes. They are a subset of the upstream types, recognizing not all options may be supported. Wherever possible, the types and names should reflect the upstream kubelet types. properties: - key: - description: The label key that the selector applies to. - type: string - operator: - description: Represents a key's relationship to a set of values. - Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and - Lt. - type: string - values: - description: An array of string values. If the operator is In - or NotIn, the values array must be non-empty. If the operator - is Exists or DoesNotExist, the values array must be empty. - If the operator is Gt or Lt, the values array must have a - single element, which will be interpreted as an integer. This - array is replaced during a strategic merge patch. + clusterDNS: + description: clusterDNS is a list of IP addresses for the cluster DNS server. Note that not all providers may use all addresses. items: type: string type: array - required: - - key - - operator + cpuCFSQuota: + description: CPUCFSQuota enables CPU CFS quota enforcement for containers that specify CPU limits. + type: boolean + evictionHard: + additionalProperties: + type: string + pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ + description: EvictionHard is the map of signal names to quantities that define hard eviction thresholds + type: object + x-kubernetes-validations: + - message: valid keys for evictionHard are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + evictionMaxPodGracePeriod: + description: EvictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use when terminating pods in response to soft eviction thresholds being met. + format: int32 + type: integer + evictionSoft: + additionalProperties: + type: string + pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ + description: EvictionSoft is the map of signal names to quantities that define soft eviction thresholds + type: object + x-kubernetes-validations: + - message: valid keys for evictionSoft are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + evictionSoftGracePeriod: + additionalProperties: + type: string + description: EvictionSoftGracePeriod is the map of signal names to quantities that define grace periods for each eviction signal + type: object + x-kubernetes-validations: + - message: valid keys for evictionSoftGracePeriod are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + imageGCHighThresholdPercent: + description: ImageGCHighThresholdPercent is the percent of disk usage after which image garbage collection is always run. The percent is calculated by dividing this field value by 100, so this field must be between 0 and 100, inclusive. When specified, the value must be greater than ImageGCLowThresholdPercent. + format: int32 + maximum: 100 + minimum: 0 + type: integer + imageGCLowThresholdPercent: + description: ImageGCLowThresholdPercent is the percent of disk usage before which image garbage collection is never run. Lowest disk usage to garbage collect to. The percent is calculated by dividing this field value by 100, so the field value must be between 0 and 100, inclusive. When specified, the value must be less than imageGCHighThresholdPercent + format: int32 + maximum: 100 + minimum: 0 + type: integer + kubeReserved: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: KubeReserved contains resources reserved for Kubernetes system components. + type: object + x-kubernetes-validations: + - message: valid keys for kubeReserved are ['cpu','memory','ephemeral-storage','pid'] + rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') + - message: kubeReserved value cannot be a negative resource quantity + rule: self.all(x, !self[x].startsWith('-')) + maxPods: + description: MaxPods is an override for the maximum number of pods that can run on a worker node instance. + format: int32 + minimum: 0 + type: integer + podsPerCore: + description: PodsPerCore is an override for the number of pods that can run on a worker node instance based on the number of cpu cores. This value cannot exceed MaxPods, so, if MaxPods is a lower value, that value will be used. + format: int32 + minimum: 0 + type: integer + systemReserved: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: SystemReserved contains resources reserved for OS system daemons and kernel memory. + type: object + x-kubernetes-validations: + - message: valid keys for systemReserved are ['cpu','memory','ephemeral-storage','pid'] + rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') + - message: systemReserved value cannot be a negative resource quantity + rule: self.all(x, !self[x].startsWith('-')) type: object - type: array - resources: - description: Resources models the resource requirements for the NodeClaim - to launch - properties: - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: Requests describes the minimum required resources - for the NodeClaim to launch - type: object - type: object - startupTaints: - description: StartupTaints are taints that are applied to nodes upon - startup which are expected to be removed automatically within a - short period of time, typically by a DaemonSet that tolerates the - taint. These are commonly used by daemonsets to allow initialization - and enforce startup ordering. StartupTaints are ignored for provisioning - purposes in that pods are not required to tolerate a StartupTaint - in order to have nodes provisioned for them. - items: - description: The node this Taint is attached to has the "effect" - on any pod that does not tolerate the Taint. + x-kubernetes-validations: + - message: imageGCHighThresholdPercent must be greater than imageGCLowThresholdPercent + rule: 'has(self.imageGCHighThresholdPercent) && has(self.imageGCLowThresholdPercent) ? self.imageGCHighThresholdPercent > self.imageGCLowThresholdPercent : true' + - message: evictionSoft OwnerKey does not have a matching evictionSoftGracePeriod + rule: has(self.evictionSoft) ? self.evictionSoft.all(e, (e in self.evictionSoftGracePeriod)):true + - message: evictionSoftGracePeriod OwnerKey does not have a matching evictionSoft + rule: has(self.evictionSoftGracePeriod) ? self.evictionSoftGracePeriod.all(e, (e in self.evictionSoft)):true + nodeClassRef: + description: NodeClassRef is a reference to an object that defines provider specific configuration properties: - effect: - description: Required. The effect of the taint on pods that - do not tolerate the taint. Valid effects are NoSchedule, PreferNoSchedule - and NoExecute. + apiVersion: + description: API version of the referent type: string - key: - description: Required. The taint key to be applied to a node. + kind: + description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' type: string - timeAdded: - description: TimeAdded represents the time at which the taint - was added. It is only written for NoExecute taints. - format: date-time - type: string - value: - description: The taint value corresponding to the taint key. + name: + description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' type: string required: - - effect - - key + - name type: object - type: array - taints: - description: Taints will be applied to the NodeClaim's node. - items: - description: The node this Taint is attached to has the "effect" - on any pod that does not tolerate the Taint. + requirements: + description: Requirements are layered with GetLabels and applied to every node. + items: + description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + resources: + description: Resources models the resource requirements for the NodeClaim to launch properties: - effect: - description: Required. The effect of the taint on pods that - do not tolerate the taint. Valid effects are NoSchedule, PreferNoSchedule - and NoExecute. - type: string - key: - description: Required. The taint key to be applied to a node. - type: string - timeAdded: - description: TimeAdded represents the time at which the taint - was added. It is only written for NoExecute taints. - format: date-time - type: string - value: - description: The taint value corresponding to the taint key. - type: string - required: - - effect - - key + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Requests describes the minimum required resources for the NodeClaim to launch + type: object type: object - type: array - required: - - nodeClassRef - - requirements - type: object - status: - description: NodeClaimStatus defines the observed state of NodeClaim - properties: - allocatable: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: Allocatable is the estimated allocatable capacity of - the node - type: object - capacity: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: Capacity is the estimated full capacity of the node - type: object - conditions: - description: Conditions contains signals for health and readiness - items: - description: 'Condition defines a readiness condition for a Knative - resource. See: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#typical-status-properties' - properties: - lastTransitionTime: - description: LastTransitionTime is the last time the condition - transitioned from one status to another. We use VolatileTime - in place of metav1.Time to exclude this from creating equality.Semantic - differences (all other things held constant). - type: string - message: - description: A human readable message indicating details about - the transition. - type: string - reason: - description: The reason for the condition's last transition. - type: string - severity: - description: Severity with which to treat failures of this type - of condition. When this is not specified, it defaults to Error. - type: string - status: - description: Status of the condition, one of True, False, Unknown. - type: string - type: - description: Type of condition. - type: string - required: - - status - - type + startupTaints: + description: StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. + items: + description: The node this Taint is attached to has the "effect" on any pod that does not tolerate the Taint. + properties: + effect: + description: Required. The effect of the taint on pods that do not tolerate the taint. Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Required. The taint key to be applied to a node. + type: string + timeAdded: + description: TimeAdded represents the time at which the taint was added. It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + required: + - effect + - key + type: object + type: array + taints: + description: Taints will be applied to the NodeClaim's node. + items: + description: The node this Taint is attached to has the "effect" on any pod that does not tolerate the Taint. + properties: + effect: + description: Required. The effect of the taint on pods that do not tolerate the taint. Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Required. The taint key to be applied to a node. + type: string + timeAdded: + description: TimeAdded represents the time at which the taint was added. It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + required: + - effect + - key + type: object + type: array + required: + - nodeClassRef + - requirements + type: object + status: + description: NodeClaimStatus defines the observed state of NodeClaim + properties: + allocatable: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Allocatable is the estimated allocatable capacity of the node + type: object + capacity: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Capacity is the estimated full capacity of the node type: object - type: array - imageID: - description: ImageID is an identifier for the image that runs on the - node - type: string - nodeName: - description: NodeName is the name of the corresponding node object - type: string - providerID: - description: ProviderID of the corresponding node object - type: string - type: object - type: object - served: true - storage: true - subresources: - status: {} + conditions: + description: Conditions contains signals for health and readiness + items: + description: 'Condition defines a readiness condition for a Knative resource. See: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#typical-status-properties' + properties: + lastTransitionTime: + description: LastTransitionTime is the last time the condition transitioned from one status to another. We use VolatileTime in place of metav1.Time to exclude this from creating equality.Semantic differences (all other things held constant). + type: string + message: + description: A human readable message indicating details about the transition. + type: string + reason: + description: The reason for the condition's last transition. + type: string + severity: + description: Severity with which to treat failures of this type of condition. When this is not specified, it defaults to Error. + type: string + status: + description: Status of the condition, one of True, False, Unknown. + type: string + type: + description: Type of condition. + type: string + required: + - status + - type + type: object + type: array + imageID: + description: ImageID is an identifier for the image that runs on the node + type: string + nodeName: + description: NodeName is the name of the corresponding node object + type: string + providerID: + description: ProviderID of the corresponding node object + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/pkg/apis/crds/karpenter.sh_nodepools.yaml b/pkg/apis/crds/karpenter.sh_nodepools.yaml index b40c7f9a4bb0..5f0083a751ef 100644 --- a/pkg/apis/crds/karpenter.sh_nodepools.yaml +++ b/pkg/apis/crds/karpenter.sh_nodepools.yaml @@ -9,386 +9,320 @@ spec: group: karpenter.sh names: categories: - - karpenter + - karpenter kind: NodePool listKind: NodePoolList plural: nodepools singular: nodepool scope: Cluster versions: - - additionalPrinterColumns: - - jsonPath: .spec.template.spec.nodeClassRef.name - name: NodeClass - type: string - - jsonPath: .spec.weight - name: Weight - priority: 1 - type: string - name: v1beta1 - schema: - openAPIV3Schema: - description: NodePool is the Schema for the NodePools API - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: NodePoolSpec is the top level provisioner specification. - Provisioners launch nodes in response to pods that are unschedulable. - A single provisioner is capable of managing a diverse set of nodes. - Node properties are determined from a combination of provisioner and - pod scheduling constraints. - properties: - disruption: - default: - consolidationPolicy: WhenUnderutilized - expireAfter: 720h - description: Disruption contains the parameters that relate to Karpenter's - disruption logic - properties: - consolidateAfter: - description: ConsolidateAfter is the duration the controller will - wait before attempting to terminate nodes that are underutilized. - Refer to ConsolidationPolicy for how underutilization is considered. - pattern: ^(([0-9]+(s|m|h))+)|(Never)$ - type: string - consolidationPolicy: - default: WhenUnderutilized - description: ConsolidationPolicy describes which nodes Karpenter - can disrupt through its consolidation algorithm. This policy - defaults to "WhenUnderutilized" if not specified - enum: - - WhenEmpty - - WhenUnderutilized - type: string - expireAfter: - default: 720h - description: ExpireAfter is the duration the controller will wait - before terminating a node, measured from when the node is created. - This is useful to implement features like eventually consistent - node upgrade, memory leak protection, and disruption testing. - pattern: ^(([0-9]+(s|m|h))+)|(Never)$ - type: string - type: object - x-kubernetes-validations: - - message: consolidateAfter cannot be combined with consolidationPolicy=WhenUnderutilized - rule: 'has(self.consolidateAfter) ? self.consolidationPolicy != - ''WhenUnderutilized'' || self.consolidateAfter == ''Never'' : - true' - - message: consolidateAfter must be specified with consolidationPolicy=WhenEmpty - rule: 'self.consolidationPolicy == ''WhenEmpty'' ? has(self.consolidateAfter) - : true' - limits: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: Limits define a set of bounds for provisioning capacity. - type: object - template: - description: Template contains the template of possibilities for the - provisioning logic to launch a NodeClaim with. NodeClaims launched - from this NodePool will often be further constrained than the template - specifies. - properties: - metadata: - properties: - annotations: - additionalProperties: - type: string - type: object - finalizers: - items: - type: string - type: array - labels: - additionalProperties: - type: string - type: object - name: - type: string - namespace: - type: string - type: object - spec: - description: NodeClaimSpec describes the desired state of the - NodeClaim - properties: - kubelet: - description: Kubelet defines args to be used when configuring - kubelet on provisioned nodes. They are a subset of the upstream - types, recognizing not all options may be supported. Wherever - possible, the types and names should reflect the upstream - kubelet types. - properties: - clusterDNS: - description: clusterDNS is a list of IP addresses for - the cluster DNS server. Note that not all providers - may use all addresses. - items: - type: string - type: array - cpuCFSQuota: - description: CPUCFSQuota enables CPU CFS quota enforcement - for containers that specify CPU limits. - type: boolean - evictionHard: - additionalProperties: - type: string - description: EvictionHard is the map of signal names to - quantities that define hard eviction thresholds - type: object - evictionMaxPodGracePeriod: - description: EvictionMaxPodGracePeriod is the maximum - allowed grace period (in seconds) to use when terminating - pods in response to soft eviction thresholds being met. - format: int32 - type: integer - evictionSoft: - additionalProperties: - type: string - description: EvictionSoft is the map of signal names to - quantities that define soft eviction thresholds - type: object - evictionSoftGracePeriod: - additionalProperties: - type: string - description: EvictionSoftGracePeriod is the map of signal - names to quantities that define grace periods for each - eviction signal - type: object - imageGCHighThresholdPercent: - description: ImageGCHighThresholdPercent is the percent - of disk usage after which image garbage collection is - always run. The percent is calculated by dividing this - field value by 100, so this field must be between 0 - and 100, inclusive. When specified, the value must be - greater than ImageGCLowThresholdPercent. - format: int32 - maximum: 100 - minimum: 0 - type: integer - imageGCLowThresholdPercent: - description: ImageGCLowThresholdPercent is the percent - of disk usage before which image garbage collection - is never run. Lowest disk usage to garbage collect to. - The percent is calculated by dividing this field value - by 100, so the field value must be between 0 and 100, - inclusive. When specified, the value must be less than - imageGCHighThresholdPercent - format: int32 - maximum: 100 - minimum: 0 - type: integer - kubeReserved: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: KubeReserved contains resources reserved - for Kubernetes system components. - type: object - maxPods: - description: MaxPods is an override for the maximum number - of pods that can run on a worker node instance. - format: int32 - minimum: 0 - type: integer - podsPerCore: - description: PodsPerCore is an override for the number - of pods that can run on a worker node instance based - on the number of cpu cores. This value cannot exceed - MaxPods, so, if MaxPods is a lower value, that value - will be used. - format: int32 - minimum: 0 - type: integer - systemReserved: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: SystemReserved contains resources reserved - for OS system daemons and kernel memory. - type: object - type: object - nodeClassRef: - description: NodeClassRef is a reference to an object that - defines provider specific configuration - properties: - apiVersion: - description: API version of the referent + - additionalPrinterColumns: + - jsonPath: .spec.template.spec.nodeClassRef.name + name: NodeClass + type: string + - jsonPath: .spec.weight + name: Weight + priority: 1 + type: string + name: v1beta1 + schema: + openAPIV3Schema: + description: NodePool is the Schema for the NodePools API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: NodePoolSpec is the top level provisioner specification. Provisioners launch nodes in response to pods that are unschedulable. A single provisioner is capable of managing a diverse set of nodes. Node properties are determined from a combination of provisioner and pod scheduling constraints. + properties: + disruption: + default: + consolidationPolicy: WhenUnderutilized + expireAfter: 720h + description: Disruption contains the parameters that relate to Karpenter's disruption logic + properties: + consolidateAfter: + description: ConsolidateAfter is the duration the controller will wait before attempting to terminate nodes that are underutilized. Refer to ConsolidationPolicy for how underutilization is considered. + pattern: ^(([0-9]+(s|m|h))+)|(Never)$ + type: string + consolidationPolicy: + default: WhenUnderutilized + description: ConsolidationPolicy describes which nodes Karpenter can disrupt through its consolidation algorithm. This policy defaults to "WhenUnderutilized" if not specified + enum: + - WhenEmpty + - WhenUnderutilized + type: string + expireAfter: + default: 720h + description: ExpireAfter is the duration the controller will wait before terminating a node, measured from when the node is created. This is useful to implement features like eventually consistent node upgrade, memory leak protection, and disruption testing. + pattern: ^(([0-9]+(s|m|h))+)|(Never)$ + type: string + type: object + x-kubernetes-validations: + - message: consolidateAfter cannot be combined with consolidationPolicy=WhenUnderutilized + rule: 'has(self.consolidateAfter) ? self.consolidationPolicy != ''WhenUnderutilized'' || self.consolidateAfter == ''Never'' : true' + - message: consolidateAfter must be specified with consolidationPolicy=WhenEmpty + rule: 'self.consolidationPolicy == ''WhenEmpty'' ? has(self.consolidateAfter) : true' + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Limits define a set of bounds for provisioning capacity. + type: object + template: + description: Template contains the template of possibilities for the provisioning logic to launch a NodeClaim with. NodeClaims launched from this NodePool will often be further constrained than the template specifies. + properties: + metadata: + properties: + annotations: + additionalProperties: type: string - kind: - description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' + type: object + finalizers: + items: type: string - name: - description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' + type: array + labels: + additionalProperties: type: string - required: - - name - type: object - requirements: - description: Requirements are layered with GetLabels and applied - to every node. - items: - description: A node selector requirement is a selector that - contains values, a key, and an operator that relates the - key and values. + type: object + name: + type: string + namespace: + type: string + type: object + spec: + description: NodeClaimSpec describes the desired state of the NodeClaim + properties: + kubelet: + description: Kubelet defines args to be used when configuring kubelet on provisioned nodes. They are a subset of the upstream types, recognizing not all options may be supported. Wherever possible, the types and names should reflect the upstream kubelet types. properties: - key: - description: The label key that the selector applies - to. - type: string - operator: - description: Represents a key's relationship to a set - of values. Valid operators are In, NotIn, Exists, - DoesNotExist. Gt, and Lt. - type: string - values: - description: An array of string values. If the operator - is In or NotIn, the values array must be non-empty. - If the operator is Exists or DoesNotExist, the values - array must be empty. If the operator is Gt or Lt, - the values array must have a single element, which - will be interpreted as an integer. This array is replaced - during a strategic merge patch. + clusterDNS: + description: clusterDNS is a list of IP addresses for the cluster DNS server. Note that not all providers may use all addresses. items: type: string type: array - required: - - key - - operator + cpuCFSQuota: + description: CPUCFSQuota enables CPU CFS quota enforcement for containers that specify CPU limits. + type: boolean + evictionHard: + additionalProperties: + type: string + pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ + description: EvictionHard is the map of signal names to quantities that define hard eviction thresholds + type: object + x-kubernetes-validations: + - message: valid keys for evictionHard are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + evictionMaxPodGracePeriod: + description: EvictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use when terminating pods in response to soft eviction thresholds being met. + format: int32 + type: integer + evictionSoft: + additionalProperties: + type: string + pattern: ^((\d{1,2}(\.\d{1,2})?|100(\.0{1,2})?)%||(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?)$ + description: EvictionSoft is the map of signal names to quantities that define soft eviction thresholds + type: object + x-kubernetes-validations: + - message: valid keys for evictionSoft are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + evictionSoftGracePeriod: + additionalProperties: + type: string + description: EvictionSoftGracePeriod is the map of signal names to quantities that define grace periods for each eviction signal + type: object + x-kubernetes-validations: + - message: valid keys for evictionSoftGracePeriod are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] + rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) + imageGCHighThresholdPercent: + description: ImageGCHighThresholdPercent is the percent of disk usage after which image garbage collection is always run. The percent is calculated by dividing this field value by 100, so this field must be between 0 and 100, inclusive. When specified, the value must be greater than ImageGCLowThresholdPercent. + format: int32 + maximum: 100 + minimum: 0 + type: integer + imageGCLowThresholdPercent: + description: ImageGCLowThresholdPercent is the percent of disk usage before which image garbage collection is never run. Lowest disk usage to garbage collect to. The percent is calculated by dividing this field value by 100, so the field value must be between 0 and 100, inclusive. When specified, the value must be less than imageGCHighThresholdPercent + format: int32 + maximum: 100 + minimum: 0 + type: integer + kubeReserved: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: KubeReserved contains resources reserved for Kubernetes system components. + type: object + x-kubernetes-validations: + - message: valid keys for kubeReserved are ['cpu','memory','ephemeral-storage','pid'] + rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') + - message: kubeReserved value cannot be a negative resource quantity + rule: self.all(x, !self[x].startsWith('-')) + maxPods: + description: MaxPods is an override for the maximum number of pods that can run on a worker node instance. + format: int32 + minimum: 0 + type: integer + podsPerCore: + description: PodsPerCore is an override for the number of pods that can run on a worker node instance based on the number of cpu cores. This value cannot exceed MaxPods, so, if MaxPods is a lower value, that value will be used. + format: int32 + minimum: 0 + type: integer + systemReserved: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: SystemReserved contains resources reserved for OS system daemons and kernel memory. + type: object + x-kubernetes-validations: + - message: valid keys for systemReserved are ['cpu','memory','ephemeral-storage','pid'] + rule: self.all(x, x=='cpu' || x=='memory' || x=='ephemeral-storage' || x=='pid') + - message: systemReserved value cannot be a negative resource quantity + rule: self.all(x, !self[x].startsWith('-')) type: object - type: array - resources: - description: Resources models the resource requirements for - the NodeClaim to launch - properties: - requests: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: Requests describes the minimum required resources - for the NodeClaim to launch - type: object - type: object - startupTaints: - description: StartupTaints are taints that are applied to - nodes upon startup which are expected to be removed automatically - within a short period of time, typically by a DaemonSet - that tolerates the taint. These are commonly used by daemonsets - to allow initialization and enforce startup ordering. StartupTaints - are ignored for provisioning purposes in that pods are not - required to tolerate a StartupTaint in order to have nodes - provisioned for them. - items: - description: The node this Taint is attached to has the - "effect" on any pod that does not tolerate the Taint. + x-kubernetes-validations: + - message: imageGCHighThresholdPercent must be greater than imageGCLowThresholdPercent + rule: 'has(self.imageGCHighThresholdPercent) && has(self.imageGCLowThresholdPercent) ? self.imageGCHighThresholdPercent > self.imageGCLowThresholdPercent : true' + - message: evictionSoft OwnerKey does not have a matching evictionSoftGracePeriod + rule: has(self.evictionSoft) ? self.evictionSoft.all(e, (e in self.evictionSoftGracePeriod)):true + - message: evictionSoftGracePeriod OwnerKey does not have a matching evictionSoft + rule: has(self.evictionSoftGracePeriod) ? self.evictionSoftGracePeriod.all(e, (e in self.evictionSoft)):true + nodeClassRef: + description: NodeClassRef is a reference to an object that defines provider specific configuration properties: - effect: - description: Required. The effect of the taint on pods - that do not tolerate the taint. Valid effects are - NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: Required. The taint key to be applied to - a node. + apiVersion: + description: API version of the referent type: string - timeAdded: - description: TimeAdded represents the time at which - the taint was added. It is only written for NoExecute - taints. - format: date-time + kind: + description: 'Kind of the referent; More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds"' type: string - value: - description: The taint value corresponding to the taint - key. + name: + description: 'Name of the referent; More info: http://kubernetes.io/docs/user-guide/identifiers#names' type: string required: - - effect - - key + - name type: object - type: array - taints: - description: Taints will be applied to the NodeClaim's node. - items: - description: The node this Taint is attached to has the - "effect" on any pod that does not tolerate the Taint. + requirements: + description: Requirements are layered with GetLabels and applied to every node. + items: + description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + type: string + values: + description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + resources: + description: Resources models the resource requirements for the NodeClaim to launch properties: - effect: - description: Required. The effect of the taint on pods - that do not tolerate the taint. Valid effects are - NoSchedule, PreferNoSchedule and NoExecute. - type: string - key: - description: Required. The taint key to be applied to - a node. - type: string - timeAdded: - description: TimeAdded represents the time at which - the taint was added. It is only written for NoExecute - taints. - format: date-time - type: string - value: - description: The taint value corresponding to the taint - key. - type: string - required: - - effect - - key + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Requests describes the minimum required resources for the NodeClaim to launch + type: object type: object - type: array - required: - - nodeClassRef - - requirements - type: object - type: object - weight: - description: Weight is the priority given to the provisioner during - scheduling. A higher numerical weight indicates that this provisioner - will be ordered ahead of other provisioners with lower weights. - A provisioner with no weight will be treated as if it is a provisioner - with a weight of 0. - format: int32 - maximum: 100 - minimum: 1 - type: integer - type: object - status: - description: NodePoolStatus defines the observed state of NodePool - properties: - resources: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: Resources is the list of resources that have been provisioned. - type: object - type: object - type: object - served: true - storage: true - subresources: - status: {} + startupTaints: + description: StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. + items: + description: The node this Taint is attached to has the "effect" on any pod that does not tolerate the Taint. + properties: + effect: + description: Required. The effect of the taint on pods that do not tolerate the taint. Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Required. The taint key to be applied to a node. + type: string + timeAdded: + description: TimeAdded represents the time at which the taint was added. It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + required: + - effect + - key + type: object + type: array + taints: + description: Taints will be applied to the NodeClaim's node. + items: + description: The node this Taint is attached to has the "effect" on any pod that does not tolerate the Taint. + properties: + effect: + description: Required. The effect of the taint on pods that do not tolerate the taint. Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + type: string + key: + description: Required. The taint key to be applied to a node. + type: string + timeAdded: + description: TimeAdded represents the time at which the taint was added. It is only written for NoExecute taints. + format: date-time + type: string + value: + description: The taint value corresponding to the taint key. + type: string + required: + - effect + - key + type: object + type: array + required: + - nodeClassRef + - requirements + type: object + type: object + weight: + description: Weight is the priority given to the provisioner during scheduling. A higher numerical weight indicates that this provisioner will be ordered ahead of other provisioners with lower weights. A provisioner with no weight will be treated as if it is a provisioner with a weight of 0. + format: int32 + maximum: 100 + minimum: 1 + type: integer + type: object + status: + description: NodePoolStatus defines the observed state of NodePool + properties: + resources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Resources is the list of resources that have been provisioned. + type: object + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/pkg/apis/settings/settings.go b/pkg/apis/settings/settings.go index 5a3d6467b225..052d6eb0341b 100644 --- a/pkg/apis/settings/settings.go +++ b/pkg/apis/settings/settings.go @@ -22,6 +22,8 @@ import ( v1 "k8s.io/api/core/v1" "knative.dev/pkg/configmap" + + coresettings "github.com/aws/karpenter-core/pkg/apis/settings" ) type settingsKeyType struct{} @@ -68,6 +70,9 @@ func (*Settings) ConfigMap() string { // Inject creates a Settings from the supplied ConfigMap func (*Settings) Inject(ctx context.Context, cm *v1.ConfigMap) (context.Context, error) { s := defaultSettings.DeepCopy() + if cm == nil { + return ToContext(ctx, s), nil + } if err := configmap.Parse(cm.Data, configmap.AsString("aws.assumeRoleARN", &s.AssumeRoleARN), @@ -92,6 +97,10 @@ func (*Settings) Inject(ctx context.Context, cm *v1.ConfigMap) (context.Context, return ToContext(ctx, s), nil } +func (*Settings) FromContext(ctx context.Context) coresettings.Injectable { + return FromContext(ctx) +} + func ToContext(ctx context.Context, s *Settings) context.Context { return context.WithValue(ctx, ContextKey, s) } diff --git a/pkg/apis/settings/settings_validation.go b/pkg/apis/settings/settings_validation.go index e8c3e6f9dbe3..467040bb84ff 100644 --- a/pkg/apis/settings/settings_validation.go +++ b/pkg/apis/settings/settings_validation.go @@ -28,7 +28,6 @@ func (s Settings) Validate() (errs *apis.FieldError) { return errs.Also( s.validateEndpoint(), s.validateTags(), - s.validateClusterName(), s.validateVMMemoryOverheadPercent(), s.validateReservedENIs(), s.validateAssumeRoleDuration(), @@ -42,13 +41,6 @@ func (s Settings) validateAssumeRoleDuration() (errs *apis.FieldError) { return nil } -func (s Settings) validateClusterName() (errs *apis.FieldError) { - if s.ClusterName == "" { - return errs.Also(apis.ErrMissingField("clusterName is required", "clusterName")) - } - return nil -} - func (s Settings) validateEndpoint() (errs *apis.FieldError) { if s.ClusterEndpoint == "" { return nil diff --git a/pkg/apis/settings/suite_test.go b/pkg/apis/settings/suite_test.go index 6bf00189afdb..af34c73a55c3 100644 --- a/pkg/apis/settings/suite_test.go +++ b/pkg/apis/settings/suite_test.go @@ -136,15 +136,6 @@ var _ = Describe("Validation", func() { Expect(s.Tags).To(HaveKeyWithValue("kubernetes.io/role/key", "value2")) Expect(s.Tags).To(HaveKeyWithValue("kubernetes.io/cluster/other-tag/hello", "value3")) }) - It("should fail validation with panic when clusterName not included", func() { - cm := &v1.ConfigMap{ - Data: map[string]string{ - "aws.clusterEndpoint": "https://00000000000000000000000.gr7.us-west-2.eks.amazonaws.com", - }, - } - _, err := (&settings.Settings{}).Inject(ctx, cm) - Expect(err).To(HaveOccurred()) - }) It("should fail validation when clusterEndpoint is invalid (not absolute)", func() { cm := &v1.ConfigMap{ Data: map[string]string{ diff --git a/pkg/apis/v1beta1/ec2nodeclass.go b/pkg/apis/v1beta1/ec2nodeclass.go index 342b75c9f82c..897bfd9326d3 100644 --- a/pkg/apis/v1beta1/ec2nodeclass.go +++ b/pkg/apis/v1beta1/ec2nodeclass.go @@ -27,12 +27,24 @@ import ( // This will contain configuration necessary to launch instances in AWS. type EC2NodeClassSpec struct { // SubnetSelectorTerms is a list of or subnet selector terms. The terms are ORed. + // +kubebuilder:validation:XValidation:message="subnetSelectorTerms cannot be empty",rule="self.size() != 0" + // +kubebuilder:validation:XValidation:message="expected at least one, got none, ['tags', 'id']",rule="self.all(x, has(x.tags) || has(x.id))" + // +kubebuilder:validation:XValidation:message="'id' is mutually exclusive, cannot be set with a combination of other fields in subnetSelectorTerms",rule="!self.all(x, has(x.id) && has(x.tags))" + // +kubebuilder:validation:MaxItems:=30 // +required SubnetSelectorTerms []SubnetSelectorTerm `json:"subnetSelectorTerms" hash:"ignore"` // SecurityGroupSelectorTerms is a list of or security group selector terms. The terms are ORed. + // +kubebuilder:validation:XValidation:message="securityGroupSelectorTerms cannot be empty",rule="self.size() != 0" + // +kubebuilder:validation:XValidation:message="expected at least one, got none, ['tags', 'id', 'name']",rule="self.all(x, has(x.tags) || has(x.id) || has(x.name))" + // +kubebuilder:validation:XValidation:message="'id' is mutually exclusive, cannot be set with a combination of other fields in securityGroupSelectorTerms",rule="!self.all(x, has(x.id) && (has(x.tags) || has(x.name)))" + // +kubebuilder:validation:XValidation:message="'name' is mutually exclusive, cannot be set with a combination of other fields in securityGroupSelectorTerms",rule="!self.all(x, has(x.name) && (has(x.tags) || has(x.id)))" + // +kubebuilder:validation:MaxItems:=30 // +required SecurityGroupSelectorTerms []SecurityGroupSelectorTerm `json:"securityGroupSelectorTerms" hash:"ignore"` // AMISelectorTerms is a list of or ami selector terms. The terms are ORed. + // +kubebuilder:validation:XValidation:message="expected at least one, got none, ['tags', 'id', 'name']",rule="self.all(x, has(x.tags) || has(x.id) || has(x.name))" + // +kubebuilder:validation:XValidation:message="'id' is mutually exclusive, cannot be set with a combination of other fields in amiSelectorTerms",rule="!self.all(x, has(x.id) && (has(x.tags) || has(x.name)) || has(x.owner))" + // +kubebuilder:validation:MaxItems:=30 // +optional AMISelectorTerms []AMISelectorTerm `json:"amiSelectorTerms,omitempty" hash:"ignore"` // AMIFamily is the AMI family that instances use. @@ -57,12 +69,20 @@ type EC2NodeClassSpec struct { // Marking this field as immutable avoids concerns around terminating managed instance profiles from running instances. // This field may be made mutable in the future, assuming the correct garbage collection and drift handling is implemented // for the old instance profiles on an update. + // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="immutable field changed" // +required Role string `json:"role"` // Tags to be applied on ec2 resources like instances and launch templates. + // +kubebuilder:validation:XValidation:message="empty tag keys aren't supported",rule="self.all(k, k != '')" + // +kubebuilder:validation:XValidation:message="tag contains a restricted tag matching kubernetes.io/cluster/",rule="self.all(k, !k.startsWith('kubernetes.io/cluster') )" + // +kubebuilder:validation:XValidation:message="tag contains a restricted tag matching karpenter.sh/provisioner-name",rule="self.all(k, k != 'karpenter.sh/provisioner-name')" + // +kubebuilder:validation:XValidation:message="tag contains a restricted tag matching karpenter.sh/nodepool",rule="self.all(k, k != 'karpenter.sh/nodepool')" + // +kubebuilder:validation:XValidation:message="tag contains a restricted tag matching karpenter.sh/managed-by",rule="self.all(k, k !='karpenter.sh/managed-by')" // +optional Tags map[string]string `json:"tags,omitempty"` // BlockDeviceMappings to be applied to provisioned nodes. + // +kubebuilder:validation:XValidation:message="must have only one blockDeviceMappings with rootVolume",rule="self.filter(x, has(x.rootVolume)?x.rootVolume==true:false).size() <= 1" + // +kubebuilder:validation:MaxItems:=50 // +optional BlockDeviceMappings []*BlockDeviceMapping `json:"blockDeviceMappings,omitempty"` // DetailedMonitoring controls if detailed monitoring is enabled for instances that are launched @@ -121,6 +141,8 @@ type EC2NodeClassSpec struct { type SubnetSelectorTerm struct { // Tags is a map of key/value tags used to select subnets // Specifying '*' for a value selects all values for a given tag key. + // +kubebuilder:validation:XValidation:message="empty tag keys or values aren't supported",rule="self.all(k, k != '' && self[k] != '')" + // +kubebuilder:validation:MaxProperties:=20 // +optional Tags map[string]string `json:"tags,omitempty"` // ID is the subnet id in EC2 @@ -134,6 +156,8 @@ type SubnetSelectorTerm struct { type SecurityGroupSelectorTerm struct { // Tags is a map of key/value tags used to select subnets // Specifying '*' for a value selects all values for a given tag key. + // +kubebuilder:validation:XValidation:message="empty tag keys or values aren't supported",rule="self.all(k, k != '' && self[k] != '')" + // +kubebuilder:validation:MaxProperties:=20 // +optional Tags map[string]string `json:"tags,omitempty"` // ID is the security group id in EC2 @@ -150,6 +174,8 @@ type SecurityGroupSelectorTerm struct { type AMISelectorTerm struct { // Tags is a map of key/value tags used to select subnets // Specifying '*' for a value selects all values for a given tag key. + // +kubebuilder:validation:XValidation:message="empty tag keys or values aren't supported",rule="self.all(k, k != '' && self[k] != '')" + // +kubebuilder:validation:MaxProperties:=20 // +optional Tags map[string]string `json:"tags,omitempty"` // ID is the ami id in EC2 @@ -200,12 +226,14 @@ type MetadataOptions struct { // If you specify a value of "disabled", instance metadata will not be accessible // on the node. // +kubebuilder:default=enabled + // +kubebuilder:validation:Enum:={enabled,disabled} // +optional HTTPEndpoint *string `json:"httpEndpoint,omitempty"` // HTTPProtocolIPv6 enables or disables the IPv6 endpoint for the instance metadata // service on provisioned nodes. If metadata options is non-nil, but this parameter // is not specified, the default state is "disabled". // +kubebuilder:default=disabled + // +kubebuilder:validation:Enum:={enabled,disabled} // +optional HTTPProtocolIPv6 *string `json:"httpProtocolIPv6,omitempty"` // HTTPPutResponseHopLimit is the desired HTTP PUT response hop limit for @@ -214,6 +242,8 @@ type MetadataOptions struct { // If metadata options is non-nil, but this parameter is not specified, the // default value is 2. // +kubebuilder:default=2 + // +kubebuilder:validation:Minimum:=1 + // +kubebuilder:validation:Maximum:=64 // +optional HTTPPutResponseHopLimit *int64 `json:"httpPutResponseHopLimit,omitempty"` // HTTPTokens determines the state of token usage for instance metadata @@ -231,16 +261,18 @@ type MetadataOptions struct { // role credentials always returns the version 2.0 credentials; the version // 1.0 credentials are not available. // +kubebuilder:default=required + // +kubebuilder:validation:Enum:={required,optional} // +optional HTTPTokens *string `json:"httpTokens,omitempty"` } type BlockDeviceMapping struct { // The device name (for example, /dev/sdh or xvdh). - // +optional + // +required DeviceName *string `json:"deviceName,omitempty"` // EBS contains parameters used to automatically set up EBS volumes when an instance is launched. - // +optional + // +kubebuilder:validation:XValidation:message="snapshotID or volumeSize must be defined",rule="has(self.snapshotID) || has(self.volumeSize)" + // +required EBS *BlockDevice `json:"ebs,omitempty"` // RootVolume is a flag indicating if this device is mounted as kubelet root dir. You can // configure at most one root volume in BlockDeviceMappings. @@ -287,7 +319,7 @@ type BlockDevice struct { // Valid Range: Minimum value of 125. Maximum value of 1000. // +optional Throughput *int64 `json:"throughput,omitempty"` - // VolumeSize in GiBs. You must specify either a snapshot ID or + // VolumeSize in `Gi`, `G`, `Ti`, or `T`. You must specify either a snapshot ID or // a volume size. The following are the supported volumes sizes for each volume // type: // @@ -298,11 +330,16 @@ type BlockDevice struct { // * st1 and sc1: 125-16,384 // // * standard: 1-1,024 + // + TODO: Add the CEL resources.quantity type after k8s 1.29 + // + https://github.com/kubernetes/apiserver/commit/b137c256373aec1c5d5810afbabb8932a19ecd2a#diff-838176caa5882465c9d6061febd456397a3e2b40fb423ed36f0cabb1847ecb4dR190 + // +kubebuilder:validation:Pattern:="^((?:[1-9][0-9]{0,3}|[1-4][0-9]{4}|[5][0-8][0-9]{3}|59000)Gi|(?:[1-9][0-9]{0,3}|[1-5][0-9]{4}|[6][0-3][0-9]{3}|64000)G|([1-9]||[1-5][0-7]|58)Ti|([1-9]||[1-5][0-9]|6[0-3]|64)T)$" + // +kubebuilder:validation:XIntOrString // +optional - VolumeSize *resource.Quantity `json:"volumeSize,omitempty" hash:"string"` + VolumeSize *resource.Quantity `json:"volumeSize,omitempty"` // VolumeType of the block device. // For more information, see Amazon EBS volume types (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html) // in the Amazon Elastic Compute Cloud User Guide. + // +kubebuilder:validation:Enum:={standard,io1,io2,gp2,sc1,st1,gp3} // +optional VolumeType *string `json:"volumeType,omitempty"` } @@ -315,6 +352,7 @@ type EC2NodeClass struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` + // +kubebuilder:validation:XValidation:message="amiSelectorTerms is required when amiFamily == 'Custom'",rule="self.amiFamily == 'Custom' ? self.amiSelectorTerms.size() != 0 : true" Spec EC2NodeClassSpec `json:"spec,omitempty"` Status EC2NodeClassStatus `json:"status,omitempty"` diff --git a/pkg/apis/v1beta1/ec2nodeclass_validation_cel_test.go b/pkg/apis/v1beta1/ec2nodeclass_validation_cel_test.go new file mode 100644 index 000000000000..a850b5545dfa --- /dev/null +++ b/pkg/apis/v1beta1/ec2nodeclass_validation_cel_test.go @@ -0,0 +1,695 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1_test + +import ( + "strings" + + "github.com/Pallinder/go-randomdata" + "github.com/imdario/mergo" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/aws/aws-sdk-go/aws" + + "github.com/aws/karpenter/pkg/apis/v1alpha1" + "github.com/aws/karpenter/pkg/apis/v1beta1" + "github.com/aws/karpenter/pkg/test" +) + +var _ = Describe("CEL/Validation", func() { + var nc *v1beta1.EC2NodeClass + + BeforeEach(func() { + env.Version.Minor() + if env.Version.Minor() < 25 { + Skip("CEL Validation is for 1.25>") + } + nc = &v1beta1.EC2NodeClass{ + ObjectMeta: metav1.ObjectMeta{Name: strings.ToLower(randomdata.SillyName())}, + Spec: v1beta1.EC2NodeClassSpec{ + AMIFamily: &v1beta1.AMIFamilyAL2, + SubnetSelectorTerms: []v1beta1.SubnetSelectorTerm{ + { + Tags: map[string]string{ + "foo": "bar", + }, + }, + }, + SecurityGroupSelectorTerms: []v1beta1.SecurityGroupSelectorTerm{ + { + Tags: map[string]string{ + "foo": "bar", + }, + }, + }, + }, + } + }) + Context("UserData", func() { + It("should succeed if user data is empty", func() { + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }) + }) + Context("Tags", func() { + It("should succeed when tags are empty", func() { + nc.Spec.Tags = map[string]string{} + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }) + It("should succeed if tags aren't in restricted tag keys", func() { + nc.Spec.Tags = map[string]string{ + "karpenter.sh/custom-key": "value", + "karpenter.sh/managed": "true", + "kubernetes.io/role/key": "value", + } + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }) + It("should fail if tags contain a restricted domain key", func() { + nc.Spec.Tags = map[string]string{ + "karpenter.sh/provisioner-name": "value", + } + Expect(env.Client.Create(ctx, nc)).To(Not(Succeed())) + nc.Spec.Tags = map[string]string{ + "kubernetes.io/cluster/test": "value", + } + Expect(env.Client.Create(ctx, nc)).To(Not(Succeed())) + nc.Spec.Tags = map[string]string{ + "karpenter.sh/managed-by": "test", + } + Expect(env.Client.Create(ctx, nc)).To(Not(Succeed())) + }) + }) + Context("SubnetSelectorTerms", func() { + It("should succeed with a valid subnet selector on tags", func() { + nc.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ + { + Tags: map[string]string{ + "test": "testvalue", + }, + }, + } + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }) + It("should succeed with a valid subnet selector on id", func() { + nc.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ + { + ID: "subnet-12345749", + }, + } + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }) + It("should fail when subnet selector terms is set to nil", func() { + nc.Spec.SubnetSelectorTerms = nil + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when no subnet selector terms exist", func() { + nc.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{} + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when a subnet selector term has no values", func() { + nc.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ + {}, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when a subnet selector term has no tag map values", func() { + nc.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ + { + Tags: map[string]string{}, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when a subnet selector term has a tag map key that is empty", func() { + nc.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ + { + Tags: map[string]string{ + "test": "", + }, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when a subnet selector term has a tag map value that is empty", func() { + nc.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ + { + Tags: map[string]string{ + "": "testvalue", + }, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when the last subnet selector is invalid", func() { + nc.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ + { + Tags: map[string]string{ + "test": "testvalue", + }, + }, + { + Tags: map[string]string{ + "test2": "testvalue2", + }, + }, + { + Tags: map[string]string{ + "test3": "testvalue3", + }, + }, + { + Tags: map[string]string{ + "": "testvalue4", + }, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when specifying id with tags", func() { + nc.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ + { + ID: "subnet-12345749", + Tags: map[string]string{ + "test": "testvalue", + }, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + }) + Context("SecurityGroupSelectorTerms", func() { + It("should succeed with a valid security group selector on tags", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ + { + Tags: map[string]string{ + "test": "testvalue", + }, + }, + } + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }) + It("should succeed with a valid security group selector on id", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ + { + ID: "sg-12345749", + }, + } + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }) + It("should succeed with a valid security group selector on name", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ + { + Name: "testname", + }, + } + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }) + It("should fail when security group selector terms is set to nil", func() { + nc.Spec.SecurityGroupSelectorTerms = nil + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when no security group selector terms exist", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{} + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when a security group selector term has no values", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ + {}, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when a security group selector term has no tag map values", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ + { + Tags: map[string]string{}, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when a security group selector term has a tag map key that is empty", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ + { + Tags: map[string]string{ + "test": "", + }, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when a security group selector term has a tag map value that is empty", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ + { + Tags: map[string]string{ + "": "testvalue", + }, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when the last security group selector is invalid", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ + { + Tags: map[string]string{ + "test": "testvalue", + }, + }, + { + Tags: map[string]string{ + "test2": "testvalue2", + }, + }, + { + Tags: map[string]string{ + "test3": "testvalue3", + }, + }, + { + Tags: map[string]string{ + "": "testvalue4", + }, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when specifying id with tags", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ + { + ID: "sg-12345749", + Tags: map[string]string{ + "test": "testvalue", + }, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when specifying id with name", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ + { + ID: "sg-12345749", + Name: "my-security-group", + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when specifying name with tags", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ + { + Name: "my-security-group", + Tags: map[string]string{ + "test": "testvalue", + }, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + }) + Context("AMISelectorTerms", func() { + It("should succeed with a valid ami selector on tags", func() { + nc.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ + { + Tags: map[string]string{ + "test": "testvalue", + }, + }, + } + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }) + It("should succeed with a valid ami selector on id", func() { + nc.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ + { + ID: "ami-12345749", + }, + } + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }) + It("should succeed with a valid ami selector on name", func() { + nc.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ + { + Name: "testname", + }, + } + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }) + It("should fail when a ami selector term has no values", func() { + nc.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ + {}, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when a ami selector term has no tag map values", func() { + nc.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ + { + Tags: map[string]string{}, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when a ami selector term has a tag map key that is empty", func() { + nc.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ + { + Tags: map[string]string{ + "test": "", + }, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when a ami selector term has a tag map value that is empty", func() { + nc.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ + { + Tags: map[string]string{ + "": "testvalue", + }, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when the last ami selector is invalid", func() { + nc.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ + { + Tags: map[string]string{ + "test": "testvalue", + }, + }, + { + Tags: map[string]string{ + "test2": "testvalue2", + }, + }, + { + Tags: map[string]string{ + "test3": "testvalue3", + }, + }, + { + Tags: map[string]string{ + "": "testvalue4", + }, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when specifying id with tags", func() { + nc.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ + { + ID: "ami-12345749", + Tags: map[string]string{ + "test": "testvalue", + }, + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when specifying id with name", func() { + nc.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ + { + ID: "ami-12345749", + Name: "my-custom-ami", + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when specifying id with owner", func() { + nc.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ + { + ID: "ami-12345749", + Owner: "123456789", + }, + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail when AMIFamily is Custom and not AMISelectorTerms", func() { + nc.Spec.AMIFamily = &v1alpha1.AMIFamilyCustom + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + }) + Context("MetadataOptions", func() { + It("should succeed for valid inputs", func() { + nc.Spec.MetadataOptions = &v1beta1.MetadataOptions{ + HTTPEndpoint: aws.String("disabled"), + HTTPProtocolIPv6: aws.String("enabled"), + HTTPPutResponseHopLimit: aws.Int64(34), + HTTPTokens: aws.String("optional"), + } + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + }) + It("should fail for invalid for HTTPEndpoint", func() { + nc.Spec.MetadataOptions = &v1beta1.MetadataOptions{ + HTTPEndpoint: aws.String("test"), + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail for invalid for HTTPProtocolIPv6", func() { + nc.Spec.MetadataOptions = &v1beta1.MetadataOptions{ + HTTPProtocolIPv6: aws.String("test"), + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail for invalid for HTTPPutResponseHopLimit", func() { + nc.Spec.MetadataOptions = &v1beta1.MetadataOptions{ + HTTPPutResponseHopLimit: aws.Int64(-5), + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + It("should fail for invalid for HTTPTokens", func() { + nc.Spec.MetadataOptions = &v1beta1.MetadataOptions{ + HTTPTokens: aws.String("test"), + } + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + }) + Context("EC2NodeClass Hash", func() { + var nodeClass *v1beta1.EC2NodeClass + BeforeEach(func() { + nodeClass = test.EC2NodeClass(v1beta1.EC2NodeClass{ + Spec: v1beta1.EC2NodeClassSpec{ + AMIFamily: aws.String(v1alpha1.AMIFamilyAL2), + Context: aws.String("context-1"), + Role: "role-1", + Tags: map[string]string{ + "keyTag-1": "valueTag-1", + "keyTag-2": "valueTag-2", + }, + MetadataOptions: &v1beta1.MetadataOptions{ + HTTPEndpoint: aws.String("test-metadata-1"), + }, + BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{ + { + DeviceName: aws.String("map-device-1"), + }, + { + DeviceName: aws.String("map-device-2"), + }, + }, + UserData: aws.String("userdata-test-1"), + DetailedMonitoring: aws.Bool(false), + }, + }) + }) + DescribeTable("should change hash when static fields are updated", func(changes v1beta1.EC2NodeClass) { + hash := nodeClass.Hash() + Expect(mergo.Merge(nodeClass, changes, mergo.WithOverride)).To(Succeed()) + updatedHash := nodeClass.Hash() + Expect(hash).ToNot(Equal(updatedHash)) + }, + Entry("InstanceProfile Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{Role: "role-2"}}), + Entry("UserData Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{UserData: aws.String("userdata-test-2")}}), + Entry("Tags Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{Tags: map[string]string{"keyTag-test-3": "valueTag-test-3"}}}), + Entry("MetadataOptions Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{MetadataOptions: &v1beta1.MetadataOptions{HTTPEndpoint: aws.String("test-metadata-2")}}}), + Entry("BlockDeviceMappings Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{DeviceName: aws.String("map-device-test-3")}}}}), + Entry("Context Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{Context: aws.String("context-2")}}), + Entry("DetailedMonitoring Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{DetailedMonitoring: aws.Bool(true)}}), + Entry("AMIFamily Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{AMIFamily: aws.String(v1alpha1.AMIFamilyBottlerocket)}}), + ) + DescribeTable("should not change hash when slices are re-ordered", func(changes v1beta1.EC2NodeClass) { + hash := nodeClass.Hash() + Expect(mergo.Merge(nodeClass, changes, mergo.WithOverride)).To(Succeed()) + updatedHash := nodeClass.Hash() + Expect(hash).To(Equal(updatedHash)) + }, + Entry("Reorder Tags", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{Tags: map[string]string{"keyTag-2": "valueTag-2", "keyTag-1": "valueTag-1"}}}), + Entry("Reorder BlockDeviceMapping", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{DeviceName: aws.String("map-device-2")}, {DeviceName: aws.String("map-device-1")}}}}), + ) + It("should not change hash when behavior/dynamic fields are updated", func() { + hash := nodeClass.Hash() + + // Update a behavior/dynamic field + nodeClass.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ + { + Tags: map[string]string{"subnet-test-key": "subnet-test-value"}, + }, + } + nodeClass.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ + { + Tags: map[string]string{"sg-test-key": "sg-test-value"}, + }, + } + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ + { + Tags: map[string]string{"ami-test-key": "ami-test-value"}, + }, + } + updatedHash := nodeClass.Hash() + Expect(hash).To(Equal(updatedHash)) + }) + It("should expect two provisioner with the same spec to have the same provisioner hash", func() { + otherNodeClass := test.EC2NodeClass(v1beta1.EC2NodeClass{ + Spec: nodeClass.Spec, + }) + Expect(nodeClass.Hash()).To(Equal(otherNodeClass.Hash())) + }) + }) + Context("BlockDeviceMappings", func() { + It("should succeed if more than one root volume is specified", func() { + nodeClass := test.EC2NodeClass(v1beta1.EC2NodeClass{ + Spec: v1beta1.EC2NodeClassSpec{ + BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{ + { + DeviceName: aws.String("map-device-1"), + EBS: &v1beta1.BlockDevice{ + VolumeSize: resource.NewScaledQuantity(500, resource.Giga), + }, + + RootVolume: true, + }, + { + DeviceName: aws.String("map-device-2"), + EBS: &v1beta1.BlockDevice{ + VolumeSize: resource.NewScaledQuantity(50, resource.Tera), + }, + + RootVolume: false, + }, + }, + }, + }) + Expect(env.Client.Create(ctx, nodeClass)).To(Succeed()) + }) + It("should succeed for valid VolumeSize in G", func() { + nodeClass := test.EC2NodeClass(v1beta1.EC2NodeClass{ + Spec: v1beta1.EC2NodeClassSpec{ + BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{ + { + DeviceName: aws.String("map-device-1"), + EBS: &v1beta1.BlockDevice{ + VolumeSize: resource.NewScaledQuantity(58, resource.Giga), + }, + RootVolume: false, + }, + }, + }, + }) + Expect(env.Client.Create(ctx, nodeClass)).To(Succeed()) + }) + It("should succeed for valid VolumeSize in T", func() { + nodeClass := test.EC2NodeClass(v1beta1.EC2NodeClass{ + Spec: v1beta1.EC2NodeClassSpec{ + BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{ + { + DeviceName: aws.String("map-device-1"), + EBS: &v1beta1.BlockDevice{ + VolumeSize: resource.NewScaledQuantity(45, resource.Tera), + }, + RootVolume: false, + }, + }, + }, + }) + Expect(env.Client.Create(ctx, nodeClass)).To(Succeed()) + }) + It("should fail if more than one root volume is specified", func() { + nodeClass := test.EC2NodeClass(v1beta1.EC2NodeClass{ + Spec: v1beta1.EC2NodeClassSpec{ + BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{ + { + DeviceName: aws.String("map-device-1"), + EBS: &v1beta1.BlockDevice{ + VolumeSize: resource.NewScaledQuantity(50, resource.Giga), + }, + RootVolume: true, + }, + { + DeviceName: aws.String("map-device-2"), + EBS: &v1beta1.BlockDevice{ + VolumeSize: resource.NewScaledQuantity(50, resource.Giga), + }, + RootVolume: true, + }, + }, + }, + }) + Expect(env.Client.Create(ctx, nodeClass)).To(Not(Succeed())) + }) + It("should fail VolumeSize is less then 1Gi/1G", func() { + nodeClass := test.EC2NodeClass(v1beta1.EC2NodeClass{ + Spec: v1beta1.EC2NodeClassSpec{ + BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{ + { + DeviceName: aws.String("map-device-1"), + EBS: &v1beta1.BlockDevice{ + VolumeSize: resource.NewScaledQuantity(1, resource.Milli), + }, + RootVolume: false, + }, + }, + }, + }) + Expect(env.Client.Create(ctx, nodeClass)).To(Not(Succeed())) + }) + It("should fail VolumeSize is greater then 64T", func() { + nodeClass := test.EC2NodeClass(v1beta1.EC2NodeClass{ + Spec: v1beta1.EC2NodeClassSpec{ + BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{ + { + DeviceName: aws.String("map-device-1"), + EBS: &v1beta1.BlockDevice{ + VolumeSize: resource.NewScaledQuantity(100, resource.Tera), + }, + RootVolume: false, + }, + }, + }, + }) + Expect(env.Client.Create(ctx, nodeClass)).To(Not(Succeed())) + }) + It("should fail for VolumeSize that do not parse into quantity values", func() { + nodeClass := test.EC2NodeClass(v1beta1.EC2NodeClass{ + Spec: v1beta1.EC2NodeClassSpec{ + BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{ + { + DeviceName: aws.String("map-device-1"), + EBS: &v1beta1.BlockDevice{ + VolumeSize: &resource.Quantity{}, + }, + RootVolume: false, + }, + }, + }, + }) + Expect(env.Client.Create(ctx, nodeClass)).To(Not(Succeed())) + }) + }) + Context("Role Immutability", func() { + It("should fail when updating the role", func() { + nc.Spec.Role = "test-role" + Expect(env.Client.Create(ctx, nc)).To(Succeed()) + + nc.Spec.Role = "test-role2" + Expect(env.Client.Create(ctx, nc)).ToNot(Succeed()) + }) + }) +}) diff --git a/pkg/apis/v1beta1/ec2nodeclass_validation_webhook_test.go b/pkg/apis/v1beta1/ec2nodeclass_validation_webhook_test.go new file mode 100644 index 000000000000..1a7f53836dd8 --- /dev/null +++ b/pkg/apis/v1beta1/ec2nodeclass_validation_webhook_test.go @@ -0,0 +1,565 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package v1beta1_test + +import ( + "strings" + + "github.com/Pallinder/go-randomdata" + "github.com/imdario/mergo" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "knative.dev/pkg/apis" + + "github.com/aws/aws-sdk-go/aws" + + "github.com/aws/karpenter/pkg/apis/v1alpha1" + "github.com/aws/karpenter/pkg/apis/v1beta1" + "github.com/aws/karpenter/pkg/test" +) + +var _ = Describe("Webhook/Validation", func() { + var nc *v1beta1.EC2NodeClass + + BeforeEach(func() { + nc = &v1beta1.EC2NodeClass{ + ObjectMeta: metav1.ObjectMeta{Name: strings.ToLower(randomdata.SillyName())}, + Spec: v1beta1.EC2NodeClassSpec{ + SubnetSelectorTerms: []v1beta1.SubnetSelectorTerm{ + { + Tags: map[string]string{ + "foo": "bar", + }, + }, + }, + SecurityGroupSelectorTerms: []v1beta1.SecurityGroupSelectorTerm{ + { + Tags: map[string]string{ + "foo": "bar", + }, + }, + }, + }, + } + }) + + Context("UserData", func() { + It("should succeed if user data is empty", func() { + Expect(nc.Validate(ctx)).To(Succeed()) + }) + }) + Context("Tags", func() { + It("should succeed when tags are empty", func() { + nc.Spec.Tags = map[string]string{} + Expect(nc.Validate(ctx)).To(Succeed()) + }) + It("should succeed if tags aren't in restricted tag keys", func() { + nc.Spec.Tags = map[string]string{ + "karpenter.sh/custom-key": "value", + "karpenter.sh/managed": "true", + "kubernetes.io/role/key": "value", + } + Expect(nc.Validate(ctx)).To(Succeed()) + }) + It("should succeed by validating that regex is properly escaped", func() { + nc.Spec.Tags = map[string]string{ + "karpenterzsh/provisioner-name": "value", + } + Expect(nc.Validate(ctx)).To(Succeed()) + nc.Spec.Tags = map[string]string{ + "kubernetesbio/cluster/test": "value", + } + Expect(nc.Validate(ctx)).To(Succeed()) + nc.Spec.Tags = map[string]string{ + "karpenterzsh/managed-by": "test", + } + Expect(nc.Validate(ctx)).To(Succeed()) + }) + It("should fail if tags contain a restricted domain key", func() { + nc.Spec.Tags = map[string]string{ + "karpenter.sh/provisioner-name": "value", + } + Expect(nc.Validate(ctx)).To(Not(Succeed())) + nc.Spec.Tags = map[string]string{ + "kubernetes.io/cluster/test": "value", + } + Expect(nc.Validate(ctx)).To(Not(Succeed())) + nc.Spec.Tags = map[string]string{ + "karpenter.sh/managed-by": "test", + } + Expect(nc.Validate(ctx)).To(Not(Succeed())) + }) + }) + Context("SubnetSelectorTerms", func() { + It("should succeed with a valid subnet selector on tags", func() { + nc.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ + { + Tags: map[string]string{ + "test": "testvalue", + }, + }, + } + Expect(nc.Validate(ctx)).To(Succeed()) + }) + It("should succeed with a valid subnet selector on id", func() { + nc.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ + { + ID: "subnet-12345749", + }, + } + Expect(nc.Validate(ctx)).To(Succeed()) + }) + It("should fail when subnet selector terms is set to nil", func() { + nc.Spec.SubnetSelectorTerms = nil + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + It("should fail when no subnet selector terms exist", func() { + nc.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{} + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + It("should fail when a subnet selector term has no values", func() { + nc.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ + {}, + } + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + It("should fail when a subnet selector term has no tag map values", func() { + nc.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ + { + Tags: map[string]string{}, + }, + } + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + It("should fail when a subnet selector term has a tag map key that is empty", func() { + nc.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ + { + Tags: map[string]string{ + "test": "", + }, + }, + } + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + It("should fail when a subnet selector term has a tag map value that is empty", func() { + nc.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ + { + Tags: map[string]string{ + "": "testvalue", + }, + }, + } + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + It("should fail when the last subnet selector is invalid", func() { + nc.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ + { + Tags: map[string]string{ + "test": "testvalue", + }, + }, + { + Tags: map[string]string{ + "test2": "testvalue2", + }, + }, + { + Tags: map[string]string{ + "test3": "testvalue3", + }, + }, + { + Tags: map[string]string{ + "": "testvalue4", + }, + }, + } + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + It("should fail when specifying id with tags", func() { + nc.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ + { + ID: "subnet-12345749", + Tags: map[string]string{ + "test": "testvalue", + }, + }, + } + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + }) + Context("SecurityGroupSelectorTerms", func() { + It("should succeed with a valid security group selector on tags", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ + { + Tags: map[string]string{ + "test": "testvalue", + }, + }, + } + Expect(nc.Validate(ctx)).To(Succeed()) + }) + It("should succeed with a valid security group selector on id", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ + { + ID: "sg-12345749", + }, + } + Expect(nc.Validate(ctx)).To(Succeed()) + }) + It("should succeed with a valid security group selector on name", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ + { + Name: "testname", + }, + } + Expect(nc.Validate(ctx)).To(Succeed()) + }) + It("should fail when security group selector terms is set to nil", func() { + nc.Spec.SecurityGroupSelectorTerms = nil + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + It("should fail when no security group selector terms exist", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{} + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + It("should fail when a security group selector term has no values", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ + {}, + } + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + It("should fail when a security group selector term has no tag map values", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ + { + Tags: map[string]string{}, + }, + } + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + It("should fail when a security group selector term has a tag map key that is empty", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ + { + Tags: map[string]string{ + "test": "", + }, + }, + } + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + It("should fail when a security group selector term has a tag map value that is empty", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ + { + Tags: map[string]string{ + "": "testvalue", + }, + }, + } + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + It("should fail when the last security group selector is invalid", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ + { + Tags: map[string]string{ + "test": "testvalue", + }, + }, + { + Tags: map[string]string{ + "test2": "testvalue2", + }, + }, + { + Tags: map[string]string{ + "test3": "testvalue3", + }, + }, + { + Tags: map[string]string{ + "": "testvalue4", + }, + }, + } + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + It("should fail when specifying id with tags", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ + { + ID: "sg-12345749", + Tags: map[string]string{ + "test": "testvalue", + }, + }, + } + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + It("should fail when specifying id with name", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ + { + ID: "sg-12345749", + Name: "my-security-group", + }, + } + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + It("should fail when specifying name with tags", func() { + nc.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ + { + Name: "my-security-group", + Tags: map[string]string{ + "test": "testvalue", + }, + }, + } + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + }) + Context("AMISelectorTerms", func() { + It("should succeed with a valid ami selector on tags", func() { + nc.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ + { + Tags: map[string]string{ + "test": "testvalue", + }, + }, + } + Expect(nc.Validate(ctx)).To(Succeed()) + }) + It("should succeed with a valid ami selector on id", func() { + nc.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ + { + ID: "sg-12345749", + }, + } + Expect(nc.Validate(ctx)).To(Succeed()) + }) + It("should succeed with a valid ami selector on name", func() { + nc.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ + { + Name: "testname", + }, + } + Expect(nc.Validate(ctx)).To(Succeed()) + }) + It("should fail when a ami selector term has no values", func() { + nc.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ + {}, + } + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + It("should fail when a ami selector term has no tag map values", func() { + nc.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ + { + Tags: map[string]string{}, + }, + } + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + It("should fail when a ami selector term has a tag map key that is empty", func() { + nc.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ + { + Tags: map[string]string{ + "test": "", + }, + }, + } + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + It("should fail when a ami selector term has a tag map value that is empty", func() { + nc.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ + { + Tags: map[string]string{ + "": "testvalue", + }, + }, + } + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + It("should fail when the last ami selector is invalid", func() { + nc.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ + { + Tags: map[string]string{ + "test": "testvalue", + }, + }, + { + Tags: map[string]string{ + "test2": "testvalue2", + }, + }, + { + Tags: map[string]string{ + "test3": "testvalue3", + }, + }, + { + Tags: map[string]string{ + "": "testvalue4", + }, + }, + } + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + It("should fail when specifying id with tags", func() { + nc.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ + { + ID: "ami-12345749", + Tags: map[string]string{ + "test": "testvalue", + }, + }, + } + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + It("should fail when specifying id with name", func() { + nc.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ + { + ID: "ami-12345749", + Name: "my-custom-ami", + }, + } + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + It("should fail when specifying id with owner", func() { + nc.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ + { + ID: "ami-12345749", + Owner: "123456789", + }, + } + Expect(nc.Validate(ctx)).ToNot(Succeed()) + }) + }) + Context("EC2NodeClass Hash", func() { + var nodeClass *v1beta1.EC2NodeClass + BeforeEach(func() { + nodeClass = test.EC2NodeClass(v1beta1.EC2NodeClass{ + Spec: v1beta1.EC2NodeClassSpec{ + AMIFamily: aws.String(v1alpha1.AMIFamilyAL2), + Context: aws.String("context-1"), + Role: "role-1", + Tags: map[string]string{ + "keyTag-1": "valueTag-1", + "keyTag-2": "valueTag-2", + }, + MetadataOptions: &v1beta1.MetadataOptions{ + HTTPEndpoint: aws.String("test-metadata-1"), + }, + BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{ + { + DeviceName: aws.String("map-device-1"), + }, + { + DeviceName: aws.String("map-device-2"), + }, + }, + UserData: aws.String("userdata-test-1"), + DetailedMonitoring: aws.Bool(false), + }, + }) + }) + DescribeTable("should change hash when static fields are updated", func(changes v1beta1.EC2NodeClass) { + hash := nodeClass.Hash() + Expect(mergo.Merge(nodeClass, changes, mergo.WithOverride)).To(Succeed()) + updatedHash := nodeClass.Hash() + Expect(hash).ToNot(Equal(updatedHash)) + }, + Entry("InstanceProfile Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{Role: "role-2"}}), + Entry("UserData Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{UserData: aws.String("userdata-test-2")}}), + Entry("Tags Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{Tags: map[string]string{"keyTag-test-3": "valueTag-test-3"}}}), + Entry("MetadataOptions Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{MetadataOptions: &v1beta1.MetadataOptions{HTTPEndpoint: aws.String("test-metadata-2")}}}), + Entry("BlockDeviceMappings Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{DeviceName: aws.String("map-device-test-3")}}}}), + Entry("Context Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{Context: aws.String("context-2")}}), + Entry("DetailedMonitoring Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{DetailedMonitoring: aws.Bool(true)}}), + Entry("AMIFamily Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{AMIFamily: aws.String(v1alpha1.AMIFamilyBottlerocket)}}), + ) + DescribeTable("should not change hash when slices are re-ordered", func(changes v1beta1.EC2NodeClass) { + hash := nodeClass.Hash() + Expect(mergo.Merge(nodeClass, changes, mergo.WithOverride)).To(Succeed()) + updatedHash := nodeClass.Hash() + Expect(hash).To(Equal(updatedHash)) + }, + Entry("Reorder Tags", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{Tags: map[string]string{"keyTag-2": "valueTag-2", "keyTag-1": "valueTag-1"}}}), + Entry("Reorder BlockDeviceMapping", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{DeviceName: aws.String("map-device-2")}, {DeviceName: aws.String("map-device-1")}}}}), + ) + It("should not change hash when behavior/dynamic fields are updated", func() { + hash := nodeClass.Hash() + + // Update a behavior/dynamic field + nodeClass.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ + { + Tags: map[string]string{"subnet-test-key": "subnet-test-value"}, + }, + } + nodeClass.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ + { + Tags: map[string]string{"sg-test-key": "sg-test-value"}, + }, + } + nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ + { + Tags: map[string]string{"ami-test-key": "ami-test-value"}, + }, + } + updatedHash := nodeClass.Hash() + Expect(hash).To(Equal(updatedHash)) + }) + It("should expect two provisioner with the same spec to have the same provisioner hash", func() { + otherNodeClass := test.EC2NodeClass(v1beta1.EC2NodeClass{ + Spec: nodeClass.Spec, + }) + Expect(nodeClass.Hash()).To(Equal(otherNodeClass.Hash())) + }) + }) + Context("BlockDeviceMappings", func() { + It("should fail if more than one root volume is specified", func() { + nodeClass := test.EC2NodeClass(v1beta1.EC2NodeClass{ + Spec: v1beta1.EC2NodeClassSpec{ + BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{ + { + DeviceName: aws.String("map-device-1"), + EBS: &v1beta1.BlockDevice{ + VolumeSize: resource.NewScaledQuantity(50, resource.Giga), + }, + + RootVolume: true, + }, + { + DeviceName: aws.String("map-device-2"), + EBS: &v1beta1.BlockDevice{ + VolumeSize: resource.NewScaledQuantity(50, resource.Giga), + }, + + RootVolume: true, + }, + }, + }, + }) + Expect(nodeClass.Validate(ctx)).To(Not(Succeed())) + }) + }) + Context("Role Immutability", func() { + It("should fail when updating the role", func() { + nc.Spec.Role = "test-role" + Expect(nc.Validate(ctx)).To(Succeed()) + + updateCtx := apis.WithinUpdate(ctx, nc.DeepCopy()) + nc.Spec.Role = "test-role2" + Expect(nc.Validate(updateCtx)).ToNot(Succeed()) + }) + }) +}) diff --git a/pkg/apis/v1beta1/labels.go b/pkg/apis/v1beta1/labels.go index a3d0c254e548..54acdfb8fbbe 100644 --- a/pkg/apis/v1beta1/labels.go +++ b/pkg/apis/v1beta1/labels.go @@ -18,7 +18,6 @@ import ( "fmt" "regexp" - "github.com/aws/aws-sdk-go/service/ec2" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/util/sets" @@ -56,8 +55,6 @@ const ( ) var ( - CapacityTypeSpot = ec2.DefaultTargetCapacityTypeSpot - CapacityTypeOnDemand = ec2.DefaultTargetCapacityTypeOnDemand AWSToKubeArchitectures = map[string]string{ "x86_64": v1beta1.ArchitectureAmd64, v1beta1.ArchitectureArm64: v1beta1.ArchitectureArm64, @@ -78,20 +75,12 @@ var ( regexp.MustCompile(fmt.Sprintf("^%s$", regexp.QuoteMeta(v1beta1.NodePoolLabelKey))), regexp.MustCompile(fmt.Sprintf("^%s$", regexp.QuoteMeta(v1beta1.ManagedByAnnotationKey))), } - AMIFamilyBottlerocket = "Bottlerocket" - AMIFamilyAL2 = "AL2" - AMIFamilyUbuntu = "Ubuntu" - AMIFamilyWindows2019 = "Windows2019" - AMIFamilyWindows2022 = "Windows2022" - AMIFamilyCustom = "Custom" - SupportedAMIFamilies = []string{ - AMIFamilyBottlerocket, - AMIFamilyAL2, - AMIFamilyUbuntu, - AMIFamilyWindows2019, - AMIFamilyWindows2022, - AMIFamilyCustom, - } + AMIFamilyBottlerocket = "Bottlerocket" + AMIFamilyAL2 = "AL2" + AMIFamilyUbuntu = "Ubuntu" + AMIFamilyWindows2019 = "Windows2019" + AMIFamilyWindows2022 = "Windows2022" + AMIFamilyCustom = "Custom" Windows2019 = "2019" Windows2022 = "2022" WindowsCore = "Core" diff --git a/pkg/apis/v1beta1/suite_test.go b/pkg/apis/v1beta1/suite_test.go index 627d6e9e7e4a..869cf13c4775 100644 --- a/pkg/apis/v1beta1/suite_test.go +++ b/pkg/apis/v1beta1/suite_test.go @@ -16,26 +16,23 @@ package v1beta1_test import ( "context" - "strings" "testing" - "github.com/Pallinder/go-randomdata" - "github.com/imdario/mergo" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "knative.dev/pkg/apis" . "knative.dev/pkg/logging/testing" - "github.com/aws/aws-sdk-go/aws" + . "github.com/aws/karpenter-core/pkg/test/expectations" - "github.com/aws/karpenter/pkg/apis/v1alpha1" - "github.com/aws/karpenter/pkg/apis/v1beta1" + "github.com/aws/karpenter-core/pkg/operator/scheme" + coretest "github.com/aws/karpenter-core/pkg/test" + "github.com/aws/karpenter/pkg/apis" "github.com/aws/karpenter/pkg/test" ) var ctx context.Context +var env *coretest.Environment +var awsEnv *test.Environment func TestAPIs(t *testing.T) { ctx = TestContextWithLogger(t) @@ -43,534 +40,15 @@ func TestAPIs(t *testing.T) { RunSpecs(t, "Validation") } -var _ = Describe("Validation", func() { - var nc *v1beta1.EC2NodeClass - - BeforeEach(func() { - nc = &v1beta1.EC2NodeClass{ - ObjectMeta: metav1.ObjectMeta{Name: strings.ToLower(randomdata.SillyName())}, - Spec: v1beta1.EC2NodeClassSpec{ - SubnetSelectorTerms: []v1beta1.SubnetSelectorTerm{ - { - Tags: map[string]string{ - "foo": "bar", - }, - }, - }, - SecurityGroupSelectorTerms: []v1beta1.SecurityGroupSelectorTerm{ - { - Tags: map[string]string{ - "foo": "bar", - }, - }, - }, - }, - } - }) - - Context("UserData", func() { - It("should succeed if user data is empty", func() { - Expect(nc.Validate(ctx)).To(Succeed()) - }) - }) - Context("Tags", func() { - It("should succeed when tags are empty", func() { - nc.Spec.Tags = map[string]string{} - Expect(nc.Validate(ctx)).To(Succeed()) - }) - It("should succeed if tags aren't in restricted tag keys", func() { - nc.Spec.Tags = map[string]string{ - "karpenter.sh/custom-key": "value", - "karpenter.sh/managed": "true", - "kubernetes.io/role/key": "value", - } - Expect(nc.Validate(ctx)).To(Succeed()) - }) - It("should succeed by validating that regex is properly escaped", func() { - nc.Spec.Tags = map[string]string{ - "karpenterzsh/provisioner-name": "value", - } - Expect(nc.Validate(ctx)).To(Succeed()) - nc.Spec.Tags = map[string]string{ - "kubernetesbio/cluster/test": "value", - } - Expect(nc.Validate(ctx)).To(Succeed()) - nc.Spec.Tags = map[string]string{ - "karpenterzsh/managed-by": "test", - } - Expect(nc.Validate(ctx)).To(Succeed()) - }) - It("should fail if tags contain a restricted domain key", func() { - nc.Spec.Tags = map[string]string{ - "karpenter.sh/provisioner-name": "value", - } - Expect(nc.Validate(ctx)).To(Not(Succeed())) - nc.Spec.Tags = map[string]string{ - "kubernetes.io/cluster/test": "value", - } - Expect(nc.Validate(ctx)).To(Not(Succeed())) - nc.Spec.Tags = map[string]string{ - "karpenter.sh/managed-by": "test", - } - Expect(nc.Validate(ctx)).To(Not(Succeed())) - }) - }) - Context("SubnetSelectorTerms", func() { - It("should succeed with a valid subnet selector on tags", func() { - nc.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ - { - Tags: map[string]string{ - "test": "testvalue", - }, - }, - } - Expect(nc.Validate(ctx)).To(Succeed()) - }) - It("should succeed with a valid subnet selector on id", func() { - nc.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ - { - ID: "subnet-12345749", - }, - } - Expect(nc.Validate(ctx)).To(Succeed()) - }) - It("should fail when subnet selector terms is set to nil", func() { - nc.Spec.SubnetSelectorTerms = nil - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail when no subnet selector terms exist", func() { - nc.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{} - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail when a subnet selector term has no values", func() { - nc.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ - {}, - } - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail when a subnet selector term has no tag map values", func() { - nc.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ - { - Tags: map[string]string{}, - }, - } - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail when a subnet selector term has a tag map key that is empty", func() { - nc.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ - { - Tags: map[string]string{ - "test": "", - }, - }, - } - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail when a subnet selector term has a tag map value that is empty", func() { - nc.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ - { - Tags: map[string]string{ - "": "testvalue", - }, - }, - } - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail when the last subnet selector is invalid", func() { - nc.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ - { - Tags: map[string]string{ - "test": "testvalue", - }, - }, - { - Tags: map[string]string{ - "test2": "testvalue2", - }, - }, - { - Tags: map[string]string{ - "test3": "testvalue3", - }, - }, - { - Tags: map[string]string{ - "": "testvalue4", - }, - }, - } - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail when specifying id with tags", func() { - nc.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ - { - ID: "subnet-12345749", - Tags: map[string]string{ - "test": "testvalue", - }, - }, - } - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - }) - Context("SecurityGroupSelectorTerms", func() { - It("should succeed with a valid security group selector on tags", func() { - nc.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ - { - Tags: map[string]string{ - "test": "testvalue", - }, - }, - } - Expect(nc.Validate(ctx)).To(Succeed()) - }) - It("should succeed with a valid security group selector on id", func() { - nc.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ - { - ID: "sg-12345749", - }, - } - Expect(nc.Validate(ctx)).To(Succeed()) - }) - It("should succeed with a valid security group selector on name", func() { - nc.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ - { - Name: "testname", - }, - } - Expect(nc.Validate(ctx)).To(Succeed()) - }) - It("should fail when security group selector terms is set to nil", func() { - nc.Spec.SecurityGroupSelectorTerms = nil - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail when no security group selector terms exist", func() { - nc.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{} - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail when a security group selector term has no values", func() { - nc.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ - {}, - } - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail when a security group selector term has no tag map values", func() { - nc.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ - { - Tags: map[string]string{}, - }, - } - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail when a security group selector term has a tag map key that is empty", func() { - nc.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ - { - Tags: map[string]string{ - "test": "", - }, - }, - } - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail when a security group selector term has a tag map value that is empty", func() { - nc.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ - { - Tags: map[string]string{ - "": "testvalue", - }, - }, - } - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail when the last security group selector is invalid", func() { - nc.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ - { - Tags: map[string]string{ - "test": "testvalue", - }, - }, - { - Tags: map[string]string{ - "test2": "testvalue2", - }, - }, - { - Tags: map[string]string{ - "test3": "testvalue3", - }, - }, - { - Tags: map[string]string{ - "": "testvalue4", - }, - }, - } - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail when specifying id with tags", func() { - nc.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ - { - ID: "sg-12345749", - Tags: map[string]string{ - "test": "testvalue", - }, - }, - } - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail when specifying id with name", func() { - nc.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ - { - ID: "sg-12345749", - Name: "my-security-group", - }, - } - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail when specifying name with tags", func() { - nc.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ - { - Name: "my-security-group", - Tags: map[string]string{ - "test": "testvalue", - }, - }, - } - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - }) - Context("AMISelectorTerms", func() { - It("should succeed with a valid ami selector on tags", func() { - nc.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ - { - Tags: map[string]string{ - "test": "testvalue", - }, - }, - } - Expect(nc.Validate(ctx)).To(Succeed()) - }) - It("should succeed with a valid ami selector on id", func() { - nc.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ - { - ID: "sg-12345749", - }, - } - Expect(nc.Validate(ctx)).To(Succeed()) - }) - It("should succeed with a valid ami selector on name", func() { - nc.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ - { - Name: "testname", - }, - } - Expect(nc.Validate(ctx)).To(Succeed()) - }) - It("should fail when a ami selector term has no values", func() { - nc.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ - {}, - } - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail when a ami selector term has no tag map values", func() { - nc.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ - { - Tags: map[string]string{}, - }, - } - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail when a ami selector term has a tag map key that is empty", func() { - nc.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ - { - Tags: map[string]string{ - "test": "", - }, - }, - } - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail when a ami selector term has a tag map value that is empty", func() { - nc.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ - { - Tags: map[string]string{ - "": "testvalue", - }, - }, - } - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail when the last ami selector is invalid", func() { - nc.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ - { - Tags: map[string]string{ - "test": "testvalue", - }, - }, - { - Tags: map[string]string{ - "test2": "testvalue2", - }, - }, - { - Tags: map[string]string{ - "test3": "testvalue3", - }, - }, - { - Tags: map[string]string{ - "": "testvalue4", - }, - }, - } - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail when specifying id with tags", func() { - nc.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ - { - ID: "ami-12345749", - Tags: map[string]string{ - "test": "testvalue", - }, - }, - } - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail when specifying id with name", func() { - nc.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ - { - ID: "ami-12345749", - Name: "my-custom-ami", - }, - } - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - It("should fail when specifying id with owner", func() { - nc.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ - { - ID: "ami-12345749", - Owner: "123456789", - }, - } - Expect(nc.Validate(ctx)).ToNot(Succeed()) - }) - }) - Context("EC2NodeClass Hash", func() { - var nodeClass *v1beta1.EC2NodeClass - BeforeEach(func() { - nodeClass = test.EC2NodeClass(v1beta1.EC2NodeClass{ - Spec: v1beta1.EC2NodeClassSpec{ - AMIFamily: aws.String(v1alpha1.AMIFamilyAL2), - Context: aws.String("context-1"), - Role: "role-1", - Tags: map[string]string{ - "keyTag-1": "valueTag-1", - "keyTag-2": "valueTag-2", - }, - MetadataOptions: &v1beta1.MetadataOptions{ - HTTPEndpoint: aws.String("test-metadata-1"), - }, - BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{ - { - DeviceName: aws.String("map-device-1"), - }, - { - DeviceName: aws.String("map-device-2"), - }, - }, - UserData: aws.String("userdata-test-1"), - DetailedMonitoring: aws.Bool(false), - }, - }) - }) - DescribeTable("should change hash when static fields are updated", func(changes v1beta1.EC2NodeClass) { - hash := nodeClass.Hash() - Expect(mergo.Merge(nodeClass, changes, mergo.WithOverride)).To(Succeed()) - updatedHash := nodeClass.Hash() - Expect(hash).ToNot(Equal(updatedHash)) - }, - Entry("InstanceProfile Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{Role: "role-2"}}), - Entry("UserData Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{UserData: aws.String("userdata-test-2")}}), - Entry("Tags Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{Tags: map[string]string{"keyTag-test-3": "valueTag-test-3"}}}), - Entry("MetadataOptions Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{MetadataOptions: &v1beta1.MetadataOptions{HTTPEndpoint: aws.String("test-metadata-2")}}}), - Entry("BlockDeviceMappings Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{DeviceName: aws.String("map-device-test-3")}}}}), - Entry("Context Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{Context: aws.String("context-2")}}), - Entry("DetailedMonitoring Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{DetailedMonitoring: aws.Bool(true)}}), - Entry("AMIFamily Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{AMIFamily: aws.String(v1alpha1.AMIFamilyBottlerocket)}}), - ) - DescribeTable("should not change hash when slices are re-ordered", func(changes v1beta1.EC2NodeClass) { - hash := nodeClass.Hash() - Expect(mergo.Merge(nodeClass, changes, mergo.WithOverride)).To(Succeed()) - updatedHash := nodeClass.Hash() - Expect(hash).To(Equal(updatedHash)) - }, - Entry("Reorder Tags", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{Tags: map[string]string{"keyTag-2": "valueTag-2", "keyTag-1": "valueTag-1"}}}), - Entry("Reorder BlockDeviceMapping", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{DeviceName: aws.String("map-device-2")}, {DeviceName: aws.String("map-device-1")}}}}), - ) - It("should not change hash when behavior/dynamic fields are updated", func() { - hash := nodeClass.Hash() - - // Update a behavior/dynamic field - nodeClass.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ - { - Tags: map[string]string{"subnet-test-key": "subnet-test-value"}, - }, - } - nodeClass.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ - { - Tags: map[string]string{"sg-test-key": "sg-test-value"}, - }, - } - nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ - { - Tags: map[string]string{"ami-test-key": "ami-test-value"}, - }, - } - updatedHash := nodeClass.Hash() - Expect(hash).To(Equal(updatedHash)) - }) - It("should expect two provisioner with the same spec to have the same provisioner hash", func() { - otherNodeClass := test.EC2NodeClass(v1beta1.EC2NodeClass{ - Spec: nodeClass.Spec, - }) - Expect(nodeClass.Hash()).To(Equal(otherNodeClass.Hash())) - }) - }) - Context("BlockDeviceMappings", func() { - It("should fail if more than one root volume is specified", func() { - nodeClass := test.EC2NodeClass(v1beta1.EC2NodeClass{ - Spec: v1beta1.EC2NodeClassSpec{ - BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{ - { - DeviceName: aws.String("map-device-1"), - EBS: &v1beta1.BlockDevice{ - VolumeSize: resource.NewScaledQuantity(50, resource.Giga), - }, - - RootVolume: true, - }, - { - DeviceName: aws.String("map-device-2"), - EBS: &v1beta1.BlockDevice{ - VolumeSize: resource.NewScaledQuantity(50, resource.Giga), - }, +var _ = BeforeSuite(func() { + env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) + awsEnv = test.NewEnvironment(ctx, env) +}) - RootVolume: true, - }, - }, - }, - }) - Expect(nodeClass.Validate(ctx)).To(Not(Succeed())) - }) - }) - Context("Role Immutability", func() { - It("should fail when updating the role", func() { - nc.Spec.Role = "test-role" - Expect(nc.Validate(ctx)).To(Succeed()) +var _ = AfterEach(func() { + ExpectCleanedUp(ctx, env.Client) +}) - updateCtx := apis.WithinUpdate(ctx, nc.DeepCopy()) - nc.Spec.Role = "test-role2" - Expect(nc.Validate(updateCtx)).ToNot(Succeed()) - }) - }) +var _ = AfterSuite(func() { + Expect(env.Stop()).To(Succeed(), "Failed to stop environment") }) diff --git a/pkg/cloudprovider/nodeclaim_test.go b/pkg/cloudprovider/nodeclaim_test.go index 22f0dcee5af8..99b0851b4ca1 100644 --- a/pkg/cloudprovider/nodeclaim_test.go +++ b/pkg/cloudprovider/nodeclaim_test.go @@ -321,10 +321,9 @@ var _ = Describe("NodeClaim/CloudProvider", func() { Expect(err).NotTo(HaveOccurred()) Expect(isDrifted).To(Equal(cloudprovider.NodeClassDrift)) }, - Entry("InstanceProfile Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{Role: "role-2"}}), Entry("UserData Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{UserData: aws.String("userdata-test-2")}}), Entry("Tags Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{Tags: map[string]string{"keyTag-test-3": "valueTag-test-3"}}}), - Entry("MetadataOptions Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{MetadataOptions: &v1beta1.MetadataOptions{HTTPEndpoint: aws.String("test-metadata-2")}}}), + Entry("MetadataOptions Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{MetadataOptions: &v1beta1.MetadataOptions{HTTPEndpoint: aws.String("disabled")}}}), Entry("BlockDeviceMappings Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{DeviceName: aws.String("map-device-test-3")}}}}), Entry("Context Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{Context: aws.String("context-2")}}), Entry("DetailedMonitoring Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{DetailedMonitoring: aws.Bool(true)}}), @@ -345,7 +344,7 @@ var _ = Describe("NodeClaim/CloudProvider", func() { Expect(err).NotTo(HaveOccurred()) Expect(isDrifted).To(BeEmpty()) }, - Entry("AMI Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{AMISelectorTerms: []v1beta1.AMISelectorTerm{{ID: validAMI}}}}), + Entry("AMI Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{AMISelectorTerms: []v1beta1.AMISelectorTerm{{Tags: map[string]string{"*": "*"}}}}}), Entry("Subnet Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{SubnetSelectorTerms: []v1beta1.SubnetSelectorTerm{{ID: "subnet-test1"}}}}), Entry("SecurityGroup Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{SecurityGroupSelectorTerms: []v1beta1.SecurityGroupSelectorTerm{{Tags: map[string]string{"sg-key": "sg-value"}}}}}), ) diff --git a/pkg/cloudprovider/suite_test.go b/pkg/cloudprovider/suite_test.go index f520d2901452..28c9121dfb09 100644 --- a/pkg/cloudprovider/suite_test.go +++ b/pkg/cloudprovider/suite_test.go @@ -28,18 +28,17 @@ import ( . "github.com/onsi/gomega" . "knative.dev/pkg/logging/testing" - "github.com/aws/karpenter-core/pkg/operator/injection" "github.com/aws/karpenter/pkg/apis" "github.com/aws/karpenter/pkg/apis/settings" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/test" "github.com/aws/karpenter/pkg/cloudprovider" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" "github.com/aws/karpenter-core/pkg/controllers/provisioning" "github.com/aws/karpenter-core/pkg/controllers/state" "github.com/aws/karpenter-core/pkg/events" - "github.com/aws/karpenter-core/pkg/operator/options" + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/operator/scheme" coretest "github.com/aws/karpenter-core/pkg/test" . "github.com/aws/karpenter-core/pkg/test/expectations" @@ -47,7 +46,6 @@ import ( var ctx context.Context var stop context.CancelFunc -var opts options.Options var env *coretest.Environment var awsEnv *test.Environment var prov *provisioning.Provisioner @@ -64,7 +62,8 @@ func TestAWS(t *testing.T) { var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) ctx, stop = context.WithCancel(ctx) awsEnv = test.NewEnvironment(ctx, env) @@ -82,8 +81,8 @@ var _ = AfterSuite(func() { }) var _ = BeforeEach(func() { - ctx = injection.WithOptions(ctx, opts) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) cluster.Reset() diff --git a/pkg/controllers/controllers.go b/pkg/controllers/controllers.go index 31fee9fc12f5..c4067edd3f4a 100644 --- a/pkg/controllers/controllers.go +++ b/pkg/controllers/controllers.go @@ -24,43 +24,44 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "github.com/aws/karpenter-core/pkg/events" - "github.com/aws/karpenter/pkg/apis/settings" + "github.com/aws/karpenter-core/pkg/operator/controller" "github.com/aws/karpenter/pkg/cache" "github.com/aws/karpenter/pkg/cloudprovider" "github.com/aws/karpenter/pkg/controllers/interruption" nodeclaimgarbagecollection "github.com/aws/karpenter/pkg/controllers/nodeclaim/garbagecollection" nodeclaimlink "github.com/aws/karpenter/pkg/controllers/nodeclaim/link" + nodeclaimtagging "github.com/aws/karpenter/pkg/controllers/nodeclaim/tagging" "github.com/aws/karpenter/pkg/controllers/nodeclass" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/providers/amifamily" "github.com/aws/karpenter/pkg/providers/hostresourcegroup" + "github.com/aws/karpenter/pkg/providers/instance" "github.com/aws/karpenter/pkg/providers/instanceprofile" "github.com/aws/karpenter/pkg/providers/license" "github.com/aws/karpenter/pkg/providers/placementgroup" "github.com/aws/karpenter/pkg/providers/pricing" "github.com/aws/karpenter/pkg/providers/securitygroup" "github.com/aws/karpenter/pkg/providers/subnet" - "github.com/aws/karpenter/pkg/utils/project" - - "github.com/aws/karpenter-core/pkg/operator/controller" ) func NewControllers(ctx context.Context, sess *session.Session, clk clock.Clock, kubeClient client.Client, recorder events.Recorder, unavailableOfferings *cache.UnavailableOfferings, cloudProvider *cloudprovider.CloudProvider, subnetProvider *subnet.Provider, - securityGroupProvider *securitygroup.Provider, instanceProfileProvider *instanceprofile.Provider, pricingProvider *pricing.Provider, - amiProvider *amifamily.Provider, licenseProvider *license.Provider, hostResourceGroupProvider *hostresourcegroup.Provider, placementGroupProvider *placementgroup.Provider) []controller.Controller { - - logging.FromContext(ctx).With("version", project.Version).Debugf("discovered version") + securityGroupProvider *securitygroup.Provider, instanceProfileProvider *instanceprofile.Provider, instanceProvider *instance.Provider, + pricingProvider *pricing.Provider, amiProvider *amifamily.Provider, + licenseProvider *license.Provider, hostResourceGroupProvider *hostresourcegroup.Provider, placementGroupProvider *placementgroup.Provider) []controller.Controller { linkController := nodeclaimlink.NewController(kubeClient, cloudProvider) controllers := []controller.Controller{ nodeclass.NewNodeTemplateController(kubeClient, recorder, subnetProvider, securityGroupProvider, amiProvider, instanceProfileProvider, licenseProvider, hostResourceGroupProvider, placementGroupProvider), + nodeclass.NewNodeClassController(kubeClient, recorder, subnetProvider, securityGroupProvider, amiProvider, instanceProfileProvider, licenseProvider, hostResourceGroupProvider, placementGroupProvider), linkController, nodeclaimgarbagecollection.NewController(kubeClient, cloudProvider, linkController), + nodeclaimtagging.NewController(kubeClient, instanceProvider), } - if settings.FromContext(ctx).InterruptionQueueName != "" { + if options.FromContext(ctx).InterruptionQueue != "" { controllers = append(controllers, interruption.NewController(kubeClient, clk, recorder, interruption.NewSQSProvider(sqs.New(sess)), unavailableOfferings)) } - if settings.FromContext(ctx).IsolatedVPC { + if options.FromContext(ctx).IsolatedVPC { logging.FromContext(ctx).Infof("assuming isolated VPC, pricing information will not be updated") } else { controllers = append(controllers, pricing.NewController(pricingProvider)) diff --git a/pkg/controllers/interruption/controller.go b/pkg/controllers/interruption/controller.go index 305c6b918599..f6d594764755 100644 --- a/pkg/controllers/interruption/controller.go +++ b/pkg/controllers/interruption/controller.go @@ -32,12 +32,12 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1beta1" "github.com/aws/karpenter-core/pkg/utils/pretty" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" "github.com/aws/karpenter/pkg/cache" interruptionevents "github.com/aws/karpenter/pkg/controllers/interruption/events" "github.com/aws/karpenter/pkg/controllers/interruption/messages" "github.com/aws/karpenter/pkg/controllers/interruption/messages/statechange" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/utils" "github.com/aws/karpenter-core/pkg/events" @@ -80,8 +80,8 @@ func NewController(kubeClient client.Client, clk clock.Clock, recorder events.Re } func (c *Controller) Reconcile(ctx context.Context, _ reconcile.Request) (reconcile.Result, error) { - ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("queue", settings.FromContext(ctx).InterruptionQueueName)) - if c.cm.HasChanged(settings.FromContext(ctx).InterruptionQueueName, nil) { + ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("queue", options.FromContext(ctx).InterruptionQueue)) + if c.cm.HasChanged(options.FromContext(ctx).InterruptionQueue, nil) { logging.FromContext(ctx).Debugf("watching interruption queue") } sqsMessages, err := c.sqsProvider.GetSQSMessages(ctx) diff --git a/pkg/controllers/interruption/interruption_benchmark_test.go b/pkg/controllers/interruption/interruption_benchmark_test.go index 6149bdddd35c..b0459b542a46 100644 --- a/pkg/controllers/interruption/interruption_benchmark_test.go +++ b/pkg/controllers/interruption/interruption_benchmark_test.go @@ -43,6 +43,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "github.com/aws/karpenter-core/pkg/operator/scheme" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/apis/settings" awscache "github.com/aws/karpenter/pkg/cache" "github.com/aws/karpenter/pkg/controllers/interruption" @@ -50,7 +51,7 @@ import ( "github.com/aws/karpenter/pkg/fake" "github.com/aws/karpenter/pkg/test" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" coretest "github.com/aws/karpenter-core/pkg/test" ) @@ -77,12 +78,13 @@ func BenchmarkNotification100(b *testing.B) { func benchmarkNotificationController(b *testing.B, messageCount int) { ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("message-count", messageCount)) fakeClock = &clock.FakeClock{} - ctx = coresettings.ToContext(ctx, coretest.Settings()) - ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ ClusterName: lo.ToPtr("karpenter-notification-benchmarking"), IsolatedVPC: lo.ToPtr(true), InterruptionQueueName: lo.ToPtr("test-cluster"), })) + ctx = settings.ToContext(ctx, test.Settings()) env = coretest.NewEnvironment(scheme.Scheme) // Stop the coretest environment after the coretest completes defer func() { @@ -181,7 +183,7 @@ func newProviders(kubeClient client.Client) providerSet { func (p *providerSet) makeInfrastructure(ctx context.Context) error { if _, err := p.sqsAPI.CreateQueueWithContext(ctx, &sqs.CreateQueueInput{ - QueueName: lo.ToPtr(settings.FromContext(ctx).InterruptionQueueName), + QueueName: lo.ToPtr(options.FromContext(ctx).InterruptionQueueName), Attributes: map[string]*string{ sqs.QueueAttributeNameMessageRetentionPeriod: aws.String("1200"), // 20 minutes for this test }, diff --git a/pkg/controllers/interruption/sqs.go b/pkg/controllers/interruption/sqs.go index 873688801d3d..7090da489a13 100644 --- a/pkg/controllers/interruption/sqs.go +++ b/pkg/controllers/interruption/sqs.go @@ -26,8 +26,8 @@ import ( "github.com/samber/lo" "github.com/aws/karpenter-core/pkg/utils/atomic" - "github.com/aws/karpenter/pkg/apis/settings" awserrors "github.com/aws/karpenter/pkg/errors" + "github.com/aws/karpenter/pkg/operator/options" ) type SQSProvider struct { @@ -43,7 +43,7 @@ func NewSQSProvider(client sqsiface.SQSAPI) *SQSProvider { } provider.queueURL.Resolve = func(ctx context.Context) (string, error) { input := &sqs.GetQueueUrlInput{ - QueueName: aws.String(settings.FromContext(ctx).InterruptionQueueName), + QueueName: aws.String(options.FromContext(ctx).InterruptionQueue), } ret, err := provider.client.GetQueueUrlWithContext(ctx, input) if err != nil { @@ -66,12 +66,12 @@ func (s *SQSProvider) QueueExists(ctx context.Context) (bool, error) { } func (s *SQSProvider) DiscoverQueueURL(ctx context.Context) (string, error) { - if settings.FromContext(ctx).InterruptionQueueName != lo.FromPtr(s.queueName.Load()) { + if options.FromContext(ctx).InterruptionQueue != lo.FromPtr(s.queueName.Load()) { res, err := s.queueURL.TryGet(ctx, atomic.IgnoreCacheOption) if err != nil { return res, err } - s.queueName.Store(lo.ToPtr(settings.FromContext(ctx).InterruptionQueueName)) + s.queueName.Store(lo.ToPtr(options.FromContext(ctx).InterruptionQueue)) return res, nil } return s.queueURL.TryGet(ctx) diff --git a/pkg/controllers/interruption/suite_test.go b/pkg/controllers/interruption/suite_test.go index 23d3ef8ab57e..5b2bd879190a 100644 --- a/pkg/controllers/interruption/suite_test.go +++ b/pkg/controllers/interruption/suite_test.go @@ -37,10 +37,10 @@ import ( _ "knative.dev/pkg/system/testing" "sigs.k8s.io/controller-runtime/pkg/client" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" "github.com/aws/karpenter-core/pkg/events" + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/operator/scheme" coretest "github.com/aws/karpenter-core/pkg/test" . "github.com/aws/karpenter-core/pkg/test/expectations" @@ -53,6 +53,7 @@ import ( "github.com/aws/karpenter/pkg/controllers/interruption/messages/spotinterruption" "github.com/aws/karpenter/pkg/controllers/interruption/messages/statechange" "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/test" "github.com/aws/karpenter/pkg/utils" ) @@ -91,10 +92,11 @@ var _ = AfterSuite(func() { }) var _ = BeforeEach(func() { - ctx = coresettings.ToContext(ctx, coretest.Settings()) - ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ - InterruptionQueueName: lo.ToPtr("test-cluster"), + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ + InterruptionQueue: lo.ToPtr("test-cluster"), })) + ctx = settings.ToContext(ctx, test.Settings()) unavailableOfferingsCache.Flush() sqsapi.Reset() sqsProvider.Reset() diff --git a/pkg/controllers/nodeclaim/garbagecollection/machine_test.go b/pkg/controllers/nodeclaim/garbagecollection/machine_test.go index 96e2dab400b8..b9dcca9dd97e 100644 --- a/pkg/controllers/nodeclaim/garbagecollection/machine_test.go +++ b/pkg/controllers/nodeclaim/garbagecollection/machine_test.go @@ -33,9 +33,9 @@ import ( coretest "github.com/aws/karpenter-core/pkg/test" . "github.com/aws/karpenter-core/pkg/test/expectations" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/test" ) @@ -60,7 +60,7 @@ var _ = Describe("Machine/GarbageCollection", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -69,7 +69,7 @@ var _ = Describe("Machine/GarbageCollection", func() { }, { Key: aws.String(v1alpha5.MachineManagedByAnnotationKey), - Value: aws.String(settings.FromContext(ctx).ClusterName), + Value: aws.String(options.FromContext(ctx).ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), @@ -86,7 +86,7 @@ var _ = Describe("Machine/GarbageCollection", func() { }) It("should delete an instance if there is no machine owner", func() { - // Launch time was 10m ago + // Launch time was 1m ago instance.LaunchTime = aws.Time(time.Now().Add(-time.Minute)) awsEnv.EC2API.Instances.Store(aws.StringValue(instance.InstanceId), instance) @@ -96,7 +96,7 @@ var _ = Describe("Machine/GarbageCollection", func() { Expect(corecloudprovider.IsNodeClaimNotFoundError(err)).To(BeTrue()) }) It("should delete an instance along with the node if there is no machine owner (to quicken scheduling)", func() { - // Launch time was 10m ago + // Launch time was 1m ago instance.LaunchTime = aws.Time(time.Now().Add(-time.Minute)) awsEnv.EC2API.Instances.Store(aws.StringValue(instance.InstanceId), instance) @@ -125,7 +125,7 @@ var _ = Describe("Machine/GarbageCollection", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -134,7 +134,7 @@ var _ = Describe("Machine/GarbageCollection", func() { }, { Key: aws.String(v1alpha5.MachineManagedByAnnotationKey), - Value: aws.String(settings.FromContext(ctx).ClusterName), + Value: aws.String(options.FromContext(ctx).ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), @@ -179,7 +179,7 @@ var _ = Describe("Machine/GarbageCollection", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -188,14 +188,14 @@ var _ = Describe("Machine/GarbageCollection", func() { }, { Key: aws.String(v1alpha5.MachineManagedByAnnotationKey), - Value: aws.String(settings.FromContext(ctx).ClusterName), + Value: aws.String(options.FromContext(ctx).ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), Placement: &ec2.Placement{ AvailabilityZone: aws.String(fake.DefaultRegion), }, - // Launch time was 10m ago + // Launch time was 1m ago LaunchTime: aws.Time(time.Now().Add(-time.Minute)), InstanceId: aws.String(instanceID), InstanceType: aws.String("m5.large"), @@ -244,7 +244,7 @@ var _ = Describe("Machine/GarbageCollection", func() { return aws.StringValue(t.Key) == v1alpha5.MachineManagedByAnnotationKey }) - // Launch time was 10m ago + // Launch time was 1m ago instance.LaunchTime = aws.Time(time.Now().Add(-time.Minute)) awsEnv.EC2API.Instances.Store(aws.StringValue(instance.InstanceId), instance) @@ -253,7 +253,7 @@ var _ = Describe("Machine/GarbageCollection", func() { Expect(err).NotTo(HaveOccurred()) }) It("should not delete the instance or node if it already has a machine that matches it", func() { - // Launch time was 10m ago + // Launch time was 1m ago instance.LaunchTime = aws.Time(time.Now().Add(-time.Minute)) awsEnv.EC2API.Instances.Store(aws.StringValue(instance.InstanceId), instance) @@ -286,7 +286,7 @@ var _ = Describe("Machine/GarbageCollection", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -295,7 +295,7 @@ var _ = Describe("Machine/GarbageCollection", func() { }, { Key: aws.String(v1alpha5.MachineManagedByAnnotationKey), - Value: aws.String(settings.FromContext(ctx).ClusterName), + Value: aws.String(options.FromContext(ctx).ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), @@ -337,7 +337,7 @@ var _ = Describe("Machine/GarbageCollection", func() { wg.Wait() }) It("should not delete an instance if it is linked", func() { - // Launch time was 10m ago + // Launch time was 1m ago instance.LaunchTime = aws.Time(time.Now().Add(-time.Minute)) awsEnv.EC2API.Instances.Store(aws.StringValue(instance.InstanceId), instance) @@ -357,7 +357,7 @@ var _ = Describe("Machine/GarbageCollection", func() { Expect(err).NotTo(HaveOccurred()) }) It("should not delete an instance if it is recently linked but the machine doesn't exist", func() { - // Launch time was 10m ago + // Launch time was 1m ago instance.LaunchTime = aws.Time(time.Now().Add(-time.Minute)) awsEnv.EC2API.Instances.Store(aws.StringValue(instance.InstanceId), instance) diff --git a/pkg/controllers/nodeclaim/garbagecollection/nodeclaim_test.go b/pkg/controllers/nodeclaim/garbagecollection/nodeclaim_test.go index e3b4b6ca9c0e..897c83ec4b60 100644 --- a/pkg/controllers/nodeclaim/garbagecollection/nodeclaim_test.go +++ b/pkg/controllers/nodeclaim/garbagecollection/nodeclaim_test.go @@ -33,8 +33,8 @@ import ( coretest "github.com/aws/karpenter-core/pkg/test" . "github.com/aws/karpenter-core/pkg/test/expectations" "github.com/aws/karpenter/pkg/apis/v1beta1" + "github.com/aws/karpenter/pkg/operator/options" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/fake" "github.com/aws/karpenter/pkg/test" ) @@ -65,7 +65,7 @@ var _ = Describe("NodeClaim/GarbageCollection", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -74,7 +74,7 @@ var _ = Describe("NodeClaim/GarbageCollection", func() { }, { Key: aws.String(corev1beta1.ManagedByAnnotationKey), - Value: aws.String(settings.FromContext(ctx).ClusterName), + Value: aws.String(options.FromContext(ctx).ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), @@ -91,7 +91,7 @@ var _ = Describe("NodeClaim/GarbageCollection", func() { }) It("should delete an instance if there is no NodeClaim owner", func() { - // Launch time was 10m ago + // Launch time was 1m ago instance.LaunchTime = aws.Time(time.Now().Add(-time.Minute)) awsEnv.EC2API.Instances.Store(aws.StringValue(instance.InstanceId), instance) @@ -101,7 +101,7 @@ var _ = Describe("NodeClaim/GarbageCollection", func() { Expect(corecloudprovider.IsNodeClaimNotFoundError(err)).To(BeTrue()) }) It("should delete an instance along with the node if there is no NodeClaim owner (to quicken scheduling)", func() { - // Launch time was 10m ago + // Launch time was 1m ago instance.LaunchTime = aws.Time(time.Now().Add(-time.Minute)) awsEnv.EC2API.Instances.Store(aws.StringValue(instance.InstanceId), instance) @@ -130,7 +130,7 @@ var _ = Describe("NodeClaim/GarbageCollection", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -139,7 +139,7 @@ var _ = Describe("NodeClaim/GarbageCollection", func() { }, { Key: aws.String(corev1beta1.ManagedByAnnotationKey), - Value: aws.String(settings.FromContext(ctx).ClusterName), + Value: aws.String(options.FromContext(ctx).ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), @@ -184,7 +184,7 @@ var _ = Describe("NodeClaim/GarbageCollection", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -193,14 +193,14 @@ var _ = Describe("NodeClaim/GarbageCollection", func() { }, { Key: aws.String(v1alpha5.MachineManagedByAnnotationKey), - Value: aws.String(settings.FromContext(ctx).ClusterName), + Value: aws.String(options.FromContext(ctx).ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), Placement: &ec2.Placement{ AvailabilityZone: aws.String(fake.DefaultRegion), }, - // Launch time was 10m ago + // Launch time was 1m ago LaunchTime: aws.Time(time.Now().Add(-time.Minute)), InstanceId: aws.String(instanceID), InstanceType: aws.String("m5.large"), @@ -254,7 +254,7 @@ var _ = Describe("NodeClaim/GarbageCollection", func() { return aws.StringValue(t.Key) == corev1beta1.ManagedByAnnotationKey }) - // Launch time was 10m ago + // Launch time was 1m ago instance.LaunchTime = aws.Time(time.Now().Add(-time.Minute)) awsEnv.EC2API.Instances.Store(aws.StringValue(instance.InstanceId), instance) @@ -263,7 +263,7 @@ var _ = Describe("NodeClaim/GarbageCollection", func() { Expect(err).NotTo(HaveOccurred()) }) It("should not delete the instance or node if it already has a NodeClaim that matches it", func() { - // Launch time was 10m ago + // Launch time was 1m ago instance.LaunchTime = aws.Time(time.Now().Add(-time.Minute)) awsEnv.EC2API.Instances.Store(aws.StringValue(instance.InstanceId), instance) @@ -301,7 +301,7 @@ var _ = Describe("NodeClaim/GarbageCollection", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -310,7 +310,7 @@ var _ = Describe("NodeClaim/GarbageCollection", func() { }, { Key: aws.String(corev1beta1.ManagedByAnnotationKey), - Value: aws.String(settings.FromContext(ctx).ClusterName), + Value: aws.String(options.FromContext(ctx).ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), diff --git a/pkg/controllers/nodeclaim/garbagecollection/suite_test.go b/pkg/controllers/nodeclaim/garbagecollection/suite_test.go index 5a9384a3f1fb..03cf6b5fee4f 100644 --- a/pkg/controllers/nodeclaim/garbagecollection/suite_test.go +++ b/pkg/controllers/nodeclaim/garbagecollection/suite_test.go @@ -32,7 +32,6 @@ import ( "k8s.io/client-go/tools/record" . "knative.dev/pkg/logging/testing" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" corecloudprovider "github.com/aws/karpenter-core/pkg/cloudprovider" @@ -48,6 +47,7 @@ import ( "github.com/aws/karpenter/pkg/controllers/nodeclaim/garbagecollection" "github.com/aws/karpenter/pkg/controllers/nodeclaim/link" "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/test" ) @@ -65,7 +65,7 @@ func TestAPIs(t *testing.T) { } var _ = BeforeSuite(func() { - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) awsEnv = test.NewEnvironment(ctx, env) @@ -104,7 +104,7 @@ var _ = Describe("Combined/GarbageCollection", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -113,7 +113,7 @@ var _ = Describe("Combined/GarbageCollection", func() { }, { Key: aws.String(corev1beta1.ManagedByAnnotationKey), - Value: aws.String(settings.FromContext(ctx).ClusterName), + Value: aws.String(options.FromContext(ctx).ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), @@ -139,7 +139,7 @@ var _ = Describe("Combined/GarbageCollection", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -148,7 +148,7 @@ var _ = Describe("Combined/GarbageCollection", func() { }, { Key: aws.String(v1alpha5.MachineManagedByAnnotationKey), - Value: aws.String(settings.FromContext(ctx).ClusterName), + Value: aws.String(options.FromContext(ctx).ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), @@ -193,7 +193,7 @@ var _ = Describe("Combined/GarbageCollection", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -202,7 +202,7 @@ var _ = Describe("Combined/GarbageCollection", func() { }, { Key: aws.String(corev1beta1.ManagedByAnnotationKey), - Value: aws.String(settings.FromContext(ctx).ClusterName), + Value: aws.String(options.FromContext(ctx).ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), @@ -243,7 +243,7 @@ var _ = Describe("Combined/GarbageCollection", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -252,7 +252,7 @@ var _ = Describe("Combined/GarbageCollection", func() { }, { Key: aws.String(corev1beta1.ManagedByAnnotationKey), - Value: aws.String(settings.FromContext(ctx).ClusterName), + Value: aws.String(options.FromContext(ctx).ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), diff --git a/pkg/controllers/nodeclaim/link/suite_test.go b/pkg/controllers/nodeclaim/link/suite_test.go index 15e059bf15ec..7fe54382750d 100644 --- a/pkg/controllers/nodeclaim/link/suite_test.go +++ b/pkg/controllers/nodeclaim/link/suite_test.go @@ -33,10 +33,10 @@ import ( . "knative.dev/pkg/logging/testing" "sigs.k8s.io/controller-runtime/pkg/client" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/events" "github.com/aws/karpenter-core/pkg/operator/controller" + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/operator/scheme" coretest "github.com/aws/karpenter-core/pkg/test" . "github.com/aws/karpenter-core/pkg/test/expectations" @@ -47,6 +47,7 @@ import ( "github.com/aws/karpenter/pkg/cloudprovider" "github.com/aws/karpenter/pkg/controllers/nodeclaim/link" "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/test" "github.com/aws/karpenter/pkg/utils" ) @@ -64,7 +65,8 @@ func TestAPIs(t *testing.T) { } var _ = BeforeSuite(func() { - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) awsEnv = test.NewEnvironment(ctx, env) @@ -107,7 +109,7 @@ var _ = Describe("MachineLink", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -254,7 +256,7 @@ var _ = Describe("MachineLink", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -344,7 +346,7 @@ var _ = Describe("MachineLink", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, }, diff --git a/pkg/controllers/nodeclaim/tagging/suite_test.go b/pkg/controllers/nodeclaim/tagging/suite_test.go index 69324b86a03d..fefab7c37ee5 100644 --- a/pkg/controllers/nodeclaim/tagging/suite_test.go +++ b/pkg/controllers/nodeclaim/tagging/suite_test.go @@ -31,7 +31,6 @@ import ( "github.com/aws/aws-sdk-go/service/ec2" . "knative.dev/pkg/logging/testing" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" coretest "github.com/aws/karpenter-core/pkg/test" "github.com/aws/karpenter/pkg/apis" @@ -39,10 +38,12 @@ import ( "github.com/aws/karpenter/pkg/apis/v1beta1" "github.com/aws/karpenter/pkg/controllers/nodeclaim/tagging" "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/providers/instance" "github.com/aws/karpenter/pkg/test" "github.com/aws/karpenter-core/pkg/operator/controller" + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/operator/scheme" ) @@ -59,7 +60,8 @@ func TestAPIs(t *testing.T) { var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) awsEnv = test.NewEnvironment(ctx, env) taggingController = tagging.NewController(env.Client, awsEnv.InstanceProvider) @@ -86,7 +88,7 @@ var _ = Describe("TaggingController", func() { }, Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), Value: aws.String("owned"), }, { @@ -95,7 +97,7 @@ var _ = Describe("TaggingController", func() { }, { Key: aws.String(corev1beta1.ManagedByAnnotationKey), - Value: aws.String(settings.FromContext(ctx).ClusterName), + Value: aws.String(options.FromContext(ctx).ClusterName), }, }, PrivateDnsName: aws.String(fake.PrivateDNSName()), diff --git a/pkg/controllers/nodeclass/controller.go b/pkg/controllers/nodeclass/controller.go index 8a710e35d4c7..7eb3435bfa5b 100644 --- a/pkg/controllers/nodeclass/controller.go +++ b/pkg/controllers/nodeclass/controller.go @@ -20,6 +20,8 @@ import ( "sort" "time" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/samber/lo" "go.uber.org/multierr" "golang.org/x/time/rate" "k8s.io/apimachinery/pkg/api/equality" @@ -35,10 +37,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/controller-runtime/pkg/source" - - "github.com/aws/aws-sdk-go/service/ec2" - "github.com/samber/lo" corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" "github.com/aws/karpenter-core/pkg/events" @@ -120,7 +118,7 @@ func (c *Controller) Finalize(ctx context.Context, nodeClass *v1beta1.EC2NodeCla return reconcile.Result{}, nil } nodeClaimList := &corev1beta1.NodeClaimList{} - if err := c.kubeClient.List(ctx, nodeClaimList, client.MatchingFields{"spec.nodeClass.name": nodeClass.Name}); err != nil { + if err := c.kubeClient.List(ctx, nodeClaimList, client.MatchingFields{"spec.nodeClassRef.name": nodeClass.Name}); err != nil { return reconcile.Result{}, fmt.Errorf("listing nodeclaims that are using nodeclass, %w", err) } if len(nodeClaimList.Items) > 0 { @@ -279,8 +277,8 @@ func (c *NodeClassController) Builder(_ context.Context, m manager.Manager) core NewControllerManagedBy(m). For(&v1beta1.EC2NodeClass{}). Watches( - &source.Kind{Type: &corev1beta1.NodeClaim{}}, - handler.EnqueueRequestsFromMapFunc(func(o client.Object) []reconcile.Request { + &corev1beta1.NodeClaim{}, + handler.EnqueueRequestsFromMapFunc(func(_ context.Context, o client.Object) []reconcile.Request { nc := o.(*corev1beta1.NodeClaim) if nc.Spec.NodeClassRef == nil { return nil diff --git a/pkg/controllers/nodeclass/nodeclass_test.go b/pkg/controllers/nodeclass/nodeclass_test.go index 4ba7f2217753..33fc3f44d2ab 100644 --- a/pkg/controllers/nodeclass/nodeclass_test.go +++ b/pkg/controllers/nodeclass/nodeclass_test.go @@ -714,11 +714,10 @@ var _ = Describe("NodeClassController", func() { }, Entry("AMIFamily Drift", &v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{AMIFamily: aws.String(v1beta1.AMIFamilyBottlerocket)}}), Entry("UserData Drift", &v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{UserData: aws.String("userdata-test-2")}}), - Entry("Role Drift", &v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{Role: "new-role"}}), Entry("Tags Drift", &v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{Tags: map[string]string{"keyTag-test-3": "valueTag-test-3"}}}), Entry("BlockDeviceMappings Drift", &v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{DeviceName: aws.String("map-device-test-3")}}}}), Entry("DetailedMonitoring Drift", &v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{DetailedMonitoring: aws.Bool(true)}}), - Entry("MetadataOptions Drift", &v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{MetadataOptions: &v1beta1.MetadataOptions{HTTPEndpoint: aws.String("test-metadata-2")}}}), + Entry("MetadataOptions Drift", &v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{MetadataOptions: &v1beta1.MetadataOptions{HTTPEndpoint: aws.String("disabled")}}}), Entry("Context Drift", &v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{Context: aws.String("context-2")}}), ) It("should not update the static drift hash when dynamic field is updated", func() { diff --git a/pkg/controllers/nodeclass/suite_test.go b/pkg/controllers/nodeclass/suite_test.go index ba109f4905e4..f35027d37ddd 100644 --- a/pkg/controllers/nodeclass/suite_test.go +++ b/pkg/controllers/nodeclass/suite_test.go @@ -24,24 +24,22 @@ import ( . "knative.dev/pkg/logging/testing" _ "knative.dev/pkg/system/testing" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" "github.com/aws/karpenter-core/pkg/events" corecontroller "github.com/aws/karpenter-core/pkg/operator/controller" - "github.com/aws/karpenter-core/pkg/operator/injection" - "github.com/aws/karpenter-core/pkg/operator/options" + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/operator/scheme" coretest "github.com/aws/karpenter-core/pkg/test" . "github.com/aws/karpenter-core/pkg/test/expectations" "github.com/aws/karpenter/pkg/apis" "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/controllers/nodeclass" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/test" ) var ctx context.Context var env *coretest.Environment var awsEnv *test.Environment -var opts options.Options var nodeTemplateController corecontroller.Controller var nodeClassController corecontroller.Controller @@ -53,7 +51,8 @@ func TestAPIs(t *testing.T) { var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...), coretest.WithFieldIndexers(test.EC2NodeClassFieldIndexer(ctx))) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) awsEnv = test.NewEnvironment(ctx, env) @@ -66,7 +65,7 @@ var _ = AfterSuite(func() { }) var _ = BeforeEach(func() { - ctx = injection.WithOptions(ctx, opts) + ctx = coreoptions.ToContext(ctx, coretest.Options()) awsEnv.Reset() }) diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index fcbe63a10c22..5f56eebc9f0e 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -39,6 +39,7 @@ import ( "github.com/aws/aws-sdk-go/service/resourcegroups" "github.com/aws/aws-sdk-go/service/ssm" "github.com/patrickmn/go-cache" + "sigs.k8s.io/controller-runtime/pkg/client" "github.com/samber/lo" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -48,9 +49,10 @@ import ( "knative.dev/pkg/logging" "knative.dev/pkg/ptr" + corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" "github.com/aws/karpenter-core/pkg/operator" - "github.com/aws/karpenter/pkg/apis/settings" awscache "github.com/aws/karpenter/pkg/cache" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/providers/amifamily" "github.com/aws/karpenter/pkg/providers/hostresourcegroup" "github.com/aws/karpenter/pkg/providers/instance" @@ -63,7 +65,6 @@ import ( "github.com/aws/karpenter/pkg/providers/securitygroup" "github.com/aws/karpenter/pkg/providers/subnet" "github.com/aws/karpenter/pkg/providers/version" - "github.com/aws/karpenter/pkg/utils/project" ) // Operator is injected into the AWS CloudProvider's factories @@ -93,7 +94,7 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont STSRegionalEndpoint: endpoints.RegionalSTSEndpoint, } - if assumeRoleARN := settings.FromContext(ctx).AssumeRoleARN; assumeRoleARN != "" { + if assumeRoleARN := options.FromContext(ctx).AssumeRoleARN; assumeRoleARN != "" { config.Credentials = stscreds.NewCredentials(session.Must(session.NewSession()), assumeRoleARN, func(provider *stscreds.AssumeRoleProvider) { setDurationAndExpiry(ctx, provider) }) } @@ -178,6 +179,14 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont launchTemplateProvider, ) + lo.Must0(operator.Manager.GetFieldIndexer().IndexField(ctx, &corev1beta1.NodeClaim{}, "spec.nodeClassRef.name", func(o client.Object) []string { + nc := o.(*corev1beta1.NodeClaim) + if nc.Spec.NodeClassRef == nil { + return []string{} + } + return []string{nc.Spec.NodeClassRef.Name} + }), "failed to setup nodeclaim indexer") + return ctx, &Operator{ Operator: operator, Session: sess, @@ -201,7 +210,7 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont // withUserAgent adds a karpenter specific user-agent string to AWS session func withUserAgent(sess *session.Session) *session.Session { - userAgent := fmt.Sprintf("karpenter.sh-%s", project.Version) + userAgent := fmt.Sprintf("karpenter.sh-%s", operator.Version) sess.Handlers.Build.PushBack(request.MakeAddToUserAgentFreeFormHandler(userAgent)) return sess } @@ -218,12 +227,12 @@ func checkEC2Connectivity(ctx context.Context, api *ec2.EC2) error { } func ResolveClusterEndpoint(ctx context.Context, eksAPI eksiface.EKSAPI) (string, error) { - clusterEndpointFromSettings := settings.FromContext(ctx).ClusterEndpoint - if clusterEndpointFromSettings != "" { - return clusterEndpointFromSettings, nil // cluster endpoint is explicitly set + clusterEndpointFromOptions := options.FromContext(ctx).ClusterEndpoint + if clusterEndpointFromOptions != "" { + return clusterEndpointFromOptions, nil // cluster endpoint is explicitly set } out, err := eksAPI.DescribeClusterWithContext(ctx, &eks.DescribeClusterInput{ - Name: aws.String(settings.FromContext(ctx).ClusterName), + Name: aws.String(options.FromContext(ctx).ClusterName), }) if err != nil { return "", fmt.Errorf("failed to resolve cluster endpoint, %w", err) @@ -236,7 +245,7 @@ func getCABundle(ctx context.Context, restConfig *rest.Config) (*string, error) // have used the simpler client-go InClusterConfig() method. // However, that only works when Karpenter is running as a Pod // within the same cluster it's managing. - if caBundle := settings.FromContext(ctx).ClusterCABundle; caBundle != "" { + if caBundle := options.FromContext(ctx).ClusterCABundle; caBundle != "" { return lo.ToPtr(caBundle), nil } transportConfig, err := restConfig.TransportConfig() @@ -266,6 +275,6 @@ func kubeDNSIP(ctx context.Context, kubernetesInterface kubernetes.Interface) (n } func setDurationAndExpiry(ctx context.Context, provider *stscreds.AssumeRoleProvider) { - provider.Duration = settings.FromContext(ctx).AssumeRoleDuration + provider.Duration = options.FromContext(ctx).AssumeRoleDuration provider.ExpiryWindow = time.Duration(10) * time.Second } diff --git a/pkg/operator/options/options.go b/pkg/operator/options/options.go new file mode 100644 index 000000000000..9134b944662d --- /dev/null +++ b/pkg/operator/options/options.go @@ -0,0 +1,130 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "context" + "errors" + "flag" + "fmt" + "os" + "strings" + "time" + + "k8s.io/apimachinery/pkg/util/sets" + + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" + "github.com/aws/karpenter-core/pkg/utils/env" + "github.com/aws/karpenter/pkg/apis/settings" +) + +func init() { + coreoptions.Injectables = append(coreoptions.Injectables, &Options{}) +} + +type optionsKey struct{} + +type Options struct { + AssumeRoleARN string + AssumeRoleDuration time.Duration + ClusterCABundle string + ClusterName string + ClusterEndpoint string + IsolatedVPC bool + VMMemoryOverheadPercent float64 + InterruptionQueue string + ReservedENIs int + + setFlags map[string]bool +} + +func (o *Options) AddFlags(fs *coreoptions.FlagSet) { + fs.StringVar(&o.AssumeRoleARN, "assume-role-arn", env.WithDefaultString("ASSUME_ROLE_ARN", ""), "Role to assume for calling AWS services.") + fs.DurationVar(&o.AssumeRoleDuration, "assume-role-duration", env.WithDefaultDuration("ASSUME_ROLE_DURATION", 15*time.Minute), "Duration of assumed credentials in minutes. Default value is 15 minutes. Not used unless aws.assumeRole set.") + fs.StringVar(&o.ClusterCABundle, "cluster-ca-bundle", env.WithDefaultString("CLUSTER_CA_BUNDLE", ""), "Cluster CA bundle for nodes to use for TLS connections with the API server. If not set, this is taken from the controller's TLS configuration.") + fs.StringVar(&o.ClusterName, "cluster-name", env.WithDefaultString("CLUSTER_NAME", ""), "[REQUIRED] The kubernetes cluster name for resource discovery.") + fs.StringVar(&o.ClusterEndpoint, "cluster-endpoint", env.WithDefaultString("CLUSTER_ENDPOINT", ""), "The external kubernetes cluster endpoint for new nodes to connect with. If not specified, will discover the cluster endpoint using DescribeCluster API.") + fs.BoolVarWithEnv(&o.IsolatedVPC, "isolated-vpc", "ISOLATED_VPC", false, "If true, then assume we can't reach AWS services which don't have a VPC endpoint. This also has the effect of disabling look-ups to the AWS pricing endpoint.") + fs.Float64Var(&o.VMMemoryOverheadPercent, "vm-memory-overhead-percent", env.WithDefaultFloat64("VM_MEMORY_OVERHEAD_PERCENT", 0.075), "The VM memory overhead as a percent that will be subtracted from the total memory for all instance types.") + fs.StringVar(&o.InterruptionQueue, "interruption-queue", env.WithDefaultString("INTERRUPTION_QUEUE", ""), "Interruption queue is disabled if not specified. Enabling interruption handling may require additional permissions on the controller service account. Additional permissions are outlined in the docs.") + fs.IntVar(&o.ReservedENIs, "reserved-enis", env.WithDefaultInt("RESERVED_ENIS", 0), "Reserved ENIs are not included in the calculations for max-pods or kube-reserved. This is most often used in the VPC CNI custom networking setup https://docs.aws.amazon.com/eks/latest/userguide/cni-custom-network.html.") +} + +func (o *Options) Parse(fs *coreoptions.FlagSet, args ...string) error { + if err := fs.Parse(args); err != nil { + if errors.Is(err, flag.ErrHelp) { + os.Exit(0) + } + return fmt.Errorf("parsing flags, %w", err) + } + + // Check if each option has been set. This is a little brute force and better options might exist, + // but this only needs to be here for one version + o.setFlags = map[string]bool{} + cliFlags := sets.New[string]() + fs.Visit(func(f *flag.Flag) { + cliFlags.Insert(f.Name) + }) + fs.VisitAll(func(f *flag.Flag) { + envName := strings.ReplaceAll(strings.ToUpper(f.Name), "-", "_") + _, ok := os.LookupEnv(envName) + o.setFlags[f.Name] = ok || cliFlags.Has(f.Name) + }) + + if err := o.Validate(); err != nil { + return fmt.Errorf("validating options, %w", err) + } + + return nil +} + +func (o *Options) ToContext(ctx context.Context) context.Context { + return ToContext(ctx, o) +} + +func (o *Options) MergeSettings(ctx context.Context) { + s := settings.FromContext(ctx) + mergeField(&o.AssumeRoleARN, s.AssumeRoleARN, o.setFlags["assume-role-arn"]) + mergeField(&o.AssumeRoleDuration, s.AssumeRoleDuration, o.setFlags["assume-role-duration"]) + mergeField(&o.ClusterCABundle, s.ClusterCABundle, o.setFlags["cluster-ca-bundle"]) + mergeField(&o.ClusterName, s.ClusterName, o.setFlags["cluster-name"]) + mergeField(&o.ClusterEndpoint, s.ClusterEndpoint, o.setFlags["cluster-endpoint"]) + mergeField(&o.IsolatedVPC, s.IsolatedVPC, o.setFlags["isolated-vpc"]) + mergeField(&o.VMMemoryOverheadPercent, s.VMMemoryOverheadPercent, o.setFlags["vm-memory-overhead-percent"]) + mergeField(&o.InterruptionQueue, s.InterruptionQueueName, o.setFlags["interruption-queue"]) + mergeField(&o.ReservedENIs, s.ReservedENIs, o.setFlags["reserved-enis"]) + if err := o.validateRequiredFields(); err != nil { + panic(fmt.Errorf("checking required fields, %w", err)) + } +} + +func ToContext(ctx context.Context, opts *Options) context.Context { + return context.WithValue(ctx, optionsKey{}, opts) +} + +func FromContext(ctx context.Context) *Options { + retval := ctx.Value(optionsKey{}) + if retval == nil { + return nil + } + return retval.(*Options) +} + +// Note: Separated out to help with cyclomatic complexity check +func mergeField[T any](dest *T, src T, isDestSet bool) { + if !isDestSet { + *dest = src + } +} diff --git a/pkg/operator/options/options_validation.go b/pkg/operator/options/options_validation.go new file mode 100644 index 000000000000..984058d8b8e2 --- /dev/null +++ b/pkg/operator/options/options_validation.go @@ -0,0 +1,74 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options + +import ( + "fmt" + "net/url" + "time" + + "go.uber.org/multierr" +) + +func (o Options) Validate() error { + return multierr.Combine( + o.validateEndpoint(), + o.validateVMMemoryOverheadPercent(), + o.validateAssumeRoleDuration(), + o.validateReservedENIs(), + ) +} + +func (o Options) validateAssumeRoleDuration() error { + if o.AssumeRoleDuration < time.Minute*15 { + return fmt.Errorf("assume-role-duration cannot be less than 15 minutes") + } + return nil +} + +func (o Options) validateEndpoint() error { + if o.ClusterEndpoint == "" { + return nil + } + endpoint, err := url.Parse(o.ClusterEndpoint) + // url.Parse() will accept a lot of input without error; make + // sure it's a real URL + if err != nil || !endpoint.IsAbs() || endpoint.Hostname() == "" { + return fmt.Errorf("%q is not a valid cluster-endpoint URL", o.ClusterEndpoint) + } + return nil +} + +func (o Options) validateVMMemoryOverheadPercent() error { + if o.VMMemoryOverheadPercent < 0 { + return fmt.Errorf("vm-memory-overhead-percent cannot be negative") + } + return nil +} + +func (o Options) validateReservedENIs() error { + if o.ReservedENIs < 0 { + return fmt.Errorf("reserved-enis cannot be negative") + } + return nil +} + +// Note: add back to Validate when karpenter-global-settings (and merge logic) are completely removed +func (o Options) validateRequiredFields() error { + if o.ClusterName == "" { + return fmt.Errorf("missing field, cluster-name") + } + return nil +} diff --git a/pkg/operator/options/suite_test.go b/pkg/operator/options/suite_test.go new file mode 100644 index 000000000000..6703ee13598c --- /dev/null +++ b/pkg/operator/options/suite_test.go @@ -0,0 +1,265 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package options_test + +import ( + "context" + "flag" + "fmt" + "os" + "testing" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/samber/lo" + . "knative.dev/pkg/logging/testing" + + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" + "github.com/aws/karpenter/pkg/apis/settings" + "github.com/aws/karpenter/pkg/operator/options" + "github.com/aws/karpenter/pkg/test" +) + +var ctx context.Context + +func TestAPIs(t *testing.T) { + ctx = TestContextWithLogger(t) + RegisterFailHandler(Fail) + RunSpecs(t, "Options") +} + +var _ = Describe("Options", func() { + var envState map[string]string + var environmentVariables = []string{ + "ASSUME_ROLE_ARN", + "ASSUME_ROLE_DURATION", + "CLUSTER_CA_BUNDLE", + "CLUSTER_NAME", + "CLUSTER_ENDPOINT", + "ISOLATED_VPC", + "VM_MEMORY_OVERHEAD_PERCENT", + "INTERRUPTION_QUEUE", + "RESERVED_ENIS", + } + + var fs *coreoptions.FlagSet + var opts *options.Options + + BeforeEach(func() { + envState = map[string]string{} + for _, ev := range environmentVariables { + val, ok := os.LookupEnv(ev) + if ok { + envState[ev] = val + } + os.Unsetenv(ev) + } + fs = &coreoptions.FlagSet{ + FlagSet: flag.NewFlagSet("karpenter", flag.ContinueOnError), + } + opts = &options.Options{} + opts.AddFlags(fs) + + // Inject default settings + var err error + ctx, err = (&settings.Settings{}).Inject(ctx, nil) + Expect(err).To(BeNil()) + }) + + AfterEach(func() { + for _, ev := range environmentVariables { + os.Unsetenv(ev) + } + for ev, val := range envState { + os.Setenv(ev, val) + } + }) + + Context("Merging", func() { + It("shouldn't overwrite options when all are set", func() { + err := opts.Parse( + fs, + "--assume-role-arn", "options-cluster-role", + "--assume-role-duration", "20m", + "--cluster-ca-bundle", "options-bundle", + "--cluster-name", "options-cluster", + "--cluster-endpoint", "https://options-cluster", + "--isolated-vpc", + "--vm-memory-overhead-percent", "0.1", + "--interruption-queue", "options-cluster", + "--reserved-enis", "10", + ) + Expect(err).ToNot(HaveOccurred()) + ctx = settings.ToContext(ctx, &settings.Settings{ + AssumeRoleARN: "settings-cluster-role", + AssumeRoleDuration: time.Minute * 22, + ClusterCABundle: "settings-bundle", + ClusterName: "settings-cluster", + ClusterEndpoint: "https://settings-cluster", + IsolatedVPC: true, + VMMemoryOverheadPercent: 0.05, + InterruptionQueueName: "settings-cluster", + ReservedENIs: 8, + }) + opts.MergeSettings(ctx) + expectOptionsEqual(opts, test.Options(test.OptionsFields{ + AssumeRoleARN: lo.ToPtr("options-cluster-role"), + AssumeRoleDuration: lo.ToPtr(20 * time.Minute), + ClusterCABundle: lo.ToPtr("options-bundle"), + ClusterName: lo.ToPtr("options-cluster"), + ClusterEndpoint: lo.ToPtr("https://options-cluster"), + IsolatedVPC: lo.ToPtr(true), + VMMemoryOverheadPercent: lo.ToPtr[float64](0.1), + InterruptionQueue: lo.ToPtr("options-cluster"), + ReservedENIs: lo.ToPtr(10), + })) + + }) + It("should overwrite options when none are set", func() { + err := opts.Parse(fs) + Expect(err).ToNot(HaveOccurred()) + ctx = settings.ToContext(ctx, &settings.Settings{ + AssumeRoleARN: "settings-cluster-role", + AssumeRoleDuration: time.Minute * 22, + ClusterCABundle: "settings-bundle", + ClusterName: "settings-cluster", + ClusterEndpoint: "https://settings-cluster", + IsolatedVPC: true, + VMMemoryOverheadPercent: 0.05, + InterruptionQueueName: "settings-cluster", + ReservedENIs: 8, + }) + opts.MergeSettings(ctx) + expectOptionsEqual(opts, test.Options(test.OptionsFields{ + AssumeRoleARN: lo.ToPtr("settings-cluster-role"), + AssumeRoleDuration: lo.ToPtr(22 * time.Minute), + ClusterCABundle: lo.ToPtr("settings-bundle"), + ClusterName: lo.ToPtr("settings-cluster"), + ClusterEndpoint: lo.ToPtr("https://settings-cluster"), + IsolatedVPC: lo.ToPtr(true), + VMMemoryOverheadPercent: lo.ToPtr[float64](0.05), + InterruptionQueue: lo.ToPtr("settings-cluster"), + ReservedENIs: lo.ToPtr(8), + })) + + }) + It("should correctly merge options and settings when mixed", func() { + err := opts.Parse( + fs, + "--assume-role-arn", "options-cluster-role", + "--cluster-ca-bundle", "options-bundle", + "--cluster-name", "options-cluster", + "--cluster-endpoint", "https://options-cluster", + "--interruption-queue", "options-cluster", + ) + Expect(err).ToNot(HaveOccurred()) + ctx = settings.ToContext(ctx, &settings.Settings{ + AssumeRoleARN: "settings-cluster-role", + AssumeRoleDuration: time.Minute * 20, + ClusterCABundle: "settings-bundle", + ClusterName: "settings-cluster", + ClusterEndpoint: "https://settings-cluster", + IsolatedVPC: true, + VMMemoryOverheadPercent: 0.1, + InterruptionQueueName: "settings-cluster", + ReservedENIs: 10, + }) + opts.MergeSettings(ctx) + expectOptionsEqual(opts, test.Options(test.OptionsFields{ + AssumeRoleARN: lo.ToPtr("options-cluster-role"), + AssumeRoleDuration: lo.ToPtr(20 * time.Minute), + ClusterCABundle: lo.ToPtr("options-bundle"), + ClusterName: lo.ToPtr("options-cluster"), + ClusterEndpoint: lo.ToPtr("https://options-cluster"), + IsolatedVPC: lo.ToPtr(true), + VMMemoryOverheadPercent: lo.ToPtr[float64](0.1), + InterruptionQueue: lo.ToPtr("options-cluster"), + ReservedENIs: lo.ToPtr(10), + })) + }) + + It("should correctly fallback to env vars when CLI flags aren't set", func() { + os.Setenv("ASSUME_ROLE_ARN", "env-role") + os.Setenv("ASSUME_ROLE_DURATION", "20m") + os.Setenv("CLUSTER_CA_BUNDLE", "env-bundle") + os.Setenv("CLUSTER_NAME", "env-cluster") + os.Setenv("CLUSTER_ENDPOINT", "https://env-cluster") + os.Setenv("ISOLATED_VPC", "true") + os.Setenv("VM_MEMORY_OVERHEAD_PERCENT", "0.1") + os.Setenv("INTERRUPTION_QUEUE", "env-cluster") + os.Setenv("RESERVED_ENIS", "10") + fs = &coreoptions.FlagSet{ + FlagSet: flag.NewFlagSet("karpenter", flag.ContinueOnError), + } + opts.AddFlags(fs) + err := opts.Parse(fs) + Expect(err).ToNot(HaveOccurred()) + expectOptionsEqual(opts, test.Options(test.OptionsFields{ + AssumeRoleARN: lo.ToPtr("env-role"), + AssumeRoleDuration: lo.ToPtr(20 * time.Minute), + ClusterCABundle: lo.ToPtr("env-bundle"), + ClusterName: lo.ToPtr("env-cluster"), + ClusterEndpoint: lo.ToPtr("https://env-cluster"), + IsolatedVPC: lo.ToPtr(true), + VMMemoryOverheadPercent: lo.ToPtr[float64](0.1), + InterruptionQueue: lo.ToPtr("env-cluster"), + ReservedENIs: lo.ToPtr(10), + })) + }) + }) + + Context("Validation", func() { + It("should fail when cluster name is not set", func() { + err := opts.Parse(fs) + // Overwrite ClusterName since it is commonly set by environment variables in dev environments + opts.ClusterName = "" + Expect(err).ToNot(HaveOccurred()) + Expect(func() { + opts.MergeSettings(ctx) + fmt.Printf("%#v", opts) + }).To(Panic()) + }) + It("should fail when assume role duration is less than 15 minutes", func() { + err := opts.Parse(fs, "--assume-role-duration", "1s") + Expect(err).To(HaveOccurred()) + }) + It("should fail when clusterEndpoint is invalid (not absolute)", func() { + err := opts.Parse(fs, "--cluster-endpoint", "00000000000000000000000.gr7.us-west-2.eks.amazonaws.com") + Expect(err).To(HaveOccurred()) + }) + It("should fail when vmMemoryOverheadPercent is negative", func() { + err := opts.Parse(fs, "--vm-memory-overhead-percent", "-0.01") + Expect(err).To(HaveOccurred()) + }) + It("should fail when reservedENIs is negative", func() { + err := opts.Parse(fs, "--reserved-enis", "-1") + Expect(err).To(HaveOccurred()) + }) + }) +}) + +func expectOptionsEqual(optsA *options.Options, optsB *options.Options) { + GinkgoHelper() + Expect(optsA.AssumeRoleARN).To(Equal(optsB.AssumeRoleARN)) + Expect(optsA.AssumeRoleDuration).To(Equal(optsB.AssumeRoleDuration)) + Expect(optsA.ClusterCABundle).To(Equal(optsB.ClusterCABundle)) + Expect(optsA.ClusterName).To(Equal(optsB.ClusterName)) + Expect(optsA.ClusterEndpoint).To(Equal(optsB.ClusterEndpoint)) + Expect(optsA.IsolatedVPC).To(Equal(optsB.IsolatedVPC)) + Expect(optsA.VMMemoryOverheadPercent).To(Equal(optsB.VMMemoryOverheadPercent)) + Expect(optsA.InterruptionQueue).To(Equal(optsB.InterruptionQueue)) + Expect(optsA.ReservedENIs).To(Equal(optsB.ReservedENIs)) +} diff --git a/pkg/operator/suite_test.go b/pkg/operator/suite_test.go index dea32a9fedc9..5146ed6d9e9b 100644 --- a/pkg/operator/suite_test.go +++ b/pkg/operator/suite_test.go @@ -26,12 +26,11 @@ import ( . "knative.dev/pkg/logging/testing" "github.com/aws/karpenter/pkg/apis" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/fake" awscontext "github.com/aws/karpenter/pkg/operator" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/test" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" "github.com/aws/karpenter-core/pkg/operator/scheme" coretest "github.com/aws/karpenter-core/pkg/test" . "github.com/aws/karpenter-core/pkg/test/expectations" @@ -50,8 +49,6 @@ func TestAWS(t *testing.T) { var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) - ctx = coresettings.ToContext(ctx, coretest.Settings()) - ctx = settings.ToContext(ctx, test.Settings()) ctx, stop = context.WithCancel(ctx) fakeEKSAPI = &fake.EKSAPI{} @@ -72,7 +69,7 @@ var _ = AfterEach(func() { var _ = Describe("Operator", func() { It("should resolve endpoint if set via configuration", func() { - ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ ClusterEndpoint: lo.ToPtr("https://api.test-cluster.k8s.local"), })) endpoint, err := awscontext.ResolveClusterEndpoint(ctx, fakeEKSAPI) @@ -80,7 +77,7 @@ var _ = Describe("Operator", func() { Expect(endpoint).To(Equal("https://api.test-cluster.k8s.local")) }) It("should resolve endpoint if not set, via call to API", func() { - ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ ClusterEndpoint: lo.ToPtr(""), })) fakeEKSAPI.DescribeClusterBehavior.Output.Set( @@ -96,7 +93,7 @@ var _ = Describe("Operator", func() { Expect(endpoint).To(Equal("https://cluster-endpoint.test-cluster.k8s.local")) }) It("should propagate error if API fails", func() { - ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ ClusterEndpoint: lo.ToPtr(""), })) fakeEKSAPI.DescribeClusterBehavior.Error.Set(errors.New("test error")) diff --git a/pkg/providers/amifamily/ami_test.go b/pkg/providers/amifamily/ami_test.go index 9da5c2058dff..279ce74dd88b 100644 --- a/pkg/providers/amifamily/ami_test.go +++ b/pkg/providers/amifamily/ami_test.go @@ -29,14 +29,15 @@ import ( v1 "k8s.io/api/core/v1" . "knative.dev/pkg/logging/testing" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/operator/scheme" "github.com/aws/karpenter-core/pkg/scheduling" coretest "github.com/aws/karpenter-core/pkg/test" "github.com/aws/karpenter/pkg/apis" "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1beta1" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/providers/amifamily" "github.com/aws/karpenter/pkg/test" ) @@ -61,7 +62,8 @@ const ( var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) awsEnv = test.NewEnvironment(ctx, env) }) diff --git a/pkg/providers/hostresourcegroup/suite_test.go b/pkg/providers/hostresourcegroup/suite_test.go index 785b9ca060da..8d14eba806ba 100644 --- a/pkg/providers/hostresourcegroup/suite_test.go +++ b/pkg/providers/hostresourcegroup/suite_test.go @@ -30,8 +30,6 @@ import ( "github.com/aws/karpenter/pkg/test" coresettings "github.com/aws/karpenter-core/pkg/apis/settings" - "github.com/aws/karpenter-core/pkg/operator/injection" - "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/operator/scheme" coretest "github.com/aws/karpenter-core/pkg/test" . "github.com/aws/karpenter-core/pkg/test/expectations" @@ -39,7 +37,6 @@ import ( var ctx context.Context var stop context.CancelFunc -var opts options.Options var env *coretest.Environment var awsEnv *test.Environment var nodeClass *v1beta1.EC2NodeClass @@ -64,7 +61,6 @@ var _ = AfterSuite(func() { }) var _ = BeforeEach(func() { - ctx = injection.WithOptions(ctx, opts) ctx = coresettings.ToContext(ctx, coretest.Settings()) ctx = settings.ToContext(ctx, test.Settings()) nodeClass = test.EC2NodeClass(v1beta1.EC2NodeClass{ diff --git a/pkg/providers/instance/instance.go b/pkg/providers/instance/instance.go index d44f33ac659e..35690da3c5ea 100644 --- a/pkg/providers/instance/instance.go +++ b/pkg/providers/instance/instance.go @@ -39,6 +39,7 @@ import ( "github.com/aws/karpenter/pkg/batcher" "github.com/aws/karpenter/pkg/cache" awserrors "github.com/aws/karpenter/pkg/errors" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/providers/instancetype" "github.com/aws/karpenter/pkg/providers/launchtemplate" "github.com/aws/karpenter/pkg/providers/subnet" @@ -104,7 +105,7 @@ func (p *Provider) Create(ctx context.Context, nodeClass *v1beta1.EC2NodeClass, func (p *Provider) Link(ctx context.Context, id, provisionerName string) error { if err := p.CreateTags(ctx, id, map[string]string{ - v1alpha5.MachineManagedByAnnotationKey: settings.FromContext(ctx).ClusterName, + v1alpha5.MachineManagedByAnnotationKey: options.FromContext(ctx).ClusterName, v1alpha5.ProvisionerNameLabelKey: provisionerName, }); err != nil { return fmt.Errorf("linking tags, %w", err) @@ -143,7 +144,7 @@ func (p *Provider) List(ctx context.Context) ([]*Instance, error) { }, { Name: aws.String("tag-key"), - Values: aws.StringSlice([]string{fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName)}), + Values: aws.StringSlice([]string{fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)}), }, instanceStateFilter, }, @@ -257,15 +258,15 @@ func getTags(ctx context.Context, nodeClass *v1beta1.EC2NodeClass, nodeClaim *co "Name": fmt.Sprintf("%s/%s", v1alpha5.ProvisionerNameLabelKey, nodeClaim.Labels[v1alpha5.ProvisionerNameLabelKey]), } staticTags = map[string]string{ - fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName): "owned", - v1alpha5.ProvisionerNameLabelKey: nodeClaim.Labels[v1alpha5.ProvisionerNameLabelKey], - v1alpha5.MachineManagedByAnnotationKey: settings.FromContext(ctx).ClusterName, + fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName): "owned", + v1alpha5.ProvisionerNameLabelKey: nodeClaim.Labels[v1alpha5.ProvisionerNameLabelKey], + v1alpha5.MachineManagedByAnnotationKey: options.FromContext(ctx).ClusterName, } } else { staticTags = map[string]string{ - fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName): "owned", + fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName): "owned", corev1beta1.NodePoolLabelKey: nodeClaim.Labels[corev1beta1.NodePoolLabelKey], - corev1beta1.ManagedByAnnotationKey: settings.FromContext(ctx).ClusterName, + corev1beta1.ManagedByAnnotationKey: options.FromContext(ctx).ClusterName, } } return lo.Assign(overridableTags, settings.FromContext(ctx).Tags, nodeClass.Spec.Tags, staticTags) diff --git a/pkg/providers/instance/nodeclass_test.go b/pkg/providers/instance/nodeclass_test.go index f7396bebf885..a2fcaa913142 100644 --- a/pkg/providers/instance/nodeclass_test.go +++ b/pkg/providers/instance/nodeclass_test.go @@ -15,10 +15,16 @@ limitations under the License. package instance_test import ( + "fmt" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/samber/lo" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" @@ -27,6 +33,8 @@ import ( . "github.com/aws/karpenter-core/pkg/test/expectations" "github.com/aws/karpenter/pkg/apis/v1beta1" "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/operator/options" + "github.com/aws/karpenter/pkg/providers/instance" "github.com/aws/karpenter/pkg/test" ) @@ -79,4 +87,68 @@ var _ = Describe("NodeClass/InstanceProvider", func() { Expect(corecloudprovider.IsInsufficientCapacityError(err)).To(BeTrue()) Expect(instance).To(BeNil()) }) + It("should return all NodePool-owned instances from List", func() { + ids := sets.New[string]() + // Provision instances that have the karpenter.sh/nodepool key + for i := 0; i < 20; i++ { + instanceID := fake.InstanceID() + awsEnv.EC2API.Instances.Store( + instanceID, + &ec2.Instance{ + State: &ec2.InstanceState{ + Name: aws.String(ec2.InstanceStateNameRunning), + }, + Tags: []*ec2.Tag{ + { + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), + Value: aws.String("owned"), + }, + { + Key: aws.String(corev1beta1.NodePoolLabelKey), + Value: aws.String("default"), + }, + { + Key: aws.String(corev1beta1.ManagedByAnnotationKey), + Value: aws.String(options.FromContext(ctx).ClusterName), + }, + }, + PrivateDnsName: aws.String(fake.PrivateDNSName()), + Placement: &ec2.Placement{ + AvailabilityZone: aws.String(fake.DefaultRegion), + }, + // Launch time was 1m ago + LaunchTime: aws.Time(time.Now().Add(-time.Minute)), + InstanceId: aws.String(instanceID), + InstanceType: aws.String("m5.large"), + }, + ) + ids.Insert(instanceID) + } + // Provision instances that do not have this tag key + for i := 0; i < 20; i++ { + instanceID := fake.InstanceID() + awsEnv.EC2API.Instances.Store( + instanceID, + &ec2.Instance{ + State: &ec2.InstanceState{ + Name: aws.String(ec2.InstanceStateNameRunning), + }, + PrivateDnsName: aws.String(fake.PrivateDNSName()), + Placement: &ec2.Placement{ + AvailabilityZone: aws.String(fake.DefaultRegion), + }, + // Launch time was 1m ago + LaunchTime: aws.Time(time.Now().Add(-time.Minute)), + InstanceId: aws.String(instanceID), + InstanceType: aws.String("m5.large"), + }, + ) + } + instances, err := awsEnv.InstanceProvider.List(ctx) + Expect(err).To(BeNil()) + Expect(instances).To(HaveLen(20)) + + retrievedIDs := sets.New[string](lo.Map(instances, func(i *instance.Instance, _ int) string { return i.ID })...) + Expect(ids.Equal(retrievedIDs)).To(BeTrue()) + }) }) diff --git a/pkg/providers/instance/nodetemplate_test.go b/pkg/providers/instance/nodetemplate_test.go index baa4a2751a92..4894eb9bcddc 100644 --- a/pkg/providers/instance/nodetemplate_test.go +++ b/pkg/providers/instance/nodetemplate_test.go @@ -15,12 +15,17 @@ limitations under the License. package instance_test import ( + "fmt" + "time" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/samber/lo" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" corecloudprovider "github.com/aws/karpenter-core/pkg/cloudprovider" @@ -30,6 +35,8 @@ import ( nodepoolutil "github.com/aws/karpenter-core/pkg/utils/nodepool" "github.com/aws/karpenter/pkg/apis/v1alpha1" "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/operator/options" + "github.com/aws/karpenter/pkg/providers/instance" "github.com/aws/karpenter/pkg/test" nodeclassutil "github.com/aws/karpenter/pkg/utils/nodeclass" ) @@ -94,4 +101,68 @@ var _ = Describe("NodeTemplate/InstanceProvider", func() { Expect(corecloudprovider.IsInsufficientCapacityError(err)).To(BeTrue()) Expect(instance).To(BeNil()) }) + It("should return all Provisioner-owned instances from List", func() { + ids := sets.New[string]() + // Provision instances that have the karpenter.sh/provisioner-name key + for i := 0; i < 20; i++ { + instanceID := fake.InstanceID() + awsEnv.EC2API.Instances.Store( + instanceID, + &ec2.Instance{ + State: &ec2.InstanceState{ + Name: aws.String(ec2.InstanceStateNameRunning), + }, + Tags: []*ec2.Tag{ + { + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), + Value: aws.String("owned"), + }, + { + Key: aws.String(v1alpha5.ProvisionerNameLabelKey), + Value: aws.String("default"), + }, + { + Key: aws.String(v1alpha5.MachineManagedByAnnotationKey), + Value: aws.String(options.FromContext(ctx).ClusterName), + }, + }, + PrivateDnsName: aws.String(fake.PrivateDNSName()), + Placement: &ec2.Placement{ + AvailabilityZone: aws.String(fake.DefaultRegion), + }, + // Launch time was 1m ago + LaunchTime: aws.Time(time.Now().Add(-time.Minute)), + InstanceId: aws.String(instanceID), + InstanceType: aws.String("m5.large"), + }, + ) + ids.Insert(instanceID) + } + // Provision instances that do not have this tag key + for i := 0; i < 20; i++ { + instanceID := fake.InstanceID() + awsEnv.EC2API.Instances.Store( + instanceID, + &ec2.Instance{ + State: &ec2.InstanceState{ + Name: aws.String(ec2.InstanceStateNameRunning), + }, + PrivateDnsName: aws.String(fake.PrivateDNSName()), + Placement: &ec2.Placement{ + AvailabilityZone: aws.String(fake.DefaultRegion), + }, + // Launch time was 1m ago + LaunchTime: aws.Time(time.Now().Add(-time.Minute)), + InstanceId: aws.String(instanceID), + InstanceType: aws.String("m5.large"), + }, + ) + } + instances, err := awsEnv.InstanceProvider.List(ctx) + Expect(err).To(BeNil()) + Expect(instances).To(HaveLen(20)) + + retrievedIDs := sets.New[string](lo.Map(instances, func(i *instance.Instance, _ int) string { return i.ID })...) + Expect(ids.Equal(retrievedIDs)).To(BeTrue()) + }) }) diff --git a/pkg/providers/instance/suite_test.go b/pkg/providers/instance/suite_test.go index b27ca56a7cae..44d155b95004 100644 --- a/pkg/providers/instance/suite_test.go +++ b/pkg/providers/instance/suite_test.go @@ -16,27 +16,35 @@ package instance_test import ( "context" + "fmt" "testing" + "time" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ec2" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/samber/lo" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/tools/record" . "knative.dev/pkg/logging/testing" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" + "github.com/aws/karpenter-core/pkg/apis/v1alpha5" + corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" "github.com/aws/karpenter-core/pkg/events" - "github.com/aws/karpenter-core/pkg/operator/injection" - "github.com/aws/karpenter-core/pkg/operator/options" + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/operator/scheme" coretest "github.com/aws/karpenter-core/pkg/test" "github.com/aws/karpenter/pkg/apis" "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/cloudprovider" + "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/operator/options" + "github.com/aws/karpenter/pkg/providers/instance" "github.com/aws/karpenter/pkg/test" ) var ctx context.Context -var opts options.Options var env *coretest.Environment var awsEnv *test.Environment var cloudProvider *cloudprovider.CloudProvider @@ -49,7 +57,8 @@ func TestAWS(t *testing.T) { var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) awsEnv = test.NewEnvironment(ctx, env) cloudProvider = cloudprovider.New(awsEnv.InstanceTypesProvider, awsEnv.InstanceProvider, events.NewRecorder(&record.FakeRecorder{}), @@ -61,7 +70,110 @@ var _ = AfterSuite(func() { }) var _ = BeforeEach(func() { - ctx = injection.WithOptions(ctx, opts) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) + awsEnv.Reset() +}) + +var _ = Describe("Combined/InstanceProvider", func() { + It("should return both NodePool-owned instances and Provisioner-owned instances from List", func() { + ids := sets.New[string]() + // Provision instances that have the karpenter.sh/provisioner-name key + for i := 0; i < 20; i++ { + instanceID := fake.InstanceID() + awsEnv.EC2API.Instances.Store( + instanceID, + &ec2.Instance{ + State: &ec2.InstanceState{ + Name: aws.String(ec2.InstanceStateNameRunning), + }, + Tags: []*ec2.Tag{ + { + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), + Value: aws.String("owned"), + }, + { + Key: aws.String(v1alpha5.ProvisionerNameLabelKey), + Value: aws.String("default"), + }, + { + Key: aws.String(v1alpha5.MachineManagedByAnnotationKey), + Value: aws.String(options.FromContext(ctx).ClusterName), + }, + }, + PrivateDnsName: aws.String(fake.PrivateDNSName()), + Placement: &ec2.Placement{ + AvailabilityZone: aws.String(fake.DefaultRegion), + }, + // Launch time was 1m ago + LaunchTime: aws.Time(time.Now().Add(-time.Minute)), + InstanceId: aws.String(instanceID), + InstanceType: aws.String("m5.large"), + }, + ) + ids.Insert(instanceID) + } + // Provision instances that have the karpenter.sh/nodepool key + for i := 0; i < 20; i++ { + instanceID := fake.InstanceID() + awsEnv.EC2API.Instances.Store( + instanceID, + &ec2.Instance{ + State: &ec2.InstanceState{ + Name: aws.String(ec2.InstanceStateNameRunning), + }, + Tags: []*ec2.Tag{ + { + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)), + Value: aws.String("owned"), + }, + { + Key: aws.String(corev1beta1.NodePoolLabelKey), + Value: aws.String("default"), + }, + { + Key: aws.String(corev1beta1.ManagedByAnnotationKey), + Value: aws.String(options.FromContext(ctx).ClusterName), + }, + }, + PrivateDnsName: aws.String(fake.PrivateDNSName()), + Placement: &ec2.Placement{ + AvailabilityZone: aws.String(fake.DefaultRegion), + }, + // Launch time was 1m ago + LaunchTime: aws.Time(time.Now().Add(-time.Minute)), + InstanceId: aws.String(instanceID), + InstanceType: aws.String("m5.large"), + }, + ) + ids.Insert(instanceID) + } + // Provision instances that do not have this tag key + for i := 0; i < 20; i++ { + instanceID := fake.InstanceID() + awsEnv.EC2API.Instances.Store( + instanceID, + &ec2.Instance{ + State: &ec2.InstanceState{ + Name: aws.String(ec2.InstanceStateNameRunning), + }, + PrivateDnsName: aws.String(fake.PrivateDNSName()), + Placement: &ec2.Placement{ + AvailabilityZone: aws.String(fake.DefaultRegion), + }, + // Launch time was 1m ago + LaunchTime: aws.Time(time.Now().Add(-time.Minute)), + InstanceId: aws.String(instanceID), + InstanceType: aws.String("m5.large"), + }, + ) + } + instances, err := awsEnv.InstanceProvider.List(ctx) + Expect(err).To(BeNil()) + Expect(instances).To(HaveLen(40)) + + retrievedIDs := sets.New[string](lo.Map(instances, func(i *instance.Instance, _ int) string { return i.ID })...) + Expect(ids.Equal(retrievedIDs)).To(BeTrue()) + }) }) diff --git a/pkg/providers/instanceprofile/instanceprofile.go b/pkg/providers/instanceprofile/instanceprofile.go index e5b6f9437399..f9bb39d9cf8a 100644 --- a/pkg/providers/instanceprofile/instanceprofile.go +++ b/pkg/providers/instanceprofile/instanceprofile.go @@ -27,9 +27,9 @@ import ( v1 "k8s.io/api/core/v1" corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1beta1" awserrors "github.com/aws/karpenter/pkg/errors" + "github.com/aws/karpenter/pkg/operator/options" ) type Provider struct { @@ -48,10 +48,10 @@ func NewProvider(region string, iamapi iamiface.IAMAPI, cache *cache.Cache) *Pro func (p *Provider) Create(ctx context.Context, nodeClass *v1beta1.EC2NodeClass) (string, error) { tags := lo.Assign(nodeClass.Spec.Tags, map[string]string{ - fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(ctx).ClusterName): "owned", - corev1beta1.ManagedByAnnotationKey: settings.FromContext(ctx).ClusterName, - v1beta1.LabelNodeClass: nodeClass.Name, - v1.LabelTopologyRegion: p.region, + fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName): "owned", + corev1beta1.ManagedByAnnotationKey: options.FromContext(ctx).ClusterName, + v1beta1.LabelNodeClass: nodeClass.Name, + v1.LabelTopologyRegion: p.region, }) profileName := GetProfileName(ctx, p.region, nodeClass) @@ -129,5 +129,5 @@ func (p *Provider) Delete(ctx context.Context, nodeClass *v1beta1.EC2NodeClass) // GetProfileName gets the string for the profile name based on the cluster name and the NodeClass UUID. // The length of this string can never exceed the maximum instance profile name limit of 128 characters. func GetProfileName(ctx context.Context, region string, nodeClass *v1beta1.EC2NodeClass) string { - return fmt.Sprintf("%s_%d", settings.FromContext(ctx).ClusterName, lo.Must(hashstructure.Hash(fmt.Sprintf("%s%s", region, nodeClass.Name), hashstructure.FormatV2, nil))) + return fmt.Sprintf("%s_%d", options.FromContext(ctx).ClusterName, lo.Must(hashstructure.Hash(fmt.Sprintf("%s%s", region, nodeClass.Name), hashstructure.FormatV2, nil))) } diff --git a/pkg/providers/instancetype/nodeclass_test.go b/pkg/providers/instancetype/nodeclass_test.go index d2b4cdae9a00..16cb06e0f502 100644 --- a/pkg/providers/instancetype/nodeclass_test.go +++ b/pkg/providers/instancetype/nodeclass_test.go @@ -40,6 +40,7 @@ import ( "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1beta1" "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/providers/instance" "github.com/aws/karpenter/pkg/providers/instancetype" "github.com/aws/karpenter/pkg/test" @@ -645,16 +646,8 @@ var _ = Describe("NodeClass/InstanceTypes", func() { Context("Overhead", func() { var info *ec2.InstanceTypeInfo BeforeEach(func() { - ctx, err := (&settings.Settings{}).Inject(ctx, &v1.ConfigMap{ - Data: map[string]string{ - "aws.clusterName": "karpenter-cluster", - }, - }) - Expect(err).To(BeNil()) - - s := settings.FromContext(ctx) - ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ - VMMemoryOverheadPercent: &s.VMMemoryOverheadPercent, + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ + ClusterName: lo.ToPtr("karpenter-cluster"), })) var ok bool @@ -713,7 +706,7 @@ var _ = Describe("NodeClass/InstanceTypes", func() { }) Context("Eviction Thresholds", func() { BeforeEach(func() { - ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ VMMemoryOverheadPercent: lo.ToPtr[float64](0), })) }) @@ -950,7 +943,7 @@ var _ = Describe("NodeClass/InstanceTypes", func() { } }) It("should reserve ENIs when aws.reservedENIs is set and is used in max-pods calculation", func() { - ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ ReservedENIs: lo.ToPtr(1), })) @@ -970,7 +963,7 @@ var _ = Describe("NodeClass/InstanceTypes", func() { Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", maxPods)) }) It("should reserve ENIs when aws.reservedENIs is set and not go below 0 ENIs in max-pods calculation", func() { - ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ ReservedENIs: lo.ToPtr(1_000_000), })) diff --git a/pkg/providers/instancetype/nodetemplate_test.go b/pkg/providers/instancetype/nodetemplate_test.go index ba650c3b4ca7..30e03bdfa179 100644 --- a/pkg/providers/instancetype/nodetemplate_test.go +++ b/pkg/providers/instancetype/nodetemplate_test.go @@ -40,6 +40,7 @@ import ( . "github.com/aws/karpenter-core/pkg/test/expectations" nodepoolutil "github.com/aws/karpenter-core/pkg/utils/nodepool" "github.com/aws/karpenter-core/pkg/utils/resources" + "github.com/aws/karpenter/pkg/operator/options" nodeclassutil "github.com/aws/karpenter/pkg/utils/nodeclass" "github.com/aws/karpenter/pkg/apis/settings" @@ -664,16 +665,8 @@ var _ = Describe("NodeTemplate/InstanceTypes", func() { Context("Overhead", func() { var info *ec2.InstanceTypeInfo BeforeEach(func() { - ctx, err := (&settings.Settings{}).Inject(ctx, &v1.ConfigMap{ - Data: map[string]string{ - "aws.clusterName": "karpenter-cluster", - }, - }) - Expect(err).To(BeNil()) - - s := settings.FromContext(ctx) - ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ - VMMemoryOverheadPercent: &s.VMMemoryOverheadPercent, + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ + ClusterName: lo.ToPtr("karpenter-cluster"), })) var ok bool @@ -734,7 +727,7 @@ var _ = Describe("NodeTemplate/InstanceTypes", func() { }) Context("Eviction Thresholds", func() { BeforeEach(func() { - ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ VMMemoryOverheadPercent: lo.ToPtr[float64](0), })) }) @@ -990,7 +983,7 @@ var _ = Describe("NodeTemplate/InstanceTypes", func() { } }) It("should reserve ENIs when aws.reservedENIs is set and is used in max-pods calculation", func() { - ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ ReservedENIs: lo.ToPtr(1), })) @@ -1010,7 +1003,7 @@ var _ = Describe("NodeTemplate/InstanceTypes", func() { Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", maxPods)) }) It("should reserve ENIs when aws.reservedENIs is set and not go below 0 ENIs in max-pods calculation", func() { - ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ ReservedENIs: lo.ToPtr(1_000_000), })) diff --git a/pkg/providers/instancetype/suite_test.go b/pkg/providers/instancetype/suite_test.go index e0a4d03ab0ed..4a56a7bcbc62 100644 --- a/pkg/providers/instancetype/suite_test.go +++ b/pkg/providers/instancetype/suite_test.go @@ -29,14 +29,12 @@ import ( clock "k8s.io/utils/clock/testing" . "knative.dev/pkg/logging/testing" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" "github.com/aws/karpenter-core/pkg/controllers/provisioning" "github.com/aws/karpenter-core/pkg/controllers/state" "github.com/aws/karpenter-core/pkg/events" - "github.com/aws/karpenter-core/pkg/operator/injection" - "github.com/aws/karpenter-core/pkg/operator/options" + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/operator/scheme" coretest "github.com/aws/karpenter-core/pkg/test" . "github.com/aws/karpenter-core/pkg/test/expectations" @@ -44,12 +42,12 @@ import ( "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/cloudprovider" "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/providers/pricing" "github.com/aws/karpenter/pkg/test" ) var ctx context.Context -var opts options.Options var env *coretest.Environment var awsEnv *test.Environment var fakeClock *clock.FakeClock @@ -65,7 +63,8 @@ func TestAWS(t *testing.T) { var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) awsEnv = test.NewEnvironment(ctx, env) fakeClock = &clock.FakeClock{} @@ -80,8 +79,8 @@ var _ = AfterSuite(func() { }) var _ = BeforeEach(func() { - ctx = injection.WithOptions(ctx, opts) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) cluster.Reset() awsEnv.Reset() @@ -129,7 +128,7 @@ func generateSpotPricing(cp *cloudprovider.CloudProvider, nodePool *corev1beta1. func makeFakeInstances() []*ec2.InstanceTypeInfo { var instanceTypes []*ec2.InstanceTypeInfo - ctx := settings.ToContext(context.Background(), &settings.Settings{IsolatedVPC: true}) + ctx := options.ToContext(context.Background(), &options.Options{IsolatedVPC: true}) // Use keys from the static pricing data so that we guarantee pricing for the data // Create uniform instance data so all of them schedule for a given pod for _, it := range pricing.NewProvider(ctx, nil, nil, "us-east-1").InstanceTypes() { diff --git a/pkg/providers/instancetype/types.go b/pkg/providers/instancetype/types.go index a126ac84615d..5f132f1c6cb3 100644 --- a/pkg/providers/instancetype/types.go +++ b/pkg/providers/instancetype/types.go @@ -30,9 +30,10 @@ import ( "knative.dev/pkg/ptr" corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" - awssettings "github.com/aws/karpenter/pkg/apis/settings" + "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" "github.com/aws/karpenter/pkg/apis/v1beta1" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/providers/amifamily" "github.com/aws/karpenter-core/pkg/cloudprovider" @@ -201,7 +202,7 @@ func memory(ctx context.Context, info *ec2.InstanceTypeInfo) *resource.Quantity } mem := resources.Quantity(fmt.Sprintf("%dMi", sizeInMib)) // Account for VM overhead in calculation - mem.Sub(resource.MustParse(fmt.Sprintf("%dMi", int64(math.Ceil(float64(mem.Value())*awssettings.FromContext(ctx).VMMemoryOverheadPercent/1024/1024))))) + mem.Sub(resource.MustParse(fmt.Sprintf("%dMi", int64(math.Ceil(float64(mem.Value())*options.FromContext(ctx).VMMemoryOverheadPercent/1024/1024))))) return mem } @@ -240,7 +241,7 @@ func ephemeralStorage(amiFamily amifamily.AMIFamily, blockDeviceMappings []*v1be func awsPodENI(ctx context.Context, name string) *resource.Quantity { // https://docs.aws.amazon.com/eks/latest/userguide/security-groups-for-pods.html#supported-instance-types limits, ok := Limits[name] - if awssettings.FromContext(ctx).EnablePodENI && ok && limits.IsTrunkingCompatible { + if settings.FromContext(ctx).EnablePodENI && ok && limits.IsTrunkingCompatible { return resources.Quantity(fmt.Sprint(limits.BranchInterface)) } return resources.Quantity("0") @@ -308,7 +309,7 @@ func ENILimitedPods(ctx context.Context, info *ec2.InstanceTypeInfo) *resource.Q // VPC CNI only uses the default network interface // https://github.com/aws/amazon-vpc-cni-k8s/blob/3294231c0dce52cfe473bf6c62f47956a3b333b6/scripts/gen_vpc_ip_limits.go#L162 networkInterfaces := *info.NetworkInfo.NetworkCards[*info.NetworkInfo.DefaultNetworkCardIndex].MaximumNetworkInterfaces - usableNetworkInterfaces := lo.Max([]int64{(networkInterfaces - int64(awssettings.FromContext(ctx).ReservedENIs)), 0}) + usableNetworkInterfaces := lo.Max([]int64{(networkInterfaces - int64(options.FromContext(ctx).ReservedENIs)), 0}) if usableNetworkInterfaces == 0 { return resource.NewQuantity(0, resource.DecimalSI) } @@ -403,7 +404,7 @@ func pods(ctx context.Context, info *ec2.InstanceTypeInfo, amiFamily amifamily.A switch { case kc != nil && kc.MaxPods != nil: count = int64(ptr.Int32Value(kc.MaxPods)) - case awssettings.FromContext(ctx).EnableENILimitedPodDensity && amiFamily.FeatureFlags().SupportsENILimitedPodDensity: + case settings.FromContext(ctx).EnableENILimitedPodDensity && amiFamily.FeatureFlags().SupportsENILimitedPodDensity: count = ENILimitedPods(ctx, info).Value() default: count = 110 diff --git a/pkg/providers/launchtemplate/launchtemplate.go b/pkg/providers/launchtemplate/launchtemplate.go index f47b1504fc45..09a1f65f38a6 100644 --- a/pkg/providers/launchtemplate/launchtemplate.go +++ b/pkg/providers/launchtemplate/launchtemplate.go @@ -39,6 +39,7 @@ import ( "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1beta1" awserrors "github.com/aws/karpenter/pkg/errors" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/providers/amifamily" "github.com/aws/karpenter/pkg/providers/instanceprofile" "github.com/aws/karpenter/pkg/providers/securitygroup" @@ -172,7 +173,7 @@ func (p *Provider) createAMIOptions(ctx context.Context, nodeClass *v1beta1.EC2N return nil, fmt.Errorf("no security groups exist given constraints") } options := &amifamily.Options{ - ClusterName: settings.FromContext(ctx).ClusterName, + ClusterName: options.FromContext(ctx).ClusterName, ClusterEndpoint: p.ClusterEndpoint, AWSENILimitedPodDensity: settings.FromContext(ctx).EnableENILimitedPodDensity, InstanceProfile: instanceProfile, @@ -353,7 +354,7 @@ func (p *Provider) volumeSize(quantity *resource.Quantity) *int64 { // hydrateCache queries for existing Launch Templates created by Karpenter for the current cluster and adds to the LT cache. // Any error during hydration will result in a panic func (p *Provider) hydrateCache(ctx context.Context) { - clusterName := settings.FromContext(ctx).ClusterName + clusterName := options.FromContext(ctx).ClusterName ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("tag-key", karpenterManagedTagKey, "tag-value", clusterName)) if err := p.ec2api.DescribeLaunchTemplatesPagesWithContext(ctx, &ec2.DescribeLaunchTemplatesInput{ Filters: []*ec2.Filter{{Name: aws.String(fmt.Sprintf("tag:%s", karpenterManagedTagKey)), Values: []*string{aws.String(clusterName)}}}, diff --git a/pkg/providers/launchtemplate/nodeclass_test.go b/pkg/providers/launchtemplate/nodeclass_test.go index 0d1f2281aada..86a5d9941caf 100644 --- a/pkg/providers/launchtemplate/nodeclass_test.go +++ b/pkg/providers/launchtemplate/nodeclass_test.go @@ -42,6 +42,7 @@ import ( "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1beta1" "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/providers/amifamily/bootstrap" "github.com/aws/karpenter/pkg/providers/instancetype" "github.com/aws/karpenter/pkg/test" @@ -692,7 +693,10 @@ var _ = Describe("EC2NodeClass/LaunchTemplates", func() { It("should calculate memory overhead based on eni limited pods when ENI limited", func() { ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ EnableENILimitedPodDensity: lo.ToPtr(false), - VMMemoryOverheadPercent: lo.ToPtr[float64](0), + })) + + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ + VMMemoryOverheadPercent: lo.ToPtr[float64](0), })) nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 @@ -703,7 +707,10 @@ var _ = Describe("EC2NodeClass/LaunchTemplates", func() { It("should calculate memory overhead based on eni limited pods when not ENI limited", func() { ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ EnableENILimitedPodDensity: lo.ToPtr(false), - VMMemoryOverheadPercent: lo.ToPtr[float64](0), + })) + + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ + VMMemoryOverheadPercent: lo.ToPtr[float64](0), })) nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 @@ -740,9 +747,9 @@ var _ = Describe("EC2NodeClass/LaunchTemplates", func() { }) It("should calculate memory overhead based on eni limited pods when ENI limited", func() { - ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ - EnableENILimitedPodDensity: lo.ToPtr(true), - VMMemoryOverheadPercent: lo.ToPtr[float64](0), + ctx = settings.ToContext(ctx, test.Settings()) + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ + VMMemoryOverheadPercent: lo.ToPtr[float64](0), })) nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket @@ -753,7 +760,10 @@ var _ = Describe("EC2NodeClass/LaunchTemplates", func() { It("should calculate memory overhead based on max pods when not ENI limited", func() { ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ EnableENILimitedPodDensity: lo.ToPtr(false), - VMMemoryOverheadPercent: lo.ToPtr[float64](0), + })) + + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ + VMMemoryOverheadPercent: lo.ToPtr[float64](0), })) nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket @@ -877,6 +887,11 @@ var _ = Describe("EC2NodeClass/LaunchTemplates", func() { "nodefs.available": "15%", "nodefs.inodesFree": "5%", }, + EvictionSoftGracePeriod: map[string]metav1.Duration{ + "memory.available": {Duration: time.Minute}, + "nodefs.available": {Duration: time.Second * 180}, + "nodefs.inodesFree": {Duration: time.Minute * 5}, + }, } ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod() @@ -904,6 +919,11 @@ var _ = Describe("EC2NodeClass/LaunchTemplates", func() { "nodefs.available": {Duration: time.Second * 180}, "nodefs.inodesFree": {Duration: time.Minute * 5}, }, + EvictionSoft: map[string]string{ + "memory.available": "10%", + "nodefs.available": "15%", + "nodefs.inodesFree": "5%", + }, } ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod() diff --git a/pkg/providers/launchtemplate/nodetemplate_test.go b/pkg/providers/launchtemplate/nodetemplate_test.go index 84107056148a..f5ba265aaee1 100644 --- a/pkg/providers/launchtemplate/nodetemplate_test.go +++ b/pkg/providers/launchtemplate/nodetemplate_test.go @@ -39,6 +39,7 @@ import ( "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/providers/amifamily/bootstrap" "github.com/aws/karpenter/pkg/providers/instancetype" "github.com/aws/karpenter/pkg/test" @@ -707,7 +708,10 @@ var _ = Describe("NodeTemplate/LaunchTemplates", func() { It("should calculate memory overhead based on eni limited pods when ENI limited", func() { ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ EnableENILimitedPodDensity: lo.ToPtr(false), - VMMemoryOverheadPercent: lo.ToPtr[float64](0), + })) + + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ + VMMemoryOverheadPercent: lo.ToPtr[float64](0), })) nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyAL2 @@ -718,7 +722,10 @@ var _ = Describe("NodeTemplate/LaunchTemplates", func() { It("should calculate memory overhead based on eni limited pods when not ENI limited", func() { ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ EnableENILimitedPodDensity: lo.ToPtr(false), - VMMemoryOverheadPercent: lo.ToPtr[float64](0), + })) + + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ + VMMemoryOverheadPercent: lo.ToPtr[float64](0), })) nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyAL2 @@ -755,9 +762,10 @@ var _ = Describe("NodeTemplate/LaunchTemplates", func() { }) It("should calculate memory overhead based on eni limited pods when ENI limited", func() { - ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ - EnableENILimitedPodDensity: lo.ToPtr(true), - VMMemoryOverheadPercent: lo.ToPtr[float64](0), + ctx = settings.ToContext(ctx, test.Settings()) + + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ + VMMemoryOverheadPercent: lo.ToPtr[float64](0), })) nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyBottlerocket @@ -768,7 +776,10 @@ var _ = Describe("NodeTemplate/LaunchTemplates", func() { It("should calculate memory overhead based on max pods when not ENI limited", func() { ctx = settings.ToContext(ctx, test.Settings(test.SettingOptions{ EnableENILimitedPodDensity: lo.ToPtr(false), - VMMemoryOverheadPercent: lo.ToPtr[float64](0), + })) + + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ + VMMemoryOverheadPercent: lo.ToPtr[float64](0), })) nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyBottlerocket diff --git a/pkg/providers/launchtemplate/suite_test.go b/pkg/providers/launchtemplate/suite_test.go index 510bad14e700..046648db9f71 100644 --- a/pkg/providers/launchtemplate/suite_test.go +++ b/pkg/providers/launchtemplate/suite_test.go @@ -30,24 +30,22 @@ import ( clock "k8s.io/utils/clock/testing" . "knative.dev/pkg/logging/testing" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" "github.com/aws/karpenter-core/pkg/controllers/provisioning" "github.com/aws/karpenter-core/pkg/controllers/state" "github.com/aws/karpenter-core/pkg/events" - "github.com/aws/karpenter-core/pkg/operator/injection" - "github.com/aws/karpenter-core/pkg/operator/options" + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/operator/scheme" coretest "github.com/aws/karpenter-core/pkg/test" . "github.com/aws/karpenter-core/pkg/test/expectations" "github.com/aws/karpenter/pkg/apis" "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/cloudprovider" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/test" ) var ctx context.Context var stop context.CancelFunc -var opts options.Options var env *coretest.Environment var awsEnv *test.Environment var fakeClock *clock.FakeClock @@ -63,7 +61,8 @@ func TestAWS(t *testing.T) { var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) ctx, stop = context.WithCancel(ctx) awsEnv = test.NewEnvironment(ctx, env) @@ -81,8 +80,8 @@ var _ = AfterSuite(func() { }) var _ = BeforeEach(func() { - ctx = injection.WithOptions(ctx, opts) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) cluster.Reset() awsEnv.Reset() diff --git a/pkg/providers/license/suite_test.go b/pkg/providers/license/suite_test.go index 0dde5e9037b1..e2b3420e1c6a 100644 --- a/pkg/providers/license/suite_test.go +++ b/pkg/providers/license/suite_test.go @@ -31,8 +31,6 @@ import ( "github.com/aws/karpenter/pkg/test" coresettings "github.com/aws/karpenter-core/pkg/apis/settings" - "github.com/aws/karpenter-core/pkg/operator/injection" - "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/operator/scheme" coretest "github.com/aws/karpenter-core/pkg/test" . "github.com/aws/karpenter-core/pkg/test/expectations" @@ -40,7 +38,6 @@ import ( var ctx context.Context var stop context.CancelFunc -var opts options.Options var env *coretest.Environment var awsEnv *test.Environment var nodeClass *v1beta1.EC2NodeClass @@ -65,7 +62,6 @@ var _ = AfterSuite(func() { }) var _ = BeforeEach(func() { - ctx = injection.WithOptions(ctx, opts) ctx = coresettings.ToContext(ctx, coretest.Settings()) ctx = settings.ToContext(ctx, test.Settings()) nodeClass = test.EC2NodeClass(v1beta1.EC2NodeClass{ diff --git a/pkg/providers/placementgroup/suite_test.go b/pkg/providers/placementgroup/suite_test.go index 3b7f2492d6a0..c2703ccb9d73 100644 --- a/pkg/providers/placementgroup/suite_test.go +++ b/pkg/providers/placementgroup/suite_test.go @@ -30,8 +30,6 @@ import ( "github.com/aws/karpenter/pkg/test" coresettings "github.com/aws/karpenter-core/pkg/apis/settings" - "github.com/aws/karpenter-core/pkg/operator/injection" - "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/operator/scheme" coretest "github.com/aws/karpenter-core/pkg/test" . "github.com/aws/karpenter-core/pkg/test/expectations" @@ -39,7 +37,6 @@ import ( var ctx context.Context var stop context.CancelFunc -var opts options.Options var env *coretest.Environment var awsEnv *test.Environment var nodeClass *v1beta1.EC2NodeClass @@ -64,7 +61,6 @@ var _ = AfterSuite(func() { }) var _ = BeforeEach(func() { - ctx = injection.WithOptions(ctx, opts) ctx = coresettings.ToContext(ctx, coretest.Settings()) ctx = settings.ToContext(ctx, test.Settings()) nodeClass = test.EC2NodeClass(v1beta1.EC2NodeClass{ diff --git a/pkg/providers/pricing/suite_test.go b/pkg/providers/pricing/suite_test.go index 58aa3cfec975..029ed70322a5 100644 --- a/pkg/providers/pricing/suite_test.go +++ b/pkg/providers/pricing/suite_test.go @@ -29,9 +29,7 @@ import ( "k8s.io/apimachinery/pkg/types" . "knative.dev/pkg/logging/testing" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" - "github.com/aws/karpenter-core/pkg/operator/injection" - "github.com/aws/karpenter-core/pkg/operator/options" + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/operator/scheme" . "github.com/aws/karpenter-core/pkg/test/expectations" @@ -39,13 +37,13 @@ import ( "github.com/aws/karpenter/pkg/apis" "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/fake" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/providers/pricing" "github.com/aws/karpenter/pkg/test" ) var ctx context.Context var stop context.CancelFunc -var opts options.Options var env *coretest.Environment var awsEnv *test.Environment var controller *pricing.Controller @@ -58,7 +56,8 @@ func TestAWS(t *testing.T) { var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) ctx, stop = context.WithCancel(ctx) awsEnv = test.NewEnvironment(ctx, env) @@ -71,8 +70,8 @@ var _ = AfterSuite(func() { }) var _ = BeforeEach(func() { - ctx = injection.WithOptions(ctx, opts) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) awsEnv.Reset() diff --git a/pkg/providers/securitygroup/suite_test.go b/pkg/providers/securitygroup/suite_test.go index 3d71b990b07d..6cd9bd8cf88e 100644 --- a/pkg/providers/securitygroup/suite_test.go +++ b/pkg/providers/securitygroup/suite_test.go @@ -28,11 +28,10 @@ import ( "github.com/aws/karpenter/pkg/apis" "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1beta1" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/test" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" - "github.com/aws/karpenter-core/pkg/operator/injection" - "github.com/aws/karpenter-core/pkg/operator/options" + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/operator/scheme" coretest "github.com/aws/karpenter-core/pkg/test" . "github.com/aws/karpenter-core/pkg/test/expectations" @@ -40,7 +39,6 @@ import ( var ctx context.Context var stop context.CancelFunc -var opts options.Options var env *coretest.Environment var awsEnv *test.Environment var nodeClass *v1beta1.EC2NodeClass @@ -53,7 +51,8 @@ func TestAWS(t *testing.T) { var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) ctx, stop = context.WithCancel(ctx) awsEnv = test.NewEnvironment(ctx, env) @@ -65,8 +64,8 @@ var _ = AfterSuite(func() { }) var _ = BeforeEach(func() { - ctx = injection.WithOptions(ctx, opts) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) nodeClass = test.EC2NodeClass(v1beta1.EC2NodeClass{ Spec: v1beta1.EC2NodeClassSpec{ diff --git a/pkg/providers/subnet/suite_test.go b/pkg/providers/subnet/suite_test.go index 7efccecc24f4..1c8b8f240e6d 100644 --- a/pkg/providers/subnet/suite_test.go +++ b/pkg/providers/subnet/suite_test.go @@ -28,11 +28,10 @@ import ( "github.com/aws/karpenter/pkg/apis" "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1beta1" + "github.com/aws/karpenter/pkg/operator/options" "github.com/aws/karpenter/pkg/test" - coresettings "github.com/aws/karpenter-core/pkg/apis/settings" - "github.com/aws/karpenter-core/pkg/operator/injection" - "github.com/aws/karpenter-core/pkg/operator/options" + coreoptions "github.com/aws/karpenter-core/pkg/operator/options" "github.com/aws/karpenter-core/pkg/operator/scheme" coretest "github.com/aws/karpenter-core/pkg/test" . "github.com/aws/karpenter-core/pkg/test/expectations" @@ -40,7 +39,6 @@ import ( var ctx context.Context var stop context.CancelFunc -var opts options.Options var env *coretest.Environment var awsEnv *test.Environment var nodeClass *v1beta1.EC2NodeClass @@ -53,7 +51,8 @@ func TestAWS(t *testing.T) { var _ = BeforeSuite(func() { env = coretest.NewEnvironment(scheme.Scheme, coretest.WithCRDs(apis.CRDs...)) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) ctx, stop = context.WithCancel(ctx) awsEnv = test.NewEnvironment(ctx, env) @@ -65,8 +64,8 @@ var _ = AfterSuite(func() { }) var _ = BeforeEach(func() { - ctx = injection.WithOptions(ctx, opts) - ctx = coresettings.ToContext(ctx, coretest.Settings()) + ctx = coreoptions.ToContext(ctx, coretest.Options()) + ctx = options.ToContext(ctx, test.Options()) ctx = settings.ToContext(ctx, test.Settings()) nodeClass = test.EC2NodeClass(v1beta1.EC2NodeClass{ Spec: v1beta1.EC2NodeClassSpec{ diff --git a/pkg/test/nodeclass.go b/pkg/test/nodeclass.go index 8c780f497a8c..54b7f49b1e49 100644 --- a/pkg/test/nodeclass.go +++ b/pkg/test/nodeclass.go @@ -68,7 +68,7 @@ func EC2NodeClass(overrides ...v1beta1.EC2NodeClass) *v1beta1.EC2NodeClass { func EC2NodeClassFieldIndexer(ctx context.Context) func(cache.Cache) error { return func(c cache.Cache) error { - return c.IndexField(ctx, &corev1beta1.NodeClaim{}, "spec.nodeClass.name", func(obj client.Object) []string { + return c.IndexField(ctx, &corev1beta1.NodeClaim{}, "spec.nodeClassRef.name", func(obj client.Object) []string { nc := obj.(*corev1beta1.NodeClaim) if nc.Spec.NodeClassRef == nil { return []string{""} diff --git a/pkg/test/options.go b/pkg/test/options.go new file mode 100644 index 000000000000..609c118bcb2a --- /dev/null +++ b/pkg/test/options.go @@ -0,0 +1,57 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package test + +import ( + "fmt" + "time" + + "github.com/imdario/mergo" + "github.com/samber/lo" + + "github.com/aws/karpenter/pkg/operator/options" +) + +type OptionsFields struct { + AssumeRoleARN *string + AssumeRoleDuration *time.Duration + ClusterCABundle *string + ClusterName *string + ClusterEndpoint *string + IsolatedVPC *bool + VMMemoryOverheadPercent *float64 + InterruptionQueue *string + ReservedENIs *int +} + +func Options(overrides ...OptionsFields) *options.Options { + opts := OptionsFields{} + for _, override := range overrides { + if err := mergo.Merge(&opts, override, mergo.WithOverride); err != nil { + panic(fmt.Sprintf("Failed to merge settings: %s", err)) + } + } + return &options.Options{ + AssumeRoleARN: lo.FromPtrOr(opts.AssumeRoleARN, ""), + AssumeRoleDuration: lo.FromPtrOr(opts.AssumeRoleDuration, 15*time.Minute), + ClusterCABundle: lo.FromPtrOr(opts.ClusterCABundle, ""), + ClusterName: lo.FromPtrOr(opts.ClusterName, "test-cluster"), + ClusterEndpoint: lo.FromPtrOr(opts.ClusterEndpoint, "https://test-cluster"), + IsolatedVPC: lo.FromPtrOr(opts.IsolatedVPC, false), + VMMemoryOverheadPercent: lo.FromPtrOr(opts.VMMemoryOverheadPercent, 0.075), + InterruptionQueue: lo.FromPtrOr(opts.InterruptionQueue, ""), + ReservedENIs: lo.FromPtrOr(opts.ReservedENIs, 0), + } +} diff --git a/pkg/utils/project/project.go b/pkg/utils/project/project.go deleted file mode 100644 index 53d5cf3212d6..000000000000 --- a/pkg/utils/project/project.go +++ /dev/null @@ -1,19 +0,0 @@ -/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package project - -// Version is the karpenter app version injected during compilation -// when using the Makefile -var Version = "unspecified" diff --git a/pkg/webhooks/webhooks.go b/pkg/webhooks/webhooks.go index dac5c78ca289..da45794e8866 100644 --- a/pkg/webhooks/webhooks.go +++ b/pkg/webhooks/webhooks.go @@ -28,6 +28,7 @@ import ( corev1alpha5 "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter/pkg/apis/v1alpha1" "github.com/aws/karpenter/pkg/apis/v1alpha5" + "github.com/aws/karpenter/pkg/apis/v1beta1" ) func NewWebhooks() []knativeinjection.ControllerConstructor { @@ -60,4 +61,5 @@ func NewCRDValidationWebhook(ctx context.Context, _ configmap.Watcher) *controll var Resources = map[schema.GroupVersionKind]resourcesemantics.GenericCRD{ v1alpha1.SchemeGroupVersion.WithKind("AWSNodeTemplate"): &v1alpha1.AWSNodeTemplate{}, corev1alpha5.SchemeGroupVersion.WithKind("Provisioner"): &v1alpha5.Provisioner{}, + v1beta1.SchemeGroupVersion.WithKind("EC2NodeClass"): &v1beta1.EC2NodeClass{}, } diff --git a/test/cloudformation/iam_cloudformation.yaml b/test/cloudformation/iam_cloudformation.yaml index cb8a8c7a4c03..2263997cfbe2 100644 --- a/test/cloudformation/iam_cloudformation.yaml +++ b/test/cloudformation/iam_cloudformation.yaml @@ -148,8 +148,7 @@ Resources: - eks:ListFargateProfiles - eks:TagResource - eks:DescribeCluster - Resource: - - !Sub "arn:${AWS::Partition}:eks:*:${AWS::AccountId}:cluster/*" + Resource: !Sub "arn:${AWS::Partition}:eks:*:${AWS::AccountId}:cluster/*" Condition: StringEquals: aws:RequestedRegion: @@ -169,16 +168,17 @@ Resources: - eks:DeleteNodegroup - eks:DescribeNodegroup - eks:TagResource - Resource: - - !Sub "arn:${AWS::Partition}:eks:*:${AWS::AccountId}:nodegroup/*" + Resource: !Sub "arn:${AWS::Partition}:eks:*:${AWS::AccountId}:nodegroup/*" Condition: StringEquals: aws:RequestedRegion: Ref: Regions + - Effect: Allow + Action: logs:PutRetentionPolicy + Resource: !Sub "arn:aws:logs:*:${AWS::AccountId}:log-group:/aws/eks/*" - Effect: Allow Action: fis:CreateExperimentTemplate - Resource: - - !Sub "arn:${AWS::Partition}:fis:*:${AWS::AccountId}:action/*" + Resource: !Sub "arn:${AWS::Partition}:fis:*:${AWS::AccountId}:action/*" Condition: StringEquals: aws:RequestedRegion: diff --git a/test/hack/cleanup/main.go b/test/hack/cleanup/main.go index 3304d173cd02..9b5c9fe8ac71 100644 --- a/test/hack/cleanup/main.go +++ b/test/hack/cleanup/main.go @@ -372,7 +372,7 @@ func (o *oidc) Get(ctx context.Context, expirationTime time.Time) (names []strin return names, multierr.Combine(errs...) } -// Terminate any old OIDC providers that were are remaining as part of testing +// Cleanup any old OIDC providers that were are remaining as part of testing // We execute these in serial since we will most likely get rate limited if we try to delete these too aggressively func (o *oidc) Cleanup(ctx context.Context, arns []string) ([]string, error) { var errs error @@ -415,7 +415,10 @@ func (ip *instanceProfile) Get(ctx context.Context, expirationTime time.Time) (n } for _, t := range profiles.Tags { - if lo.FromPtr(t.Key) == karpenterTestingTag || lo.FromPtr(t.Key) == karpenterTestingTagLegacy && out.InstanceProfiles[i].CreateDate.Before(expirationTime) { + // Since we can only get the date of the instance profile (not the exact time the instance profile was created) + // we add a day to the time that it was created to account for the worst-case of the instance profile being created + // at 23:59:59 and being marked with a time of 00:00:00 due to only capturing the date and not the time + if lo.FromPtr(t.Key) == karpenterTestingTag && out.InstanceProfiles[i].CreateDate.Add(time.Hour*24).Before(expirationTime) { names = append(names, lo.FromPtr(out.InstanceProfiles[i].InstanceProfileName)) } } diff --git a/test/pkg/debug/monitor.go b/test/pkg/debug/monitor.go index 7230cbab36f3..08217318e165 100644 --- a/test/pkg/debug/monitor.go +++ b/test/pkg/debug/monitor.go @@ -24,6 +24,7 @@ import ( controllerruntime "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/metrics/server" "github.com/aws/karpenter-core/pkg/operator/controller" "github.com/aws/karpenter-core/pkg/operator/scheme" @@ -46,7 +47,9 @@ func New(ctx context.Context, config *rest.Config, kubeClient client.Client) *Mo logger.WithOptions() return ctx }, - MetricsBindAddress: "0", + Metrics: server.Options{ + BindAddress: "0", + }, })) for _, c := range newControllers(kubeClient) { lo.Must0(c.Builder(ctx, mgr).Complete(c), "failed to register controller") diff --git a/test/pkg/environment/aws/environment.go b/test/pkg/environment/aws/environment.go index 7961472461b2..9fc111b9759c 100644 --- a/test/pkg/environment/aws/environment.go +++ b/test/pkg/environment/aws/environment.go @@ -15,6 +15,7 @@ limitations under the License. package aws import ( + "os" "testing" "github.com/aws/aws-sdk-go/aws" @@ -54,6 +55,10 @@ type Environment struct { TimeStreamAPI timestreamwriteiface.TimestreamWriteAPI SQSProvider *interruption.SQSProvider + + ClusterName string + ClusterEndpoint string + InterruptionQueue string } func NewEnvironment(t *testing.T) *Environment { @@ -80,6 +85,10 @@ func NewEnvironment(t *testing.T) *Environment { EKSAPI: eks.New(session), SQSProvider: interruption.NewSQSProvider(sqs.New(session)), TimeStreamAPI: GetTimeStreamAPI(session), + + ClusterName: lo.Must(os.LookupEnv("CLUSTER_NAME")), + ClusterEndpoint: lo.Must(os.LookupEnv("CLUSTER_ENDPOINT")), + InterruptionQueue: lo.Must(os.LookupEnv("INTERRUPTION_QUEUE")), } } diff --git a/test/pkg/environment/aws/expectations.go b/test/pkg/environment/aws/expectations.go index 809bed9ee652..6d910e99eb85 100644 --- a/test/pkg/environment/aws/expectations.go +++ b/test/pkg/environment/aws/expectations.go @@ -123,44 +123,44 @@ func (env *Environment) ExpectInstanceProfileExists(profileName string) iam.Inst func (env *Environment) GetInstance(nodeName string) ec2.Instance { node := env.Environment.GetNode(nodeName) - return env.GetInstanceByIDWithOffset(1, env.ExpectParsedProviderID(node.Spec.ProviderID)) + return env.GetInstanceByID(env.ExpectParsedProviderID(node.Spec.ProviderID)) } func (env *Environment) ExpectInstanceStopped(nodeName string) { + GinkgoHelper() node := env.Environment.GetNode(nodeName) _, err := env.EC2API.StopInstances(&ec2.StopInstancesInput{ Force: aws.Bool(true), InstanceIds: aws.StringSlice([]string{env.ExpectParsedProviderID(node.Spec.ProviderID)}), }) - ExpectWithOffset(1, err).To(Succeed()) + Expect(err).To(Succeed()) } func (env *Environment) ExpectInstanceTerminated(nodeName string) { + GinkgoHelper() node := env.Environment.GetNode(nodeName) _, err := env.EC2API.TerminateInstances(&ec2.TerminateInstancesInput{ InstanceIds: aws.StringSlice([]string{env.ExpectParsedProviderID(node.Spec.ProviderID)}), }) - ExpectWithOffset(1, err).To(Succeed()) + Expect(err).To(Succeed()) } func (env *Environment) GetInstanceByID(instanceID string) ec2.Instance { - return env.GetInstanceByIDWithOffset(1, instanceID) -} - -func (env *Environment) GetInstanceByIDWithOffset(offset int, instanceID string) ec2.Instance { + GinkgoHelper() instance, err := env.EC2API.DescribeInstances(&ec2.DescribeInstancesInput{ InstanceIds: aws.StringSlice([]string{instanceID}), }) - ExpectWithOffset(offset+1, err).ToNot(HaveOccurred()) - ExpectWithOffset(offset+1, instance.Reservations).To(HaveLen(1)) - ExpectWithOffset(offset+1, instance.Reservations[0].Instances).To(HaveLen(1)) + Expect(err).ToNot(HaveOccurred()) + Expect(instance.Reservations).To(HaveLen(1)) + Expect(instance.Reservations[0].Instances).To(HaveLen(1)) return *instance.Reservations[0].Instances[0] } func (env *Environment) GetVolume(volumeID *string) ec2.Volume { + GinkgoHelper() dvo, err := env.EC2API.DescribeVolumes(&ec2.DescribeVolumesInput{VolumeIds: []*string{volumeID}}) - ExpectWithOffset(1, err).ToNot(HaveOccurred()) - ExpectWithOffset(1, len(dvo.Volumes)).To(Equal(1)) + Expect(err).ToNot(HaveOccurred()) + Expect(len(dvo.Volumes)).To(Equal(1)) return *dvo.Volumes[0] } @@ -244,12 +244,14 @@ func (env *Environment) GetSecurityGroups(tags map[string]string) []SecurityGrou } func (env *Environment) ExpectQueueExists() { + GinkgoHelper() exists, err := env.SQSProvider.QueueExists(env.Context) - ExpectWithOffset(1, err).ToNot(HaveOccurred()) - ExpectWithOffset(1, exists).To(BeTrue()) + Expect(err).ToNot(HaveOccurred()) + Expect(exists).To(BeTrue()) } func (env *Environment) ExpectMessagesCreated(msgs ...interface{}) { + GinkgoHelper() wg := &sync.WaitGroup{} mu := &sync.Mutex{} @@ -268,12 +270,13 @@ func (env *Environment) ExpectMessagesCreated(msgs ...interface{}) { }(msg) } wg.Wait() - ExpectWithOffset(1, err).To(Succeed()) + Expect(err).To(Succeed()) } func (env *Environment) ExpectParsedProviderID(providerID string) string { + GinkgoHelper() providerIDSplit := strings.Split(providerID, "/") - ExpectWithOffset(1, len(providerIDSplit)).ToNot(Equal(0)) + Expect(len(providerIDSplit)).ToNot(Equal(0)) return providerIDSplit[len(providerIDSplit)-1] } diff --git a/test/pkg/environment/aws/setup.go b/test/pkg/environment/aws/setup.go index 9cfacdc52469..461264f3d7fc 100644 --- a/test/pkg/environment/aws/setup.go +++ b/test/pkg/environment/aws/setup.go @@ -22,7 +22,7 @@ import ( "github.com/aws/karpenter/pkg/apis/v1alpha1" ) -var persistedSettings *v1.ConfigMap +var persistedSettings = &v1.ConfigMap{} var ( CleanableObjects = []client.Object{ diff --git a/test/pkg/environment/common/environment.go b/test/pkg/environment/common/environment.go index bc41d081d3f5..026aa71f8368 100644 --- a/test/pkg/environment/common/environment.go +++ b/test/pkg/environment/common/environment.go @@ -37,9 +37,9 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" coreapis "github.com/aws/karpenter-core/pkg/apis" + "github.com/aws/karpenter-core/pkg/operator" "github.com/aws/karpenter-core/pkg/operator/injection" "github.com/aws/karpenter/pkg/apis" - "github.com/aws/karpenter/pkg/utils/project" ) type ContextKey string @@ -91,7 +91,7 @@ func (env *Environment) Stop() { func NewConfig() *rest.Config { config := controllerruntime.GetConfigOrDie() - config.UserAgent = fmt.Sprintf("testing-%s", project.Version) + config.UserAgent = fmt.Sprintf("testing-%s", operator.Version) config.QPS = 1e6 config.Burst = 1e6 return config @@ -116,10 +116,9 @@ func NewClient(ctx context.Context, config *rest.Config) client.Client { node := o.(*v1.Node) return []string{strconv.FormatBool(node.Spec.Unschedulable)} })) - c := lo.Must(client.NewDelegatingClient(client.NewDelegatingClientInput{ - CacheReader: cache, - Client: lo.Must(client.New(config, client.Options{Scheme: scheme})), - })) + + c := lo.Must(client.New(config, client.Options{Scheme: scheme, Cache: &client.CacheOptions{Reader: cache}})) + go func() { lo.Must0(cache.Start(ctx)) }() diff --git a/test/pkg/environment/common/expectations.go b/test/pkg/environment/common/expectations.go index 8f22b8e433e9..ab856d96e318 100644 --- a/test/pkg/environment/common/expectations.go +++ b/test/pkg/environment/common/expectations.go @@ -188,13 +188,13 @@ func (env *Environment) ExpectPodENIDisabled() { func (env *Environment) ExpectPrefixDelegationEnabled() { GinkgoHelper() env.ExpectDaemonSetEnvironmentVariableUpdated(types.NamespacedName{Namespace: "kube-system", Name: "aws-node"}, - "ENABLE_PREFIX_DELEGATION", "true") + "ENABLE_PREFIX_DELEGATION", "true", "aws-node") } func (env *Environment) ExpectPrefixDelegationDisabled() { GinkgoHelper() env.ExpectDaemonSetEnvironmentVariableUpdated(types.NamespacedName{Namespace: "kube-system", Name: "aws-node"}, - "ENABLE_PREFIX_DELEGATION", "false") + "ENABLE_PREFIX_DELEGATION", "false", "aws-node") } func (env *Environment) ExpectExists(obj client.Object) { @@ -299,7 +299,8 @@ func (env *Environment) ExpectActiveKarpenterPod() *v1.Pod { } func (env *Environment) EventuallyExpectPendingPodCount(selector labels.Selector, numPods int) { - EventuallyWithOffset(1, func(g Gomega) { + GinkgoHelper() + Eventually(func(g Gomega) { g.Expect(env.Monitor.PendingPodsCount(selector)).To(Equal(numPods)) }).Should(Succeed()) } @@ -312,7 +313,7 @@ func (env *Environment) EventuallyExpectHealthyPodCount(selector labels.Selector func (env *Environment) EventuallyExpectHealthyPodCountWithTimeout(timeout time.Duration, selector labels.Selector, numPods int) { GinkgoHelper() - EventuallyWithOffset(1, func(g Gomega) { + Eventually(func(g Gomega) { g.Expect(env.Monitor.RunningPodsCount(selector)).To(Equal(numPods)) }).WithTimeout(timeout).Should(Succeed()) } @@ -326,35 +327,30 @@ func (env *Environment) ExpectPodsMatchingSelector(selector labels.Selector) []* } func (env *Environment) ExpectUniqueNodeNames(selector labels.Selector, uniqueNames int) { + GinkgoHelper() pods := env.Monitor.RunningPods(selector) nodeNames := sets.NewString() for _, pod := range pods { nodeNames.Insert(pod.Spec.NodeName) } - ExpectWithOffset(1, len(nodeNames)).To(BeNumerically("==", uniqueNames)) + Expect(len(nodeNames)).To(BeNumerically("==", uniqueNames)) } func (env *Environment) eventuallyExpectScaleDown() { - EventuallyWithOffset(1, func(g Gomega) { + GinkgoHelper() + Eventually(func(g Gomega) { // expect the current node count to be what it was when the test started g.Expect(env.Monitor.NodeCount()).To(Equal(env.StartingNodeCount)) }).Should(Succeed(), fmt.Sprintf("expected scale down to %d nodes, had %d", env.StartingNodeCount, env.Monitor.NodeCount())) } func (env *Environment) EventuallyExpectNotFound(objects ...client.Object) { - env.EventuallyExpectNotFoundWithOffset(1, objects...) -} - -func (env *Environment) EventuallyExpectNotFoundWithOffset(offset int, objects ...client.Object) { - env.EventuallyExpectNotFoundAssertionWithOffset(offset+1, objects...).Should(Succeed()) + GinkgoHelper() + env.EventuallyExpectNotFoundAssertion(objects...).Should(Succeed()) } func (env *Environment) EventuallyExpectNotFoundAssertion(objects ...client.Object) AsyncAssertion { - return env.EventuallyExpectNotFoundAssertionWithOffset(1, objects...) -} - -func (env *Environment) EventuallyExpectNotFoundAssertionWithOffset(offset int, objects ...client.Object) AsyncAssertion { - return EventuallyWithOffset(offset, func(g Gomega) { + return Eventually(func(g Gomega) { for _, object := range objects { err := env.Client.Get(env, client.ObjectKeyFromObject(object), object) g.Expect(errors.IsNotFound(err)).To(BeTrue()) @@ -363,8 +359,9 @@ func (env *Environment) EventuallyExpectNotFoundAssertionWithOffset(offset int, } func (env *Environment) ExpectCreatedNodeCount(comparator string, count int) []*v1.Node { + GinkgoHelper() createdNodes := env.Monitor.CreatedNodes() - ExpectWithOffset(1, len(createdNodes)).To(BeNumerically(comparator, count), + Expect(len(createdNodes)).To(BeNumerically(comparator, count), fmt.Sprintf("expected %d created nodes, had %d (%v)", count, len(createdNodes), NodeNames(createdNodes))) return createdNodes } @@ -454,9 +451,10 @@ func (env *Environment) EventuallyExpectNodeCountWithSelector(comparator string, } func (env *Environment) EventuallyExpectCreatedNodeCount(comparator string, count int) []*v1.Node { + GinkgoHelper() By(fmt.Sprintf("waiting for created nodes to be %s to %d", comparator, count)) var createdNodes []*v1.Node - EventuallyWithOffset(1, func(g Gomega) { + Eventually(func(g Gomega) { createdNodes = env.Monitor.CreatedNodes() g.Expect(len(createdNodes)).To(BeNumerically(comparator, count), fmt.Sprintf("expected %d created nodes, had %d (%v)", count, len(createdNodes), NodeNames(createdNodes))) @@ -492,9 +490,10 @@ func (env *Environment) EventuallyExpectDeletedNodeCountWithSelector(comparator } func (env *Environment) EventuallyExpectInitializedNodeCount(comparator string, count int) []*v1.Node { + GinkgoHelper() By(fmt.Sprintf("waiting for initialized nodes to be %s to %d", comparator, count)) var nodes []*v1.Node - EventuallyWithOffset(1, func(g Gomega) { + Eventually(func(g Gomega) { nodes = env.Monitor.CreatedNodes() nodes = lo.Filter(nodes, func(n *v1.Node, _ int) bool { return n.Labels[v1alpha5.LabelNodeInitialized] == "true" @@ -505,9 +504,10 @@ func (env *Environment) EventuallyExpectInitializedNodeCount(comparator string, } func (env *Environment) EventuallyExpectCreatedMachineCount(comparator string, count int) []*v1alpha5.Machine { + GinkgoHelper() By(fmt.Sprintf("waiting for created machines to be %s to %d", comparator, count)) machineList := &v1alpha5.MachineList{} - EventuallyWithOffset(1, func(g Gomega) { + Eventually(func(g Gomega) { g.Expect(env.Client.List(env.Context, machineList)).To(Succeed()) g.Expect(len(machineList.Items)).To(BeNumerically(comparator, count)) }).Should(Succeed()) @@ -527,16 +527,18 @@ func (env *Environment) EventuallyExpectMachinesReady(machines ...*v1alpha5.Mach } func (env *Environment) GetNode(nodeName string) v1.Node { + GinkgoHelper() var node v1.Node - ExpectWithOffset(1, env.Client.Get(env.Context, types.NamespacedName{Name: nodeName}, &node)).To(Succeed()) + Expect(env.Client.Get(env.Context, types.NamespacedName{Name: nodeName}, &node)).To(Succeed()) return node } func (env *Environment) ExpectNoCrashes() { + GinkgoHelper() _, crashed := lo.Find(lo.Values(env.Monitor.RestartCount()), func(restartCount int) bool { return restartCount > 0 }) - ExpectWithOffset(1, crashed).To(BeFalse(), "expected karpenter containers to not crash") + Expect(crashed).To(BeFalse(), "expected karpenter containers to not crash") } var ( @@ -573,13 +575,15 @@ func (env *Environment) printControllerLogs(options *v1.PodLogOptions) { } func (env *Environment) EventuallyExpectMinUtilization(resource v1.ResourceName, comparator string, value float64) { - EventuallyWithOffset(1, func(g Gomega) { + GinkgoHelper() + Eventually(func(g Gomega) { g.Expect(env.Monitor.MinUtilization(resource)).To(BeNumerically(comparator, value)) }).Should(Succeed()) } func (env *Environment) EventuallyExpectAvgUtilization(resource v1.ResourceName, comparator string, value float64) { - EventuallyWithOffset(1, func(g Gomega) { + GinkgoHelper() + Eventually(func(g Gomega) { g.Expect(env.Monitor.AvgUtilization(resource)).To(BeNumerically(comparator, value)) }, 10*time.Minute).Should(Succeed()) } @@ -616,10 +620,11 @@ func (env *Environment) ExpectCABundle() string { // have used the simpler client-go InClusterConfig() method. // However, that only works when Karpenter is running as a Pod // within the same cluster it's managing. + GinkgoHelper() transportConfig, err := env.Config.TransportConfig() - ExpectWithOffset(1, err).ToNot(HaveOccurred()) + Expect(err).ToNot(HaveOccurred()) _, err = transport.TLSConfigFor(transportConfig) // fills in CAData! - ExpectWithOffset(1, err).ToNot(HaveOccurred()) + Expect(err).ToNot(HaveOccurred()) logging.FromContext(env.Context).Debugf("Discovered caBundle, length %d", len(transportConfig.TLS.CAData)) return base64.StdEncoding.EncodeToString(transportConfig.TLS.CAData) } diff --git a/test/suites/chaos/suite_test.go b/test/suites/chaos/suite_test.go index 1f12d51e6afd..b2d2dedb01b5 100644 --- a/test/suites/chaos/suite_test.go +++ b/test/suites/chaos/suite_test.go @@ -38,19 +38,18 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" nodeutils "github.com/aws/karpenter-core/pkg/utils/node" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" "github.com/aws/karpenter/test/pkg/debug" - "github.com/aws/karpenter/test/pkg/environment/common" + "github.com/aws/karpenter/test/pkg/environment/aws" ) -var env *common.Environment +var env *aws.Environment func TestChaos(t *testing.T) { RegisterFailHandler(Fail) BeforeSuite(func() { - env = common.NewEnvironment(t) + env = aws.NewEnvironment(t) }) AfterSuite(func() { env.Stop() @@ -69,8 +68,8 @@ var _ = Describe("Chaos", func() { defer cancel() provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ Requirements: []v1.NodeSelectorRequirement{ @@ -114,8 +113,8 @@ var _ = Describe("Chaos", func() { defer cancel() provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ Requirements: []v1.NodeSelectorRequirement{ diff --git a/test/suites/consolidation/suite_test.go b/test/suites/consolidation/suite_test.go index 1aed325442f8..6be4afe97e3f 100644 --- a/test/suites/consolidation/suite_test.go +++ b/test/suites/consolidation/suite_test.go @@ -28,7 +28,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" "github.com/aws/karpenter/test/pkg/debug" @@ -60,8 +59,8 @@ var _ = AfterEach(func() { env.AfterEach() }) var _ = Describe("Consolidation", func() { It("should consolidate nodes (delete)", Label(debug.NoWatch), Label(debug.NoEvents), func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ Requirements: []v1.NodeSelectorRequirement{ @@ -126,8 +125,8 @@ var _ = Describe("Consolidation", func() { }) It("should consolidate on-demand nodes (replace)", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ Requirements: []v1.NodeSelectorRequirement{ @@ -242,8 +241,8 @@ var _ = Describe("Consolidation", func() { }) It("should consolidate on-demand nodes to spot (replace)", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ Requirements: []v1.NodeSelectorRequirement{ diff --git a/test/suites/drift/suite_test.go b/test/suites/drift/suite_test.go index 3612229d1483..baf91ee74faa 100644 --- a/test/suites/drift/suite_test.go +++ b/test/suites/drift/suite_test.go @@ -39,7 +39,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" "github.com/aws/karpenter/test/pkg/environment/aws" @@ -73,8 +72,8 @@ var _ = Describe("Drift", Label("AWS"), func() { BeforeEach(func() { customAMI = env.GetCustomAMI("/aws/service/eks/optimized-ami/%s/amazon-linux-2/recommended/image_id", 1) nodeTemplate = awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner = test.Provisioner(test.ProvisionerOptions{ Requirements: []v1.NodeSelectorRequirement{{Key: v1alpha5.LabelCapacityType, Operator: v1.NodeSelectorOpIn, Values: []string{v1alpha5.CapacityTypeOnDemand}}}, @@ -101,7 +100,7 @@ var _ = Describe("Drift", Label("AWS"), func() { oldCustomAMI := *parameter.Parameter.Value nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyCustom nodeTemplate.Spec.AMISelector = map[string]string{"aws-ids": oldCustomAMI} - nodeTemplate.Spec.UserData = awssdk.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", settings.FromContext(env.Context).ClusterName)) + nodeTemplate.Spec.UserData = awssdk.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", env.ClusterName)) env.ExpectCreated(pod, nodeTemplate, provisioner) env.EventuallyExpectHealthy(pod) @@ -112,7 +111,7 @@ var _ = Describe("Drift", Label("AWS"), func() { nodeTemplate.Spec.AMISelector = map[string]string{"aws-ids": customAMI} env.ExpectCreatedOrUpdated(nodeTemplate) - EventuallyWithOffset(1, func(g Gomega) { + Eventually(func(g Gomega) { g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(machine), machine)).To(Succeed()) g.Expect(machine.StatusConditions().GetCondition(v1alpha5.MachineDrifted)).ToNot(BeNil()) g.Expect(machine.StatusConditions().GetCondition(v1alpha5.MachineDrifted).IsTrue()).To(BeTrue()) @@ -134,7 +133,7 @@ var _ = Describe("Drift", Label("AWS"), func() { oldCustomAMI := *parameter.Parameter.Value nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyCustom nodeTemplate.Spec.AMISelector = map[string]string{"aws-ids": oldCustomAMI} - nodeTemplate.Spec.UserData = awssdk.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", settings.FromContext(env.Context).ClusterName)) + nodeTemplate.Spec.UserData = awssdk.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", env.ClusterName)) env.ExpectCreated(pod, nodeTemplate, provisioner) env.EventuallyExpectHealthy(pod) @@ -151,7 +150,7 @@ var _ = Describe("Drift", Label("AWS"), func() { }) It("should deprovision nodes that have drifted due to securitygroup", func() { By("getting the cluster vpc id") - output, err := env.EKSAPI.DescribeCluster(&eks.DescribeClusterInput{Name: awssdk.String(settings.FromContext(env.Context).ClusterName)}) + output, err := env.EKSAPI.DescribeCluster(&eks.DescribeClusterInput{Name: awssdk.String(env.ClusterName)}) Expect(err).To(BeNil()) By("creating new security group") @@ -165,11 +164,11 @@ var _ = Describe("Drift", Label("AWS"), func() { Tags: []*ec2.Tag{ { Key: awssdk.String("karpenter.sh/discovery"), - Value: awssdk.String(settings.FromContext(env.Context).ClusterName), + Value: awssdk.String(env.ClusterName), }, { Key: awssdk.String(test.DiscoveryLabel), - Value: awssdk.String(settings.FromContext(env.Context).ClusterName), + Value: awssdk.String(env.ClusterName), }, { Key: awssdk.String("creation-date"), @@ -185,7 +184,7 @@ var _ = Describe("Drift", Label("AWS"), func() { var securitygroups []aws.SecurityGroup var testSecurityGroup aws.SecurityGroup Eventually(func(g Gomega) { - securitygroups = env.GetSecurityGroups(map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}) + securitygroups = env.GetSecurityGroups(map[string]string{"karpenter.sh/discovery": env.ClusterName}) testSecurityGroup, _ = lo.Find(securitygroups, func(sg aws.SecurityGroup) bool { return awssdk.StringValue(sg.GroupName) == "security-group-drift" }) @@ -211,7 +210,7 @@ var _ = Describe("Drift", Label("AWS"), func() { env.ExpectCreatedOrUpdated(nodeTemplate) By("validating the drifted status condition has propagated") - EventuallyWithOffset(1, func(g Gomega) { + Eventually(func(g Gomega) { g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(machine), machine)).To(Succeed()) g.Expect(machine.StatusConditions().GetCondition(v1alpha5.MachineDrifted)).ToNot(BeNil()) g.Expect(machine.StatusConditions().GetCondition(v1alpha5.MachineDrifted).IsTrue()).To(BeTrue()) @@ -222,7 +221,7 @@ var _ = Describe("Drift", Label("AWS"), func() { env.EventuallyExpectNotFound(pod, machine, node) }) It("should deprovision nodes that have drifted due to subnets", func() { - subnets := env.GetSubnetNameAndIds(map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}) + subnets := env.GetSubnetNameAndIds(map[string]string{"karpenter.sh/discovery": env.ClusterName}) Expect(len(subnets)).To(BeNumerically(">", 1)) nodeTemplate.Spec.SubnetSelector = map[string]string{"aws-ids": subnets[0].ID} @@ -236,7 +235,7 @@ var _ = Describe("Drift", Label("AWS"), func() { env.ExpectCreatedOrUpdated(nodeTemplate) By("validating the drifted status condition has propagated") - EventuallyWithOffset(1, func(g Gomega) { + Eventually(func(g Gomega) { g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(machine), machine)).To(Succeed()) g.Expect(machine.StatusConditions().GetCondition(v1alpha5.MachineDrifted)).ToNot(BeNil()) g.Expect(machine.StatusConditions().GetCondition(v1alpha5.MachineDrifted).IsTrue()).To(BeTrue()) @@ -261,7 +260,7 @@ var _ = Describe("Drift", Label("AWS"), func() { env.ExpectCreatedOrUpdated(updatedProvisioner) By("validating the drifted status condition has propagated") - EventuallyWithOffset(1, func(g Gomega) { + Eventually(func(g Gomega) { g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(machine), machine)).To(Succeed()) g.Expect(machine.StatusConditions().GetCondition(v1alpha5.MachineDrifted)).ToNot(BeNil()) g.Expect(machine.StatusConditions().GetCondition(v1alpha5.MachineDrifted).IsTrue()).To(BeTrue()) @@ -300,7 +299,7 @@ var _ = Describe("Drift", Label("AWS"), func() { ) DescribeTable("AWSNodeTemplate Drift", func(fieldName string, nodeTemplateSpec v1alpha1.AWSNodeTemplateSpec) { if fieldName == "InstanceProfile" { - nodeTemplateSpec.AWS.InstanceProfile = awssdk.String(fmt.Sprintf("KarpenterNodeInstanceProfile-Drift-%s", settings.FromContext(env.Context).ClusterName)) + nodeTemplateSpec.AWS.InstanceProfile = awssdk.String(fmt.Sprintf("KarpenterNodeInstanceProfile-Drift-%s", env.ClusterName)) ExpectInstanceProfileCreated(nodeTemplateSpec.AWS.InstanceProfile) } @@ -316,7 +315,7 @@ var _ = Describe("Drift", Label("AWS"), func() { env.ExpectCreatedOrUpdated(updatedAWSNodeTemplate) By("validating the drifted status condition has propagated") - EventuallyWithOffset(1, func(g Gomega) { + Eventually(func(g Gomega) { g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(machine), machine)).To(Succeed()) g.Expect(machine.StatusConditions().GetCondition(v1alpha5.MachineDrifted)).ToNot(BeNil()) g.Expect(machine.StatusConditions().GetCondition(v1alpha5.MachineDrifted).IsTrue()).To(BeTrue()) @@ -476,7 +475,7 @@ func ExpectInstanceProfileCreated(instanceProfileName *string) { Tags: []*iam.Tag{ { Key: awssdk.String(test.DiscoveryLabel), - Value: awssdk.String(settings.FromContext(env.Context).ClusterName), + Value: awssdk.String(env.ClusterName), }, }, } @@ -485,7 +484,7 @@ func ExpectInstanceProfileCreated(instanceProfileName *string) { Expect(ignoreAlreadyExists(err)).ToNot(HaveOccurred()) addInstanceProfile := &iam.AddRoleToInstanceProfileInput{ InstanceProfileName: instanceProfileName, - RoleName: awssdk.String(fmt.Sprintf("KarpenterNodeRole-%s", settings.FromContext(env.Context).ClusterName)), + RoleName: awssdk.String(fmt.Sprintf("KarpenterNodeRole-%s", env.ClusterName)), } _, err = env.IAMAPI.AddRoleToInstanceProfile(addInstanceProfile) Expect(ignoreAlreadyContainsRole(err)).ToNot(HaveOccurred()) diff --git a/test/suites/expiration/expiration_test.go b/test/suites/expiration/expiration_test.go index 16c381f9e268..9c3c7ff9ab96 100644 --- a/test/suites/expiration/expiration_test.go +++ b/test/suites/expiration/expiration_test.go @@ -34,7 +34,6 @@ import ( "github.com/aws/aws-sdk-go/service/ssm" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" - "github.com/aws/karpenter/pkg/apis/settings" awstest "github.com/aws/karpenter/pkg/test" "github.com/aws/karpenter/test/pkg/environment/aws" @@ -67,8 +66,8 @@ var _ = Describe("Expiration", func() { var provisioner *v1alpha5.Provisioner BeforeEach(func() { nodeTemplate = awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner = test.Provisioner(test.ProvisionerOptions{ ProviderRef: &v1alpha5.MachineTemplateRef{Name: nodeTemplate.Name}, @@ -101,7 +100,7 @@ var _ = Describe("Expiration", func() { env.Monitor.Reset() // Reset the monitor so that we can expect a single node to be spun up after expiration // Expect that the Machine will get an expired status condition - EventuallyWithOffset(1, func(g Gomega) { + Eventually(func(g Gomega) { g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(machine), machine)).To(Succeed()) g.Expect(machine.StatusConditions().GetCondition(v1alpha5.MachineExpired)).ToNot(BeNil()) g.Expect(machine.StatusConditions().GetCondition(v1alpha5.MachineExpired).IsTrue()).To(BeTrue()) @@ -169,7 +168,7 @@ var _ = Describe("Expiration", func() { env.ExpectUpdated(provisioner) // Expect that the Machine will get an expired status condition - EventuallyWithOffset(1, func(g Gomega) { + Eventually(func(g Gomega) { g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(machine), machine)).To(Succeed()) g.Expect(machine.StatusConditions().GetCondition(v1alpha5.MachineExpired)).ToNot(BeNil()) g.Expect(machine.StatusConditions().GetCondition(v1alpha5.MachineExpired).IsTrue()).To(BeTrue()) diff --git a/test/suites/integration/ami_test.go b/test/suites/integration/ami_test.go index 358c5f5dd526..44625235af31 100644 --- a/test/suites/integration/ami_test.go +++ b/test/suites/integration/ami_test.go @@ -32,7 +32,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" @@ -48,8 +47,8 @@ var _ = Describe("AMI", func() { It("should use the AMI defined by the AMI Selector", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, AMIFamily: &v1alpha1.AMIFamilyAL2, }, AMISelector: map[string]string{"aws-ids": customAMI}, @@ -71,12 +70,12 @@ var _ = Describe("AMI", func() { Expect(err).To(BeNil()) oldCustomAMI := *parameter.Parameter.Value provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, AMIFamily: &v1alpha1.AMIFamilyCustom, }, AMISelector: map[string]string{"aws-ids": fmt.Sprintf("%s,%s", customAMI, oldCustomAMI)}, - UserData: aws.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", settings.FromContext(env.Context).ClusterName)), + UserData: aws.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", env.ClusterName)), }) provisioner := test.Provisioner(test.ProvisionerOptions{ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}}) pod := test.Pod() @@ -95,12 +94,12 @@ var _ = Describe("AMI", func() { Expect(output.Images).To(HaveLen(1)) provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, AMIFamily: &v1alpha1.AMIFamilyCustom, }, AMISelector: map[string]string{"aws::name": *output.Images[0].Name, "aws::owners": "fakeOwnerValue"}, - UserData: aws.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", settings.FromContext(env.Context).ClusterName)), + UserData: aws.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", env.ClusterName)), }) provisioner := test.Provisioner(test.ProvisionerOptions{ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}}) @@ -118,12 +117,12 @@ var _ = Describe("AMI", func() { Expect(output.Images).To(HaveLen(1)) provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, AMIFamily: &v1alpha1.AMIFamilyCustom, }, AMISelector: map[string]string{"aws::name": *output.Images[0].Name}, - UserData: aws.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", settings.FromContext(env.Context).ClusterName)), + UserData: aws.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", env.ClusterName)), }) provisioner := test.Provisioner(test.ProvisionerOptions{ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}}) @@ -137,12 +136,12 @@ var _ = Describe("AMI", func() { }) It("should support ami selector aws::ids", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, AMIFamily: &v1alpha1.AMIFamilyCustom, }, AMISelector: map[string]string{"aws::ids": customAMI}, - UserData: aws.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", settings.FromContext(env.Context).ClusterName)), + UserData: aws.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", env.ClusterName)), }) provisioner := test.Provisioner(test.ProvisionerOptions{ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}}) pod := test.Pod() @@ -157,8 +156,8 @@ var _ = Describe("AMI", func() { Context("AMIFamily", func() { It("should provision a node using the AL2 family", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}, @@ -170,8 +169,8 @@ var _ = Describe("AMI", func() { }) It("should provision a node using the Bottlerocket family", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, AMIFamily: &v1alpha1.AMIFamilyBottlerocket, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ @@ -184,8 +183,8 @@ var _ = Describe("AMI", func() { }) It("should provision a node using the Ubuntu family", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, AMIFamily: &v1alpha1.AMIFamilyUbuntu, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ @@ -212,12 +211,12 @@ var _ = Describe("AMI", func() { }) It("should support Custom AMIFamily with AMI Selectors", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, AMIFamily: &v1alpha1.AMIFamilyCustom, }, AMISelector: map[string]string{"aws-ids": customAMI}, - UserData: aws.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", settings.FromContext(env.Context).ClusterName)), + UserData: aws.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", env.ClusterName)), }) provisioner := test.Provisioner(test.ProvisionerOptions{ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}}) pod := test.Pod() @@ -231,8 +230,8 @@ var _ = Describe("AMI", func() { It("should have the AWSNodeTemplateStatus for AMIs using wildcard", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }, AMISelector: map[string]string{"aws::name": "*"}, }) @@ -244,8 +243,8 @@ var _ = Describe("AMI", func() { It("should have the AWSNodeTemplateStatus for AMIs using tags", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }, AMISelector: map[string]string{"aws-ids": customAMI}, }) @@ -263,8 +262,8 @@ var _ = Describe("AMI", func() { content, err := os.ReadFile("testdata/al2_userdata_input.sh") Expect(err).ToNot(HaveOccurred()) provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, AMIFamily: &v1alpha1.AMIFamilyAL2, }, UserData: aws.String(string(content)), @@ -292,8 +291,8 @@ var _ = Describe("AMI", func() { content, err := os.ReadFile("testdata/al2_no_mime_userdata_input.sh") Expect(err).ToNot(HaveOccurred()) provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, AMIFamily: &v1alpha1.AMIFamilyAL2, }, UserData: aws.String(string(content)), @@ -321,8 +320,8 @@ var _ = Describe("AMI", func() { content, err := os.ReadFile("testdata/br_userdata_input.sh") Expect(err).ToNot(HaveOccurred()) provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, AMIFamily: &v1alpha1.AMIFamilyBottlerocket, }, UserData: aws.String(string(content)), @@ -353,8 +352,8 @@ var _ = Describe("AMI", func() { content, err := os.ReadFile("testdata/windows_userdata_input.ps1") Expect(err).ToNot(HaveOccurred()) provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, AMIFamily: &v1alpha1.AMIFamilyWindows2022, }, UserData: aws.String(string(content)), diff --git a/test/suites/integration/aws_metadata_test.go b/test/suites/integration/aws_metadata_test.go index 018d29010c9a..157fb3dc1348 100644 --- a/test/suites/integration/aws_metadata_test.go +++ b/test/suites/integration/aws_metadata_test.go @@ -22,7 +22,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" @@ -32,8 +31,8 @@ var _ = Describe("MetadataOptions", func() { It("should use specified metadata options", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, LaunchTemplate: v1alpha1.LaunchTemplate{ MetadataOptions: &v1alpha1.MetadataOptions{ HTTPEndpoint: aws.String("enabled"), diff --git a/test/suites/integration/backwards_compatability_test.go b/test/suites/integration/backwards_compatability_test.go index 87c609a201a5..17d707277f56 100644 --- a/test/suites/integration/backwards_compatability_test.go +++ b/test/suites/integration/backwards_compatability_test.go @@ -21,7 +21,6 @@ import ( "github.com/samber/lo" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" ) @@ -30,8 +29,8 @@ var _ = Describe("BackwardsCompatability", func() { provisioner := test.Provisioner( test.ProvisionerOptions{ Provider: &v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, Tags: map[string]string{ "custom-tag": "custom-value", "custom-tag2": "custom-value2", diff --git a/test/suites/integration/block_device_mappings_test.go b/test/suites/integration/block_device_mappings_test.go index 3e11d74e5154..7d8f4ffb6a64 100644 --- a/test/suites/integration/block_device_mappings_test.go +++ b/test/suites/integration/block_device_mappings_test.go @@ -22,7 +22,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" "github.com/aws/karpenter-core/pkg/utils/resources" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" @@ -32,8 +31,8 @@ var _ = Describe("BlockDeviceMappings", func() { It("should use specified block device mappings", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, LaunchTemplate: v1alpha1.LaunchTemplate{ BlockDeviceMappings: []*v1alpha1.BlockDeviceMapping{ { diff --git a/test/suites/integration/cni_test.go b/test/suites/integration/cni_test.go index d686de9b2707..85c78ff9b64e 100644 --- a/test/suites/integration/cni_test.go +++ b/test/suites/integration/cni_test.go @@ -26,7 +26,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" @@ -39,8 +38,8 @@ var _ = Describe("CNITests", func() { }) provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, AMIFamily: &v1alpha1.AMIFamilyAL2, }, }) @@ -58,8 +57,8 @@ var _ = Describe("CNITests", func() { It("should set eni-limited maxPods when AWSENILimited when AWS_ENI_LIMITED_POD_DENSITY is true", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, AMIFamily: &v1alpha1.AMIFamilyAL2, }, }) @@ -79,8 +78,8 @@ var _ = Describe("CNITests", func() { }) provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, AMIFamily: &v1alpha1.AMIFamilyAL2, }, }) diff --git a/test/suites/integration/daemonset_test.go b/test/suites/integration/daemonset_test.go index 3498429f1e0c..ba89b43d28c5 100644 --- a/test/suites/integration/daemonset_test.go +++ b/test/suites/integration/daemonset_test.go @@ -26,7 +26,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" @@ -45,8 +44,8 @@ var _ = Describe("DaemonSet", func() { BeforeEach(func() { provider = awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner = test.Provisioner(test.ProvisionerOptions{ ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}, diff --git a/test/suites/integration/emptiness_test.go b/test/suites/integration/emptiness_test.go index 73a80afae68f..837d0b8bdba3 100644 --- a/test/suites/integration/emptiness_test.go +++ b/test/suites/integration/emptiness_test.go @@ -25,7 +25,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" ) @@ -33,8 +32,8 @@ import ( var _ = Describe("Emptiness", func() { It("should terminate an empty node", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}, @@ -56,7 +55,7 @@ var _ = Describe("Emptiness", func() { Expect(env.Client.Patch(env, deployment, client.MergeFrom(persisted))).To(Succeed()) By("waiting for the machine emptiness status condition to propagate") - EventuallyWithOffset(1, func(g Gomega) { + Eventually(func(g Gomega) { g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(machine), machine)).To(Succeed()) g.Expect(machine.StatusConditions().GetCondition(v1alpha5.MachineEmpty)).ToNot(BeNil()) g.Expect(machine.StatusConditions().GetCondition(v1alpha5.MachineEmpty).IsTrue()).To(BeTrue()) diff --git a/test/suites/integration/extended_resources_test.go b/test/suites/integration/extended_resources_test.go index 93850bdc87f8..7e222743d734 100644 --- a/test/suites/integration/extended_resources_test.go +++ b/test/suites/integration/extended_resources_test.go @@ -30,7 +30,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" @@ -41,8 +40,8 @@ var _ = Describe("Extended Resources", func() { ExpectNvidiaDevicePluginCreated() provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}, @@ -80,8 +79,8 @@ var _ = Describe("Extended Resources", func() { // For Bottlerocket, we are testing that resources are initialized without needing a device plugin provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ AMIFamily: &v1alpha1.AMIFamilyBottlerocket, - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}, @@ -124,8 +123,8 @@ var _ = Describe("Extended Resources", func() { "aws.enablePodENI": "true", }) provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}, @@ -178,8 +177,8 @@ var _ = Describe("Extended Resources", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ AMIFamily: &v1alpha1.AMIFamilyCustom, - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }, AMISelector: map[string]string{ "aws-ids": customAMI, @@ -195,8 +194,8 @@ var _ = Describe("Extended Resources", func() { }, }, }) - provider.Spec.UserData = lo.ToPtr(fmt.Sprintf(string(rawContent), settings.FromContext(env.Context).ClusterName, - settings.FromContext(env.Context).ClusterEndpoint, env.ExpectCABundle(), provisioner.Name)) + provider.Spec.UserData = lo.ToPtr(fmt.Sprintf(string(rawContent), env.ClusterName, + env.ClusterEndpoint, env.ExpectCABundle(), provisioner.Name)) numPods := 1 dep := test.Deployment(test.DeploymentOptions{ @@ -231,8 +230,8 @@ var _ = Describe("Extended Resources", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }, AMISelector: map[string]string{"aws-ids": "ami-0fae925f94979981f"}, }) diff --git a/test/suites/integration/hash_test.go b/test/suites/integration/hash_test.go index 84fa5302c209..08e6f6dbc365 100644 --- a/test/suites/integration/hash_test.go +++ b/test/suites/integration/hash_test.go @@ -21,7 +21,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" @@ -31,8 +30,8 @@ var _ = Describe("CRD Hash", func() { It("should have Provisioner hash", func() { nodeTemplate := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }, }) provisioner := test.Provisioner(test.ProvisionerOptions{ @@ -54,8 +53,8 @@ var _ = Describe("CRD Hash", func() { It("should have AWSNodeTemplate hash", func() { nodeTemplate := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }, }) env.ExpectCreated(nodeTemplate) diff --git a/test/suites/integration/instance_profile_test.go b/test/suites/integration/instance_profile_test.go index bdb19893d9f6..5e8e29292d5d 100644 --- a/test/suites/integration/instance_profile_test.go +++ b/test/suites/integration/instance_profile_test.go @@ -25,7 +25,6 @@ import ( corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" coretest "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1beta1" awserrors "github.com/aws/karpenter/pkg/errors" "github.com/aws/karpenter/pkg/providers/instanceprofile" @@ -49,7 +48,7 @@ var _ = Describe("InstanceProfile Generation", func() { Tags: map[string]string{"*": "*"}, }, }, - Role: fmt.Sprintf("KarpenterNodeRole-%s", settings.FromContext(env.Context).ClusterName), + Role: fmt.Sprintf("KarpenterNodeRole-%s", env.ClusterName), }, }) }) diff --git a/test/suites/integration/kubelet_config_test.go b/test/suites/integration/kubelet_config_test.go index 02eba693982f..aec27cb931c9 100644 --- a/test/suites/integration/kubelet_config_test.go +++ b/test/suites/integration/kubelet_config_test.go @@ -26,7 +26,6 @@ import ( "knative.dev/pkg/ptr" "github.com/aws/karpenter-core/pkg/apis/v1alpha5" - "github.com/aws/karpenter/pkg/apis/settings" awstest "github.com/aws/karpenter/pkg/test" "github.com/aws/karpenter/test/pkg/environment/aws" @@ -41,8 +40,8 @@ var _ = Describe("KubeletConfiguration Overrides", func() { BeforeEach(func() { nodeTemplate = awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) // MaxPods needs to account for the daemonsets that will run on the nodes @@ -176,8 +175,8 @@ var _ = Describe("KubeletConfiguration Overrides", func() { }) It("should schedule pods onto separate nodes when maxPods is set", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) // MaxPods needs to account for the daemonsets that will run on the nodes @@ -219,8 +218,8 @@ var _ = Describe("KubeletConfiguration Overrides", func() { }) It("should schedule pods onto separate nodes when podsPerCore is set", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) // PodsPerCore needs to account for the daemonsets that will run on the nodes // This will have 4 pods available on each node (2 taken by daemonset pods) @@ -273,8 +272,8 @@ var _ = Describe("KubeletConfiguration Overrides", func() { }) It("should ignore podsPerCore value when Bottlerocket is used", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, AMIFamily: &v1alpha1.AMIFamilyBottlerocket, }}) // All pods should schedule to a single node since we are ignoring podsPerCore value diff --git a/test/suites/integration/scheduling_test.go b/test/suites/integration/scheduling_test.go index 72dd94c75ff4..352b0a526f85 100644 --- a/test/suites/integration/scheduling_test.go +++ b/test/suites/integration/scheduling_test.go @@ -29,7 +29,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" "github.com/aws/karpenter/test/pkg/debug" "github.com/aws/karpenter/test/pkg/environment/aws" @@ -44,8 +43,8 @@ var _ = Describe("Scheduling", Ordered, ContinueOnFailure, func() { BeforeEach(func() { provider = awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner = test.Provisioner(test.ProvisionerOptions{ ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}, diff --git a/test/suites/integration/security_group_test.go b/test/suites/integration/security_group_test.go index 17775726209f..e13e994919ea 100644 --- a/test/suites/integration/security_group_test.go +++ b/test/suites/integration/security_group_test.go @@ -28,7 +28,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" "github.com/aws/karpenter/test/pkg/environment/aws" @@ -37,14 +36,14 @@ import ( var _ = Describe("SecurityGroups", func() { It("should use the security-group-id selector", func() { - securityGroups := env.GetSecurityGroups(map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}) + securityGroups := env.GetSecurityGroups(map[string]string{"karpenter.sh/discovery": env.ClusterName}) Expect(len(securityGroups)).To(BeNumerically(">", 1)) ids := strings.Join([]string{*securityGroups[0].GroupId, *securityGroups[1].GroupId}, ",") provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ SecurityGroupSelector: map[string]string{"aws-ids": ids}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }, }) provisioner := test.Provisioner(test.ProvisionerOptions{ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}}) @@ -58,7 +57,7 @@ var _ = Describe("SecurityGroups", func() { }) It("should use the security group selector with multiple tag values", func() { - securityGroups := env.GetSecurityGroups(map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}) + securityGroups := env.GetSecurityGroups(map[string]string{"karpenter.sh/discovery": env.ClusterName}) Expect(len(securityGroups)).To(BeNumerically(">", 1)) first := securityGroups[0] last := securityGroups[len(securityGroups)-1] @@ -69,7 +68,7 @@ var _ = Describe("SecurityGroups", func() { lo.FromPtr(lo.FindOrElse(first.Tags, &ec2.Tag{}, func(tag *ec2.Tag) bool { return lo.FromPtr(tag.Key) == "Name" }).Value), lo.FromPtr(lo.FindOrElse(last.Tags, &ec2.Tag{}, func(tag *ec2.Tag) bool { return lo.FromPtr(tag.Key) == "Name" }).Value), )}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }, }) provisioner := test.Provisioner(test.ProvisionerOptions{ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}}) @@ -85,8 +84,8 @@ var _ = Describe("SecurityGroups", func() { It("should update the AWSNodeTemplateStatus for security groups", func() { nodeTemplate := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }, }) @@ -96,7 +95,7 @@ var _ = Describe("SecurityGroups", func() { }) func EventuallyExpectSecurityGroups(env *aws.Environment, nodeTemplate *v1alpha1.AWSNodeTemplate) { - securityGroups := env.GetSecurityGroups(map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}) + securityGroups := env.GetSecurityGroups(map[string]string{"karpenter.sh/discovery": env.ClusterName}) Expect(securityGroups).ToNot(HaveLen(0)) ids := sets.New(lo.Map(securityGroups, func(s aws.SecurityGroup, _ int) string { diff --git a/test/suites/integration/storage_test.go b/test/suites/integration/storage_test.go index 5dd8acf231ae..f0fc0322883e 100644 --- a/test/suites/integration/storage_test.go +++ b/test/suites/integration/storage_test.go @@ -28,7 +28,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" @@ -53,8 +52,8 @@ var _ = Describe("Dynamic PVC", func() { } provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}}) @@ -91,8 +90,8 @@ var _ = Describe("Dynamic PVC", func() { var _ = Describe("Static PVC", func() { It("should run a pod with a static persistent volume", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}}) diff --git a/test/suites/integration/subnet_test.go b/test/suites/integration/subnet_test.go index 2158c483ce6e..45901a433c64 100644 --- a/test/suites/integration/subnet_test.go +++ b/test/suites/integration/subnet_test.go @@ -29,7 +29,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" "github.com/aws/karpenter/test/pkg/environment/aws" @@ -37,14 +36,14 @@ import ( var _ = Describe("Subnets", func() { It("should use the subnet-id selector", func() { - subnets := env.GetSubnets(map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}) + subnets := env.GetSubnets(map[string]string{"karpenter.sh/discovery": env.ClusterName}) Expect(len(subnets)).ToNot(Equal(0)) shuffledAZs := lo.Shuffle(lo.Keys(subnets)) firstSubnet := subnets[shuffledAZs[0]][0] provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, SubnetSelector: map[string]string{"aws-ids": firstSubnet}, }, }) @@ -58,7 +57,7 @@ var _ = Describe("Subnets", func() { env.ExpectInstance(pod.Spec.NodeName).To(HaveField("SubnetId", HaveValue(Equal(firstSubnet)))) }) It("should use resource based naming as node names", func() { - subnets := env.GetSubnets(map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}) + subnets := env.GetSubnets(map[string]string{"karpenter.sh/discovery": env.ClusterName}) Expect(len(subnets)).ToNot(Equal(0)) allSubnets := lo.Flatten(lo.Values(subnets)) @@ -70,8 +69,8 @@ var _ = Describe("Subnets", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }, }) provisioner := test.Provisioner(test.ProvisionerOptions{ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}}) @@ -85,14 +84,14 @@ var _ = Describe("Subnets", func() { }) It("should use the subnet tag selector with multiple tag values", func() { // Get all the subnets for the cluster - subnets := env.GetSubnetNameAndIds(map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}) + subnets := env.GetSubnetNameAndIds(map[string]string{"karpenter.sh/discovery": env.ClusterName}) Expect(len(subnets)).To(BeNumerically(">", 1)) firstSubnet := subnets[0] lastSubnet := subnets[len(subnets)-1] provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, SubnetSelector: map[string]string{"Name": fmt.Sprintf("%s,%s", firstSubnet.Name, lastSubnet.Name)}, }, }) @@ -107,14 +106,14 @@ var _ = Describe("Subnets", func() { }) It("should use a subnet within the AZ requested", func() { - subnets := env.GetSubnets(map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}) + subnets := env.GetSubnets(map[string]string{"karpenter.sh/discovery": env.ClusterName}) Expect(len(subnets)).ToNot(Equal(0)) shuffledAZs := lo.Shuffle(lo.Keys(subnets)) provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }, }) provisioner := test.Provisioner(test.ProvisionerOptions{ @@ -141,8 +140,8 @@ var _ = Describe("Subnets", func() { It("should have the AWSNodeTemplateStatus for subnets", func() { nodeTemplate := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }, }) @@ -198,7 +197,7 @@ type SubnetInfo struct { } func EventuallyExpectSubnets(env *aws.Environment, nodeTemplate *v1alpha1.AWSNodeTemplate) { - subnets := env.GetSubnets(map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}) + subnets := env.GetSubnets(map[string]string{"karpenter.sh/discovery": env.ClusterName}) Expect(subnets).ToNot(HaveLen(0)) ids := sets.New(lo.Flatten(lo.Values(subnets))...) diff --git a/test/suites/integration/tags_test.go b/test/suites/integration/tags_test.go index fb958b140b51..cd97aea53bf5 100644 --- a/test/suites/integration/tags_test.go +++ b/test/suites/integration/tags_test.go @@ -29,7 +29,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" coretest "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" "github.com/aws/karpenter/pkg/apis/v1beta1" "github.com/aws/karpenter/pkg/providers/instance" @@ -41,8 +40,8 @@ var _ = Describe("Tags", func() { It("should tag all associated resources", func() { provider := test.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, Tags: map[string]string{"TestTag": "TestVal"}, }, }) @@ -62,8 +61,8 @@ var _ = Describe("Tags", func() { It("should tag all associated resources with global tags", func() { provider := test.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }, }) @@ -94,10 +93,10 @@ var _ = Describe("Tags", func() { BeforeEach(func() { nodeClass = test.EC2NodeClass(v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{ SecurityGroupSelectorTerms: []v1beta1.SecurityGroupSelectorTerm{{ - Tags: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + Tags: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}, SubnetSelectorTerms: []v1beta1.SubnetSelectorTerm{{ - Tags: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + Tags: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}, }}) @@ -169,8 +168,8 @@ var _ = Describe("Tags", func() { It("shouldn't tag nodes provisioned by v1alpha5 provisioner", func() { nodeTemplate := test.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner := coretest.Provisioner(coretest.ProvisionerOptions{ diff --git a/test/suites/integration/termination_test.go b/test/suites/integration/termination_test.go index 9eb99453016f..3fac773e06eb 100644 --- a/test/suites/integration/termination_test.go +++ b/test/suites/integration/termination_test.go @@ -23,7 +23,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" ) @@ -31,8 +30,8 @@ import ( var _ = Describe("Termination", func() { It("should terminate the node and the instance on deletion", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}, diff --git a/test/suites/integration/webhook_test.go b/test/suites/integration/webhook_test.go index a946f0be33bb..996666d7633a 100644 --- a/test/suites/integration/webhook_test.go +++ b/test/suites/integration/webhook_test.go @@ -25,7 +25,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" ) @@ -133,8 +132,8 @@ var _ = Describe("Webhooks", func() { It("should error when provider and providerRef are combined", func() { Expect(env.Client.Create(env, test.Provisioner(test.ProvisionerOptions{ Provider: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }, ProviderRef: &v1alpha5.MachineTemplateRef{Name: "test"}, }))).ToNot(Succeed()) @@ -266,15 +265,15 @@ var _ = Describe("Webhooks", func() { It("should error when amiSelector is not defined for amiFamily Custom", func() { Expect(env.Client.Create(env, awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ AMIFamily: &v1alpha1.AMIFamilyCustom, - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}))).ToNot(Succeed()) }) It("should fail if both userdata and launchTemplate are set", func() { Expect(env.Client.Create(env, awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ LaunchTemplate: v1alpha1.LaunchTemplate{LaunchTemplateName: ptr.String("lt")}, - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }, UserData: ptr.String("data"), }))).ToNot(Succeed()) @@ -282,47 +281,47 @@ var _ = Describe("Webhooks", func() { It("should fail if both amiSelector and launchTemplate are set", func() { Expect(env.Client.Create(env, awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ LaunchTemplate: v1alpha1.LaunchTemplate{LaunchTemplateName: ptr.String("lt")}, - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }, AMISelector: map[string]string{"foo": "bar"}, }))).ToNot(Succeed()) }) It("should fail for poorly formatted aws-ids", func() { Expect(env.Client.Create(env, awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }, AMISelector: map[string]string{"aws-ids": "must-start-with-ami"}, }))).ToNot(Succeed()) }) It("should succeed when tags don't contain restricted keys", func() { Expect(env.Client.Create(env, awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, Tags: map[string]string{"karpenter.sh/custom-key": "custom-value", "kubernetes.io/role/key": "custom-value"}, }, }))).To(Succeed()) }) It("should error when tags contains a restricted key", func() { Expect(env.Client.Create(env, awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, Tags: map[string]string{"karpenter.sh/provisioner-name": "custom-value"}, }, }))).ToNot(Succeed()) Expect(env.Client.Create(env, awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - Tags: map[string]string{"karpenter.sh/managed-by": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + Tags: map[string]string{"karpenter.sh/managed-by": env.ClusterName}, }, }))).ToNot(Succeed()) Expect(env.Client.Create(env, awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - Tags: map[string]string{fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(env.Context).ClusterName): "owned"}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + Tags: map[string]string{fmt.Sprintf("kubernetes.io/cluster/%s", env.ClusterName): "owned"}, }, }))).ToNot(Succeed()) }) @@ -330,22 +329,22 @@ var _ = Describe("Webhooks", func() { Expect(env.Client.Create(env, awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ SecurityGroupSelector: map[string]string{ - "karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName, + "karpenter.sh/discovery": env.ClusterName, "aws-ids": "sg-12345", }, SubnetSelector: map[string]string{ - "karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName, + "karpenter.sh/discovery": env.ClusterName, }, }, }))).ToNot(Succeed()) Expect(env.Client.Create(env, awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ SecurityGroupSelector: map[string]string{ - "karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName, + "karpenter.sh/discovery": env.ClusterName, "aws-ids": "sg-12345", }, SubnetSelector: map[string]string{ - "karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName, + "karpenter.sh/discovery": env.ClusterName, }, }, }))).ToNot(Succeed()) @@ -354,10 +353,10 @@ var _ = Describe("Webhooks", func() { Expect(env.Client.Create(env, awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ SecurityGroupSelector: map[string]string{ - "karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName, + "karpenter.sh/discovery": env.ClusterName, }, SubnetSelector: map[string]string{ - "karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName, + "karpenter.sh/discovery": env.ClusterName, "aws-ids": "subnet-12345", }, }, @@ -365,10 +364,10 @@ var _ = Describe("Webhooks", func() { Expect(env.Client.Create(env, awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ SecurityGroupSelector: map[string]string{ - "karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName, + "karpenter.sh/discovery": env.ClusterName, }, SubnetSelector: map[string]string{ - "karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName, + "karpenter.sh/discovery": env.ClusterName, "aws-ids": "subnet-12345", }, }, @@ -378,10 +377,10 @@ var _ = Describe("Webhooks", func() { Expect(env.Client.Create(env, awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ SecurityGroupSelector: map[string]string{ - "karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName, + "karpenter.sh/discovery": env.ClusterName, }, SubnetSelector: map[string]string{ - "karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName, + "karpenter.sh/discovery": env.ClusterName, }, }, AMISelector: map[string]string{ @@ -392,10 +391,10 @@ var _ = Describe("Webhooks", func() { Expect(env.Client.Create(env, awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ SecurityGroupSelector: map[string]string{ - "karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName, + "karpenter.sh/discovery": env.ClusterName, }, SubnetSelector: map[string]string{ - "karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName, + "karpenter.sh/discovery": env.ClusterName, }, }, AMISelector: map[string]string{ diff --git a/test/suites/interruption/suite_test.go b/test/suites/interruption/suite_test.go index a7b4f6128a3b..0e1fc563efd5 100644 --- a/test/suites/interruption/suite_test.go +++ b/test/suites/interruption/suite_test.go @@ -21,6 +21,7 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "github.com/samber/lo" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -29,10 +30,10 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" "github.com/aws/karpenter/pkg/controllers/interruption/messages" "github.com/aws/karpenter/pkg/controllers/interruption/messages/scheduledchange" + "github.com/aws/karpenter/pkg/operator/options" awstest "github.com/aws/karpenter/pkg/test" "github.com/aws/karpenter/pkg/utils" "github.com/aws/karpenter/test/pkg/environment/aws" @@ -53,6 +54,9 @@ func TestInterruption(t *testing.T) { } var _ = BeforeEach(func() { + env.Context = options.ToContext(env.Context, awstest.Options(awstest.OptionsFields{ + InterruptionQueue: lo.ToPtr(env.InterruptionQueue), + })) env.BeforeEach() env.ExpectQueueExists() }) @@ -63,8 +67,8 @@ var _ = Describe("Interruption", Label("AWS"), func() { It("should terminate the spot instance and spin-up a new node on spot interruption warning", func() { By("Creating a single healthy node with a healthy deployment") provider = awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ Requirements: []v1.NodeSelectorRequirement{ @@ -111,8 +115,8 @@ var _ = Describe("Interruption", Label("AWS"), func() { It("should terminate the node at the API server when the EC2 instance is stopped", func() { By("Creating a single healthy node with a healthy deployment") provider = awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ Requirements: []v1.NodeSelectorRequirement{ @@ -151,8 +155,8 @@ var _ = Describe("Interruption", Label("AWS"), func() { It("should terminate the node at the API server when the EC2 instance is terminated", func() { By("Creating a single healthy node with a healthy deployment") provider = awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ Requirements: []v1.NodeSelectorRequirement{ @@ -191,8 +195,8 @@ var _ = Describe("Interruption", Label("AWS"), func() { It("should terminate the node when receiving a scheduled change health event", func() { By("Creating a single healthy node with a healthy deployment") provider = awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ Requirements: []v1.NodeSelectorRequirement{ diff --git a/test/suites/ipv6/suite_test.go b/test/suites/ipv6/suite_test.go index 2813c51c171c..7e6bad6f0e95 100644 --- a/test/suites/ipv6/suite_test.go +++ b/test/suites/ipv6/suite_test.go @@ -25,7 +25,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" "github.com/aws/karpenter/test/pkg/environment/aws" @@ -52,8 +51,8 @@ var _ = Describe("IPv6", func() { It("should provision an IPv6 node by discovering kube-dns IPv6", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}, Requirements: []v1.NodeSelectorRequirement{ { @@ -82,8 +81,8 @@ var _ = Describe("IPv6", func() { clusterDNSAddr := env.ExpectIPv6ClusterDNS() provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{ AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}, Requirements: []v1.NodeSelectorRequirement{ { diff --git a/test/suites/machine/garbage_collection_test.go b/test/suites/machine/garbage_collection_test.go index 4062a12f7f9e..74247796d631 100644 --- a/test/suites/machine/garbage_collection_test.go +++ b/test/suites/machine/garbage_collection_test.go @@ -43,8 +43,8 @@ var _ = Describe("NodeClaimGarbageCollection", func() { BeforeEach(func() { provisioner = test.Provisioner() - securityGroups := env.GetSecurityGroups(map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}) - subnets := env.GetSubnetNameAndIds(map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}) + securityGroups := env.GetSecurityGroups(map[string]string{"karpenter.sh/discovery": env.ClusterName}) + subnets := env.GetSubnetNameAndIds(map[string]string{"karpenter.sh/discovery": env.ClusterName}) Expect(securityGroups).ToNot(HaveLen(0)) Expect(subnets).ToNot(HaveLen(0)) @@ -75,7 +75,7 @@ var _ = Describe("NodeClaimGarbageCollection", func() { ResourceType: aws.String(ec2.ResourceTypeInstance), Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(env.Context).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", env.ClusterName)), Value: aws.String("owned"), }, { @@ -93,8 +93,8 @@ var _ = Describe("NodeClaimGarbageCollection", func() { // Update the userData for the instance input with the correct provisionerName rawContent, err := os.ReadFile("testdata/al2_userdata_input.sh") Expect(err).ToNot(HaveOccurred()) - instanceInput.UserData = lo.ToPtr(base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(string(rawContent), settings.FromContext(env.Context).ClusterName, - settings.FromContext(env.Context).ClusterEndpoint, env.ExpectCABundle(), provisioner.Name)))) + instanceInput.UserData = lo.ToPtr(base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(string(rawContent), env.ClusterName, + env.ClusterEndpoint, env.ExpectCABundle(), provisioner.Name)))) // Create an instance manually to mock Karpenter launching an instance out := env.ExpectRunInstances(instanceInput) @@ -120,7 +120,7 @@ var _ = Describe("NodeClaimGarbageCollection", func() { Tags: []*ec2.Tag{ { Key: aws.String(v1alpha5.MachineManagedByAnnotationKey), - Value: aws.String(settings.FromContext(env.Context).ClusterName), + Value: aws.String(env.ClusterName), }, }, }) @@ -139,8 +139,8 @@ var _ = Describe("NodeClaimGarbageCollection", func() { }) provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}, diff --git a/test/suites/machine/link_test.go b/test/suites/machine/link_test.go index 87b1c0739192..18958b98daf5 100644 --- a/test/suites/machine/link_test.go +++ b/test/suites/machine/link_test.go @@ -42,8 +42,8 @@ var _ = Describe("MachineLink", func() { var instanceInput *ec2.RunInstancesInput BeforeEach(func() { - securityGroups := env.GetSecurityGroups(map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}) - subnets := env.GetSubnetNameAndIds(map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}) + securityGroups := env.GetSecurityGroups(map[string]string{"karpenter.sh/discovery": env.ClusterName}) + subnets := env.GetSubnetNameAndIds(map[string]string{"karpenter.sh/discovery": env.ClusterName}) Expect(securityGroups).ToNot(HaveLen(0)) Expect(subnets).ToNot(HaveLen(0)) @@ -74,7 +74,7 @@ var _ = Describe("MachineLink", func() { ResourceType: aws.String(ec2.ResourceTypeInstance), Tags: []*ec2.Tag{ { - Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", settings.FromContext(env.Context).ClusterName)), + Key: aws.String(fmt.Sprintf("kubernetes.io/cluster/%s", env.ClusterName)), Value: aws.String("owned"), }, }, @@ -87,8 +87,8 @@ var _ = Describe("MachineLink", func() { It("should succeed to link a Machine for an existing instance launched by Karpenter", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ AMIFamily: &v1alpha1.AMIFamilyAL2, - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}, @@ -102,8 +102,8 @@ var _ = Describe("MachineLink", func() { Key: aws.String(v1alpha5.ProvisionerNameLabelKey), Value: aws.String(provisioner.Name), }) - instanceInput.UserData = lo.ToPtr(base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(string(rawContent), settings.FromContext(env.Context).ClusterName, - settings.FromContext(env.Context).ClusterEndpoint, env.ExpectCABundle(), provisioner.Name)))) + instanceInput.UserData = lo.ToPtr(base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(string(rawContent), env.ClusterName, + env.ClusterEndpoint, env.ExpectCABundle(), provisioner.Name)))) // Create an instance manually to mock Karpenter launching an instance out := env.ExpectRunInstances(instanceInput) @@ -141,15 +141,15 @@ var _ = Describe("MachineLink", func() { return aws.StringValue(t.Key) == v1alpha5.MachineManagedByAnnotationKey }) g.Expect(ok).To(BeTrue()) - g.Expect(aws.StringValue(tag.Value)).To(Equal(settings.FromContext(env.Context).ClusterName)) + g.Expect(aws.StringValue(tag.Value)).To(Equal(env.ClusterName)) }, time.Minute, time.Second).Should(Succeed()) }) It("should succeed to link a Machine for an existing instance launched by Karpenter with provider", func() { provisioner := test.Provisioner(test.ProvisionerOptions{ Provider: &v1alpha1.AWS{ AMIFamily: &v1alpha1.AMIFamilyAL2, - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }, }) env.ExpectCreated(provisioner) @@ -161,8 +161,8 @@ var _ = Describe("MachineLink", func() { Key: aws.String(v1alpha5.ProvisionerNameLabelKey), Value: aws.String(provisioner.Name), }) - instanceInput.UserData = lo.ToPtr(base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(string(rawContent), settings.FromContext(env.Context).ClusterName, - settings.FromContext(env.Context).ClusterEndpoint, env.ExpectCABundle(), provisioner.Name)))) + instanceInput.UserData = lo.ToPtr(base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(string(rawContent), env.ClusterName, + env.ClusterEndpoint, env.ExpectCABundle(), provisioner.Name)))) // Create an instance manually to mock Karpenter launching an instance out := env.ExpectRunInstances(instanceInput) @@ -200,14 +200,14 @@ var _ = Describe("MachineLink", func() { return aws.StringValue(t.Key) == v1alpha5.MachineManagedByAnnotationKey }) g.Expect(ok).To(BeTrue()) - g.Expect(aws.StringValue(tag.Value)).To(Equal(settings.FromContext(env.Context).ClusterName)) + g.Expect(aws.StringValue(tag.Value)).To(Equal(env.ClusterName)) }, time.Minute, time.Second).Should(Succeed()) }) It("should succeed to link a Machine for an existing instance re-owned by Karpenter", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ AMIFamily: &v1alpha1.AMIFamilyAL2, - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}, @@ -219,8 +219,8 @@ var _ = Describe("MachineLink", func() { Expect(err).ToNot(HaveOccurred()) // No tag specifications since we're mocking an instance not launched by Karpenter - instanceInput.UserData = lo.ToPtr(base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(string(rawContent), settings.FromContext(env.Context).ClusterName, - settings.FromContext(env.Context).ClusterEndpoint, env.ExpectCABundle(), provisioner.Name)))) + instanceInput.UserData = lo.ToPtr(base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(string(rawContent), env.ClusterName, + env.ClusterEndpoint, env.ExpectCABundle(), provisioner.Name)))) // Create an instance manually to mock Karpenter launching an instance out := env.ExpectRunInstances(instanceInput) @@ -263,7 +263,7 @@ var _ = Describe("MachineLink", func() { return aws.StringValue(t.Key) == v1alpha5.MachineManagedByAnnotationKey }) g.Expect(ok).To(BeTrue()) - g.Expect(aws.StringValue(tag.Value)).To(Equal(settings.FromContext(env.Context).ClusterName)) + g.Expect(aws.StringValue(tag.Value)).To(Equal(env.ClusterName)) tag, ok = lo.Find(instance.Tags, func(t *ec2.Tag) bool { return aws.StringValue(t.Key) == v1alpha5.ProvisionerNameLabelKey }) diff --git a/test/suites/machine/machine_test.go b/test/suites/machine/machine_test.go index f92d4ec81734..b86c5d51917f 100644 --- a/test/suites/machine/machine_test.go +++ b/test/suites/machine/machine_test.go @@ -31,7 +31,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" "github.com/aws/karpenter-core/pkg/utils/resources" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" ) @@ -40,8 +39,8 @@ var _ = Describe("StandaloneMachine", func() { var nodeTemplate *v1alpha1.AWSNodeTemplate BeforeEach(func() { nodeTemplate = awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) }) It("should create a standard machine within the 'c' instance family", func() { @@ -273,8 +272,8 @@ var _ = Describe("StandaloneMachine", func() { // Create userData that adds custom labels through the --kubelet-extra-args nodeTemplate.Spec.AMIFamily = &v1alpha1.AMIFamilyCustom nodeTemplate.Spec.AMISelector = map[string]string{"aws-ids": customAMI} - nodeTemplate.Spec.UserData = lo.ToPtr(base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(string(rawContent), settings.FromContext(env.Context).ClusterName, - settings.FromContext(env.Context).ClusterEndpoint, env.ExpectCABundle())))) + nodeTemplate.Spec.UserData = lo.ToPtr(base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf(string(rawContent), env.ClusterName, + env.ClusterEndpoint, env.ExpectCABundle())))) machine := test.Machine(v1alpha5.Machine{ Spec: v1alpha5.MachineSpec{ diff --git a/test/suites/scale/deprovisioning_test.go b/test/suites/scale/deprovisioning_test.go index e4f818297530..1ae51e001155 100644 --- a/test/suites/scale/deprovisioning_test.go +++ b/test/suites/scale/deprovisioning_test.go @@ -32,7 +32,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" "github.com/aws/karpenter/pkg/controllers/interruption/messages" "github.com/aws/karpenter/pkg/controllers/interruption/messages/scheduledchange" @@ -86,8 +85,8 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), Label(debug.NoEvents), "featureGates.driftEnabled": "true", }) nodeTemplate = awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisionerOptions = test.ProvisionerOptions{ ProviderRef: &v1alpha5.MachineTemplateRef{ diff --git a/test/suites/scale/provisioning_test.go b/test/suites/scale/provisioning_test.go index b59667ce00fd..0ebf0268815b 100644 --- a/test/suites/scale/provisioning_test.go +++ b/test/suites/scale/provisioning_test.go @@ -28,7 +28,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" awstest "github.com/aws/karpenter/pkg/test" "github.com/aws/karpenter/test/pkg/debug" @@ -46,8 +45,8 @@ var _ = Describe("Provisioning", Label(debug.NoWatch), Label(debug.NoEvents), fu BeforeEach(func() { nodeTemplate = awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner = test.Provisioner(test.ProvisionerOptions{ ProviderRef: &v1alpha5.MachineTemplateRef{ diff --git a/test/suites/utilization/suite_test.go b/test/suites/utilization/suite_test.go index 9504da9fa598..9c09f49b66b4 100644 --- a/test/suites/utilization/suite_test.go +++ b/test/suites/utilization/suite_test.go @@ -25,7 +25,6 @@ import ( "github.com/aws/karpenter-core/pkg/apis/v1alpha5" "github.com/aws/karpenter-core/pkg/test" - "github.com/aws/karpenter/pkg/apis/settings" "github.com/aws/karpenter/pkg/apis/v1alpha1" "github.com/aws/karpenter/test/pkg/debug" @@ -53,8 +52,8 @@ var _ = AfterEach(func() { env.AfterEach() }) var _ = Describe("Utilization", Label(debug.NoWatch), Label(debug.NoEvents), func() { It("should provision one pod per node", func() { provider := awstest.AWSNodeTemplate(v1alpha1.AWSNodeTemplateSpec{AWS: v1alpha1.AWS{ - SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, - SubnetSelector: map[string]string{"karpenter.sh/discovery": settings.FromContext(env.Context).ClusterName}, + SecurityGroupSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, + SubnetSelector: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }}) provisioner := test.Provisioner(test.ProvisionerOptions{ProviderRef: &v1alpha5.MachineTemplateRef{Name: provider.Name}, Requirements: []v1.NodeSelectorRequirement{{ Key: v1.LabelInstanceTypeStable, diff --git a/tools/allocatable-diff/go.mod b/tools/allocatable-diff/go.mod index 040414d5407b..ddfeda4d5d07 100644 --- a/tools/allocatable-diff/go.mod +++ b/tools/allocatable-diff/go.mod @@ -67,12 +67,12 @@ require ( go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.25.0 // indirect golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 // indirect - golang.org/x/net v0.12.0 // indirect + golang.org/x/net v0.17.0 // indirect golang.org/x/oauth2 v0.8.0 // indirect golang.org/x/sync v0.3.0 // indirect - golang.org/x/sys v0.10.0 // indirect - golang.org/x/term v0.10.0 // indirect - golang.org/x/text v0.12.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/term v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect golang.org/x/time v0.3.0 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/api v0.124.0 // indirect diff --git a/tools/allocatable-diff/go.sum b/tools/allocatable-diff/go.sum index aaa20cd1b9ae..96fcd58ca328 100644 --- a/tools/allocatable-diff/go.sum +++ b/tools/allocatable-diff/go.sum @@ -431,8 +431,8 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= -golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -498,13 +498,13 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= -golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= -golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -513,8 +513,8 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= -golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/tools/karpenter-convert/README.md b/tools/karpenter-convert/README.md index 574b79a71f88..e83eb1ebcd55 100644 --- a/tools/karpenter-convert/README.md +++ b/tools/karpenter-convert/README.md @@ -9,7 +9,20 @@ It converts `v1alpha5/Provisioner` to `v1beta1/NodePool` and `v1alpha1/AWSNodeTe go install github.com/aws/karpenter/tools/karpenter-convert/cmd/karpenter-convert@latest ``` -## Usage: +## Usage +```console +Usage: + karpenter-convert [flags] + +Flags: + -f, --filename strings Filename, directory, or URL to files to need to get converted. + -h, --help help for karpenter-convert + --ignore-defaults Ignore defining default requirements when migrating Provisioners to NodePool. + -o, --output string Output format. One of: (json, yaml, name, go-template, go-template-file, template, templatefile, jsonpath, jsonpath-as-json, jsonpath-file). (default "yaml") + -R, --recursive Process the directory used in -f, --filename recursively. Useful when you want to manage related manifests organized within the same directory. +``` + +## Examples: ```console # Convert a single Provisioner file to NodePool @@ -23,9 +36,14 @@ karpenter-convert -f . > output.yaml # Convert a single provisioner and apply directly to the cluster karpenter-convert -f provisioner.yaml | kubectl apply -f - + +# Convert a single provisioner ignoring the default requirements +# Karpenter provisioners had logic to default Instance Families, OS, Architecture and Cpacity type when these were not provided. +# NodePool drops these defaults, and you can avoid that the conversion tools applies them for you during the conversion +karpenter-convert --ignore-defaults -f provisioner.yaml > nodepool.yaml ``` ## Usage notes When converting an AWSNodeTemplate to EC2NodeClass, the newly introduced field `role` can't be mapped automatically. -The tool leaves a placeholder `` which needs to be manually updated. +The tool leaves a placeholder `$KARPENTER_NODE_ROLE` which needs to be manually updated. diff --git a/tools/karpenter-convert/go.mod b/tools/karpenter-convert/go.mod index fb38f85ceeda..45b70a1c78c7 100644 --- a/tools/karpenter-convert/go.mod +++ b/tools/karpenter-convert/go.mod @@ -7,6 +7,7 @@ require ( github.com/aws/karpenter-core v0.31.1-0.20231003200119-b5ccaf30be9c github.com/onsi/ginkgo/v2 v2.12.1 github.com/onsi/gomega v1.28.0 + github.com/samber/lo v1.38.1 github.com/spf13/cobra v1.7.0 k8s.io/apimachinery v0.28.2 k8s.io/cli-runtime v0.28.2 @@ -21,6 +22,7 @@ require ( github.com/aws/aws-sdk-go v1.45.15 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/blang/semver/v4 v4.0.0 // indirect + github.com/blendle/zapdriver v1.3.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chai2010/gettext-go v1.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -68,18 +70,18 @@ require ( github.com/prometheus/common v0.44.0 // indirect github.com/prometheus/procfs v0.11.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect - github.com/samber/lo v1.38.1 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stretchr/testify v1.8.2 // indirect github.com/xlab/treeprint v1.2.0 // indirect go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.26.0 // indirect golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e // indirect - golang.org/x/net v0.14.0 // indirect + golang.org/x/net v0.17.0 // indirect golang.org/x/oauth2 v0.8.0 // indirect golang.org/x/sync v0.3.0 // indirect - golang.org/x/sys v0.12.0 // indirect - golang.org/x/term v0.11.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/term v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.12.0 // indirect @@ -90,6 +92,8 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/api v0.28.2 // indirect k8s.io/apiextensions-apiserver v0.26.6 // indirect + k8s.io/cloud-provider v0.26.6 // indirect + k8s.io/csi-translation-lib v0.26.6 // indirect k8s.io/klog/v2 v2.100.1 // indirect k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect diff --git a/tools/karpenter-convert/go.sum b/tools/karpenter-convert/go.sum index ac88781b0f81..45236adb1e10 100644 --- a/tools/karpenter-convert/go.sum +++ b/tools/karpenter-convert/go.sum @@ -209,8 +209,13 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= go.starlark.net v0.0.0-20230525235612-a134d8f9ddca h1:VdD38733bfYv5tUZwEIskMM93VanwNIi5bIKnDrJdEY= go.starlark.net v0.0.0-20230525235612-a134d8f9ddca/go.mod h1:jxU+3+j+71eXOW14274+SmmuW82qJzl6iZSeqEtTGds= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= +go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -240,8 +245,8 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= -golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8= golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= @@ -266,14 +271,14 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20220526004731-065cf7ba2467/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= -golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= diff --git a/tools/karpenter-convert/pkg/convert/convert.go b/tools/karpenter-convert/pkg/convert/convert.go index d49685efda54..53fc8fd07b11 100644 --- a/tools/karpenter-convert/pkg/convert/convert.go +++ b/tools/karpenter-convert/pkg/convert/convert.go @@ -15,9 +15,14 @@ limitations under the License. package convert import ( + "bytes" + "context" "fmt" + "io" "regexp" + "strings" + "github.com/samber/lo" "github.com/spf13/cobra" "github.com/aws/karpenter/pkg/apis" @@ -32,13 +37,17 @@ import ( "k8s.io/cli-runtime/pkg/printers" "k8s.io/cli-runtime/pkg/resource" - "github.com/aws/karpenter-core/pkg/apis/v1alpha5" + corev1alpha5 "github.com/aws/karpenter-core/pkg/apis/v1alpha5" corev1beta1 "github.com/aws/karpenter-core/pkg/apis/v1beta1" nodepoolutil "github.com/aws/karpenter-core/pkg/utils/nodepool" cmdutil "k8s.io/kubectl/pkg/cmd/util" + + "github.com/aws/karpenter/pkg/apis/v1alpha5" ) +const karpenterNodeRolePlaceholder string = "$KARPENTER_NODE_ROLE" + type Context struct { PrintFlags *genericclioptions.PrintFlags Printer printers.ResourcePrinter @@ -47,6 +56,8 @@ type Context struct { resource.FilenameOptions genericiooptions.IOStreams + + IgnoreDefaults bool } func NewCmd(f cmdutil.Factory, ioStreams genericiooptions.IOStreams) *cobra.Command { @@ -63,33 +74,37 @@ func NewCmd(f cmdutil.Factory, ioStreams genericiooptions.IOStreams) *cobra.Comm }, } - cmdutil.AddFilenameOptionFlags(rootCmd, &o.FilenameOptions, "to need to get converted.") + rootCmd.Flags().BoolVar(&o.IgnoreDefaults, "ignore-defaults", o.IgnoreDefaults, "Ignore defining default requirements when migrating Provisioners to NodePool.") + cmdutil.AddJsonFilenameFlag(rootCmd.Flags(), &o.Filenames, "Filename, directory, or URL to files to need to get converted.") + rootCmd.Flags().BoolVarP(&o.Recursive, "recursive", "R", o.Recursive, "Process the directory used in -f, --filename recursively. Useful when you want to manage related manifests organized within the same directory.") + o.PrintFlags.AddFlags(rootCmd) return rootCmd } func (o *Context) Complete(f cmdutil.Factory, _ *cobra.Command) (err error) { - err = o.FilenameOptions.RequireFilenameOrKustomize() - if err != nil { - return err + if len(o.Filenames) == 0 { + return fmt.Errorf("must specify -f") } + o.builder = f.NewBuilder o.Printer, err = o.PrintFlags.ToPrinter() return err } +//nolint:gocyclo func (o *Context) RunConvert() error { scheme := runtime.NewScheme() if err := apis.AddToScheme(scheme); err != nil { return err } - if err := v1alpha5.SchemeBuilder.AddToScheme(scheme); err != nil { + if err := corev1alpha5.SchemeBuilder.AddToScheme(scheme); err != nil { return err } b := o.builder(). - WithScheme(scheme, v1alpha1.SchemeGroupVersion, v1alpha5.SchemeGroupVersion). + WithScheme(scheme, v1alpha1.SchemeGroupVersion, corev1alpha5.SchemeGroupVersion). LocalParam(true) r := b. @@ -128,52 +143,108 @@ func (o *Context) RunConvert() error { continue } - obj, err := Process(info.Object) + obj, err := convert(info.Object, o) if err != nil { fmt.Fprintln(o.IOStreams.ErrOut, err.Error()) } else { - if err := o.Printer.PrintObj(obj, o.Out); err != nil { + var buffer bytes.Buffer + writer := io.Writer(&buffer) + + if err := o.Printer.PrintObj(obj, writer); err != nil { + fmt.Fprintln(o.IOStreams.ErrOut, err.Error()) + } + + output := dropFields(buffer) + + if _, err := o.Out.Write([]byte(output)); err != nil { fmt.Fprintln(o.IOStreams.ErrOut, err.Error()) } } } - return nil } -func Process(resource runtime.Object) (runtime.Object, error) { +func dropFields(buffer bytes.Buffer) string { + output := buffer.String() + output = strings.Replace(output, "status: {}\n", "", -1) + output = strings.Replace(output, " creationTimestamp: null\n", "", -1) + output = strings.Replace(output, " creationTimestamp: null\n", "", -1) + output = strings.Replace(output, " resources: {}\n", "", -1) + + return output +} + +// Convert a Provisioner into a NodePool and an AWSNodeTemplate into a NodeClass. +// If the input is of a different kind, returns an error +func convert(resource runtime.Object, o *Context) (runtime.Object, error) { kind := resource.GetObjectKind().GroupVersionKind().Kind switch kind { case "Provisioner": - return processProvisioner(resource), nil + return convertProvisioner(resource, o), nil case "AWSNodeTemplate": - return processNodeTemplate(resource), nil + return convertNodeTemplate(resource), nil default: return nil, fmt.Errorf("unknown kind. expected one of Provisioner, AWSNodeTemplate. got %s", kind) } } -func processNodeTemplate(resource runtime.Object) runtime.Object { +func convertNodeTemplate(resource runtime.Object) runtime.Object { nodetemplate := resource.(*v1alpha1.AWSNodeTemplate) + // If the AMIFamily wasn't specified, then we know that it should be AL2 for the conversion + if nodetemplate.Spec.AMIFamily == nil { + nodetemplate.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 + } nodeclass := nodeclassutil.New(nodetemplate) nodeclass.TypeMeta = metav1.TypeMeta{ Kind: "EC2NodeClass", APIVersion: v1beta1.SchemeGroupVersion.String(), } - nodeclass.Spec.Role = "" + // From the input NodeTemplate, keep only name, labels, annotations and finalizers + nodeclass.ObjectMeta = metav1.ObjectMeta{ + Name: nodetemplate.Name, + Labels: nodetemplate.Labels, + Annotations: nodetemplate.Annotations, + Finalizers: nodetemplate.Finalizers, + } + + // Cleanup the status provided in input + nodeclass.Status = v1beta1.EC2NodeClassStatus{} + + // Leave a placeholder for the role. This can be substituted with `envsubst` or other means + nodeclass.Spec.Role = karpenterNodeRolePlaceholder return nodeclass } -func processProvisioner(resource runtime.Object) runtime.Object { - provisioner := resource.(*v1alpha5.Provisioner) +func convertProvisioner(resource runtime.Object, o *Context) runtime.Object { + coreprovisioner := resource.(*corev1alpha5.Provisioner) - nodepool := nodepoolutil.New(provisioner) + if !o.IgnoreDefaults { + provisioner := lo.ToPtr(v1alpha5.Provisioner(lo.FromPtr(coreprovisioner))) + provisioner.SetDefaults(context.Background()) + coreprovisioner = lo.ToPtr(corev1alpha5.Provisioner(lo.FromPtr(provisioner))) + } + + nodepool := nodepoolutil.New(coreprovisioner) nodepool.TypeMeta = metav1.TypeMeta{ Kind: "NodePool", APIVersion: corev1beta1.SchemeGroupVersion.String(), } + // From the input Provisioner, keep only name, labels, annotations and finalizers + nodepool.ObjectMeta = metav1.ObjectMeta{ + Name: coreprovisioner.Name, + Labels: coreprovisioner.Labels, + Annotations: coreprovisioner.Annotations, + Finalizers: coreprovisioner.Finalizers, + } + + // Reset timestamp if present + nodepool.Spec.Template.CreationTimestamp = metav1.Time{} + + // Cleanup the status provided in input + nodepool.Status = corev1beta1.NodePoolStatus{} + return nodepool } diff --git a/tools/karpenter-convert/pkg/convert/convert_test.go b/tools/karpenter-convert/pkg/convert/convert_test.go index 11b6e39a4252..a5c4433d30cd 100644 --- a/tools/karpenter-convert/pkg/convert/convert_test.go +++ b/tools/karpenter-convert/pkg/convert/convert_test.go @@ -19,6 +19,7 @@ import ( "fmt" "net/http" "os" + "strconv" "testing" . "github.com/onsi/ginkgo/v2" @@ -38,9 +39,10 @@ func TestConvert(t *testing.T) { var _ = Describe("ConvertObject", func() { type testcase struct { - name string - file string - outputFile string + name string + ignoreDefaults bool + file string + outputFile string } DescribeTable("conversion tests", @@ -63,13 +65,16 @@ var _ = Describe("ConvertObject", func() { if err := cmd.Flags().Set("output", "yaml"); err != nil { Expect(err).To(BeNil()) } + if err := cmd.Flags().Set("ignore-defaults", strconv.FormatBool(tc.ignoreDefaults)); err != nil { + Expect(err).To(BeNil()) + } cmd.Run(cmd, []string{}) bytes, _ := os.ReadFile(tc.outputFile) content := string(bytes) - Expect(buf.String()).To(ContainSubstring(content), fmt.Sprintf("unexpected output when converting %s to %q, expected: %q, but got %q", tc.file, tc.outputFile, content, buf.String())) + Expect(buf.String()).To(Equal(content), fmt.Sprintf("unexpected output when converting %s to %q, expected: %q, but got %q", tc.file, tc.outputFile, content, buf.String())) }, Entry("provisioner to nodepool", @@ -80,6 +85,33 @@ var _ = Describe("ConvertObject", func() { }, ), + Entry("provisioner (set defaults) to nodepool", + testcase{ + name: "provisioner to nodepool", + file: "./testdata/provisioner_defaults.yaml", + outputFile: "./testdata/nodepool_defaults.yaml", + ignoreDefaults: false, + }, + ), + + Entry("provisioner (no set defaults) to nodepool", + testcase{ + name: "provisioner to nodepool", + file: "./testdata/provisioner_no_defaults.yaml", + outputFile: "./testdata/nodepool_no_defaults.yaml", + ignoreDefaults: true, + }, + ), + + Entry("provisioner (kubectl output) to nodepool", + testcase{ + name: "provisioner to nodepool", + file: "./testdata/provisioner_kubectl_output.yaml", + outputFile: "./testdata/nodepool_kubectl_output.yaml", + ignoreDefaults: true, + }, + ), + Entry("nodetemplate to nodeclass", testcase{ name: "nodetemplate to nodeclass", @@ -87,5 +119,21 @@ var _ = Describe("ConvertObject", func() { outputFile: "./testdata/nodeclass.yaml", }, ), + + Entry("nodetemplate (empty amifamily) to nodeclass", + testcase{ + name: "nodetemplate (empty amifamily) to nodeclass", + file: "./testdata/nodetemplate_no_amifamily.yaml", + outputFile: "./testdata/nodeclass_no_amifamily.yaml", + }, + ), + + Entry("nodetemplate (kubectl output) to nodeclass", + testcase{ + name: "nodetemplate to nodeclass", + file: "./testdata/nodetemplate_kubectl_output.yaml", + outputFile: "./testdata/nodeclass_kubectl_output.yaml", + }, + ), ) }) diff --git a/tools/karpenter-convert/pkg/convert/testdata/nodeclass.yaml b/tools/karpenter-convert/pkg/convert/testdata/nodeclass.yaml index fad3593d8f9c..d4ce60f377bc 100644 --- a/tools/karpenter-convert/pkg/convert/testdata/nodeclass.yaml +++ b/tools/karpenter-convert/pkg/convert/testdata/nodeclass.yaml @@ -1,7 +1,6 @@ apiVersion: karpenter.k8s.aws/v1beta1 kind: EC2NodeClass metadata: - creationTimestamp: null name: default spec: amiFamily: Bottlerocket @@ -32,7 +31,7 @@ spec: deleteOnTermination: true volumeSize: 100Gi volumeType: gp3 - role: + role: $KARPENTER_NODE_ROLE securityGroupSelectorTerms: - tags: karpenter.sh/discovery: $MY_CLUSTER_NAME @@ -47,4 +46,3 @@ spec: kube-api-qps = 30 [settings.kubernetes.eviction-hard] "memory.available" = "10%" -status: {} diff --git a/tools/karpenter-convert/pkg/convert/testdata/nodeclass_kubectl_output.yaml b/tools/karpenter-convert/pkg/convert/testdata/nodeclass_kubectl_output.yaml new file mode 100644 index 000000000000..bedfe9304dff --- /dev/null +++ b/tools/karpenter-convert/pkg/convert/testdata/nodeclass_kubectl_output.yaml @@ -0,0 +1,16 @@ +apiVersion: karpenter.k8s.aws/v1beta1 +kind: EC2NodeClass +metadata: + name: default +spec: + amiFamily: AL2 + role: $KARPENTER_NODE_ROLE + securityGroupSelectorTerms: + - tags: + karpenter.sh/discovery: eks-workshop-camigration + subnetSelectorTerms: + - tags: + karpenter.sh/discovery: eks-workshop-camigration + tags: + app.kubernetes.io/created-by: eks-workshop + aws-node-termination-handler/managed: "true" diff --git a/tools/karpenter-convert/pkg/convert/testdata/nodeclass_no_amifamily.yaml b/tools/karpenter-convert/pkg/convert/testdata/nodeclass_no_amifamily.yaml new file mode 100644 index 000000000000..32f7c304b15e --- /dev/null +++ b/tools/karpenter-convert/pkg/convert/testdata/nodeclass_no_amifamily.yaml @@ -0,0 +1,27 @@ +apiVersion: karpenter.k8s.aws/v1beta1 +kind: EC2NodeClass +metadata: + name: default +spec: + amiFamily: AL2 + blockDeviceMappings: + - deviceName: /dev/xvdb + ebs: + deleteOnTermination: true + volumeSize: 100Gi + volumeType: gp3 + role: $KARPENTER_NODE_ROLE + securityGroupSelectorTerms: + - tags: + karpenter.sh/discovery: karpenter-demo + subnetSelectorTerms: + - tags: + karpenter.sh/discovery: karpenter-demo + tags: + MyBackupTag: "yes" + MyTag: "1234" + userData: | + [settings.kubernetes] + kube-api-qps = 30 + [settings.kubernetes.eviction-hard] + "memory.available" = "10%" diff --git a/tools/karpenter-convert/pkg/convert/testdata/nodepool.yaml b/tools/karpenter-convert/pkg/convert/testdata/nodepool.yaml index b7509a48717b..7831c6969171 100644 --- a/tools/karpenter-convert/pkg/convert/testdata/nodepool.yaml +++ b/tools/karpenter-convert/pkg/convert/testdata/nodepool.yaml @@ -1,7 +1,6 @@ apiVersion: karpenter.sh/v1beta1 kind: NodePool metadata: - creationTimestamp: null name: default spec: disruption: @@ -15,7 +14,6 @@ spec: metadata: annotations: example.com/owner: my-team - creationTimestamp: null labels: billing-team: my-team spec: @@ -87,7 +85,10 @@ spec: values: - spot - on-demand - resources: {} + - key: kubernetes.io/os + operator: In + values: + - linux startupTaints: - effect: NoSchedule key: example.com/another-taint @@ -95,4 +96,3 @@ spec: - effect: NoSchedule key: example.com/special-taint weight: 10 -status: {} diff --git a/tools/karpenter-convert/pkg/convert/testdata/nodepool_defaults.yaml b/tools/karpenter-convert/pkg/convert/testdata/nodepool_defaults.yaml new file mode 100644 index 000000000000..7617d9c7f861 --- /dev/null +++ b/tools/karpenter-convert/pkg/convert/testdata/nodepool_defaults.yaml @@ -0,0 +1,51 @@ +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: default +spec: + disruption: + consolidateAfter: 30s + consolidationPolicy: WhenEmpty + expireAfter: 720h0m0s + limits: + cpu: 1k + memory: 1000Gi + template: + metadata: + annotations: + example.com/owner: my-team + labels: + billing-team: my-team + spec: + nodeClassRef: + name: default + requirements: + - key: kubernetes.io/os + operator: In + values: + - linux + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: karpenter.sh/capacity-type + operator: In + values: + - on-demand + - key: karpenter.k8s.aws/instance-category + operator: In + values: + - c + - m + - r + - key: karpenter.k8s.aws/instance-generation + operator: Gt + values: + - "2" + startupTaints: + - effect: NoSchedule + key: example.com/another-taint + taints: + - effect: NoSchedule + key: example.com/special-taint + weight: 10 diff --git a/tools/karpenter-convert/pkg/convert/testdata/nodepool_kubectl_output.yaml b/tools/karpenter-convert/pkg/convert/testdata/nodepool_kubectl_output.yaml new file mode 100644 index 000000000000..e8f5cf2e3756 --- /dev/null +++ b/tools/karpenter-convert/pkg/convert/testdata/nodepool_kubectl_output.yaml @@ -0,0 +1,46 @@ +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: provisioner1 +spec: + disruption: + expireAfter: Never + limits: + cpu: "100" + memory: 1000Gi + template: + metadata: + labels: + type: app1 + spec: + nodeClassRef: + name: default + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: + - on-demand + - key: node.kubernetes.io/instance-family + operator: In + values: + - c + - r + - m + - key: kubernetes.io/os + operator: In + values: + - linux + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: karpenter.k8s.aws/instance-category + operator: In + values: + - c + - m + - r + - key: karpenter.k8s.aws/instance-generation + operator: Gt + values: + - "2" diff --git a/tools/karpenter-convert/pkg/convert/testdata/nodepool_no_defaults.yaml b/tools/karpenter-convert/pkg/convert/testdata/nodepool_no_defaults.yaml new file mode 100644 index 000000000000..795149212e2c --- /dev/null +++ b/tools/karpenter-convert/pkg/convert/testdata/nodepool_no_defaults.yaml @@ -0,0 +1,29 @@ +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: default +spec: + disruption: + consolidateAfter: 30s + consolidationPolicy: WhenEmpty + expireAfter: 720h0m0s + limits: + cpu: 1k + memory: 1000Gi + template: + metadata: + annotations: + example.com/owner: my-team + labels: + billing-team: my-team + spec: + nodeClassRef: + name: default + requirements: null + startupTaints: + - effect: NoSchedule + key: example.com/another-taint + taints: + - effect: NoSchedule + key: example.com/special-taint + weight: 10 diff --git a/tools/karpenter-convert/pkg/convert/testdata/nodetemplate_kubectl_output.yaml b/tools/karpenter-convert/pkg/convert/testdata/nodetemplate_kubectl_output.yaml new file mode 100644 index 000000000000..a4327e5d3108 --- /dev/null +++ b/tools/karpenter-convert/pkg/convert/testdata/nodetemplate_kubectl_output.yaml @@ -0,0 +1,32 @@ +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +metadata: + creationTimestamp: "2023-05-08T13:37:43Z" + generation: 2 + name: default + resourceVersion: "44860087" + uid: 1da65734-30f3-47c2-9bec-8b671a71fb67 +spec: + securityGroupSelector: + karpenter.sh/discovery: eks-workshop-camigration + subnetSelector: + karpenter.sh/discovery: eks-workshop-camigration + tags: + app.kubernetes.io/created-by: eks-workshop + aws-node-termination-handler/managed: "true" +status: + securityGroups: + - id: sg-0d92f39ac96c4b1b4 + subnets: + - id: subnet-0537532d0cfbdb536 + zone: eu-west-1b + - id: subnet-0f038ea98c29b03a8 + zone: eu-west-1c + - id: subnet-0ce70ec5f86ceedc8 + zone: eu-west-1a + - id: subnet-0b42ad5aea229d2af + zone: eu-west-1b + - id: subnet-065094b81faaa208a + zone: eu-west-1c + - id: subnet-09d0686695ae92fd3 + zone: eu-west-1a diff --git a/tools/karpenter-convert/pkg/convert/testdata/nodetemplate_no_amifamily.yaml b/tools/karpenter-convert/pkg/convert/testdata/nodetemplate_no_amifamily.yaml new file mode 100644 index 000000000000..cbcd79974c7a --- /dev/null +++ b/tools/karpenter-convert/pkg/convert/testdata/nodetemplate_no_amifamily.yaml @@ -0,0 +1,23 @@ +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +metadata: + name: default +spec: + subnetSelector: + karpenter.sh/discovery: karpenter-demo + securityGroupSelector: + karpenter.sh/discovery: karpenter-demo + blockDeviceMappings: + - deviceName: /dev/xvdb + ebs: + volumeSize: 100Gi + volumeType: gp3 + deleteOnTermination: true + tags: + MyTag: "1234" + MyBackupTag: "yes" + userData: | + [settings.kubernetes] + kube-api-qps = 30 + [settings.kubernetes.eviction-hard] + "memory.available" = "10%" \ No newline at end of file diff --git a/tools/karpenter-convert/pkg/convert/testdata/provisioner.yaml b/tools/karpenter-convert/pkg/convert/testdata/provisioner.yaml index ec196355e6d6..a548d265557d 100644 --- a/tools/karpenter-convert/pkg/convert/testdata/provisioner.yaml +++ b/tools/karpenter-convert/pkg/convert/testdata/provisioner.yaml @@ -3,35 +3,18 @@ kind: Provisioner metadata: name: default spec: - # References cloud provider-specific custom resource, see your cloud provider specific documentation providerRef: name: default - - # Provisioned nodes will have these taints - # Taints may prevent pods from scheduling if they are not tolerated by the pod. taints: - key: example.com/special-taint effect: NoSchedule - - # Provisioned nodes will have these taints, but pods do not need to tolerate these taints to be provisioned by this - # provisioner. These taints are expected to be temporary and some other entity (e.g. a DaemonSet) is responsible for - # removing the taint after it has finished initializing the node. startupTaints: - key: example.com/another-taint effect: NoSchedule - - # Labels are arbitrary key-values that are applied to all nodes labels: billing-team: my-team - - # Annotations are arbitrary key-values that are applied to all nodes annotations: example.com/owner: "my-team" - - # Requirements that constrain the parameters of provisioned nodes. - # These requirements are combined with pod.spec.affinity.nodeAffinity rules. - # Operators { In, NotIn, Exists, DoesNotExist, Gt, and Lt } are supported. - # https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#operators requirements: - key: "karpenter.k8s.aws/instance-category" operator: In @@ -54,9 +37,6 @@ spec: - key: "karpenter.sh/capacity-type" # If not included, the webhook for the AWS cloud provider will default to on-demand operator: In values: ["spot", "on-demand"] - - # Karpenter provides the ability to specify a few additional Kubelet args. - # These are all optional and provide support for additional customization and use cases. kubeletConfiguration: clusterDNS: ["10.0.1.100"] containerRuntime: containerd @@ -86,28 +66,12 @@ spec: cpuCFSQuota: true podsPerCore: 2 maxPods: 20 - - - # Resource limits constrain the total size of the cluster. - # Limits prevent Karpenter from creating new instances once the limit is exceeded. limits: resources: cpu: "1000" memory: 1000Gi - - # Enables consolidation which attempts to reduce cluster cost by both removing un-needed nodes and down-sizing those - # that can't be removed. Mutually exclusive with the ttlSecondsAfterEmpty parameter. consolidation: enabled: false - - # If omitted, the feature is disabled and nodes will never expire. If set to less time than it requires for a node - # to become ready, the node may expire before any pods successfully start. ttlSecondsUntilExpired: 2592000 # 30 Days = 60 * 60 * 24 * 30 Seconds; - - # If omitted, the feature is disabled, nodes will never scale down due to low utilization ttlSecondsAfterEmpty: 30 - - # Priority given to the provisioner when the scheduler considers which provisioner - # to select. Higher weights indicate higher priority when comparing provisioners. - # Specifying no weight is equivalent to specifying a weight of 0. weight: 10 \ No newline at end of file diff --git a/tools/karpenter-convert/pkg/convert/testdata/provisioner_defaults.yaml b/tools/karpenter-convert/pkg/convert/testdata/provisioner_defaults.yaml new file mode 100644 index 000000000000..602e7f9f3452 --- /dev/null +++ b/tools/karpenter-convert/pkg/convert/testdata/provisioner_defaults.yaml @@ -0,0 +1,26 @@ +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: default +spec: + providerRef: + name: default + taints: + - key: example.com/special-taint + effect: NoSchedule + startupTaints: + - key: example.com/another-taint + effect: NoSchedule + labels: + billing-team: my-team + annotations: + example.com/owner: "my-team" + limits: + resources: + cpu: "1000" + memory: 1000Gi + consolidation: + enabled: false + ttlSecondsUntilExpired: 2592000 # 30 Days = 60 * 60 * 24 * 30 Seconds; + ttlSecondsAfterEmpty: 30 + weight: 10 \ No newline at end of file diff --git a/tools/karpenter-convert/pkg/convert/testdata/provisioner_kubectl_output.yaml b/tools/karpenter-convert/pkg/convert/testdata/provisioner_kubectl_output.yaml new file mode 100644 index 000000000000..3ab7703b4ace --- /dev/null +++ b/tools/karpenter-convert/pkg/convert/testdata/provisioner_kubectl_output.yaml @@ -0,0 +1,47 @@ +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + creationTimestamp: "2023-07-10T13:41:16Z" + generation: 1 + name: provisioner1 + resourceVersion: "44858469" + uid: 71fc8dfd-7ae5-452e-9d72-57e1fca75fdb +spec: + labels: + type: app1 + limits: + resources: + cpu: "100" + memory: 1000Gi + providerRef: + name: default + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: + - on-demand + - key: node.kubernetes.io/instance-family + operator: In + values: + - c + - r + - m + - key: kubernetes.io/os + operator: In + values: + - linux + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: karpenter.k8s.aws/instance-category + operator: In + values: + - c + - m + - r + - key: karpenter.k8s.aws/instance-generation + operator: Gt + values: + - "2" +status: {} \ No newline at end of file diff --git a/tools/karpenter-convert/pkg/convert/testdata/provisioner_no_defaults.yaml b/tools/karpenter-convert/pkg/convert/testdata/provisioner_no_defaults.yaml new file mode 100644 index 000000000000..602e7f9f3452 --- /dev/null +++ b/tools/karpenter-convert/pkg/convert/testdata/provisioner_no_defaults.yaml @@ -0,0 +1,26 @@ +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +metadata: + name: default +spec: + providerRef: + name: default + taints: + - key: example.com/special-taint + effect: NoSchedule + startupTaints: + - key: example.com/another-taint + effect: NoSchedule + labels: + billing-team: my-team + annotations: + example.com/owner: "my-team" + limits: + resources: + cpu: "1000" + memory: 1000Gi + consolidation: + enabled: false + ttlSecondsUntilExpired: 2592000 # 30 Days = 60 * 60 * 24 * 30 Seconds; + ttlSecondsAfterEmpty: 30 + weight: 10 \ No newline at end of file diff --git a/website/content/en/docs/concepts/provisioners.md b/website/content/en/docs/concepts/provisioners.md index a65c84eee24e..5881eb6046f8 100644 --- a/website/content/en/docs/concepts/provisioners.md +++ b/website/content/en/docs/concepts/provisioners.md @@ -58,7 +58,7 @@ spec: example.com/owner: "my-team" # Requirements that constrain the parameters of provisioned nodes. - # These requirements are combined with pod.spec.affinity.nodeAffinity rules. + # These requirements are combined with pod.spec.topologySpreadConstraints, pod.spec.affinity.nodeAffinity, pod.spec.affinity.podAffinity, and pod.spec.nodeSelector rules. # Operators { In, NotIn, Exists, DoesNotExist, Gt, and Lt } are supported. # https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#operators requirements: diff --git a/website/content/en/preview/concepts/_index.md b/website/content/en/preview/concepts/_index.md index 90a5aa85f98e..67d573135369 100755 --- a/website/content/en/preview/concepts/_index.md +++ b/website/content/en/preview/concepts/_index.md @@ -1,92 +1,66 @@ --- title: "Concepts" linkTitle: "Concepts" -weight: 10 +weight: 20 description: > Understand key concepts of Karpenter --- -Users fall under two basic roles: Kubernetes cluster administrators and application developers. -This document describes Karpenter concepts through the lens of those two types of users. +Users fall under two basic roles: [Kubernetes cluster administrators]({{}}) and [application developers]({{}}). This document describes Karpenter concepts through the lens of those two types of users. -## Cluster administrator +## Cluster Administrator As a Kubernetes cluster administrator, you can engage with Karpenter to: * Install Karpenter -* Configure provisioners to set constraints and other features for managing nodes -* Deprovision nodes -* Upgrade nodes +* Configure NodePools to set constraints and other features for managing nodes +* Disrupting nodes Concepts associated with this role are described below. ### Installing Karpenter -Karpenter is designed to run on a node in your Kubernetes cluster. -As part of the installation process, you need credentials from the underlying cloud provider to allow nodes to be started up and added to the cluster as they are needed. +Karpenter is designed to run on a node in your Kubernetes cluster. As part of the installation process, you need credentials from the underlying cloud provider to allow nodes to be started up and added to the cluster as they are needed. -[Getting Started with Karpenter on AWS](../getting-started) -describes the process of installing Karpenter on an AWS cloud provider. -Because requests to add and delete nodes and schedule pods are made through Kubernetes, AWS IAM Roles for Service Accounts (IRSA) are needed by your Kubernetes cluster to make privileged requests to AWS. -For example, Karpenter uses AWS IRSA roles to grant the permissions needed to describe EC2 instance types and create EC2 instances. +[Getting Started with Karpenter]({{}}) describes the process of installing Karpenter. Because requests to add and delete nodes and schedule pods are made through Kubernetes, AWS IAM Roles for Service Accounts (IRSA) are needed by your Kubernetes cluster to make privileged requests to AWS. For example, Karpenter uses AWS IRSA roles to grant the permissions needed to describe EC2 instance types and create EC2 instances. Once privileges are in place, Karpenter is deployed with a Helm chart. -### Configuring provisioners +### Configuring NodePools -Karpenter's job is to add nodes to handle unschedulable pods, schedule pods on those nodes, and remove the nodes when they are not needed. -To configure Karpenter, you create *provisioners* that define how Karpenter manages unschedulable pods and expires nodes. -Here are some things to know about the Karpenter provisioner: +Karpenter's job is to add nodes to handle unschedulable pods, schedule pods on those nodes, and remove the nodes when they are not needed. To configure Karpenter, you create [NodePools]({{}}) that define how Karpenter manages unschedulable pods and configures nodes. You will also define behaviors for your NodePools, capturing details like how Karpenter handles disruption of nodes and setting limits and weights for each NodePool + +Here are some things to know about Karpenter's NodePools: * **Unschedulable pods**: Karpenter only attempts to schedule pods that have a status condition `Unschedulable=True`, which the kube scheduler sets when it fails to schedule the pod to existing capacity. -* **Provisioner CR**: Karpenter defines a Custom Resource called a Provisioner to specify provisioning configuration. -Each provisioner manages a distinct set of nodes, but pods can be scheduled to any provisioner that supports its scheduling constraints. -A provisioner contains constraints that impact the nodes that can be provisioned and attributes of those nodes (such timers for removing nodes). -See [Provisioning]({{}}) docs for a description of settings and provisioner examples. +* [**Defining Constraints**]({{}}): Karpenter defines a Custom Resource called a NodePool to specify configuration. Each NodePool manages a distinct set of nodes, but pods can be scheduled to any NodePool that supports its scheduling constraints. A NodePool contains constraints that impact the nodes that can be provisioned and attributes of those nodes. See the [NodePools Documentation]({{}}) docs for a description of configuration and NodePool examples. -* **Well-known labels**: The provisioner can use well-known Kubernetes labels to allow pods to request only certain instance types, architectures, operating systems, or other attributes when creating nodes. -See [Well-Known Labels, Annotations and Taints](https://kubernetes.io/docs/reference/labels-annotations-taints/) for details. -Keep in mind that only a subset of these labels are supported in Karpenter, as described later. +* [**Defining Disruption**]({{}}): A NodePool can also include values to indicate when nodes should be disrupted. This includes configuration around concepts like [Consolidation]({{}}), [Drift]({{}}), and [Expiration]({{}}). -* **Deprovisioning nodes**: A provisioner can also include time-to-live values to indicate when nodes should be deprovisioned after a set amount of time from when they were created or after they becomes empty of deployed pods. +* **Well-known labels**: The NodePool can use well-known Kubernetes labels to allow pods to request only certain instance types, architectures, operating systems, or other attributes when creating nodes. See [Well-Known Labels, Annotations and Taints](https://kubernetes.io/docs/reference/labels-annotations-taints/) for details. Keep in mind that only a subset of these labels are supported in Karpenter, as described later. -* **Multiple provisioners**: Multiple provisioners can be configured on the same cluster. -For example, you might want to configure different teams on the same cluster to run on completely separate capacity. -One team could run on nodes using BottleRocket, while another uses EKSOptimizedAMI. +* **Multiple NodePools**: Multiple NodePools can be configured on the same cluster. For example, you might want to configure different teams on the same cluster to run on completely separate capacity. One team could run on nodes using BottleRocket, while another uses EKSOptimizedAMI. -Although most use cases are addressed with a single provisioner for multiple teams, multiple provisioners are useful to isolate nodes for billing, use different node constraints (such as no GPUs for a team), or use different deprovisioning settings. +Although most use cases are addressed with a single NodePool for multiple teams, multiple NodePools are useful to isolate nodes for billing, use different node constraints (such as no GPUs for a team), or use different disruption settings. -### Deprovisioning nodes +### Disrupting nodes Karpenter deletes nodes when they are no longer needed. -* **Finalizer**: Karpenter places a finalizer bit on each node it creates. +* [**Finalizer**]({{}}): Karpenter places a finalizer bit on each node it creates. When a request comes in to delete one of those nodes (such as a TTL or a manual `kubectl delete node`), Karpenter will cordon the node, drain all the pods, terminate the EC2 instance, and delete the node object. Karpenter handles all clean-up work needed to properly delete the node. -* **Node Expiry**: If a node expiry time-to-live value (`ttlSecondsUntilExpired`) is reached, that node is drained of pods and deleted (even if it is still running workloads). -* **Empty nodes**: When the last workload pod running on a Karpenter-managed node is gone, the node is annotated with an emptiness timestamp. -Once that "node empty" time-to-live (`ttlSecondsAfterEmpty`) is reached, finalization is triggered. -* **Consolidation**: If enabled, Karpenter will work to actively reduce cluster cost by identifying when nodes can be removed as their workloads will run on other nodes in the cluster and when nodes can be replaced with cheaper variants due to a change in the workloads. -* **Interruption**: If enabled, Karpenter will watch for upcoming involuntary interruption events that could affect your nodes (health events, spot interruption, etc.) and will cordon, drain, and terminate the node(s) ahead of the event to reduce workload disruption. - -For more details on how Karpenter deletes nodes, see [Deprovisioning nodes](./deprovisioning) for details. - -### Upgrading nodes - -A straight-forward way to upgrade nodes is to set `ttlSecondsUntilExpired`. -Nodes will be terminated after a set period of time and will be replaced with newer nodes using the latest discovered AMI. -See more in [AWSNodeTemplate](./node-templates). - -### Constraints - -The concept of layered constraints is key to using Karpenter. -With no constraints defined in provisioners and none requested from pods being deployed, Karpenter chooses from the entire universe of features available to your cloud provider. -Nodes can be created using any instance type and run in any zones. +* [**Expiration**]({{}}): Karpenter will mark nodes as expired and disrupt them after they have lived a set number of seconds, based on the NodePool's `spec.disruption.expireAfter` value. You can use node expiry to periodically recycle nodes due to security concerns. +* [**Consolidation**]({{}}): Karpenter works to actively reduce cluster cost by identifying when: + * Nodes can be removed because the node is empty + * Nodes can be removed as their workloads will run on other nodes in the cluster. + * Nodes can be replaced with cheaper variants due to a change in the workloads. +* [**Drift**]({{}}): Karpenter will mark nodes as drifted and disrupt nodes that have drifted from their desired specification. See [Drift]({{}}) to see which fields are considered. +* [**Interruption**]({{}}): Karpenter will watch for upcoming interruption events that could affect your nodes (health events, spot interruption, etc.) and will cordon, drain, and terminate the node(s) ahead of the event to reduce workload disruption. -An application developer can tighten the constraints defined in a provisioner by the cluster administrator by defining additional scheduling constraints in their pod spec. -Refer to the description of Karpenter constraints in the Application Developer section below for details. +For more details on how Karpenter deletes nodes, see the [Disruption Documentation]({{}}). ### Scheduling @@ -94,81 +68,43 @@ Karpenter launches nodes in response to pods that the Kubernetes scheduler has m Once Karpenter brings up a node, that node is available for the Kubernetes scheduler to schedule pods on it as well. -### Cloud provider -Karpenter makes requests to provision new nodes to the associated cloud provider. -The first supported cloud provider is AWS, although Karpenter is designed to work with other cloud providers. -Separating Kubernetes and AWS-specific settings allows Karpenter a clean path to integrating with other cloud providers. - -While using Kubernetes well-known labels, the provisioner can set some values that are specific to the cloud provider. -So, for example, to include a certain instance type, you could use the Kubernetes label `node.kubernetes.io/instance-type`, but set its value to an AWS instance type (such as `m5.large` or `m5.2xlarge`). - -### Consolidation - -If consolidation is enabled for a provisioner, Karpenter attempts to reduce the overall cost of the nodes launched by that provisioner if workloads have changed in two ways: -- Node Deletion -- Node Replacement - -To perform these actions, Karpenter simulates all pods being evicted from a candidate node and then looks at the results of the scheduling simulation to determine if those pods can run on a combination of existing nodes in the cluster and a new cheaper node. This operation takes into consideration all scheduling constraints placed on your workloads and provisioners (e.g. taints, tolerations, node selectors, inter-pod affinity, etc). - -If as a result of the scheduling simulation all pods can run on existing nodes, the candidate node is simply deleted. If all pods can run on a combination of existing nodes and a cheaper node, we launch the cheaper node and delete the candidate node which causes the pods to be evicted and re-created by their controllers in order to be rescheduled. +#### Constraints -For Node Replacement to work well, your provisioner must allow selecting from a variety of instance types with varying amounts of allocatable resources. Consolidation will only consider launching nodes using instance types which are allowed by your provisioner. +The concept of layered constraints is key to using Karpenter. With no constraints defined in NodePools and none requested from pods being deployed, Karpenter chooses from the entire universe of features available to your cloud provider. Nodes can be created using any instance type and run in any zones. -### Interruption +An application developer can tighten the constraints defined in a NodePool by the cluster administrator by defining additional scheduling constraints in their pod spec. Refer to the description of Karpenter constraints in the Application Developer section below for details. -If interruption-handling is enabled for the controller, Karpenter will watch for upcoming involuntary interruption events that would cause disruption to your workloads. These interruption events include: +### Cloud Provider -* Spot Interruption Warnings -* Scheduled Change Health Events (Maintenance Events) -* Instance Terminating Events -* Instance Stopping Events +Karpenter makes requests to provision new nodes to the associated cloud provider. The first supported cloud provider is AWS, although Karpenter is designed to work with other cloud providers. Separating Kubernetes and AWS-specific settings allows Karpenter a clean path to integrating with other cloud providers. -When Karpenter detects one of these events will occur to your nodes, it automatically cordons, drains, and terminates the node(s) ahead of the interruption event to give the maximum amount of time for workload cleanup prior to compute disruption. This enables scenarios where the `terminationGracePeriod` for your workloads may be long or cleanup for your workloads is critical, and you want enough time to be able to gracefully clean-up your pods. +While using Kubernetes well-known labels, the NodePool can set some values that are specific to the cloud provider. For example, to include a certain instance type, you could use the Kubernetes label `node.kubernetes.io/instance-type`, but set its value to an AWS instance type (such as `m5.large` or `m5.2xlarge`). -{{% alert title="Note" color="warning" %}} -Karpenter publishes Kubernetes events to the node for all events listed above in addition to __Spot Rebalance Recommendations__. Karpenter does not currently support cordon, drain, and terminate logic for Spot Rebalance Recommendations. -{{% /alert %}} +### Kubernetes Cluster Autoscaler -### Kubernetes cluster autoscaler -Like Karpenter, [Kubernetes Cluster Autoscaler](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler) is -designed to add nodes when requests come in to run pods that cannot be met by current capacity. -Cluster autoscaler is part of the Kubernetes project, with implementations by most major Kubernetes cloud providers. -By taking a fresh look at provisioning, Karpenter offers the following improvements: +Like Karpenter, [Kubernetes Cluster Autoscaler](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler) is designed to add nodes when requests come in to run pods that cannot be met by current capacity. Cluster autoscaler is part of the Kubernetes project, with implementations by most major Kubernetes cloud providers. By taking a fresh look at provisioning, Karpenter offers the following improvements: -* **Designed to handle the full flexibility of the cloud**: -Karpenter has the ability to efficiently address the full range of instance types available through AWS. -Cluster autoscaler was not originally built with the flexibility to handle hundreds of instance types, zones, and purchase options. +* **Designed to handle the full flexibility of the cloud**: Karpenter has the ability to efficiently address the full range of instance types available through AWS. Cluster autoscaler was not originally built with the flexibility to handle hundreds of instance types, zones, and purchase options. -* **Group-less node provisioning**: Karpenter manages each instance directly, without use of additional orchestration mechanisms like node groups. -This enables it to retry in milliseconds instead of minutes when capacity is unavailable. -It also allows Karpenter to leverage diverse instance types, availability zones, and purchase options without the creation of hundreds of node groups. +* **Quick node provisioning**: Karpenter manages each instance directly, without use of additional orchestration mechanisms like node groups. This enables it to retry in milliseconds instead of minutes when capacity is unavailable. It also allows Karpenter to leverage diverse instance types, availability zones, and purchase options without the creation of hundreds of node groups. -## Application developer +## Application Developer -As someone deploying pods that might be evaluated by Karpenter, you should know how to request the properties that your pods need of its compute resources. -Karpenter's job is to efficiently assess and choose compute assets based on requests from pod deployments. -These can include basic Kubernetes features or features that are specific to the cloud provider (such as AWS). +As someone deploying pods that might be evaluated by Karpenter, you should know how to request the properties that your pods need of its compute resources. Karpenter's job is to efficiently assess and choose compute assets based on requests from pod deployments. These can include basic Kubernetes features or features that are specific to the cloud provider (such as AWS). -Layered *constraints* are applied when a pod makes requests for compute resources that cannot be met by current capacity. -A pod can specify `nodeAffinity` (to run in a particular zone or instance type) or a `topologySpreadConstraints` spread (to cause a set of pods to be balanced across multiple nodes). +Layered *constraints* are applied when a pod makes requests for compute resources that cannot be met by current capacity. A pod can specify `nodeAffinity` (to run in a particular zone or instance type) or a `topologySpreadConstraints` spread (to cause a set of pods to be balanced across multiple nodes). The pod can specify a `nodeSelector` to run only on nodes with a particular label and `resource.requests` to ensure that the node has enough available memory. -The Kubernetes scheduler tries to match those constraints with available nodes. -If the pod is unschedulable, Karpenter creates compute resources that match its needs. -When Karpenter tries to provision a node, it analyzes scheduling constraints before choosing the node to create. +The Kubernetes scheduler tries to match those constraints with available nodes. If the pod is unschedulable, Karpenter creates compute resources that match its needs. When Karpenter tries to provision a node, it analyzes scheduling constraints before choosing the node to create. -As long as the requests are not outside of the provisioner's constraints, -Karpenter will look to best match the request, comparing the same well-known labels defined by the pod's scheduling constraints. -Note that if the constraints are such that a match is not possible, the pod will remain unscheduled. +As long as the requests are not outside the NodePool's constraints, Karpenter will look to best match the request, comparing the same well-known labels defined by the pod's scheduling constraints. Note that if the constraints are such that a match is not possible, the pod will remain unscheduled. So, what constraints can you use as an application developer deploying pods that could be managed by Karpenter? Kubernetes features that Karpenter supports for scheduling pods include nodeAffinity and [nodeSelector](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector). It also supports [PodDisruptionBudget](https://kubernetes.io/docs/tasks/run-application/configure-pdb/), [topologySpreadConstraints](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/), and [inter-pod affinity and anti-affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity). -From the Kubernetes [Well-Known Labels, Annotations and Taints](https://kubernetes.io/docs/reference/labels-annotations-taints/) page, -you can see a full list of Kubernetes labels, annotations and taints that determine scheduling. -Those that are implemented in Karpenter include: +From the Kubernetes [Well-Known Labels, Annotations and Taints](https://kubernetes.io/docs/reference/labels-annotations-taints/) page, you can see a full list of Kubernetes labels, annotations and taints that determine scheduling. Those that are implemented in Karpenter include: * **kubernetes.io/arch**: For example, kubernetes.io/arch=amd64 * **node.kubernetes.io/instance-type**: For example, node.kubernetes.io/instance-type=m3.medium diff --git a/website/content/en/preview/concepts/disruption.md b/website/content/en/preview/concepts/disruption.md index 8854f0b85ec5..1c0693925053 100644 --- a/website/content/en/preview/concepts/disruption.md +++ b/website/content/en/preview/concepts/disruption.md @@ -7,152 +7,123 @@ description: > --- ## Control Flow + Karpenter sets a Kubernetes [finalizer](https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/) on each node it provisions. -The finalizer blocks deletion of the node object while the Termination Controller cordons and drains the node, before removing the underlying machine. Deprovisioning is triggered by the Deprovisioning Controller, by the user through manual deprovisioning, or through an external system that sends a delete request to the node object. - -### Deprovisioning Controller -Karpenter automatically discovers deprovisionable nodes and spins up replacements when needed. Karpenter deprovisions nodes by executing one [automatic method](#methods) at a time, in order of Expiration, Drift, Emptiness, and then Consolidation. Each method varies slightly but they all follow the standard deprovisioning process: -1. Identify a list of prioritized candidates for the deprovisioning method. - * If there are [pods that cannot be evicted](#pod-eviction) on the node, Karpenter will ignore the node and try deprovisioning it later. - * If there are no deprovisionable nodes, continue to the next deprovisioning method. -2. For each deprovisionable node, execute a scheduling simulation with the pods on the node to find if any replacement nodes are needed. +The finalizer blocks deletion of the node object while the Termination Controller cordons and drains the node, before removing the underlying NodeClaim. Disruption is triggered by the Disruption Controller, by the user through manual disruption, or through an external system that sends a delete request to the node object. + +### Disruption Controller + +Karpenter automatically discovers disruptable nodes and spins up replacements when needed. Karpenter disrupts nodes by executing one [automatic method](#automatic-methods) at a time, in order of Expiration, Drift, and then Consolidation. Each method varies slightly, but they all follow the standard disruption process: +1. Identify a list of prioritized candidates for the disruption method. + * If there are [pods that cannot be evicted](#pod-eviction) on the node, Karpenter will ignore the node and try disrupting it later. + * If there are no disruptable nodes, continue to the next disruption method. +2. For each disruptable node, execute a scheduling simulation with the pods on the node to find if any replacement nodes are needed. 3. Cordon the node(s) to prevent pods from scheduling to it. 4. Pre-spin any replacement nodes needed as calculated in Step (2), and wait for them to become ready. - * If a replacement node fails to initialize, un-cordon the node(s), and restart from Step (1), starting at the first deprovisioning method again. + * If a replacement node fails to initialize, un-cordon the node(s), and restart from Step (1), starting at the first disruption method again. 5. Delete the node(s) and wait for the Termination Controller to gracefully shutdown the node(s). -6. Once the Termination Controller terminates the node, go back to Step (1), starting at the the first deprovisioning method again. +6. Once the Termination Controller terminates the node, go back to Step (1), starting at the first disruption method again. ### Termination Controller -When a Karpenter node is deleted, the Karpenter finalizer will block deletion and the APIServer will set the `DeletionTimestamp` on the node, allowing Karpenter to gracefully shutdown the node, modeled after [K8s Graceful Node Shutdown](https://kubernetes.io/docs/concepts/architecture/nodes/#graceful-node-shutdown). Karpenter's graceful shutdown process will: + +When a Karpenter node is deleted, the Karpenter finalizer will block deletion and the APIServer will set the `DeletionTimestamp` on the node, allowing Karpenter to gracefully shutdown the node, modeled after [Kubernetes Graceful Node Shutdown](https://kubernetes.io/docs/concepts/architecture/nodes/#graceful-node-shutdown). Karpenter's graceful shutdown process will: 1. Cordon the node to prevent pods from scheduling to it. -2. Begin evicting the pods on the node with the [K8s Eviction API](https://kubernetes.io/docs/concepts/scheduling-eviction/api-eviction/) to respect PDBs, while ignoring all non-daemonset pods and [static pods](https://kubernetes.io/docs/tasks/configure-pod-container/static-pod/). Wait for the node to be fully drained before proceeding to Step (3). - * While waiting, if the underlying machine for the node no longer exists, remove the finalizer to allow the APIServer to delete the node, completing termination. -3. Terminate the machine in the Cloud Provider. +2. Begin evicting the pods on the node with the [Kubernetes Eviction API](https://kubernetes.io/docs/concepts/scheduling-eviction/api-eviction/) to respect PDBs, while ignoring all non-daemonset pods and [static pods](https://kubernetes.io/docs/tasks/configure-pod-container/static-pod/). Wait for the node to be fully drained before proceeding to Step (3). + * While waiting, if the underlying NodeClaim for the node no longer exists, remove the finalizer to allow the APIServer to delete the node, completing termination. +3. Terminate the NodeClaim in the Cloud Provider. 4. Remove the finalizer from the node to allow the APIServer to delete the node, completing termination. -## Methods - -There are both automated and manual ways of deprovisioning nodes provisioned by Karpenter: - -### Manual Methods +## Manual Methods * **Node Deletion**: You could use `kubectl` to manually remove a single Karpenter node: ```bash # Delete a specific node kubectl delete node $NODE_NAME - # Delete all nodes owned any provisioner - kubectl delete nodes -l karpenter.sh/provisioner-name + # Delete all nodes owned by any nodepool + kubectl delete nodes -l karpenter.sh/nodepool - # Delete all nodes owned by a specific provisioner - kubectl delete nodes -l karpenter.sh/provisioner-name=$PROVISIONER_NAME + # Delete all nodes owned by a specific nodepool + kubectl delete nodes -l karpenter.sh/nodepool=$NODEPOOL_NAME ``` -* **Provisioner Deletion**: Nodes are owned by the Provisioner through an [owner reference](https://kubernetes.io/docs/concepts/overview/working-with-objects/owners-dependents/#owner-references-in-object-specifications) that launched them. Karpenter will gracefully terminate nodes through cascading deletion when the owning provisioner is deleted. - -### Automated Methods -* **Emptiness**: Karpenter notes when the last workload (non-daemonset) pod stops running on a node. From that point, Karpenter waits the number of seconds set by `ttlSecondsAfterEmpty` in the provisioner, then Karpenter requests to delete the node. This feature can keep costs down by removing nodes that are no longer being used for workloads. -* **Expiration**: Karpenter will annotate nodes as expired and deprovision nodes after they have lived a set number of seconds, based on the provisioner `ttlSecondsUntilExpired` value. One use case for node expiry is to periodically recycle nodes. Old nodes (with a potentially outdated Kubernetes version or operating system) are deleted, and replaced with nodes on the current version (assuming that you requested the latest version, rather than a specific version). -* **Consolidation**: Karpenter works to actively reduce cluster cost by identifying when: - * Nodes can be removed as their workloads will run on other nodes in the cluster. - * Nodes can be replaced with cheaper variants due to a change in the workloads. -* **Drift**: Karpenter will annotate nodes as drifted and deprovision nodes that have drifted from their desired specification. See [Drift]({{}}) to see which fields are considered. -* **Interruption**: If enabled, Karpenter will watch for upcoming involuntary interruption events that could affect your nodes (health events, spot interruption, etc.) and will cordon, drain, and terminate the node(s) ahead of the event to reduce workload disruption. +* **NodePool Deletion**: Nodes are owned by the NodePool through an [owner reference](https://kubernetes.io/docs/concepts/overview/working-with-objects/owners-dependents/#owner-references-in-object-specifications) that launched them. Karpenter will gracefully terminate nodes through cascading deletion when the owning NodePool is deleted. {{% alert title="Note" color="primary" %}} -- Automated deprovisioning is configured through the ProvisionerSpec `.ttlSecondsAfterEmpty`, `.ttlSecondsUntilExpired` and `.consolidation.enabled` fields. If these are not configured, Karpenter will not set default values for them and will not terminate nodes for that purpose. - -- Keep in mind that a small `ttlSecondsUntilExpired` results in a higher churn in cluster activity. For a small enough `ttlSecondsUntilExpired`, nodes may expire faster than Karpenter can safely deprovision them, resulting in constant node deprovisioning. +By adding the finalizer, Karpenter improves the default Kubernetes process of node deletion. +When you run `kubectl delete node` on a node without a finalizer, the node is deleted without triggering the finalization logic. The instance will continue running in EC2, even though there is no longer a node object for it. The kubelet isn’t watching for its own existence, so if a node is deleted, the kubelet doesn’t terminate itself. All the pod objects get deleted by a garbage collection process later, because the pods’ node is gone. +{{% /alert %}} -- Pods without an ownerRef (also called "controllerless" or "naked" pods) will be evicted during automatic node disruption, besides [Interruption](#interruption). A pod with the annotation `karpenter.sh/do-not-evict: "true"` will cause its node to be opted out from the same deprovisioning methods. +## Automated Methods -- Using preferred anti-affinity and topology spreads can reduce the effectiveness of consolidation. At node launch, Karpenter attempts to satisfy affinity and topology spread preferences. In order to reduce node churn, consolidation must also attempt to satisfy these constraints to avoid immediately consolidating nodes after they launch. This means that consolidation may not deprovision nodes in order to avoid violating preferences, even if kube-scheduler can fit the host pods elsewhere. Karpenter reports these pods via logging to bring awareness to the possible issues they can cause (e.g. `pod default/inflate-anti-self-55894c5d8b-522jd has a preferred Anti-Affinity which can prevent consolidation`). +* **Expiration**: Karpenter will mark nodes as expired and disrupt them after they have lived a set number of seconds, based on the NodePool's `spec.disruption.expireAfter` value. You can use node expiry to periodically recycle nodes due to security concerns. +* [**Consolidation**]({{}}): Karpenter works to actively reduce cluster cost by identifying when: + * Nodes can be removed because the node is empty + * Nodes can be removed as their workloads will run on other nodes in the cluster. + * Nodes can be replaced with cheaper variants due to a change in the workloads. +* [**Drift**]({{}}): Karpenter will mark nodes as drifted and disrupt nodes that have drifted from their desired specification. See [Drift]({{}}) to see which fields are considered. +* [**Interruption**]({{}}): Karpenter will watch for upcoming interruption events that could affect your nodes (health events, spot interruption, etc.) and will cordon, drain, and terminate the node(s) ahead of the event to reduce workload disruption. -- By adding the finalizer, Karpenter improves the default Kubernetes process of node deletion. -When you run `kubectl delete node` on a node without a finalizer, the node is deleted without triggering the finalization logic. The machine will continue running in EC2, even though there is no longer a node object for it. -The kubelet isn’t watching for its own existence, so if a node is deleted, the kubelet doesn’t terminate itself. -All the pod objects get deleted by a garbage collection process later, because the pods’ node is gone. +{{% alert title="Defaults" color="secondary" %}} +Disruption is configured through the NodePool's disruption block by the `consolidationPolicy`, `expireAfter` and `consolidateAfter` fields. Karpenter will configure these fields with the following values by default if they are not set: +```yaml +spec: + disruption: + consolidationPolicy: WhenUnderutilized + expireAfter: 720h +``` {{% /alert %}} -## Consolidation +### Consolidation Karpenter has two mechanisms for cluster consolidation: -- Deletion - A node is eligible for deletion if all of its pods can run on free capacity of other nodes in the cluster. -- Replace - A node can be replaced if all of its pods can run on a combination of free capacity of other nodes in the cluster and a single cheaper replacement node. +1. **Deletion** - A node is eligible for deletion if all of its pods can run on free capacity of other nodes in the cluster. +2. **Replace** - A node can be replaced if all of its pods can run on a combination of free capacity of other nodes in the cluster and a single cheaper replacement node. Consolidation has three mechanisms that are performed in order to attempt to identify a consolidation action: -1) Empty Node Consolidation - Delete any entirely empty nodes in parallel -2) Multi-Node Consolidation - Try to delete two or more nodes in parallel, possibly launching a single replacement that is cheaper than the price of all nodes being removed -3) Single-Node Consolidation - Try to delete any single node, possibly launching a single replacement that is cheaper than the price of that node +1. **Empty Node Consolidation** - Delete any entirely empty nodes in parallel +2. **Multi Node Consolidation** - Try to delete two or more nodes in parallel, possibly launching a single replacement that is cheaper than the price of all nodes being removed +3. **Single Node Consolidation** - Try to delete any single node, possibly launching a single replacement that is cheaper than the price of that node It's impractical to examine all possible consolidation options for multi-node consolidation, so Karpenter uses a heuristic to identify a likely set of nodes that can be consolidated. For single-node consolidation we consider each node in the cluster individually. -When there are multiple nodes that could be potentially deleted or replaced, Karpenter choose to consolidate the node that overall disrupts your workloads the least by preferring to terminate: +When there are multiple nodes that could be potentially deleted or replaced, Karpenter chooses to consolidate the node that overall disrupts your workloads the least by preferring to terminate: -* nodes running fewer pods -* nodes that will expire soon -* nodes with lower priority pods - -{{% alert title="Note" color="primary" %}} -For spot nodes, Karpenter only uses the deletion consolidation mechanism. It will not replace a spot node with a cheaper spot node. Spot instance types are selected with the `price-capacity-optimized` strategy and often the cheapest spot instance type is not launched due to the likelihood of interruption. Consolidation would then replace the spot instance with a cheaper instance negating the `price-capacity-optimized` strategy entirely and increasing interruption rate. -{{% /alert %}} +* Nodes running fewer pods +* Nodes that will expire soon +* Nodes with lower priority pods If consolidation is enabled, Karpenter periodically reports events against nodes that indicate why the node can't be consolidated. These events can be used to investigate nodes that you expect to have been consolidated, but still remain in your cluster. -``` +```bash Events: Type Reason Age From Message ---- ------ ---- ---- ------- Normal Unconsolidatable 66s karpenter pdb default/inflate-pdb prevents pod evictions Normal Unconsolidatable 33s (x3 over 30m) karpenter can't replace with a cheaper node - ``` - -## Interruption - -If interruption-handling is enabled, Karpenter will watch for upcoming involuntary interruption events that would cause disruption to your workloads. These interruption events include: - -* Spot Interruption Warnings -* Scheduled Change Health Events (Maintenance Events) -* Instance Terminating Events -* Instance Stopping Events - -When Karpenter detects one of these events will occur to your nodes, it automatically cordons, drains, and terminates the node(s) ahead of the interruption event to give the maximum amount of time for workload cleanup prior to compute disruption. This enables scenarios where the `terminationGracePeriod` for your workloads may be long or cleanup for your workloads is critical, and you want enough time to be able to gracefully clean-up your pods. +``` -For Spot interruptions, the provisioner will start a new machine as soon as it sees the Spot interruption warning. Spot interruptions have a __2 minute notice__ before Amazon EC2 reclaims the instance. Karpenter's average node startup time means that, generally, there is sufficient time for the new node to become ready and to move the pods to the new node before the machine is reclaimed. +{{% alert title="Warning" color="warning" %}} +Using preferred anti-affinity and topology spreads can reduce the effectiveness of consolidation. At node launch, Karpenter attempts to satisfy affinity and topology spread preferences. In order to reduce node churn, consolidation must also attempt to satisfy these constraints to avoid immediately consolidating nodes after they launch. This means that consolidation may not disrupt nodes in order to avoid violating preferences, even if kube-scheduler can fit the host pods elsewhere. Karpenter reports these pods via logging to bring awareness to the possible issues they can cause (e.g. `pod default/inflate-anti-self-55894c5d8b-522jd has a preferred Anti-Affinity which can prevent consolidation`). +{{% /alert %}} {{% alert title="Note" color="primary" %}} -Karpenter publishes Kubernetes events to the node for all events listed above in addition to __Spot Rebalance Recommendations__. Karpenter does not currently support cordon, drain, and terminate logic for Spot Rebalance Recommendations. +For spot nodes, Karpenter only uses the deletion consolidation mechanism. It will not replace a spot node with a cheaper spot node. Spot instance types are selected with the `price-capacity-optimized` strategy and often the cheapest spot instance type is not launched due to the likelihood of interruption. Consolidation would then replace the spot instance with a cheaper instance negating the `price-capacity-optimized` strategy entirely and increasing interruption rate. {{% /alert %}} -Karpenter enables this feature by watching an SQS queue which receives critical events from AWS services which may affect your nodes. Karpenter requires that an SQS queue be provisioned and EventBridge rules and targets be added that forward interruption events from AWS services to the SQS queue. Karpenter provides details for provisioning this infrastructure in the [CloudFormation template in the Getting Started Guide](../../getting-started/getting-started-with-karpenter/#create-the-karpenter-infrastructure-and-iam-roles). - -To enable the interruption handling feature flag, configure the `karpenter-global-settings` ConfigMap with the following value mapped to the name of the interruption queue that handles interruption events. - -```yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: karpenter-global-settings - namespace: karpenter -data: - ... - aws.interruptionQueueName: karpenter-cluster - ... -``` - -## Drift +### Drift -Drift on most fields are only triggered by changes to the owning CustomResource. Some special cases will be reconciled two-ways, triggered by Machine/Node/Instance changes or Provisioner/AWSNodeTemplate changes. For one-way reconciliation, values in the CustomResource are reflected in the Machine in the same way that they’re set. A machine will be detected as drifted if the values in the CRDs do not match the values in the Machine. By default, fields are drifted using one-way reconciliation. +Drift on most fields are only triggered by changes to the owning CustomResource. Some special cases will be reconciled two-ways, triggered by NodeClaim/Node/Instance changes or NodePool/EC2NodeClass changes. For one-way reconciliation, values in the CustomResource are reflected in the NodeClaim in the same way that they’re set. A NodeClaim will be detected as drifted if the values in the CRDs do not match the values in the NodeClaim. By default, fields are drifted using one-way reconciliation. -### Two-way Reconciliation -Two-way reconciliation can correspond to multiple values and must be handled differently. Two-way reconciliation can create cases where drift occurs without changes to CRDs, or where CRD changes do not result in drift. For example, if a machine has `node.kubernetes.io/instance-type: m5.large`, and requirements change from `node.kubernetes.io/instance-type In [m5.large]` to `node.kubernetes.io/instance-type In [m5.large, m5.2xlarge]`, the machine will not be drifted because its value is still compatible with the new requirements. Conversely, for an AWS Installation, if a machine is using a machine image `ami: ami-abc`, but a new image is published, Karpenter's `AWSNodeTemplate.amiSelector` will discover that the new correct value is `ami: ami-xyz`, and detect the machine as drifted. +#### Two-way Reconciliation +Two-way reconciliation can correspond to multiple values and must be handled differently. Two-way reconciliation can create cases where drift occurs without changes to CRDs, or where CRD changes do not result in drift. For example, if a NodeClaim has `node.kubernetes.io/instance-type: m5.large`, and requirements change from `node.kubernetes.io/instance-type In [m5.large]` to `node.kubernetes.io/instance-type In [m5.large, m5.2xlarge]`, the NodeClaim will not be drifted because its value is still compatible with the new requirements. Conversely, for an AWS Installation, if a NodeClaim is using a NodeClaim image `ami: ami-abc`, but a new image is published, Karpenter's `AWSNodeTemplate.amiSelector` will discover that the new correct value is `ami: ami-xyz`, and detect the NodeClaim as drifted. -### Behavioral Fields -Behavioral Fields are treated as over-arching settings on the Provisioner to dictate how Karpenter behaves. These fields don’t correspond to settings on the machine or instance. They’re set by the user to control Karpenter’s Provisioning and Deprovisioning logic. Since these don’t map to a desired state of machines, __behavioral fields are not considered for Drift__. +#### Behavioral Fields +Behavioral Fields are treated as over-arching settings on the NodePool to dictate how Karpenter behaves. These fields don’t correspond to settings on the NodeClaim or instance. They’re set by the user to control Karpenter’s Provisioning and disruption logic. Since these don’t map to a desired state of NodeClaims, __behavioral fields are not considered for Drift__. Read the [Drift Design](https://github.com/aws/karpenter-core/blob/main/designs/drift.md) for more. -#### Provisioner +##### NodePool | Fields | One-way | Two-way | |----------------------------| :---: | :---: | | Startup Taints | x | | @@ -165,39 +136,55 @@ Read the [Drift Design](https://github.com/aws/karpenter-core/blob/main/designs/ __Behavioral Fields__ - Weight - Limits -- Consolidation -- TTLSecondsUntilExpired -- TTLSecondsAfterEmpty +- ConsolidationPolicy +- ConsolidateAfter +- ExpireAfter --- -#### AWSNodeTemplate -| Fields | One-way | Two-way | -|----------------------------| :---: | :---: | -| Subnet Selector | | x | -| Security Group Selector | | x | -| Instance Profile | x | | -| AMI Family | x | | -| AMI Selector | | x | -| UserData | x | | -| Tags | x | | -| Metadata Options | x | | -| Block Device Mappings | x | | -| Detailed Monitoring | x | | - -To enable the drift feature flag, refer to the [Settings Feature Gates]({{}}). - -Karpenter will add `MachineDrifted` status condition on the machines if the machine is drifted, and does not have the status condition, - -Karpenter will remove the `MachineDrifted` status condition for the following these scenarios: -1. The `featureGates.driftEnabled` is not enabled but the machine is drifted, karpenter will remove the status condition. -2. The machine isn't drifted, but has the status condition, karpenter will remove it. - -If the node is marked as drifted by another controller, karpenter will do nothing. +##### EC2NodeClass +| Fields | One-way | Two-way | +|-------------------------------|:-------:|:-------:| +| Subnet Selector Terms | | x | +| Security Group Selector Terms | | x | +| AMI Family | x | | +| AMI Selector Terms | | x | +| UserData | x | | +| Tags | x | | +| Metadata Options | x | | +| Block Device Mappings | x | | +| Detailed Monitoring | x | | + +To enable the drift feature flag, refer to the [Feature Gates]({{}}). + +Karpenter will add the `Drifted` status condition on NodeClaims if the NodeClaim is drifted from its owning NodePool. Karpenter will also remove the `Drifted` status condition if either: +1. The `Drift` feature gate is not enabled but the NodeClaim is drifted, Karpenter will remove the status condition. +2. The NodeClaim isn't drifted, but has the status condition, Karpenter will remove it. + +### Interruption + +If interruption-handling is enabled, Karpenter will watch for upcoming involuntary interruption events that would cause disruption to your workloads. These interruption events include: + +* Spot Interruption Warnings +* Scheduled Change Health Events (Maintenance Events) +* Instance Terminating Events +* Instance Stopping Events + +When Karpenter detects one of these events will occur to your nodes, it automatically cordons, drains, and terminates the node(s) ahead of the interruption event to give the maximum amount of time for workload cleanup prior to compute disruption. This enables scenarios where the `terminationGracePeriod` for your workloads may be long or cleanup for your workloads is critical, and you want enough time to be able to gracefully clean-up your pods. + +For Spot interruptions, the NodePool will start a new node as soon as it sees the Spot interruption warning. Spot interruptions have a __2 minute notice__ before Amazon EC2 reclaims the instance. Karpenter's average node startup time means that, generally, there is sufficient time for the new node to become ready and to move the pods to the new node before the NodeClaim is reclaimed. + +{{% alert title="Note" color="primary" %}} +Karpenter publishes Kubernetes events to the node for all events listed above in addition to __Spot Rebalance Recommendations__. Karpenter does not currently support cordon, drain, and terminate logic for Spot Rebalance Recommendations. +{{% /alert %}} + +Karpenter enables this feature by watching an SQS queue which receives critical events from AWS services which may affect your nodes. Karpenter requires that an SQS queue be provisioned and EventBridge rules and targets be added that forward interruption events from AWS services to the SQS queue. Karpenter provides details for provisioning this infrastructure in the [CloudFormation template in the Getting Started Guide](../../getting-started/getting-started-with-karpenter/#create-the-karpenter-infrastructure-and-iam-roles). + +To enable interruption handling, configure the `--interruption-queue-name` CLI argument with the name of the interruption queue provisioned to handle interruption events. ## Controls ### Pod-Level Controls -You can block Karpenter from voluntarily choosing to disrupt certain pods by setting the `karpenter.sh/do-not-evict: "true"` annotation on the pod. This is useful for pods that you want to run from start to finish without disruption. By opting pods out of this disruption, you are telling Karpenter that it should not voluntarily remove a node containing this pod. +You can block Karpenter from voluntarily choosing to disrupt certain pods by setting the `karpenter.sh/do-not-disrupt: "true"` annotation on the pod. This is useful for pods that you want to run from start to finish without disruption. By opting pods out of this disruption, you are telling Karpenter that it should not voluntarily remove a node containing this pod. Examples of pods that you might want to opt-out of disruption include an interactive game that you don't want to interrupt or a long batch job (such as you might have with machine learning) that would need to start over if it were interrupted. @@ -208,7 +195,7 @@ spec: template: metadata: annotations: - karpenter.sh/do-not-evict: "true" + karpenter.sh/do-not-disrupt: "true" ``` {{% alert title="Note" color="primary" %}} @@ -218,7 +205,6 @@ This annotation will be ignored for [terminating pods](https://kubernetes.io/doc Examples of voluntary node removal that will be prevented by this annotation include: - [Consolidation]({{}}) - [Drift]({{}}) -- Emptiness - Expiration {{% alert title="Note" color="primary" %}} @@ -227,26 +213,28 @@ Voluntary node removal does not include [Interruption]({{} ### Node-Level Controls -Nodes can be opted out of consolidation deprovisioning by setting the annotation `karpenter.sh/do-not-consolidate: "true"` on the node. +Nodes can be opted out of consolidation disruption by setting the annotation `karpenter.sh/do-not-disrupt: "true"` on the node. ```yaml apiVersion: v1 kind: Node metadata: annotations: - karpenter.sh/do-not-consolidate: "true" + karpenter.sh/do-not-disrupt: "true" ``` -#### Example: Disable Consolidation on Provisioner +#### Example: Disable Disruption on a NodePool -Provisioner `.spec.annotations` allow you to set annotations that will be applied to all nodes launched by this provisioner. By setting the annotation `karpenter.sh/do-not-consolidate: "true"` on the provisioner, you will selectively prevent all nodes launched by this Provisioner from being considered in consolidation calculations. +NodePool `.spec.annotations` allow you to set annotations that will be applied to all nodes launched by this NodePool. By setting the annotation `karpenter.sh/do-not-disrupt: "true"` on the NodePool, you will selectively prevent all nodes launched by this NodePool from being considered in consolidation calculations. ```yaml -apiVersion: karpenter.sh/v1alpha5 -kind: Provisioner +apiVersion: karpenter.sh/v1beta1 +kind: NodePool metadata: name: default spec: - annotations: # will be applied to all nodes - karpenter.sh/do-not-consolidate: "true" + template: + metadata: + annotations: # will be applied to all nodes + karpenter.sh/do-not-disrupt: "true" ``` diff --git a/website/content/en/preview/concepts/nodeclasses.md b/website/content/en/preview/concepts/nodeclasses.md index 20b608bc89bd..0121fd2de2bb 100644 --- a/website/content/en/preview/concepts/nodeclasses.md +++ b/website/content/en/preview/concepts/nodeclasses.md @@ -140,15 +140,15 @@ status: placementGroup: - arn:aws:ec2:us-west-2:111122223333:group/my-placement-group ``` -Refer to the [NodePool docs]({{}}) for settings applicable to all providers. To explore various `EC2NodeClass` configurations, refer to the examples provided [in the Karpenter Github repository](https://github.com/aws/karpenter/blob/main/examples/nodepool/). +Refer to the [NodePool docs]({{}}) for settings applicable to all providers. To explore various `EC2NodeClass` configurations, refer to the examples provided [in the Karpenter Github repository](https://github.com/aws/karpenter/blob/main/examples/v1beta1/). ## spec.amiFamily -AMIFamily is a required field, dictating both the default bootstrapping logic for nodes provisioned through this `EC2NodeClass` but also selecting a group of recommended, latest AMIs by default. Currently, Karpenter supports `amiFamily` values `AL2`, `Bottlerocket`, `Ubuntu`, `Windows2019`, `Windows2022` and `Custom`. GPUs are only supported by default with `AL2` and `Bottlerocket`. The `AL2` amiFamily does not support ARM64 GPU instance types unless you specify a custom amiSelector. Default bootstrapping logic is shown below for each of the supported families. +AMIFamily is a required field, dictating both the default bootstrapping logic for nodes provisioned through this `EC2NodeClass` but also selecting a group of recommended, latest AMIs by default. Currently, Karpenter supports `amiFamily` values `AL2`, `Bottlerocket`, `Ubuntu`, `Windows2019`, `Windows2022` and `Custom`. GPUs are only supported by default with `AL2` and `Bottlerocket`. The `AL2` amiFamily does not support ARM64 GPU instance types unless you specify custom [`amiSelectorTerms`]({{}}). Default bootstrapping logic is shown below for each of the supported families. ### AL2 -```console +```bash MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="//" @@ -161,7 +161,7 @@ exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 --dns-cluster-ip '10.100.0.10' \ --use-max-pods false \ --container-runtime containerd \ ---kubelet-extra-args '--node-labels=karpenter.sh/capacity-type=on-demand,karpenter.sh/provisioner-name=test --max-pods=110' +--kubelet-extra-args '--node-labels=karpenter.sh/capacity-type=on-demand,karpenter.sh/nodepool=test --max-pods=110' --//-- ``` @@ -178,12 +178,12 @@ max-pods = 110 [settings.kubernetes.node-labels] 'karpenter.sh/capacity-type' = 'on-demand' -'karpenter.sh/provisioner-name' = 'test' +'karpenter.sh/nodepool' = 'test' ``` ### Ubuntu -```console +```bash MIME-Version: 1.0 Content-Type: multipart/mixed; boundary="//" @@ -195,7 +195,7 @@ exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 /etc/eks/bootstrap.sh 'test-cluster' --apiserver-endpoint 'https://test-cluster' --b64-cluster-ca 'ca-bundle' \ --dns-cluster-ip '10.100.0.10' \ --use-max-pods false \ ---kubelet-extra-args '--node-labels="karpenter.sh/capacity-type=on-demand,karpenter.sh/provisioner-name=test" --max-pods=110' +--kubelet-extra-args '--node-labels="karpenter.sh/capacity-type=on-demand,karpenter.sh/nodepool=test" --max-pods=110' --//-- ``` @@ -204,7 +204,7 @@ exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 ```powershell [string]$EKSBootstrapScriptFile = "$env:ProgramFiles\Amazon\EKS\Start-EKSBootstrap.ps1" -& $EKSBootstrapScriptFile -EKSClusterName 'test-cluster' -APIServerEndpoint 'https://test-cluster' -Base64ClusterCA 'ca-bundle' -KubeletExtraArgs '--node-labels="karpenter.sh/capacity-type=on-demand,karpenter.sh/provisioner-name=test" --max-pods=110' -DNSClusterIP '10.100.0.10' +& $EKSBootstrapScriptFile -EKSClusterName 'test-cluster' -APIServerEndpoint 'https://test-cluster' -Base64ClusterCA 'ca-bundle' -KubeletExtraArgs '--node-labels="karpenter.sh/capacity-type=on-demand,karpenter.sh/nodepool=test" --max-pods=110' -DNSClusterIP '10.100.0.10' ``` @@ -213,7 +213,7 @@ exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 ```powershell [string]$EKSBootstrapScriptFile = "$env:ProgramFiles\Amazon\EKS\Start-EKSBootstrap.ps1" -& $EKSBootstrapScriptFile -EKSClusterName 'test-cluster' -APIServerEndpoint 'https://test-cluster' -Base64ClusterCA 'ca-bundle' -KubeletExtraArgs '--node-labels="karpenter.sh/capacity-type=on-demand,karpenter.sh/provisioner-name=test" --max-pods=110' -DNSClusterIP '10.100.0.10' +& $EKSBootstrapScriptFile -EKSClusterName 'test-cluster' -APIServerEndpoint 'https://test-cluster' -Base64ClusterCA 'ca-bundle' -KubeletExtraArgs '--node-labels="karpenter.sh/capacity-type=on-demand,karpenter.sh/nodepool=test" --max-pods=110' -DNSClusterIP '10.100.0.10' ``` @@ -298,7 +298,7 @@ CLUSTER_VPC_ID="$(aws eks describe-cluster --name $CLUSTER_NAME --query cluster. aws ec2 describe-security-groups --filters Name=vpc-id,Values=$CLUSTER_VPC_ID Name=tag-key,Values=kubernetes.io/cluster/$CLUSTER_NAME --query 'SecurityGroups[].[GroupName]' --output text ``` -If multiple securityGroups are printed, you will need a more specific securityGroupSelector. We generally recommend that you use the `karpenter.sh/discovery: $CLUSTER_NAME` tag selector instead. +If multiple securityGroups are printed, you will need more specific securityGroupSelectorTerms. We generally recommend that you use the `karpenter.sh/discovery: $CLUSTER_NAME` tag selector instead. {{% /alert %}} #### Examples @@ -348,14 +348,14 @@ spec: Select using ids: ```yaml spec: - securityGroupSelector: + securityGroupSelectorTerms: - id: "sg-063d7acfb4b06c82c" - id: "sg-06e0cf9c198874591" ``` ## spec.amiSelectorTerms -AMISelector is used to configure custom AMIs for Karpenter to use, where the AMIs are discovered through ids, owners, name, and [tags](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html). This field is optional, and Karpenter will use the latest EKS-optimized AMIs for the AMIFamily if no amiSelectorTerms are specified. To select an AMI by name, use the `name` field in the selector term. To select an AMI by id, use the `id` field in the selector term. To ensure that AMIs are owned by the expected owner, use the `owner` field - you can use a combination of account aliases (e.g. `self` `amazon`, `your-aws-account-name`) and account IDs. If this is not set, it defaults to `self,amazon`. +AMISelectorTerms are used to configure custom AMIs for Karpenter to use, where the AMIs are discovered through ids, owners, name, and [tags](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html). This field is optional, and Karpenter will use the latest EKS-optimized AMIs for the AMIFamily if no amiSelectorTerms are specified. To select an AMI by name, use the `name` field in the selector term. To select an AMI by id, use the `id` field in the selector term. To ensure that AMIs are owned by the expected owner, use the `owner` field - you can use a combination of account aliases (e.g. `self` `amazon`, `your-aws-account-name`) and account IDs. If this is not set, it defaults to `self,amazon`. {{% alert title="Tip" color="secondary" %}} AMIs may be specified by any AWS tag, including `Name`. Selecting by tag or by name using wildcards (`*`) is supported. @@ -386,14 +386,14 @@ Select by name: Select by `Name` tag: ```yaml - amiSelector: + amiSelectorTerms: - tags: Name: my-ami ``` Select by name and owner: ```yaml - amiSelector: + amiSelectorTerms: - name: my-ami owner: self - name: my-ami @@ -417,7 +417,7 @@ spec: Specify using ids: ```yaml - amiSelector: + amiSelectorTerms: - id: "ami-123" - id: "ami-456" ``` @@ -457,11 +457,11 @@ Karpenter allows overrides of the default "Name" tag but does not allow override ## spec.metadataOptions -Control the exposure of [Instance Metadata Service](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) on EC2 Instances launched by this NodePool using a generated launch template. +Control the exposure of [Instance Metadata Service](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) on EC2 Instances launched by this EC2NodeClass using a generated launch template. Refer to [recommended, security best practices](https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node) for limiting exposure of Instance Metadata and User Data to pods. -If metadataOptions are omitted from this provisioner, the following default settings are applied to the `EC2NodeClass`. +If metadataOptions are omitted from this EC2NodeClass, the following default settings are applied: ```yaml spec: @@ -595,7 +595,7 @@ spec: chown -R ec2-user ~ec2-user/.ssh ``` -For more examples on configuring fields for different AMI families, see the [examples here](https://github.com/aws/karpenter/blob/main/examples/provisioner/launchtemplates). +For more examples on configuring fields for different AMI families, see the [examples here](https://github.com/aws/karpenter/blob/main/examples/v1beta1). Karpenter will merge the userData you specify with the default userData for that AMIFamily. See the [AMIFamily]({{< ref "#specamifamily" >}}) section for more details on these defaults. View the sections below to understand the different merge strategies for each AMIFamily. @@ -649,7 +649,7 @@ exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 /etc/eks/bootstrap.sh 'test-cluster' --apiserver-endpoint 'https://test-cluster' --b64-cluster-ca 'ca-bundle' \ --use-max-pods false \ --container-runtime containerd \ ---kubelet-extra-args '--node-labels=karpenter.sh/capacity-type=on-demand,karpenter.sh/provisioner-name=test --max-pods=110' +--kubelet-extra-args '--node-labels=karpenter.sh/capacity-type=on-demand,karpenter.sh/nodepool=test --max-pods=110' --//-- ``` @@ -703,7 +703,7 @@ cluster-name = 'cluster' [settings.kubernetes.node-labels] 'karpenter.sh/capacity-type' = 'on-demand' -'karpenter.sh/provisioner-name' = 'provisioner' +'karpenter.sh/nodepool' = 'default' [settings.kubernetes.node-taints] @@ -731,7 +731,7 @@ Write-Host "Running custom user data script" Write-Host "Running custom user data script" [string]$EKSBootstrapScriptFile = "$env:ProgramFiles\Amazon\EKS\Start-EKSBootstrap.ps1" -& $EKSBootstrapScriptFile -EKSClusterName 'test-cluster' -APIServerEndpoint 'https://test-cluster' -Base64ClusterCA 'ca-bundle' -KubeletExtraArgs '--node-labels="karpenter.sh/capacity-type=spot,karpenter.sh/provisioner-name=windows2022" --max-pods=110' -DNSClusterIP '10.0.100.10' +& $EKSBootstrapScriptFile -EKSClusterName 'test-cluster' -APIServerEndpoint 'https://test-cluster' -Base64ClusterCA 'ca-bundle' -KubeletExtraArgs '--node-labels="karpenter.sh/capacity-type=spot,karpenter.sh/nodepool=windows2022" --max-pods=110' -DNSClusterIP '10.0.100.10' ``` diff --git a/website/content/en/preview/concepts/nodepools.md b/website/content/en/preview/concepts/nodepools.md index 73a0704d227a..169ca8328c31 100644 --- a/website/content/en/preview/concepts/nodepools.md +++ b/website/content/en/preview/concepts/nodepools.md @@ -6,153 +6,162 @@ description: > Configure Karpenter with NodePools --- -When you first installed Karpenter, you set up a default Provisioner. -The Provisioner sets constraints on the nodes that can be created by Karpenter and the pods that can run on those nodes. -The Provisioner can be set to do things like: +When you first installed Karpenter, you set up a default NodePool. +The NodePool sets constraints on the nodes that can be created by Karpenter and the pods that can run on those nodes. +The NodePool can be set to do things like: * Define taints to limit the pods that can run on nodes Karpenter creates * Define any startup taints to inform Karpenter that it should taint the node initially, but that the taint is temporary. * Limit node creation to certain zones, instance types, and computer architectures * Set defaults for node expiration -You can change your Provisioner or add other Provisioners to Karpenter. -Here are things you should know about Provisioners: +You can change your NodePool or add other NodePools to Karpenter. +Here are things you should know about NodePools: -* Karpenter won't do anything if there is not at least one Provisioner configured. -* Each Provisioner that is configured is looped through by Karpenter. -* If Karpenter encounters a taint in the Provisioner that is not tolerated by a Pod, Karpenter won't use that Provisioner to provision the pod. -* If Karpenter encounters a startup taint in the Provisioner it will be applied to nodes that are provisioned, but pods do not need to tolerate the taint. Karpenter assumes that the taint is temporary and some other system will remove the taint. -* It is recommended to create Provisioners that are mutually exclusive. So no Pod should match multiple Provisioners. If multiple Provisioners are matched, Karpenter will use the Provisioner with the highest [weight](#specweight). +* Karpenter won't do anything if there is not at least one NodePool configured. +* Each NodePool that is configured is looped through by Karpenter. +* If Karpenter encounters a taint in the NodePool that is not tolerated by a Pod, Karpenter won't use that NodePool to provision the pod. +* If Karpenter encounters a startup taint in the NodePool it will be applied to nodes that are provisioned, but pods do not need to tolerate the taint. Karpenter assumes that the taint is temporary and some other system will remove the taint. +* It is recommended to create NodePools that are mutually exclusive. So no Pod should match multiple NodePools. If multiple NodePools are matched, Karpenter will use the NodePool with the highest [weight](#specweight). -For some example `Provisioner` configurations, see the [examples in the Karpenter GitHub repository](https://github.com/aws/karpenter/blob/main/examples/provisioner/). +For some example `NodePool` configurations, see the [examples in the Karpenter GitHub repository](https://github.com/aws/karpenter/blob/main/examples/v1beta1/). ```yaml -apiVersion: karpenter.sh/v1alpha5 -kind: Provisioner +apiVersion: karpenter.sh/v1beta1 +kind: NodePool metadata: name: default spec: - # References cloud provider-specific custom resource, see your cloud provider specific documentation - providerRef: - name: default - - # Provisioned nodes will have these taints - # Taints may prevent pods from scheduling if they are not tolerated by the pod. - taints: - - key: example.com/special-taint - effect: NoSchedule - - # Provisioned nodes will have these taints, but pods do not need to tolerate these taints to be provisioned by this - # provisioner. These taints are expected to be temporary and some other entity (e.g. a DaemonSet) is responsible for - # removing the taint after it has finished initializing the node. - startupTaints: - - key: example.com/another-taint - effect: NoSchedule - - # Labels are arbitrary key-values that are applied to all nodes - labels: - billing-team: my-team - - # Annotations are arbitrary key-values that are applied to all nodes - annotations: - example.com/owner: "my-team" - - # Requirements that constrain the parameters of provisioned nodes. - # These requirements are combined with pod.spec.affinity.nodeAffinity rules. - # Operators { In, NotIn, Exists, DoesNotExist, Gt, and Lt } are supported. - # https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#operators - requirements: - - key: "karpenter.k8s.aws/instance-category" - operator: In - values: ["c", "m", "r"] - - key: "karpenter.k8s.aws/instance-cpu" - operator: In - values: ["4", "8", "16", "32"] - - key: "karpenter.k8s.aws/instance-hypervisor" - operator: In - values: ["nitro"] - - key: "karpenter.k8s.aws/instance-generation" - operator: Gt - values: ["2"] - - key: "topology.kubernetes.io/zone" - operator: In - values: ["us-west-2a", "us-west-2b"] - - key: "kubernetes.io/arch" - operator: In - values: ["arm64", "amd64"] - - key: "karpenter.sh/capacity-type" # If not included, the webhook for the AWS cloud provider will default to on-demand - operator: In - values: ["spot", "on-demand"] - - # Karpenter provides the ability to specify a few additional Kubelet args. - # These are all optional and provide support for additional customization and use cases. - kubeletConfiguration: - clusterDNS: ["10.0.1.100"] - containerRuntime: containerd - systemReserved: - cpu: 100m - memory: 100Mi - ephemeral-storage: 1Gi - kubeReserved: - cpu: 200m - memory: 100Mi - ephemeral-storage: 3Gi - evictionHard: - memory.available: 5% - nodefs.available: 10% - nodefs.inodesFree: 10% - evictionSoft: - memory.available: 500Mi - nodefs.available: 15% - nodefs.inodesFree: 15% - evictionSoftGracePeriod: - memory.available: 1m - nodefs.available: 1m30s - nodefs.inodesFree: 2m - evictionMaxPodGracePeriod: 60 - imageGCHighThresholdPercent: 85 - imageGCLowThresholdPercent: 80 - cpuCFSQuota: true - podsPerCore: 2 - maxPods: 20 - + # Template section that describes how to template out NodeClaim resources that Karpenter will provision + # Karpenter will consider this template to be the minimum requirements needed to provision a Node using this NodePool + # It will overlay this NodePool with Pods that need to schedule to further constrain the NodeClaims + # Karpenter will provision to launch new Nodes for the cluster + template: + metadata: + # Labels are arbitrary key-values that are applied to all nodes + labels: + billing-team: my-team + + # Annotations are arbitrary key-values that are applied to all nodes + annotations: + example.com/owner: "my-team" + spec: + # References the Cloud Provider's NodeClass resource, see your cloud provider specific documentation + nodeClassRef: + name: default + + # Provisioned nodes will have these taints + # Taints may prevent pods from scheduling if they are not tolerated by the pod. + taints: + - key: example.com/special-taint + effect: NoSchedule + + # Provisioned nodes will have these taints, but pods do not need to tolerate these taints to be provisioned by this + # NodePool. These taints are expected to be temporary and some other entity (e.g. a DaemonSet) is responsible for + # removing the taint after it has finished initializing the node. + startupTaints: + - key: example.com/another-taint + effect: NoSchedule + + # Requirements that constrain the parameters of provisioned nodes. + # These requirements are combined with pod.spec.topologySpreadConstraints, pod.spec.affinity.nodeAffinity, pod.spec.affinity.podAffinity, and pod.spec.nodeSelector rules. + # Operators { In, NotIn, Exists, DoesNotExist, Gt, and Lt } are supported. + # https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#operators + requirements: + - key: "karpenter.k8s.aws/instance-category" + operator: In + values: ["c", "m", "r"] + - key: "karpenter.k8s.aws/instance-cpu" + operator: In + values: ["4", "8", "16", "32"] + - key: "karpenter.k8s.aws/instance-hypervisor" + operator: In + values: ["nitro"] + - key: "karpenter.k8s.aws/instance-generation" + operator: Gt + values: ["2"] + - key: "topology.kubernetes.io/zone" + operator: In + values: ["us-west-2a", "us-west-2b"] + - key: "kubernetes.io/arch" + operator: In + values: ["arm64", "amd64"] + - key: "karpenter.sh/capacity-type" # If not included, the webhook for the AWS cloud provider will default to on-demand + operator: In + values: ["spot", "on-demand"] + + # Karpenter provides the ability to specify a few additional Kubelet args. + # These are all optional and provide support for additional customization and use cases. + kubelet: + clusterDNS: ["10.0.1.100"] + containerRuntime: containerd + systemReserved: + cpu: 100m + memory: 100Mi + ephemeral-storage: 1Gi + kubeReserved: + cpu: 200m + memory: 100Mi + ephemeral-storage: 3Gi + evictionHard: + memory.available: 5% + nodefs.available: 10% + nodefs.inodesFree: 10% + evictionSoft: + memory.available: 500Mi + nodefs.available: 15% + nodefs.inodesFree: 15% + evictionSoftGracePeriod: + memory.available: 1m + nodefs.available: 1m30s + nodefs.inodesFree: 2m + evictionMaxPodGracePeriod: 60 + imageGCHighThresholdPercent: 85 + imageGCLowThresholdPercent: 80 + cpuCFSQuota: true + podsPerCore: 2 + maxPods: 20 + + # Disruption section which describes the ways in which Karpenter can disrupt and replace Nodes + # Configuration in this section constrains how aggressive Karpenter can be with performing operations + # like rolling Nodes due to them hitting their maximum lifetime (expiry) or scaling down nodes to reduce cluster cost + disruption: + # Describes which types of Nodes Karpenter should consider for consolidation + # If using 'WhenUnderutilized', Karpenter will consider all nodes for consolidation and attempt to remove or replace Nodes when it discovers that the Node is underutilized and could be changed to reduce cost + # If using `WhenEmpty`, Karpenter will only consider nodes for consolidation that contain no workload pods + consolidationPolicy: WhenUnderutilized | WhenEmpty + + # The amount of time Karpenter should wait after discovering a consolidation decision + # This value can currently only be set when the consolidationPolicy is 'WhenEmpty' + # You can choose to disable consolidation entirely by setting the string value 'Never' here + consolidateAfter: 30s + + # The amount of time a Node can live on the cluster before being removed + # Avoiding long-running Nodes helps to reduce security vulnerabilities as well as to reduce the chance of issues that can plague Nodes with long uptimes such as file fragmentation or memory leaks from system processes + # You can choose to disable expiration entirely by setting the string value 'Never' here + expireAfter: 720h # Resource limits constrain the total size of the cluster. # Limits prevent Karpenter from creating new instances once the limit is exceeded. limits: - resources: - cpu: "1000" - memory: 1000Gi - - # Enables consolidation which attempts to reduce cluster cost by both removing un-needed nodes and down-sizing those - # that can't be removed. Mutually exclusive with the ttlSecondsAfterEmpty parameter. - consolidation: - enabled: true - - # If omitted, the feature is disabled and nodes will never expire. If set to less time than it requires for a node - # to become ready, the node may expire before any pods successfully start. - ttlSecondsUntilExpired: 2592000 # 30 Days = 60 * 60 * 24 * 30 Seconds; + cpu: "1000" + memory: 1000Gi - # If omitted, the feature is disabled, nodes will never scale down due to low utilization - ttlSecondsAfterEmpty: 30 - - # Priority given to the provisioner when the scheduler considers which provisioner - # to select. Higher weights indicate higher priority when comparing provisioners. + # Priority given to the NodePool when the scheduler considers which NodePool + # to select. Higher weights indicate higher priority when comparing NodePools. # Specifying no weight is equivalent to specifying a weight of 0. weight: 10 ``` -## spec.requirements +## spec.template.spec.requirements -Kubernetes defines the following [Well-Known Labels](https://kubernetes.io/docs/reference/labels-annotations-taints/), and cloud providers (e.g., AWS) implement them. They are defined at the "spec.requirements" section of the Provisioner API. +Kubernetes defines the following [Well-Known Labels](https://kubernetes.io/docs/reference/labels-annotations-taints/), and cloud providers (e.g., AWS) implement them. They are defined at the "spec.requirements" section of the NodePool API. In addition to the well-known labels from Kubernetes, Karpenter supports AWS-specific labels for more advanced scheduling. See the full list [here](../scheduling/#well-known-labels). -These well-known labels may be specified at the provisioner level, or in a workload definition (e.g., nodeSelector on a pod.spec). Nodes are chosen using both the provisioner's and pod's requirements. If there is no overlap, nodes will not be launched. In other words, a pod's requirements must be within the provisioner's requirements. If a requirement is not defined for a well known label, any value available to the cloud provider may be chosen. - -For example, an instance type may be specified using a nodeSelector in a pod spec. If the instance type requested is not included in the provisioner list and the provisioner has instance type requirements, Karpenter will not create a node or schedule the pod. +These well-known labels may be specified at the NodePool level, or in a workload definition (e.g., nodeSelector on a pod.spec). Nodes are chosen using both the NodePool's and pod's requirements. If there is no overlap, nodes will not be launched. In other words, a pod's requirements must be within the NodePool's requirements. If a requirement is not defined for a well known label, any value available to the cloud provider may be chosen. -📝 None of these values are required. +For example, an instance type may be specified using a nodeSelector in a pod spec. If the instance type requested is not included in the NodePool list and the NodePool has instance type requirements, Karpenter will not create a node or schedule the pod. ### Instance Types @@ -165,20 +174,6 @@ Generally, instance types should be a list and not a single value. Leaving these Review [AWS instance types](../instance-types). Most instance types are supported with the exclusion of [non-HVM](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/virtualization_types.html). -{{% alert title="Defaults" color="secondary" %}} -If no instance type constraints are defined, Karpenter will set default instance type constraints on your Provisioner that supports most common user workloads: - -```yaml -requirements: - - key: karpenter.k8s.aws/instance-category - operator: In - values: ["c", "m", "r"] - - key: karpenter.k8s.aws/instance-generation - operator: Gt - values: ["2"] -``` -{{% /alert %}} - ### Availability Zones - key: `topology.kubernetes.io/zone` @@ -199,17 +194,6 @@ IDs.](https://docs.aws.amazon.com/ram/latest/userguide/working-with-az-ids.html) Karpenter supports `amd64` nodes, and `arm64` nodes. -{{% alert title="Defaults" color="secondary" %}} -If no architecture constraint is defined, Karpenter will set the default architecture constraint on your Provisioner that supports most common user workloads: - -```yaml -requirements: - - key: kubernetes.io/arch - operator: In - values: ["amd64"] -``` -{{% /alert %}} - ### Operating System - key: `kubernetes.io/os` - values @@ -218,17 +202,6 @@ requirements: Karpenter supports `linux` and `windows` operating systems. -{{% alert title="Defaults" color="secondary" %}} -If no operating system constraint is defined, Karpenter will set the default operating system constraint on your Provisioner that supports most common user workloads: - -```yaml -requirements: - - key: kubernetes.io/os - operator: In - values: ["linux"] -``` -{{% /alert %}} - ### Capacity Type - key: `karpenter.sh/capacity-type` @@ -238,64 +211,78 @@ requirements: Karpenter supports specifying capacity type, which is analogous to [EC2 purchase options](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-purchasing-options.html). -Karpenter prioritizes Spot offerings if the provisioner allows Spot and on-demand instances. If the provider API (e.g. EC2 Fleet's API) indicates Spot capacity is unavailable, Karpenter caches that result across all attempts to provision EC2 capacity for that instance type and zone for the next 45 seconds. If there are no other possible offerings available for Spot, Karpenter will attempt to provision on-demand instances, generally within milliseconds. +Karpenter prioritizes Spot offerings if the NodePool allows Spot and on-demand instances. If the provider API (e.g. EC2 Fleet's API) indicates Spot capacity is unavailable, Karpenter caches that result across all attempts to provision EC2 capacity for that instance type and zone for the next 45 seconds. If there are no other possible offerings available for Spot, Karpenter will attempt to provision on-demand instances, generally within milliseconds. Karpenter also allows `karpenter.sh/capacity-type` to be used as a topology key for enforcing topology-spread. -{{% alert title="Defaults" color="secondary" %}} -If no capacity type constraint is defined, Karpenter will set the default capacity type constraint on your Provisioner that supports most common user workloads: +{{% alert title="Recommended" color="primary" %}} +Karpenter allows you to be extremely flexible with your NodePools by only constraining your instance types in ways that are absolutely necessary for your cluster. By default, Karpenter will enforce that you specify the `spec.template.spec.requirements` field, but will not enforce that you specify any requirements within the field. If you choose to specify `requirements: []`, this means that you will completely flexible to _all_ instance types that your cloud provider supports. + +Though Karpenter doesn't enforce these defaults, for most use-cases, we recommend that you specify _some_ requirements to avoid odd behavior or exotic instance types. Below, is a high-level recommendation for requirements that should fit the majority of use-cases for generic workloads ```yaml -requirements: - - key: karpenter.sh/capacity-type - operator: In - values: ["on-demand"] +spec: + template: + spec: + requirements: + - key: kubernetes.io/arch + operator: In + values: ["amd64"] + - key: kubernetes.io/os + operator: In + values: ["linux"] + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand"] + - key: karpenter.k8s.aws/instance-category + operator: In + values: ["c", "m", "r"] + - key: karpenter.k8s.aws/instance-generation + operator: Gt + values: ["2"] ``` -{{% /alert %}} -## spec.weight +{{% /alert %}} -Karpenter allows you to describe provisioner preferences through a `weight` mechanism similar to how weight is described with [pod and node affinities](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity). +## spec.template.spec.nodeClassRef -For more information on weighting Provisioners, see the [Weighting Provisioners section](../scheduling#weighting-provisioners) in the scheduling details. +This field points to the Cloud Provider NodeClass resource. Learn more about [EC2NodeClasses]({{}}). -## spec.kubeletConfiguration +## spec.template.spec.kubelet Karpenter provides the ability to specify a few additional Kubelet args. These are all optional and provide support for additional customization and use cases. Adjust these only if you know you need to do so. For more details on kubelet configuration arguments, [see the KubeletConfiguration API specification docs](https://kubernetes.io/docs/reference/config-api/kubelet-config.v1beta1/). The implemented fields are a subset of the full list of upstream kubelet configuration arguments. Please cut an issue if you'd like to see another field implemented. ```yaml -spec: - ... - kubeletConfiguration: - clusterDNS: ["10.0.1.100"] - containerRuntime: containerd - systemReserved: - cpu: 100m - memory: 100Mi - ephemeral-storage: 1Gi - kubeReserved: - cpu: 200m - memory: 100Mi - ephemeral-storage: 3Gi - evictionHard: - memory.available: 5% - nodefs.available: 10% - nodefs.inodesFree: 10% - evictionSoft: - memory.available: 500Mi - nodefs.available: 15% - nodefs.inodesFree: 15% - evictionSoftGracePeriod: - memory.available: 1m - nodefs.available: 1m30s - nodefs.inodesFree: 2m - evictionMaxPodGracePeriod: 60 - imageGCHighThresholdPercent: 85 - imageGCLowThresholdPercent: 80 - cpuCFSQuota: true - podsPerCore: 2 - maxPods: 20 +kubelet: + clusterDNS: ["10.0.1.100"] + containerRuntime: containerd + systemReserved: + cpu: 100m + memory: 100Mi + ephemeral-storage: 1Gi + kubeReserved: + cpu: 200m + memory: 100Mi + ephemeral-storage: 3Gi + evictionHard: + memory.available: 5% + nodefs.available: 10% + nodefs.inodesFree: 10% + evictionSoft: + memory.available: 500Mi + nodefs.available: 15% + nodefs.inodesFree: 15% + evictionSoftGracePeriod: + memory.available: 1m + nodefs.available: 1m30s + nodefs.inodesFree: 2m + evictionMaxPodGracePeriod: 60 + imageGCHighThresholdPercent: 85 + imageGCLowThresholdPercent: 80 + cpuCFSQuota: true + podsPerCore: 2 + maxPods: 20 ``` You can specify the container runtime to be either `dockerd` or `containerd`. By default, `containerd` is used. @@ -304,7 +291,7 @@ You can specify the container runtime to be either `dockerd` or `containerd`. By ### Reserved Resources -Karpenter will automatically configure the system and kube reserved resource requests on the fly on your behalf. These requests are used to configure your node and to make scheduling decisions for your pods. If you have specific requirements or know that you will have additional capacity requirements, you can optionally override the `--system-reserved` configuration defaults with the `.spec.kubeletConfiguration.systemReserved` values and the `--kube-reserved` configuration defaults with the `.spec.kubeletConfiguration.kubeReserved` values. +Karpenter will automatically configure the system and kube reserved resource requests on the fly on your behalf. These requests are used to configure your node and to make scheduling decisions for your pods. If you have specific requirements or know that you will have additional capacity requirements, you can optionally override the `--system-reserved` configuration defaults with the `.spec.template.spec.kubelet.systemReserved` values and the `--kube-reserved` configuration defaults with the `.spec.template.spec.kubelet.kubeReserved` values. For more information on the default `--system-reserved` and `--kube-reserved` configuration refer to the [Kubelet Docs](https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/#kube-reserved) @@ -314,38 +301,36 @@ The kubelet supports eviction thresholds by default. When enough memory or file Kubelet has the notion of [hard evictions](https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#hard-eviction-thresholds) and [soft evictions](https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#soft-eviction-thresholds). In hard evictions, pods are evicted as soon as a threshold is met, with no grace period to terminate. Soft evictions, on the other hand, provide an opportunity for pods to be terminated gracefully. They do so by sending a termination signal to pods that are planning to be evicted and allowing those pods to terminate up to their grace period. -Karpenter supports [hard evictions](https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#hard-eviction-thresholds) through the `.spec.kubeletConfiguration.evictionHard` field and [soft evictions](https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#soft-eviction-thresholds) through the `.spec.kubeletConfiguration.evictionSoft` field. `evictionHard` and `evictionSoft` are configured by listing [signal names](https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#eviction-signals) with either percentage values or resource values. +Karpenter supports [hard evictions](https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#hard-eviction-thresholds) through the `.spec.template.spec.kubelet.evictionHard` field and [soft evictions](https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#soft-eviction-thresholds) through the `.spec.template.spec.kubelet.evictionSoft` field. `evictionHard` and `evictionSoft` are configured by listing [signal names](https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#eviction-signals) with either percentage values or resource values. ```yaml -spec: - ... - kubeletConfiguration: - evictionHard: - memory.available: 500Mi - nodefs.available: 10% - nodefs.inodesFree: 10% - imagefs.available: 5% - imagefs.inodesFree: 5% - pid.available: 7% - evictionSoft: - memory.available: 1Gi - nodefs.available: 15% - nodefs.inodesFree: 15% - imagefs.available: 10% - imagefs.inodesFree: 10% - pid.available: 10% +kubelet: + evictionHard: + memory.available: 500Mi + nodefs.available: 10% + nodefs.inodesFree: 10% + imagefs.available: 5% + imagefs.inodesFree: 5% + pid.available: 7% + evictionSoft: + memory.available: 1Gi + nodefs.available: 15% + nodefs.inodesFree: 15% + imagefs.available: 10% + imagefs.inodesFree: 10% + pid.available: 10% ``` #### Supported Eviction Signals -| Eviction Signal | Description | -| --------------- | ----------- | -| memory.available | memory.available := node.status.capacity[memory] - node.stats.memory.workingSet | -| nodefs.available | nodefs.available := node.stats.fs.available | -| nodefs.inodesFree | nodefs.inodesFree := node.stats.fs.inodesFree | -| imagefs.available | imagefs.available := node.stats.runtime.imagefs.available | -| imagefs.inodesFree | imagefs.inodesFree := node.stats.runtime.imagefs.inodesFree | -| pid.available | pid.available := node.stats.rlimit.maxpid - node.stats.rlimit.curproc | +| Eviction Signal | Description | +|--------------------|---------------------------------------------------------------------------------| +| memory.available | memory.available := node.status.capacity[memory] - node.stats.memory.workingSet | +| nodefs.available | nodefs.available := node.stats.fs.available | +| nodefs.inodesFree | nodefs.inodesFree := node.stats.fs.inodesFree | +| imagefs.available | imagefs.available := node.stats.runtime.imagefs.available | +| imagefs.inodesFree | imagefs.inodesFree := node.stats.runtime.imagefs.inodesFree | +| pid.available | pid.available := node.stats.rlimit.maxpid - node.stats.rlimit.curproc | For more information on eviction thresholds, view the [Node-pressure Eviction](https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction) section of the official Kubernetes docs. @@ -356,64 +341,83 @@ Soft eviction pairs an eviction threshold with a specified grace period. With so Optionally, you can specify an `evictionMaxPodGracePeriod` which defines the administrator-specified maximum pod termination grace period to use during soft eviction. If a namespace-owner had specified a pod `terminationGracePeriodInSeconds` on pods in their namespace, the minimum of `evictionPodGracePeriod` and `terminationGracePeriodInSeconds` would be used. ```yaml -spec: - ... - kubeletConfiguration: - evictionSoftGracePeriod: - memory.available: 1m - nodefs.available: 1m30s - nodefs.inodesFree: 2m - imagefs.available: 1m30s - imagefs.inodesFree: 2m - pid.available: 2m - evictionMaxPodGracePeriod: 60 +kubelet: + evictionSoftGracePeriod: + memory.available: 1m + nodefs.available: 1m30s + nodefs.inodesFree: 2m + imagefs.available: 1m30s + imagefs.inodesFree: 2m + pid.available: 2m + evictionMaxPodGracePeriod: 60 ``` ### Pod Density +By default, the number of pods on a node is limited by both the number of networking interfaces (ENIs) that may be attached to an instance type and the number of IP addresses that can be assigned to each ENI. See [IP addresses per network interface per instance type](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-eni.html#AvailableIpPerENI) for a more detailed information on these instance types' limits. + +{{% alert title="Note" color="primary" %}} +By default, the VPC CNI allocates IPs for a node and pods from the same subnet. With [VPC CNI Custom Networking](https://aws.github.io/aws-eks-best-practices/networking/custom-networking), the pods will receive IP addresses from another subnet dedicated to pod IPs. This approach makes it easier to manage IP addresses and allows for separate Network Access Control Lists (NACLs) applied to your pods. VPC CNI Custom Networking reduces the pod density of a node since one of the ENI attachments will be used for the node and cannot share the allocated IPs on the interface to pods. Karpenter supports VPC CNI Custom Networking and similar CNI setups where the primary node interface is separated from the pods interfaces through a global [setting](./settings.md#configmap) within the karpenter-global-settings configmap: `aws.reservedENIs`. In the common case, `aws.reservedENIs` should be set to `"1"` if using Custom Networking. +{{% /alert %}} + +{{% alert title="Windows Support Notice" color="warning" %}} +It's currently not possible to specify custom networking with Windows nodes. +{{% /alert %}} + #### Max Pods -By default, AWS will configure the maximum density of pods on a node [based on the node instance type](https://github.com/awslabs/amazon-eks-ami/blob/master/files/eni-max-pods.txt). For small instances that require an increased pod density or large instances that require a reduced pod density, you can override this default value with `.spec.kubeletConfiguration.maxPods`. This value will be used during Karpenter pod scheduling and passed through to `--max-pods` on kubelet startup. +For small instances that require an increased pod density or large instances that require a reduced pod density, you can override this default value with `.spec.template.spec.kubelet.maxPods`. This value will be used during Karpenter pod scheduling and passed through to `--max-pods` on kubelet startup. {{% alert title="Note" color="primary" %}} When using small instance types, it may be necessary to enable [prefix assignment mode](https://aws.amazon.com/blogs/containers/amazon-vpc-cni-increases-pods-per-node-limits/) in the AWS VPC CNI plugin to support a higher pod density per node. Prefix assignment mode was introduced in AWS VPC CNI v1.9 and allows ENIs to manage a broader set of IP addresses. Much higher pod densities are supported as a result. {{% /alert %}} +{{% alert title="Windows Support Notice" color="warning" %}} +Presently, Windows worker nodes do not support using more than one ENI. +As a consequence, the number of IP addresses, and subsequently, the number of pods that a Windows worker node can support is limited by the number of IPv4 addresses available on the primary ENI. +Currently, Karpenter will only consider individual secondary IP addresses when calculating the pod density limit. +{{% /alert %}} + #### Pods Per Core -An alternative way to dynamically set the maximum density of pods on a node is to use the `.spec.kubeletConfiguration.podsPerCore` value. Karpenter will calculate the pod density during scheduling by multiplying this value by the number of logical cores (vCPUs) on an instance type. This value will also be passed through to the `--pods-per-core` value on kubelet startup to configure the number of allocatable pods the kubelet can assign to the node instance. +An alternative way to dynamically set the maximum density of pods on a node is to use the `.spec.template.spec.kubelet.podsPerCore` value. Karpenter will calculate the pod density during scheduling by multiplying this value by the number of logical cores (vCPUs) on an instance type. This value will also be passed through to the `--pods-per-core` value on kubelet startup to configure the number of allocatable pods the kubelet can assign to the node instance. The value generated from `podsPerCore` cannot exceed `maxPods`, meaning, if both are set, the minimum of the `podsPerCore` dynamic pod density and the static `maxPods` value will be used for scheduling. {{% alert title="Note" color="primary" %}} -`maxPods` may not be set in the `kubeletConfiguration` of a Provisioner, but may still be restricted by the `ENI_LIMITED_POD_DENSITY` value. You may want to ensure that the `podsPerCore` value that will be used for instance families associated with the Provisioner will not cause unexpected behavior by exceeding the `maxPods` value. +`maxPods` may not be set in the `kubelet` of a NodePool, but may still be restricted by the `ENI_LIMITED_POD_DENSITY` value. You may want to ensure that the `podsPerCore` value that will be used for instance families associated with the NodePool will not cause unexpected behavior by exceeding the `maxPods` value. {{% /alert %}} {{% alert title="Pods Per Core on Bottlerocket" color="warning" %}} -Bottlerocket AMIFamily currently does not support `podsPerCore` configuration. If a Provisioner contains a `provider` or `providerRef` to a node template that will launch a Bottlerocket instance, the `podsPerCore` value will be ignored for scheduling and for configuring the kubelet. +Bottlerocket AMIFamily currently does not support `podsPerCore` configuration. If a NodePool contains a `provider` or `providerRef` to a node template that will launch a Bottlerocket instance, the `podsPerCore` value will be ignored for scheduling and for configuring the kubelet. {{% /alert %}} -## spec.limits.resources +## spec.disruption + +You can configure Karpenter to disrupt Nodes through your NodePool in multiple ways. You can use `spec.disruption.consolidationPolicy`, `spec.disruption.consolidateAfter` or `spec.disruption.expireAfter`. Read [Disruption]({{}}) for more. + +## spec.limits -The provisioner spec includes a limits section (`spec.limits.resources`), which constrains the maximum amount of resources that the provisioner will manage. +The NodePool spec includes a limits section (`spec.limits`), which constrains the maximum amount of resources that the NodePool will manage. Karpenter supports limits of any resource type reported by your cloudprovider. It limits instance types when scheduling to those that will not exceed the specified limits. If a limit has been exceeded, nodes provisioning is prevented until some nodes have been terminated. ```yaml -apiVersion: karpenter.sh/v1alpha5 -kind: Provisioner +apiVersion: karpenter.sh/v1beta1 +kind: NodePool metadata: name: default spec: - requirements: - - key: karpenter.sh/capacity-type - operator: In - values: ["spot"] + template: + spec: + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["spot"] limits: - resources: - cpu: 1000 - memory: 1000Gi - nvidia.com/gpu: 2 + cpu: 1000 + memory: 1000Gi + nvidia.com/gpu: 2 ``` {{% alert title="Note" color="primary" %}} @@ -426,59 +430,61 @@ Memory limits are described with a [`BinarySI` value, such as 1000Gi.](https://k You can view the current consumption of cpu and memory on your cluster by running: ``` -kubectl get provisioner -o=jsonpath='{.items[0].status}' +kubectl get nodepool -o=jsonpath='{.items[0].status}' ``` Review the [Kubernetes core API](https://github.com/kubernetes/api/blob/37748cca582229600a3599b40e9a82a951d8bbbf/core/v1/resource.go#L23) (`k8s.io/api/core/v1`) for more information on `resources`. -## spec.providerRef - -This field points to the cloud provider-specific custom resource. Learn more about [AWSNodeTemplates](../node-templates/). +## spec.weight -## spec.consolidation +Karpenter allows you to describe NodePool preferences through a `weight` mechanism similar to how weight is described with [pod and node affinities](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity). -You can configure Karpenter to deprovision instances through your Provisioner in multiple ways. You can use `spec.ttlSecondsAfterEmpty`, `spec.ttlSecondsUntilExpired` or `spec.consolidation.enabled`. Read [Deprovisioning]({{}}) for more. +For more information on weighting NodePools, see the [Weighting NodePools section]({{}}) in the scheduling details. -## Example Use-Cases +## Examples ### Isolating Expensive Hardware -A provisioner can be set up to only provision nodes on particular processor types. +A NodePool can be set up to only provision nodes on particular processor types. The following example sets a taint that only allows pods with tolerations for Nvidia GPUs to be scheduled: ```yaml -apiVersion: karpenter.sh/v1alpha5 -kind: Provisioner +apiVersion: karpenter.sh/v1beta1 +kind: NodePool metadata: name: gpu spec: - consolidation: - enabled: true - requirements: - - key: node.kubernetes.io/instance-type - operator: In - values: ["p3.8xlarge", "p3.16xlarge"] - taints: - - key: nvidia.com/gpu - value: "true" - effect: NoSchedule + disruption: + consolidationPolicy: WhenUnderutilized + template: + spec: + requirements: + - key: node.kubernetes.io/instance-type + operator: In + values: ["p3.8xlarge", "p3.16xlarge"] + taints: + - key: nvidia.com/gpu + value: "true" + effect: NoSchedule ``` -In order for a pod to run on a node defined in this provisioner, it must tolerate `nvidia.com/gpu` in its pod spec. +In order for a pod to run on a node defined in this NodePool, it must tolerate `nvidia.com/gpu` in its pod spec. ### Cilium Startup Taint Per the Cilium [docs](https://docs.cilium.io/en/stable/installation/taints/#taint-effects), it's recommended to place a taint of `node.cilium.io/agent-not-ready=true:NoExecute` on nodes to allow Cilium to configure networking prior to other pods starting. This can be accomplished via the use of Karpenter `startupTaints`. These taints are placed on the node, but pods aren't required to tolerate these taints to be considered for provisioning. ```yaml -apiVersion: karpenter.sh/v1alpha5 -kind: Provisioner +apiVersion: karpenter.sh/v1beta1 +kind: NodePool metadata: name: cilium-startup spec: - consolidation: - enabled: true - startupTaints: - - key: node.cilium.io/agent-not-ready - value: "true" - effect: NoExecute + disruption: + consolidationPolicy: WhenUnderutilized + template: + spec: + startupTaints: + - key: node.cilium.io/agent-not-ready + value: "true" + effect: NoExecute ``` diff --git a/website/content/en/preview/concepts/pod-density.md b/website/content/en/preview/concepts/pod-density.md deleted file mode 100644 index 7d1c12f5d326..000000000000 --- a/website/content/en/preview/concepts/pod-density.md +++ /dev/null @@ -1,91 +0,0 @@ ---- -title: "Control Pod Density" -linkTitle: "Control Pod Density" -weight: 6 -description: > - Learn ways to specify pod density with Karpenter ---- - -Pod density is the number of pods per node. - -Kubernetes has a default limit of 110 pods per node. If you are using the EKS Optimized AMI on AWS, the [number of pods is limited by instance type](https://github.com/awslabs/amazon-eks-ami/blob/master/files/eni-max-pods.txt) in the default configuration. - -## Increase Pod Density - -### Networking Limitations - -*☁️ AWS Specific* - -By default, the number of pods on a node is limited by both the number of networking interfaces (ENIs) that may be attached to an instance type and the number of IP addresses that can be assigned to each ENI. See [IP addresses per network interface per instance type](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-eni.html#AvailableIpPerENI) for a more detailed information on these instance types' limits. - -Karpenter can be configured to disable nodes' ENI-based pod density. This is especially useful for small to medium instance types which have a lower ENI-based pod density. - -{{% alert title="Note" color="primary" %}} -When using small instance types, it may be necessary to enable [prefix assignment mode](https://aws.amazon.com/blogs/containers/amazon-vpc-cni-increases-pods-per-node-limits/) in the AWS VPC CNI plugin to more pods per node. Prefix assignment mode was introduced in AWS VPC CNI v1.9 and allows ENIs to manage a broader set of IP addresses. Much higher pod densities are supported as a result. -{{% /alert %}} - -{{% alert title="Windows Support Notice" color="warning" %}} -Presently, Windows worker nodes do not support using more than one ENI. -As a consequence, the number of IP addresses, and subsequently, the number of pods that a Windows worker node can support is limited by the number of IPv4 addresses available on the primary ENI. -At the moment, Karpenter will only consider individual secondary IP addresses when calculating the pod density limit. -{{% /alert %}} - -### Provisioner-Specific Pod Density - -#### Static Pod Density - -Static pod density can be configured at the provisioner level by specifying `maxPods` within the `.spec.kubeletConfiguration`. All nodes spawned by this provisioner will set this `maxPods` value on their kubelet and will account for this value during scheduling. - -See [Provisioner API Kubelet Configuration](../provisioners/#max-pods) for more details. - -#### Dynamic Pod Density - -Dynamic pod density (density that scales with the instance size) can be configured at the provisioner level by specifying `podsPerCore` within the `.spec.kubeletConfiguration`. Karpenter will calculate the expected pod density for each instance based on the instance's number of logical cores (vCPUs) and will account for this during scheduling. - -See [Provisioner API Kubelet Configuration](../provisioners/#pod-density) for more details. - -### Controller-Wide Pod Density - -{{% alert title="Deprecation Warning" color="warning" %}} -`AWS_ENI_LIMITED_POD_DENSITY` is deprecated in favor of the `.spec.kubeletConfiguration.maxPods` set at the Provisioner-level -{{% /alert %}} - -Set the environment variable `AWS_ENI_LIMITED_POD_DENSITY: "false"` (or the argument `--aws-eni-limited-pod-density=false`) in the Karpenter controller to allow nodes to host up to 110 pods by default. - -Environment variables for the Karpenter controller may be specified as [helm chart values](https://github.com/aws/karpenter/blob/c73f425e924bb64c3f898f30ca5035a1d8591183/charts/karpenter/values.yaml#L15). - -### VPC CNI Custom Networking - -By default, the VPC CNI allocates IPs for a node and pods from the same subnet. With [VPC CNI Custom Networking](https://aws.github.io/aws-eks-best-practices/networking/custom-networking), the pods will receive IP addresses from another subnet dedicated to pod IPs. This approach makes it easier to manage IP addresses and allows for separate Network Access Control Lists (NACLs) applied to your pods. VPC CNI Custom Networking reduces the pod density of a node since one of the ENI attachments will be used for the node and cannot share the allocated IPs on the interface to pods. Karpenter supports VPC CNI Custom Networking and similar CNI setups where the primary node interface is separated from the pods interfaces through a global [setting](../reference/settings.md#configmap) within the karpenter-global-settings configmap: `aws.reservedENIs`. In the common case, `aws.reservedENIs` should be set to `"1"` if using Custom Networking. - -{{% alert title="Windows Support Notice" color="warning" %}} -It's currently not possible to specify custom networking with Windows nodes. -{{% /alert %}} - -## Limit Pod Density - -Generally, increasing pod density is more efficient. However, some use cases exist for limiting pod density. - -### Topology Spread - -You can use [topology spread]({{< relref "scheduling.md#topology-spread" >}}) features to reduce blast radius. For example, spreading workloads across EC2 Availability Zones. - - -### Restrict Instance Types - -Exclude large instance sizes to reduce the blast radius of an EC2 instance failure. - -Consider setting up upper or lower boundaries on target instance sizes with the node.kubernetes.io/instance-type key. - -The following example shows how to avoid provisioning large Graviton instances in order to reduce the impact of individual instance failures: - -``` --key: node.kubernetes.io/instance-type - operator: NotIn - values: - 'm6g.16xlarge' - 'm6gd.16xlarge' - 'r6g.16xlarge' - 'r6gd.16xlarge' - 'c6g.16xlarge' -``` diff --git a/website/content/en/preview/concepts/scheduling.md b/website/content/en/preview/concepts/scheduling.md index 9e7e2a4402b4..dc7af6a933e1 100755 --- a/website/content/en/preview/concepts/scheduling.md +++ b/website/content/en/preview/concepts/scheduling.md @@ -453,8 +453,7 @@ metadata: spec: weight: 50 limits: - resources: - cpu: 100 + cpu: 100 template: spec: requirements: diff --git a/website/content/en/preview/contributing/_index.md b/website/content/en/preview/contributing/_index.md index 10bb749d39dc..6ec2c3df504e 100644 --- a/website/content/en/preview/contributing/_index.md +++ b/website/content/en/preview/contributing/_index.md @@ -1,7 +1,7 @@ --- title: "Contributing" linkTitle: "Contributing" -weight: 100 +weight: 40 description: > Learn about how to contribute to Karpenter --- diff --git a/website/content/en/preview/faq.md b/website/content/en/preview/faq.md index d29e497896aa..7f91d19a341a 100644 --- a/website/content/en/preview/faq.md +++ b/website/content/en/preview/faq.md @@ -1,34 +1,32 @@ --- title: "FAQs" linkTitle: "FAQs" -weight: 90 +weight: 60 description: > Review Karpenter Frequently Asked Questions --- ## General -### How does a provisioner decide to manage a particular node? -See [Configuring provisioners]({{< ref "./concepts/#configuring-provisioners" >}}) for information on how Karpenter provisions and manages nodes. +### How does a NodePool decide to manage a particular node? +See [Configuring NodePools]({{< ref "./concepts/#configuring-nodepools" >}}) for information on how Karpenter configures and manages nodes. ### What cloud providers are supported? AWS is the first cloud provider supported by Karpenter, although it is designed to be used with other cloud providers as well. ### Can I write my own cloud provider for Karpenter? -Yes, but there is no documentation yet for it. -Start with Karpenter's GitHub [cloudprovider](https://github.com/aws/karpenter-core/tree{{< githubRelRef >}}pkg/cloudprovider) documentation to see how the AWS provider is built, but there are other sections of the code that will require changes too. +Yes, but there is no documentation yet for it. Start with Karpenter's GitHub [cloudprovider](https://github.com/aws/karpenter-core/tree{{< githubRelRef >}}pkg/cloudprovider) documentation to see how the AWS provider is built, but there are other sections of the code that will require changes too. ### What operating system nodes does Karpenter deploy? By default, Karpenter uses Amazon Linux 2 images. ### Can I provide my own custom operating system images? -Karpenter has multiple mechanisms for configuring the [operating system]({{< ref "./concepts/nodeclasses/#spec-amiselector" >}}) for your nodes. +Karpenter has multiple mechanisms for configuring the [operating system]({{< ref "./concepts/nodeclasses/#specamiselectorterms" >}}) for your nodes. ### Can Karpenter deal with workloads for mixed architecture cluster (arm vs. amd)? -Karpenter is flexible to multi architecture configurations using [well known labels]({{< ref "./concepts/scheduling/#supported-labels">}}). +Karpenter is flexible to multi-architecture configurations using [well known labels]({{< ref "./concepts/scheduling/#supported-labels">}}). ### What RBAC access is required? -All of the required RBAC rules can be found in the helm chart template. -See [clusterrolebinding.yaml](https://github.com/aws/karpenter/blob{{< githubRelRef >}}charts/karpenter/templates/clusterrolebinding.yaml), [clusterrole.yaml](https://github.com/aws/karpenter/blob{{< githubRelRef >}}charts/karpenter/templates/clusterrole.yaml), [rolebinding.yaml](https://github.com/aws/karpenter/blob{{< githubRelRef >}}charts/karpenter/templates/rolebinding.yaml), and [role.yaml](https://github.com/aws/karpenter/blob{{< githubRelRef >}}charts/karpenter/templates/role.yaml) files for details. +All the required RBAC rules can be found in the helm chart template. See [clusterrole-core.yaml](https://github.com/aws/karpenter/blob{{< githubRelRef >}}charts/karpenter/templates/clusterrole-core.yaml), [clusterrole.yaml](https://github.com/aws/karpenter/blob{{< githubRelRef >}}charts/karpenter/templates/clusterrole.yaml), [rolebinding.yaml](https://github.com/aws/karpenter/blob{{< githubRelRef >}}charts/karpenter/templates/rolebinding.yaml), and [role.yaml](https://github.com/aws/karpenter/blob{{< githubRelRef >}}charts/karpenter/templates/role.yaml) files for details. ### Can I run Karpenter outside of a Kubernetes cluster? Yes, as long as the controller has network and IAM/RBAC access to the Kubernetes API and your provider API. @@ -36,66 +34,49 @@ Yes, as long as the controller has network and IAM/RBAC access to the Kubernetes ## Compatibility ### Which versions of Kubernetes does Karpenter support? -See the [Compatibility Matrix in the Upgrade Guide]({{< ref "./upgrade-guide#compatibility-matrix" >}}) to view the supported Kubernetes versions per Karpenter released version. +See the [Compatibility Matrix in the Upgrade Section]({{< ref "./upgrading/compatibility#compatibility-matrix" >}}) to view the supported Kubernetes versions per Karpenter released version. ### What Kubernetes distributions are supported? -Karpenter documents integration with a fresh or existing install of the latest AWS Elastic Kubernetes Service (EKS). -Other Kubernetes distributions (KOPs, etc.) can be used, but setting up cloud provider permissions for those distributions has not been documented. +Karpenter documents integration with a fresh or existing installation of the latest AWS Elastic Kubernetes Service (EKS). Other Kubernetes distributions (KOPs, etc.) can be used, but setting up cloud provider permissions for those distributions has not been documented. ### How does Karpenter interact with AWS node group features? -Provisioners are designed to work alongside static capacity management solutions like EKS Managed Node Groups and EC2 Auto Scaling Groups. -You can manage all capacity using provisioners, use a mixed model with dynamic and statically managed capacity, or use a fully static approach. -We expect most users will use a mixed approach in the near term and provisioner-managed in the long term. +NodePools are designed to work alongside static capacity management solutions like EKS Managed Node Groups and EC2 Auto Scaling Groups. You can manage all capacity using NodePools, use a mixed model with dynamic and statically managed capacity, or use a fully static approach. We expect most users will use a mixed approach in the near term and NodePool-managed in the long term. ### How does Karpenter interact with Kubernetes features? -* Kubernetes Cluster Autoscaler: Karpenter can work alongside cluster autoscaler. -See [Kubernetes cluster autoscaler]({{< ref "./concepts/#kubernetes-cluster-autoscaler" >}}) for details. -* Kubernetes Scheduler: Karpenter focuses on scheduling pods that the Kubernetes scheduler has marked as unschedulable. -See [Scheduling]({{< ref "./concepts/scheduling" >}}) for details on how Karpenter interacts with the Kubernetes scheduler. +* Kubernetes Cluster Autoscaler: Karpenter can work alongside Cluster Autoscaler. See [Kubernetes Cluster Autoscaler]({{< ref "./concepts/#kubernetes-cluster-autoscaler" >}}) for details. +* Kubernetes Scheduler: Karpenter focuses on scheduling pods that the Kubernetes scheduler has marked as unschedulable. See [Scheduling]({{< ref "./concepts/scheduling" >}}) for details on how Karpenter interacts with the Kubernetes scheduler. ## Provisioning -### What features does the Karpenter provisioner support? -See [Provisioner API]({{< ref "./concepts/nodepools" >}}) for provisioner examples and descriptions of features. +### What features does the Karpenter NodePool support? +See the [NodePool API docs]({{< ref "./concepts/nodepools" >}}) for NodePool examples and descriptions of features. -### Can I create multiple (team-based) provisioners on a cluster? -Yes, provisioners can identify multiple teams based on labels. -See [Provisioner API]({{< ref "./concepts/nodepools" >}}) for details. +### Can I create multiple (team-based) NodePools on a cluster? +Yes, NodePools can identify multiple teams based on labels. See the [NodePool API docs]({{< ref "./concepts/nodepools" >}}) for details. -### If multiple provisioners are defined, which will my pod use? +### If multiple NodePools are defined, which will my pod use? -Pending pods will be handled by any Provisioner that matches the requirements of the pod. -There is no ordering guarantee if multiple provisioners match pod requirements. -We recommend that Provisioners are setup to be mutually exclusive. -Read more about this recommendation in the [EKS Best Practices Guide for Karpenter](https://aws.github.io/aws-eks-best-practices/karpenter/#create-provisioners-that-are-mutually-exclusive). -To select a specific provisioner, use the node selector `karpenter.sh/provisioner-name: my-provisioner`. +Pending pods will be handled by any NodePools that matches the requirements of the pod. There is no ordering guarantee if multiple NodePools match pod requirements. We recommend that NodePools are set-up to be mutually exclusive. To select a specific NodePool, use the node selector `karpenter.sh/nodepool: my-nodepool`. ### How can I configure Karpenter to only provision pods for a particular namespace? -There is no native support for namespaced based provisioning. -Karpenter can be configured to provision a subset of pods based on a combination of taints/tolerations and node selectors. -This allows Karpenter to work in concert with the `kube-scheduler` in that the same mechanisms that `kube-scheduler` uses to determine if a pod can schedule to an existing node are also used for provisioning new nodes. -This avoids scenarios where pods are bound to nodes that were provisioned by Karpenter which Karpenter would not have bound itself. -If this were to occur, a node could remain non-empty and have its lifetime extended due to a pod that wouldn't have caused the node to be provisioned had the pod been unschedulable. +There is no native support for namespaced-based provisioning. Karpenter can be configured to provision a subset of pods based on a combination of taints/tolerations and node selectors. This allows Karpenter to work in concert with the `kube-scheduler` using the same mechanisms to determine if a pod can schedule to an existing node are also used for provisioning new nodes. This avoids scenarios where pods are bound to nodes that were provisioned by Karpenter which Karpenter would not have bound itself. If this were to occur, a node could remain non-empty and have its lifetime extended due to a pod that wouldn't have caused the node to be provisioned had the pod been unschedulable. -We recommend using Kubernetes native scheduling constraints to achieve namespace based scheduling segregation. Using native scheduling constraints ensures that Karpenter, `kube-scheduler` and any other scheduling or auto-provisioning mechanism all have an identical understanding of which pods can be scheduled on which nodes. This can be enforced via policy agents, an example of which can be seen [here](https://blog.mikesir87.io/2022/01/creating-tenant-node-pools-with-karpenter/). +We recommend using Kubernetes native scheduling constraints to achieve namespace-based scheduling segregation. Using native scheduling constraints ensures that Karpenter, `kube-scheduler` and any other scheduling or auto-provisioning mechanism all have an identical understanding of which pods can be scheduled on which nodes. This can be enforced via policy agents, an example of which can be seen [here](https://blog.mikesir87.io/2022/01/creating-tenant-node-pools-with-karpenter/). -### Can I add SSH keys to a provisioner? +### Can I add SSH keys to a NodePool? -Karpenter does not offer a way to add SSH keys via provisioners or secrets to the nodes it manages. -However, you can use Session Manager (SSM) or EC2 Instance Connect to gain shell access to Karpenter nodes. -See [Node NotReady]({{< ref "./troubleshooting/#node-notready" >}}) troubleshooting for an example of starting an SSM session from the command line or [EC2 Instance Connect](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-connect-set-up.html) documentation to connect to nodes using SSH. +Karpenter does not offer a way to add SSH keys via NodePools or secrets to the nodes it manages. +However, you can use Session Manager (SSM) or EC2 Instance Connect to gain shell access to Karpenter nodes. See [Node NotReady]({{< ref "./troubleshooting/#node-notready" >}}) troubleshooting for an example of starting an SSM session from the command line or [EC2 Instance Connect](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-connect-set-up.html) documentation to connect to nodes using SSH. -Though not recommended, if you need to access Karpenter-managed nodes without AWS credentials, you can add SSH keys using AWSNodeTemplate. -See [Custom User Data]({{< ref "./concepts/nodeclasses/#spec-userdata" >}}) for details. +Though not recommended, if you need to access Karpenter-managed nodes without AWS credentials, you can add SSH keys using EC2NodeClass User Data. See the [User Data section in the EC2NodeClass documentation]({{< ref "./concepts/nodeclasses/#specuserdata" >}}) for details. -### Can I set total limits of CPU and memory for a provisioner? -Yes, the setting is provider-specific. -See examples in [Accelerators, GPU]({{< ref "./concepts/scheduling/#accelerators-gpu-resources" >}}) Karpenter documentation. +### Can I set limits of CPU and memory for a NodePool? +Yes. View the [NodePool API docs]({{< ref "./concepts/nodepools#speclimits" >}}) for NodePool examples and descriptions of how to configure limits. ### Can I mix spot and on-demand EC2 run types? -Yes, see [Provisioning]({{< ref "./concepts/nodepools#examples" >}}) for an example. +Yes, see the [NodePool API docs]({{< ref "./concepts/nodepools#examples" >}}) for an example. ### Can I restrict EC2 instance types? @@ -104,53 +85,48 @@ Yes, see [Provisioning]({{< ref "./concepts/nodepools#examples" >}}) for an exam ### Can I use Bare Metal instance types? -Yes, Karpenter supports provisioning metal instance types when a Provisioner's `node.kubernetes.io/instance-type` Requirements only include `metal` instance types. If other instance types fulfill pod requirements, then Karpenter will prioritize all non-metal instance types before metal ones are provisioned. +Yes, Karpenter supports provisioning metal instance types when a NodePool's `node.kubernetes.io/instance-type` Requirements only include `metal` instance types. If other instance types fulfill pod requirements, then Karpenter will prioritize all non-metal instance types before metal ones are provisioned. ### How does Karpenter dynamically select instance types? -Karpenter batches pending pods and then binpacks them based on CPU, memory, and GPUs required, taking into account node overhead, VPC CNI resources required, and daemonsets that will be packed when bringing up a new node. -By default Karpenter uses C, M, and R >= Gen 3 instance types, but it can be constrained in the provisioner spec with the [instance-type](https://kubernetes.io/docs/reference/labels-annotations-taints/#nodekubernetesioinstance-type) well-known label in the requirements section. -After the pods are binpacked on the most efficient instance type (i.e. the smallest instance type that can fit the pod batch), Karpenter takes 59 other instance types that are larger than the most efficient packing, and passes all 60 instance type options to an API called Amazon EC2 Fleet. -The EC2 fleet API attempts to provision the instance type based on an allocation strategy. -If you are using the on-demand capacity type, then Karpenter uses the `lowest-price` allocation strategy. -So fleet will provision the lowest priced instance type it can get from the 60 instance types Karpenter passed to the EC2 fleet API. -If the instance type is unavailable for some reason, then fleet will move on to the next cheapest instance type. -If you are using the spot capacity type, Karpenter uses the price-capacity-optimized allocation strategy. This tells fleet to find the instance type that EC2 has the most capacity for while also considering price. This allocation strategy will balance cost and decrease the probability of a spot interruption happening in the near term. -See [Choose the appropriate allocation strategy](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-fleet-allocation-strategy.html#ec2-fleet-allocation-use-cases) for information on fleet optimization. +Karpenter batches pending pods and then binpacks them based on CPU, memory, and GPUs required, taking into account node overhead, VPC CNI resources required, and daemonsets that will be packed when bringing up a new node. Karpenter [recommends the use of C, M, and R >= Gen 3 instance types]({{< ref "./concepts/nodepools#spectemplatespecrequirements" >}}) for most generic workloads, but it can be constrained in the NodePool spec with the [instance-type](https://kubernetes.io/docs/reference/labels-annotations-taints/#nodekubernetesioinstance-type) well-known label in the requirements section. + +After the pods are binpacked on the most efficient instance type (i.e. the smallest instance type that can fit the pod batch), Karpenter takes 59 other instance types that are larger than the most efficient packing, and passes all 60 instance type options to an API called Amazon EC2 Fleet. + + +The EC2 fleet API attempts to provision the instance type based on the [Price Capacity Optimized allocation strategy](https://aws.amazon.com/blogs/compute/introducing-price-capacity-optimized-allocation-strategy-for-ec2-spot-instances/). For the on-demand capacity type, this is effectively equivalent to the `lowest-price` allocation strategy. For the spot capacity type, Fleet will determine an instance type that has both the lowest price combined with the lowest chance of being interrupted. Note that this may not give you the instance type with the strictly lowest price for spot. ### How does Karpenter calculate the resource usage of Daemonsets when simulating scheduling? -Karpenter currently calculates the applicable daemonsets at the provisioner level with label selectors/taints, etc. It does not look to see if there are requirements on the daemonsets that would exclude it from running on particular instances that the provisioner could or couldn't launch. -The recommendation for now is to use multiple provisioners with taints/tolerations or label selectors to limit daemonsets to only nodes launched from specific provisioners. +Karpenter currently calculates the applicable daemonsets at the NodePool level with label selectors/taints, etc. It does not look to see if there are requirements on the daemonsets that would exclude it from running on particular instances that the NodePool could or couldn't launch. +The recommendation for now is to use multiple NodePools with taints/tolerations or label selectors to limit daemonsets to only nodes launched from specific NodePoools. ### What if there is no Spot capacity? Will Karpenter use On-Demand? -The best defense against running out of Spot capacity is to allow Karpenter to provision as many different instance types as possible. -Even instance types that have higher specs, e.g. vCPU, memory, etc., than what you need can still be cheaper in the Spot market than using On-Demand instances. -When Spot capacity is constrained, On-Demand capacity can also be constrained since Spot is fundamentally spare On-Demand capacity. -Allowing Karpenter to provision nodes from a large, diverse set of instance types will help you to stay on Spot longer and lower your costs due to Spot’s discounted pricing. -Moreover, if Spot capacity becomes constrained, this diversity will also increase the chances that you’ll be able to continue to launch On-Demand capacity for your workloads. +The best defense against running out of Spot capacity is to allow Karpenter to provision as many distinct instance types as possible. Even instance types that have higher specs (e.g. vCPU, memory, etc.) than what you need can still be cheaper in the Spot market than using On-Demand instances. When Spot capacity is constrained, On-Demand capacity can also be constrained since Spot is fundamentally spare On-Demand capacity. -If your Karpenter Provisioner specifies flexibility to both Spot and On-Demand capacity, Karpenter will attempt to provision On-Demand capacity if there is no Spot capacity available. -However, it’s strongly recommended that you specify at least 20 instance types in your Provisioner (or none and allow Karpenter to pick the best instance types) as our research indicates that this additional diversity increases the chances that your workloads will not need to launch On-Demand capacity at all. -Today, Karpenter will warn you if the number of instances in your Provisioner isn’t sufficiently diverse. +Allowing Karpenter to provision nodes from a large, diverse set of instance types will help you to stay on Spot longer and lower your costs due to Spot’s discounted pricing. Moreover, if Spot capacity becomes constrained, this diversity will also increase the chances that you’ll be able to continue to launch On-Demand capacity for your workloads. -Technically, Karpenter has a concept of an “offering” for each instance type, which is a combination of zone and capacity type (equivalent in the AWS cloud provider to an EC2 purchase option – Spot or On-Demand). -Whenever the Fleet API returns an insufficient capacity error for Spot instances, those particular offerings are temporarily removed from consideration (across the entire provisioner) so that Karpenter can make forward progress with different options. +If your Karpenter NodePool specifies allows both Spot and On-Demand capacity, Karpenter will fallback to provision On-Demand capacity if there is no Spot capacity available. However, it’s strongly recommended that you allow at least 20 instance types in your NodePool since this additional diversity increases the chances that your workloads will not need to launch On-Demand capacity at all. + +Karpenter has a concept of an “offering” for each instance type, which is a combination of zone and capacity type. Whenever the Fleet API returns an insufficient capacity error for Spot instances, those particular offerings are temporarily removed from consideration (across the entire NodePool) so that Karpenter can make forward progress with different options. ### Does Karpenter support IPv6? -Yes! Karpenter dynamically discovers if you are running in an IPv6 cluster by checking the kube-dns service's cluster-ip. When using an AMI Family such as `AL2`, Karpenter will automatically configure the EKS Bootstrap script for IPv6. Some EC2 instance types do not support IPv6 and the Amazon VPC CNI only supports instance types that run on the Nitro hypervisor. It's best to add a requirement to your Provisioner to only allow Nitro instance types: +Yes! Karpenter dynamically discovers if you are running in an IPv6 cluster by checking the kube-dns service's cluster-ip. When using an AMI Family such as `AL2`, Karpenter will automatically configure the EKS Bootstrap script for IPv6. Some EC2 instance types do not support IPv6 and the Amazon VPC CNI only supports instance types that run on the Nitro hypervisor. It's best to add a requirement to your NodePool to only allow Nitro instance types: ``` -kind: Provisioner +apiVersion: karpenter.sh/v1beta1 +kind: NodePool ... spec: - requirements: - - key: karpenter.k8s.aws/instance-hypervisor - operator: In - values: - - nitro + template: + spec: + requirements: + - key: karpenter.k8s.aws/instance-hypervisor + operator: In + values: + - nitro ``` For more documentation on enabling IPv6 with the Amazon VPC CNI, see the [docs](https://docs.aws.amazon.com/eks/latest/userguide/cni-ipv6.html). @@ -165,7 +141,7 @@ Windows nodes do not support IPv6. `kube-scheduler` is responsible for the scheduling of pods, while Karpenter launches the capacity. When using any sort of preferred scheduling constraint, `kube-scheduler` will schedule pods to nodes anytime it is possible. -As an example, suppose you scale up a deployment with a preferred zonal topology spread and none of the newly created pods can run on your existing cluster. Karpenter will then launch multiple nodes to satisfy that preference. If a) one of the nodes becomes ready slightly faster than other nodes and b) has enough capacity for multiple pods, `kube-scheduler` will schedule as many pods as possible to the single ready node so they won't remain unschedulable. It doesn't consider the in-flight capacity that will be ready in a few seconds. If all of the pods fit on the single node, the remaining nodes that Karpenter has launched aren't needed when they become ready and consolidation will delete them. +As an example, suppose you scale up a deployment with a preferred zonal topology spread and none of the newly created pods can run on your existing cluster. Karpenter will then launch multiple nodes to satisfy that preference. If a) one of the nodes becomes ready slightly faster than other nodes and b) has enough capacity for multiple pods, `kube-scheduler` will schedule as many pods as possible to the single ready node, so they won't remain unschedulable. It doesn't consider the in-flight capacity that will be ready in a few seconds. If all the pods fit on the single node, the remaining nodes that Karpenter has launched aren't needed when they become ready and consolidation will delete them. ### When deploying an additional DaemonSet to my cluster, why does Karpenter not scale-up my nodes to support the extra DaemonSet? @@ -194,7 +170,7 @@ See [Application developer]({{< ref "./concepts/#application-developer" >}}) for Yes. See [Persistent Volume Topology]({{< ref "./concepts/scheduling#persistent-volume-topology" >}}) for details. ### Can I set `--max-pods` on my nodes? -Yes, see the [KubeletConfiguration Section in the Provisioners Documentation]({{}}) to learn more. +Yes, see the [KubeletConfiguration Section in the NodePool docs]({{}}) to learn more. ### Why do the Windows2019 and Windows2022 AMI families only support Windows Server Core? The difference between the Core and Full variants is that Core is a minimal OS with less components and no graphic user interface (GUI) or desktop experience. @@ -221,21 +197,19 @@ To upgrade Karpenter to version `$VERSION`, make sure that the `KarpenterNode IA Next, locate `KarpenterController IAM Role` ARN (i.e., ARN of the resource created in [Create the KarpenterController IAM Role](../getting-started/getting-started-with-karpenter/#create-the-karpentercontroller-iam-role)) and pass them to the helm upgrade command. {{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step08-apply-helm-chart.sh" language="bash"%}} -For information on upgrading Karpenter, see the [Upgrade Guide]({{< ref "./upgrade-guide/" >}}). - -### Why do I get an `unknown field "startupTaints"` error when creating a provisioner with startupTaints? - -```bash -error: error validating "provisioner.yaml": error validating data: ValidationError(Provisioner.spec): unknown field "startupTaints" in sh.karpenter.v1alpha5.Provisioner.spec; if you choose to ignore these errors, turn validation off with --validate=false -``` - -The `startupTaints` parameter was added in v0.10.0. Helm upgrades do not upgrade the CRD describing the provisioner, so it must be done manually. For specific details, see the [Upgrade Guide]({{< ref "./upgrade-guide/#upgrading-to-v0100" >}}) +For information on upgrading Karpenter, see the [Upgrade Guide]({{< ref "./upgrading/upgrade-guide/" >}}). ## Upgrading Kubernetes Cluster ### How do I upgrade an EKS Cluster with Karpenter? -When upgrading an Amazon EKS cluster, [Karpenter's Drift feature]({{}}) can automatically upgrade the Karpenter-provisioned nodes to stay in-sync with the EKS control plane. Karpenter Drift currently needs to be enabled using a [feature gate]({{}}). Karpenter's default [AWSNodeTemplate `amiFamily` configuration]({{}}) uses the latest EKS Optimized AL2 AMI for the same major and minor version as the EKS cluster's control plane. Karpenter's AWSNodeTemplate can be configured to not use the EKS optimized AL2 AMI in favor of a custom AMI by configuring the [`amiSelector`]({{}}). If using a custom AMI, you will need to trigger the rollout of this new worker node image through the publication of a new AMI with tags matching the [`amiSelector`]({{}}), or a change to the [`amiSelector`]({{}}) field. +When upgrading an Amazon EKS cluster, [Karpenter's Drift feature]({{}}) can automatically upgrade the Karpenter-provisioned nodes to stay in-sync with the EKS control plane. Karpenter Drift currently needs to be enabled using a [feature gate]({{}}). + +{{% alert title="Note" color="primary" %}} +Karpenter's default [EC2NodeClass `amiFamily` configuration]({{}}) uses the latest EKS Optimized AL2 AMI for the same major and minor version as the EKS cluster's control plane, meaning that an upgrade of the control plane will cause Karpenter to auto-discover the new AMIs for that version. + +If using a custom AMI, you will need to trigger the rollout of this new worker node image through the publication of a new AMI with tags matching the [`amiSelector`]({{}}), or a change to the [`amiSelector`]({{}}) field. +{{% /alert %}} Start by [upgrading the EKS Cluster control plane](https://docs.aws.amazon.com/eks/latest/userguide/update-cluster.html). After the EKS Cluster upgrade completes, Karpenter's Drift feature will detect that the Karpenter-provisioned nodes are using EKS Optimized AMIs for the previous cluster version, and [automatically cordon, drain, and replace those nodes]({{}}). To support pods moving to new nodes, follow Kubernetes best practices by setting appropriate pod [Resource Quotas](https://kubernetes.io/docs/concepts/policy/resource-quotas/), and using [Pod Disruption Budgets](https://kubernetes.io/docs/concepts/workloads/pods/disruptions/) (PDB). Karpenter's Drift feature will spin up replacement nodes based on the pod resource requests, and will respect the PDBs when deprovisioning nodes. @@ -249,7 +223,7 @@ Karpenter's native interruption handling offers two main benefits over the stand 1. You don't have to manage and maintain a separate component to exclusively handle interruption events. 2. Karpenter's native interruption handling coordinates with other deprovisioning so that consolidation, expiration, etc. can be aware of interruption events and vice-versa. -### Why am I receiving QueueNotFound errors when I set `aws.interruptionQueueName`? +### Why am I receiving QueueNotFound errors when I set `--interruption-queue-name`? Karpenter requires a queue to exist that receives event messages from EC2 and health services in order to handle interruption messages properly for nodes. Details on the types of events that Karpenter handles can be found in the [Interruption Handling Docs]({{< ref "./concepts/disruption/#interruption" >}}). diff --git a/website/content/en/preview/getting-started/_index.md b/website/content/en/preview/getting-started/_index.md index 77e7a529548f..12fd2a9f9fd2 100644 --- a/website/content/en/preview/getting-started/_index.md +++ b/website/content/en/preview/getting-started/_index.md @@ -1,18 +1,16 @@ --- title: "Getting Started" linkTitle: "Getting Started" -weight: 1 +weight: 10 description: > Choose from different methods to get started with Karpenter -cascade: - type: docs --- To get started with Karpenter, the [Getting Started with Karpenter]({{< relref "getting-started-with-karpenter" >}}) guide provides an end-to-end procedure for creating a cluster (with `eksctl`) and adding Karpenter. If you prefer, the following instructions use Terraform to create a cluster and add Karpenter: -* [Amazon EKS Blueprints for Terraform](https://aws-ia.github.io/terraform-aws-eks-blueprints): Follow a basic [Getting Started](https://aws-ia.github.io/terraform-aws-eks-blueprints/v4.18.0/getting-started/) guide and also add modules and add-ons. This includes a [Karpenter](https://aws-ia.github.io/terraform-aws-eks-blueprints/v4.18.0/add-ons/karpenter/) add-on that lets you bypass the instructions in this guide for setting up Karpenter. +* [Amazon EKS Blueprints for Terraform](https://aws-ia.github.io/terraform-aws-eks-blueprints): Follow a basic [Getting Started](https://aws-ia.github.io/terraform-aws-eks-blueprints/getting-started/) guide and also add modules and add-ons. This includes a [Karpenter](https://aws-ia.github.io/terraform-aws-eks-blueprints/patterns/karpenter/) add-on that lets you bypass the instructions in this guide for setting up Karpenter. Although not supported, you could also try Karpenter on other Kubernetes distributions running on AWS. For example: diff --git a/website/content/en/preview/getting-started/getting-started-with-karpenter/_index.md b/website/content/en/preview/getting-started/getting-started-with-karpenter/_index.md index efdf7d929b38..924e9a74577d 100644 --- a/website/content/en/preview/getting-started/getting-started-with-karpenter/_index.md +++ b/website/content/en/preview/getting-started/getting-started-with-karpenter/_index.md @@ -92,32 +92,23 @@ See [Enabling Windows support](https://docs.aws.amazon.com/eks/latest/userguide/ Karpenter creates a mapping between CloudProvider machines and CustomResources in the cluster for capacity tracking. To ensure this mapping is consistent, Karpenter utilizes the following tag keys: * `karpenter.sh/managed-by` -* `karpenter.sh/provisioner-name` +* `karpenter.sh/nodepool` * `kubernetes.io/cluster/${CLUSTER_NAME}` Because Karpenter takes this dependency, any user that has the ability to Create/Delete these tags on CloudProvider machines will have the ability to orchestrate Karpenter to Create/Delete CloudProvider machines as a side effect. We recommend that you [enforce tag-based IAM policies](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_tags.html) on these tags against any EC2 instance resource (`i-*`) for any users that might have [CreateTags](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateTags.html)/[DeleteTags](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DeleteTags.html) permissions but should not have [RunInstances](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_RunInstances.html)/[TerminateInstances](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_TerminateInstances.html) permissions. {{% /alert %}} -### 5. Create Provisioner +### 5. Create NodePool -A single Karpenter provisioner is capable of handling many different pod -shapes. Karpenter makes scheduling and provisioning decisions based on pod -attributes such as labels and affinity. In other words, Karpenter eliminates -the need to manage many different node groups. +A single Karpenter NodePool is capable of handling many different pod shapes. Karpenter makes scheduling and provisioning decisions based on pod attributes such as labels and affinity. In other words, Karpenter eliminates the need to manage many different node groups. -Create a default provisioner using the command below. -This provisioner uses `securityGroupSelector` and `subnetSelector` to discover resources used to launch nodes. -We applied the tag `karpenter.sh/discovery` in the `eksctl` command above. -Depending how these resources are shared between clusters, you may need to use different tagging schemes. +Create a default NodePool using the command below. This NodePool uses `securityGroupSelectorTerms` and `subnetSelectorTerms` to discover resources used to launch nodes. We applied the tag `karpenter.sh/discovery` in the `eksctl` command above. Depending on how these resources are shared between clusters, you may need to use different tagging schemes. -The `consolidation` value configures Karpenter to reduce cluster cost by removing and replacing nodes. As a result, consolidation will terminate any empty nodes on the cluster. This behavior can be disabled by leaving the value undefined or setting `consolidation.enabled` to `false`. Review the [provisioner CRD]({{}}) for more information. +The `consolidationPolicy` set to `WhenUnderutilized` in the `disruption` block configures Karpenter to reduce cluster cost by removing and replacing nodes. As a result, consolidation will terminate any empty nodes on the cluster. This behavior can be disabled by setting `consolidateAfter` to `Never`, telling Karpenter that it should never consolidate nodes. Review the [NodePool API docs]({{}}) for more information. -Review the [provisioner CRD]({{}}) for more information. For example, -`ttlSecondsUntilExpired` configures Karpenter to terminate nodes when a maximum age is reached. +Note: This NodePool will create capacity as long as the sum of all created capacity is less than the specified limit. -Note: This provisioner will create capacity as long as the sum of all created capacity is less than the specified limit. - -{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step12-add-provisioner.sh" language="bash"%}} +{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step12-add-nodepool.sh" language="bash"%}} Karpenter is now active and ready to begin provisioning nodes. diff --git a/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step08-apply-helm-chart.sh b/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step08-apply-helm-chart.sh index 99fa927b12b6..b6c86db990e4 100755 --- a/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step08-apply-helm-chart.sh +++ b/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step08-apply-helm-chart.sh @@ -3,9 +3,8 @@ helm registry logout public.ecr.aws helm upgrade --install karpenter oci://public.ecr.aws/karpenter/karpenter --version ${KARPENTER_VERSION} --namespace karpenter --create-namespace \ --set serviceAccount.annotations."eks\.amazonaws\.com/role-arn"=${KARPENTER_IAM_ROLE_ARN} \ - --set settings.aws.clusterName=${CLUSTER_NAME} \ - --set settings.aws.defaultInstanceProfile=KarpenterNodeInstanceProfile-${CLUSTER_NAME} \ - --set settings.aws.interruptionQueueName=${CLUSTER_NAME} \ + --set settings.clusterName=${CLUSTER_NAME} \ + --set settings.interruptionQueueName=${CLUSTER_NAME} \ --set controller.resources.requests.cpu=1 \ --set controller.resources.requests.memory=1Gi \ --set controller.resources.limits.cpu=1 \ diff --git a/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step12-add-nodepool.sh b/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step12-add-nodepool.sh new file mode 100755 index 000000000000..8c518e9e74b8 --- /dev/null +++ b/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step12-add-nodepool.sh @@ -0,0 +1,46 @@ +cat <}} We can now generate a full Karpenter deployment yaml from the helm chart. -{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step09-generate-chart.sh" language="bash" %}} +{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step08-generate-chart.sh" language="bash" %}} Modify the following lines in the karpenter.yaml file. @@ -115,7 +111,7 @@ affinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - - key: karpenter.sh/provisioner-name + - key: karpenter.sh/nodepool operator: DoesNotExist - matchExpressions: - key: eks.amazonaws.com/nodegroup @@ -127,16 +123,15 @@ affinity: - topologyKey: "kubernetes.io/hostname" ``` -Now that our deployment is ready we can create the karpenter namespace, create the provisioner CRD, and then deploy the rest of the karpenter resources. +Now that our deployment is ready we can create the karpenter namespace, create the NodePool CRD, and then deploy the rest of the karpenter resources. -{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step10-deploy.sh" language="bash" %}} +{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step09-deploy.sh" language="bash" %}} -## Create default provisioner +## Create default NodePool -We need to create a default provisioner so Karpenter knows what types of nodes we want for unscheduled workloads. -You can refer to some of the [example provisioners](https://github.com/aws/karpenter/tree{{< githubRelRef >}}examples/provisioner) for specific needs. +We need to create a default NodePool so Karpenter knows what types of nodes we want for unscheduled workloads. You can refer to some of the [example NodePool](https://github.com/aws/karpenter/tree{{< githubRelRef >}}examples/v1beta1) for specific needs. -{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step11-create-provisioner.sh" language="bash" %}} +{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step10-create-nodepool.sh" language="bash" %}} ## Set nodeAffinity for critical workloads (optional) @@ -167,7 +162,7 @@ affinity: Now that karpenter is running we can disable the cluster autoscaler. To do that we will scale the number of replicas to zero. -{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step12-scale-cas.sh" language="bash" %}} +{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step11-scale-cas.sh" language="bash" %}} To get rid of the instances that were added from the node group we can scale our nodegroup down to a minimum size to support Karpenter and other critical services. @@ -175,11 +170,11 @@ To get rid of the instances that were added from the node group we can scale our If you have a single multi-AZ node group, we suggest a minimum of 2 instances. -{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step13-scale-single-ng.sh" language="bash" %}} +{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step12-scale-single-ng.sh" language="bash" %}} Or, if you have multiple single-AZ node groups, we suggest a minimum of 1 instance each. -{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step13-scale-multiple-ng.sh" language="bash" %}} +{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step12-scale-multiple-ng.sh" language="bash" %}} {{% alert title="Note" color="warning" %}} If you have a lot of nodes or workloads you may want to slowly scale down your node groups by a few instances at a time. It is recommended to watch the transition carefully for workloads that may not have enough replicas running or disruption budgets configured. diff --git a/website/content/en/preview/getting-started/migrating-from-cas/scripts/step05-controller-iam.sh b/website/content/en/preview/getting-started/migrating-from-cas/scripts/step04-controller-iam.sh similarity index 100% rename from website/content/en/preview/getting-started/migrating-from-cas/scripts/step05-controller-iam.sh rename to website/content/en/preview/getting-started/migrating-from-cas/scripts/step04-controller-iam.sh diff --git a/website/content/en/preview/getting-started/migrating-from-cas/scripts/step04-instance-profile.sh b/website/content/en/preview/getting-started/migrating-from-cas/scripts/step04-instance-profile.sh deleted file mode 100644 index 3c112c819588..000000000000 --- a/website/content/en/preview/getting-started/migrating-from-cas/scripts/step04-instance-profile.sh +++ /dev/null @@ -1,6 +0,0 @@ -aws iam create-instance-profile \ - --instance-profile-name "KarpenterNodeInstanceProfile-${CLUSTER_NAME}" - -aws iam add-role-to-instance-profile \ - --instance-profile-name "KarpenterNodeInstanceProfile-${CLUSTER_NAME}" \ - --role-name "KarpenterNodeRole-${CLUSTER_NAME}" diff --git a/website/content/en/preview/getting-started/migrating-from-cas/scripts/step06-tag-subnets.sh b/website/content/en/preview/getting-started/migrating-from-cas/scripts/step05-tag-subnets.sh similarity index 100% rename from website/content/en/preview/getting-started/migrating-from-cas/scripts/step06-tag-subnets.sh rename to website/content/en/preview/getting-started/migrating-from-cas/scripts/step05-tag-subnets.sh diff --git a/website/content/en/preview/getting-started/migrating-from-cas/scripts/step07-tag-security-groups.sh b/website/content/en/preview/getting-started/migrating-from-cas/scripts/step06-tag-security-groups.sh similarity index 100% rename from website/content/en/preview/getting-started/migrating-from-cas/scripts/step07-tag-security-groups.sh rename to website/content/en/preview/getting-started/migrating-from-cas/scripts/step06-tag-security-groups.sh diff --git a/website/content/en/preview/getting-started/migrating-from-cas/scripts/step08-edit-aws-auth.sh b/website/content/en/preview/getting-started/migrating-from-cas/scripts/step07-edit-aws-auth.sh similarity index 100% rename from website/content/en/preview/getting-started/migrating-from-cas/scripts/step08-edit-aws-auth.sh rename to website/content/en/preview/getting-started/migrating-from-cas/scripts/step07-edit-aws-auth.sh diff --git a/website/content/en/preview/getting-started/migrating-from-cas/scripts/step09-generate-chart.sh b/website/content/en/preview/getting-started/migrating-from-cas/scripts/step08-generate-chart.sh similarity index 77% rename from website/content/en/preview/getting-started/migrating-from-cas/scripts/step09-generate-chart.sh rename to website/content/en/preview/getting-started/migrating-from-cas/scripts/step08-generate-chart.sh index f2bc603e0eeb..bc66ee53e387 100644 --- a/website/content/en/preview/getting-started/migrating-from-cas/scripts/step09-generate-chart.sh +++ b/website/content/en/preview/getting-started/migrating-from-cas/scripts/step08-generate-chart.sh @@ -1,6 +1,5 @@ helm template karpenter oci://public.ecr.aws/karpenter/karpenter --version ${KARPENTER_VERSION} --namespace karpenter \ - --set settings.aws.defaultInstanceProfile=KarpenterNodeInstanceProfile-${CLUSTER_NAME} \ - --set settings.aws.clusterName=${CLUSTER_NAME} \ + --set settings.clusterName=${CLUSTER_NAME} \ --set serviceAccount.annotations."eks\.amazonaws\.com/role-arn"="arn:${AWS_PARTITION}:iam::${AWS_ACCOUNT_ID}:role/KarpenterControllerRole-${CLUSTER_NAME}" \ --set controller.resources.requests.cpu=1 \ --set controller.resources.requests.memory=1Gi \ diff --git a/website/content/en/preview/getting-started/migrating-from-cas/scripts/step10-deploy.sh b/website/content/en/preview/getting-started/migrating-from-cas/scripts/step09-deploy.sh similarity index 65% rename from website/content/en/preview/getting-started/migrating-from-cas/scripts/step10-deploy.sh rename to website/content/en/preview/getting-started/migrating-from-cas/scripts/step09-deploy.sh index 45322127f591..f35e9513d247 100644 --- a/website/content/en/preview/getting-started/migrating-from-cas/scripts/step10-deploy.sh +++ b/website/content/en/preview/getting-started/migrating-from-cas/scripts/step09-deploy.sh @@ -1,8 +1,8 @@ kubectl create namespace karpenter kubectl create -f \ - https://raw.githubusercontent.com/aws/karpenter/${KARPENTER_VERSION}/pkg/apis/crds/karpenter.sh_provisioners.yaml + https://raw.githubusercontent.com/aws/karpenter/${KARPENTER_VERSION}/pkg/apis/crds/karpenter.sh_nodepools.yaml kubectl create -f \ - https://raw.githubusercontent.com/aws/karpenter/${KARPENTER_VERSION}/pkg/apis/crds/karpenter.k8s.aws_awsnodetemplates.yaml + https://raw.githubusercontent.com/aws/karpenter/${KARPENTER_VERSION}/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml kubectl create -f \ - https://raw.githubusercontent.com/aws/karpenter/${KARPENTER_VERSION}/pkg/apis/crds/karpenter.sh_machines.yaml + https://raw.githubusercontent.com/aws/karpenter/${KARPENTER_VERSION}/pkg/apis/crds/karpenter.sh_nodeclaims.yaml kubectl apply -f karpenter.yaml diff --git a/website/content/en/preview/getting-started/migrating-from-cas/scripts/step10-create-nodepool.sh b/website/content/en/preview/getting-started/migrating-from-cas/scripts/step10-create-nodepool.sh new file mode 100644 index 000000000000..8c518e9e74b8 --- /dev/null +++ b/website/content/en/preview/getting-started/migrating-from-cas/scripts/step10-create-nodepool.sh @@ -0,0 +1,46 @@ +cat < Reference documentation for Karpenter --- \ No newline at end of file diff --git a/website/content/en/preview/reference/instance-types.md b/website/content/en/preview/reference/instance-types.md index 95752c33498f..a7af742e2edb 100644 --- a/website/content/en/preview/reference/instance-types.md +++ b/website/content/en/preview/reference/instance-types.md @@ -3192,6 +3192,283 @@ below are the resources available with some assumptions and after the instance o |memory|238333Mi| |pods|345| |vpc.amazonaws.com/pod-eni|108| +## c7a Family +### `c7a.medium` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|1| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7a| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|2048| + |karpenter.k8s.aws/instance-pods|8| + |karpenter.k8s.aws/instance-size|medium| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7a.medium| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|940m| + |ephemeral-storage|17Gi| + |memory|1451Mi| + |pods|8| +### `c7a.large` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7a| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|4096| + |karpenter.k8s.aws/instance-pods|29| + |karpenter.k8s.aws/instance-size|large| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7a.large| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|1930m| + |ephemeral-storage|17Gi| + |memory|3114Mi| + |pods|29| +### `c7a.xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7a| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|8192| + |karpenter.k8s.aws/instance-pods|58| + |karpenter.k8s.aws/instance-size|xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7a.xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|3920m| + |ephemeral-storage|17Gi| + |memory|6584Mi| + |pods|58| +### `c7a.2xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7a| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|16384| + |karpenter.k8s.aws/instance-pods|58| + |karpenter.k8s.aws/instance-size|2xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7a.2xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|7910m| + |ephemeral-storage|17Gi| + |memory|14162Mi| + |pods|58| +### `c7a.4xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7a| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|32768| + |karpenter.k8s.aws/instance-pods|234| + |karpenter.k8s.aws/instance-size|4xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7a.4xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|15890m| + |ephemeral-storage|17Gi| + |memory|27381Mi| + |pods|234| +### `c7a.8xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7a| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|65536| + |karpenter.k8s.aws/instance-pods|234| + |karpenter.k8s.aws/instance-size|8xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7a.8xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|31850m| + |ephemeral-storage|17Gi| + |memory|57691Mi| + |pods|234| +### `c7a.12xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7a| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|98304| + |karpenter.k8s.aws/instance-pods|234| + |karpenter.k8s.aws/instance-size|12xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7a.12xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|47810m| + |ephemeral-storage|17Gi| + |memory|88002Mi| + |pods|234| +### `c7a.16xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7a| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|131072| + |karpenter.k8s.aws/instance-pods|737| + |karpenter.k8s.aws/instance-size|16xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7a.16xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|63770m| + |ephemeral-storage|17Gi| + |memory|112779Mi| + |pods|737| +### `c7a.24xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7a| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|196608| + |karpenter.k8s.aws/instance-pods|737| + |karpenter.k8s.aws/instance-size|24xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7a.24xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|95690m| + |ephemeral-storage|17Gi| + |memory|173400Mi| + |pods|737| +### `c7a.32xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7a| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|262144| + |karpenter.k8s.aws/instance-pods|737| + |karpenter.k8s.aws/instance-size|32xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7a.32xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|127610m| + |ephemeral-storage|17Gi| + |memory|234021Mi| + |pods|737| +### `c7a.48xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|192| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7a| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|393216| + |karpenter.k8s.aws/instance-pods|737| + |karpenter.k8s.aws/instance-size|48xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7a.48xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|191450m| + |ephemeral-storage|17Gi| + |memory|355262Mi| + |pods|737| +### `c7a.metal-48xl` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|192| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7a| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|| + |karpenter.k8s.aws/instance-memory|393216| + |karpenter.k8s.aws/instance-pods|737| + |karpenter.k8s.aws/instance-size|metal-48xl| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7a.metal-48xl| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|191450m| + |ephemeral-storage|17Gi| + |memory|355262Mi| + |pods|737| ## c7g Family ### `c7g.medium` #### Labels @@ -4401,6 +4678,38 @@ below are the resources available with some assumptions and after the instance o |memory|180528Mi| |pods|89| |vpc.amazonaws.com/pod-eni|119| +## dl1 Family +### `dl1.24xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|dl| + |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|dl1| + |karpenter.k8s.aws/instance-generation|1| + |karpenter.k8s.aws/instance-gpu-count|8| + |karpenter.k8s.aws/instance-gpu-manufacturer|habana| + |karpenter.k8s.aws/instance-gpu-memory|32768| + |karpenter.k8s.aws/instance-gpu-name|gaudi-hl-205| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-local-nvme|4000| + |karpenter.k8s.aws/instance-memory|786432| + |karpenter.k8s.aws/instance-network-bandwidth|400000| + |karpenter.k8s.aws/instance-pods|737| + |karpenter.k8s.aws/instance-size|24xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|dl1.24xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|95690m| + |ephemeral-storage|17Gi| + |habana.ai/gaudi|8| + |memory|718987Mi| + |pods|737| + |vpc.amazonaws.com/pod-eni|62| ## f1 Family ### `f1.2xlarge` #### Labels @@ -4477,63 +4786,6 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|919778Mi| |pods|394| -## g2 Family -### `g2.2xlarge` -#### Labels - | Label | Value | - |--|--| - |karpenter.k8s.aws/instance-category|g| - |karpenter.k8s.aws/instance-cpu|8| - |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| - |karpenter.k8s.aws/instance-family|g2| - |karpenter.k8s.aws/instance-generation|2| - |karpenter.k8s.aws/instance-gpu-count|1| - |karpenter.k8s.aws/instance-gpu-manufacturer|nvidia| - |karpenter.k8s.aws/instance-gpu-memory|4096| - |karpenter.k8s.aws/instance-gpu-name|k520| - |karpenter.k8s.aws/instance-hypervisor|xen| - |karpenter.k8s.aws/instance-memory|15360| - |karpenter.k8s.aws/instance-pods|58| - |karpenter.k8s.aws/instance-size|2xlarge| - |kubernetes.io/arch|amd64| - |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|g2.2xlarge| -#### Resources - | Resource | Quantity | - |--|--| - |cpu|7910m| - |ephemeral-storage|17Gi| - |memory|13215Mi| - |nvidia.com/gpu|1| - |pods|58| -### `g2.8xlarge` -#### Labels - | Label | Value | - |--|--| - |karpenter.k8s.aws/instance-category|g| - |karpenter.k8s.aws/instance-cpu|32| - |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| - |karpenter.k8s.aws/instance-family|g2| - |karpenter.k8s.aws/instance-generation|2| - |karpenter.k8s.aws/instance-gpu-count|4| - |karpenter.k8s.aws/instance-gpu-manufacturer|nvidia| - |karpenter.k8s.aws/instance-gpu-memory|4096| - |karpenter.k8s.aws/instance-gpu-name|k520| - |karpenter.k8s.aws/instance-hypervisor|xen| - |karpenter.k8s.aws/instance-memory|61440| - |karpenter.k8s.aws/instance-pods|234| - |karpenter.k8s.aws/instance-size|8xlarge| - |kubernetes.io/arch|amd64| - |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|g2.8xlarge| -#### Resources - | Resource | Quantity | - |--|--| - |cpu|31850m| - |ephemeral-storage|17Gi| - |memory|53903Mi| - |nvidia.com/gpu|4| - |pods|234| ## g3 Family ### `g3.4xlarge` #### Labels @@ -5552,6 +5804,79 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|234021Mi| |pods|737| +## hpc7g Family +### `hpc7g.4xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|hpc| + |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|hpc7g| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|131072| + |karpenter.k8s.aws/instance-network-bandwidth|200000| + |karpenter.k8s.aws/instance-pods|198| + |karpenter.k8s.aws/instance-size|4xlarge| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|hpc7g.4xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|15890m| + |ephemeral-storage|17Gi| + |memory|118649Mi| + |pods|198| +### `hpc7g.8xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|hpc| + |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|hpc7g| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|131072| + |karpenter.k8s.aws/instance-network-bandwidth|200000| + |karpenter.k8s.aws/instance-pods|198| + |karpenter.k8s.aws/instance-size|8xlarge| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|hpc7g.8xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|31850m| + |ephemeral-storage|17Gi| + |memory|118649Mi| + |pods|198| +### `hpc7g.16xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|hpc| + |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|hpc7g| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|131072| + |karpenter.k8s.aws/instance-network-bandwidth|200000| + |karpenter.k8s.aws/instance-pods|198| + |karpenter.k8s.aws/instance-size|16xlarge| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|hpc7g.16xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|63770m| + |ephemeral-storage|17Gi| + |memory|118649Mi| + |pods|198| ## i2 Family ### `i2.xlarge` #### Labels @@ -11733,59 +12058,245 @@ below are the resources available with some assumptions and after the instance o | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|p| - |karpenter.k8s.aws/instance-cpu|32| - |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| - |karpenter.k8s.aws/instance-family|p2| - |karpenter.k8s.aws/instance-generation|2| + |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| + |karpenter.k8s.aws/instance-family|p2| + |karpenter.k8s.aws/instance-generation|2| + |karpenter.k8s.aws/instance-gpu-count|8| + |karpenter.k8s.aws/instance-gpu-manufacturer|nvidia| + |karpenter.k8s.aws/instance-gpu-memory|12288| + |karpenter.k8s.aws/instance-gpu-name|k80| + |karpenter.k8s.aws/instance-hypervisor|xen| + |karpenter.k8s.aws/instance-memory|499712| + |karpenter.k8s.aws/instance-network-bandwidth|10000| + |karpenter.k8s.aws/instance-pods|234| + |karpenter.k8s.aws/instance-size|8xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|p2.8xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|31850m| + |ephemeral-storage|17Gi| + |memory|459304Mi| + |nvidia.com/gpu|8| + |pods|234| +### `p2.16xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|p| + |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| + |karpenter.k8s.aws/instance-family|p2| + |karpenter.k8s.aws/instance-generation|2| + |karpenter.k8s.aws/instance-gpu-count|16| + |karpenter.k8s.aws/instance-gpu-manufacturer|nvidia| + |karpenter.k8s.aws/instance-gpu-memory|12288| + |karpenter.k8s.aws/instance-gpu-name|k80| + |karpenter.k8s.aws/instance-hypervisor|xen| + |karpenter.k8s.aws/instance-memory|749568| + |karpenter.k8s.aws/instance-network-bandwidth|25000| + |karpenter.k8s.aws/instance-pods|234| + |karpenter.k8s.aws/instance-size|16xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|p2.16xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|63770m| + |ephemeral-storage|17Gi| + |memory|690421Mi| + |nvidia.com/gpu|16| + |pods|234| +## p3 Family +### `p3.2xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|p| + |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| + |karpenter.k8s.aws/instance-family|p3| + |karpenter.k8s.aws/instance-generation|3| + |karpenter.k8s.aws/instance-gpu-count|1| + |karpenter.k8s.aws/instance-gpu-manufacturer|nvidia| + |karpenter.k8s.aws/instance-gpu-memory|16384| + |karpenter.k8s.aws/instance-gpu-name|v100| + |karpenter.k8s.aws/instance-hypervisor|xen| + |karpenter.k8s.aws/instance-memory|62464| + |karpenter.k8s.aws/instance-pods|58| + |karpenter.k8s.aws/instance-size|2xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|p3.2xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|7910m| + |ephemeral-storage|17Gi| + |memory|56786Mi| + |nvidia.com/gpu|1| + |pods|58| + |vpc.amazonaws.com/pod-eni|38| +### `p3.8xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|p| + |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| + |karpenter.k8s.aws/instance-family|p3| + |karpenter.k8s.aws/instance-generation|3| + |karpenter.k8s.aws/instance-gpu-count|4| + |karpenter.k8s.aws/instance-gpu-manufacturer|nvidia| + |karpenter.k8s.aws/instance-gpu-memory|16384| + |karpenter.k8s.aws/instance-gpu-name|v100| + |karpenter.k8s.aws/instance-hypervisor|xen| + |karpenter.k8s.aws/instance-memory|249856| + |karpenter.k8s.aws/instance-network-bandwidth|10000| + |karpenter.k8s.aws/instance-pods|234| + |karpenter.k8s.aws/instance-size|8xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|p3.8xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|31850m| + |ephemeral-storage|17Gi| + |memory|228187Mi| + |nvidia.com/gpu|4| + |pods|234| + |vpc.amazonaws.com/pod-eni|54| +### `p3.16xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|p| + |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| + |karpenter.k8s.aws/instance-family|p3| + |karpenter.k8s.aws/instance-generation|3| + |karpenter.k8s.aws/instance-gpu-count|8| + |karpenter.k8s.aws/instance-gpu-manufacturer|nvidia| + |karpenter.k8s.aws/instance-gpu-memory|16384| + |karpenter.k8s.aws/instance-gpu-name|v100| + |karpenter.k8s.aws/instance-hypervisor|xen| + |karpenter.k8s.aws/instance-memory|499712| + |karpenter.k8s.aws/instance-network-bandwidth|25000| + |karpenter.k8s.aws/instance-pods|234| + |karpenter.k8s.aws/instance-size|16xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|p3.16xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|63770m| + |ephemeral-storage|17Gi| + |memory|459304Mi| + |nvidia.com/gpu|8| + |pods|234| + |vpc.amazonaws.com/pod-eni|114| +## p3dn Family +### `p3dn.24xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|p| + |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|p3dn| + |karpenter.k8s.aws/instance-generation|3| + |karpenter.k8s.aws/instance-gpu-count|8| + |karpenter.k8s.aws/instance-gpu-manufacturer|nvidia| + |karpenter.k8s.aws/instance-gpu-memory|32768| + |karpenter.k8s.aws/instance-gpu-name|v100| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-local-nvme|1800| + |karpenter.k8s.aws/instance-memory|786432| + |karpenter.k8s.aws/instance-network-bandwidth|100000| + |karpenter.k8s.aws/instance-pods|737| + |karpenter.k8s.aws/instance-size|24xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|p3dn.24xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|95690m| + |ephemeral-storage|17Gi| + |memory|718987Mi| + |nvidia.com/gpu|8| + |pods|737| + |vpc.amazonaws.com/pod-eni|107| +## p4d Family +### `p4d.24xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|p| + |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|p4d| + |karpenter.k8s.aws/instance-generation|4| |karpenter.k8s.aws/instance-gpu-count|8| |karpenter.k8s.aws/instance-gpu-manufacturer|nvidia| - |karpenter.k8s.aws/instance-gpu-memory|12288| - |karpenter.k8s.aws/instance-gpu-name|k80| - |karpenter.k8s.aws/instance-hypervisor|xen| - |karpenter.k8s.aws/instance-memory|499712| - |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| - |karpenter.k8s.aws/instance-size|8xlarge| + |karpenter.k8s.aws/instance-gpu-memory|40960| + |karpenter.k8s.aws/instance-gpu-name|a100| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-local-nvme|8000| + |karpenter.k8s.aws/instance-memory|1179648| + |karpenter.k8s.aws/instance-network-bandwidth|400000| + |karpenter.k8s.aws/instance-pods|737| + |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|p2.8xlarge| + |node.kubernetes.io/instance-type|p4d.24xlarge| #### Resources | Resource | Quantity | |--|--| - |cpu|31850m| + |cpu|95690m| |ephemeral-storage|17Gi| - |memory|459304Mi| + |memory|1082712Mi| |nvidia.com/gpu|8| - |pods|234| -### `p2.16xlarge` + |pods|737| + |vpc.amazonaws.com/pod-eni|62| +## p5 Family +### `p5.48xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|p| - |karpenter.k8s.aws/instance-cpu|64| - |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| - |karpenter.k8s.aws/instance-family|p2| - |karpenter.k8s.aws/instance-generation|2| - |karpenter.k8s.aws/instance-gpu-count|16| + |karpenter.k8s.aws/instance-cpu|192| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|p5| + |karpenter.k8s.aws/instance-generation|5| + |karpenter.k8s.aws/instance-gpu-count|8| |karpenter.k8s.aws/instance-gpu-manufacturer|nvidia| - |karpenter.k8s.aws/instance-gpu-memory|12288| - |karpenter.k8s.aws/instance-gpu-name|k80| - |karpenter.k8s.aws/instance-hypervisor|xen| - |karpenter.k8s.aws/instance-memory|749568| - |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|234| - |karpenter.k8s.aws/instance-size|16xlarge| + |karpenter.k8s.aws/instance-gpu-memory|81920| + |karpenter.k8s.aws/instance-gpu-name|h100| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-local-nvme|30400| + |karpenter.k8s.aws/instance-memory|2097152| + |karpenter.k8s.aws/instance-network-bandwidth|3200000| + |karpenter.k8s.aws/instance-pods|100| + |karpenter.k8s.aws/instance-size|48xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|p2.16xlarge| + |node.kubernetes.io/instance-type|p5.48xlarge| #### Resources | Resource | Quantity | |--|--| - |cpu|63770m| + |cpu|191450m| |ephemeral-storage|17Gi| - |memory|690421Mi| - |nvidia.com/gpu|16| - |pods|234| + |memory|1938410Mi| + |nvidia.com/gpu|8| + |pods|100| + |vpc.amazonaws.com/pod-eni|120| ## r3 Family ### `r3.large` #### Labels @@ -15632,6 +16143,29 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|1446437Mi| |pods|737| +### `r7a.metal-48xl` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|192| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7a| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|| + |karpenter.k8s.aws/instance-memory|1572864| + |karpenter.k8s.aws/instance-pods|737| + |karpenter.k8s.aws/instance-size|metal-48xl| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7a.metal-48xl| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|191450m| + |ephemeral-storage|17Gi| + |memory|1446437Mi| + |pods|737| ## r7g Family ### `r7g.medium` #### Labels @@ -15864,209 +16398,417 @@ below are the resources available with some assumptions and after the instance o | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| - |karpenter.k8s.aws/instance-cpu|1| + |karpenter.k8s.aws/instance-cpu|1| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7gd| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-local-nvme|59| + |karpenter.k8s.aws/instance-memory|8192| + |karpenter.k8s.aws/instance-network-bandwidth|520| + |karpenter.k8s.aws/instance-pods|8| + |karpenter.k8s.aws/instance-size|medium| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7gd.medium| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|940m| + |ephemeral-storage|17Gi| + |memory|7075Mi| + |pods|8| + |vpc.amazonaws.com/pod-eni|4| +### `r7gd.large` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7gd| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-local-nvme|118| + |karpenter.k8s.aws/instance-memory|16384| + |karpenter.k8s.aws/instance-network-bandwidth|937| + |karpenter.k8s.aws/instance-pods|29| + |karpenter.k8s.aws/instance-size|large| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7gd.large| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|1930m| + |ephemeral-storage|17Gi| + |memory|14422Mi| + |pods|29| + |vpc.amazonaws.com/pod-eni|9| +### `r7gd.xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7gd| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-local-nvme|237| + |karpenter.k8s.aws/instance-memory|32768| + |karpenter.k8s.aws/instance-network-bandwidth|1876| + |karpenter.k8s.aws/instance-pods|58| + |karpenter.k8s.aws/instance-size|xlarge| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7gd.xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|3920m| + |ephemeral-storage|17Gi| + |memory|29258Mi| + |pods|58| + |vpc.amazonaws.com/pod-eni|18| +### `r7gd.2xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7gd| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-local-nvme|474| + |karpenter.k8s.aws/instance-memory|65536| + |karpenter.k8s.aws/instance-network-bandwidth|3750| + |karpenter.k8s.aws/instance-pods|58| + |karpenter.k8s.aws/instance-size|2xlarge| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7gd.2xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|7910m| + |ephemeral-storage|17Gi| + |memory|59568Mi| + |pods|58| + |vpc.amazonaws.com/pod-eni|38| +### `r7gd.4xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7gd| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-local-nvme|950| + |karpenter.k8s.aws/instance-memory|131072| + |karpenter.k8s.aws/instance-network-bandwidth|7500| + |karpenter.k8s.aws/instance-pods|234| + |karpenter.k8s.aws/instance-size|4xlarge| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7gd.4xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|15890m| + |ephemeral-storage|17Gi| + |memory|118253Mi| + |pods|234| + |vpc.amazonaws.com/pod-eni|54| +### `r7gd.8xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7gd| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-local-nvme|1900| + |karpenter.k8s.aws/instance-memory|196608| + |karpenter.k8s.aws/instance-network-bandwidth|15000| + |karpenter.k8s.aws/instance-pods|234| + |karpenter.k8s.aws/instance-size|8xlarge| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7gd.8xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|31850m| + |ephemeral-storage|17Gi| + |memory|178874Mi| + |pods|234| + |vpc.amazonaws.com/pod-eni|54| +### `r7gd.12xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7gd| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-local-nvme|2850| + |karpenter.k8s.aws/instance-memory|262144| + |karpenter.k8s.aws/instance-network-bandwidth|22500| + |karpenter.k8s.aws/instance-pods|234| + |karpenter.k8s.aws/instance-size|12xlarge| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7gd.12xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|47810m| + |ephemeral-storage|17Gi| + |memory|239495Mi| + |pods|234| + |vpc.amazonaws.com/pod-eni|54| +### `r7gd.16xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7gd| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| - |karpenter.k8s.aws/instance-local-nvme|59| - |karpenter.k8s.aws/instance-memory|8192| - |karpenter.k8s.aws/instance-network-bandwidth|520| - |karpenter.k8s.aws/instance-pods|8| - |karpenter.k8s.aws/instance-size|medium| + |karpenter.k8s.aws/instance-local-nvme|3800| + |karpenter.k8s.aws/instance-memory|524288| + |karpenter.k8s.aws/instance-network-bandwidth|30000| + |karpenter.k8s.aws/instance-pods|737| + |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|r7gd.medium| + |node.kubernetes.io/instance-type|r7gd.16xlarge| #### Resources | Resource | Quantity | |--|--| - |cpu|940m| + |cpu|63770m| |ephemeral-storage|17Gi| - |memory|7075Mi| - |pods|8| - |vpc.amazonaws.com/pod-eni|4| -### `r7gd.large` + |memory|476445Mi| + |pods|737| + |vpc.amazonaws.com/pod-eni|107| +## r7i Family +### `r7i.large` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| - |karpenter.k8s.aws/instance-family|r7gd| + |karpenter.k8s.aws/instance-family|r7i| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| - |karpenter.k8s.aws/instance-local-nvme|118| |karpenter.k8s.aws/instance-memory|16384| - |karpenter.k8s.aws/instance-network-bandwidth|937| |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| - |kubernetes.io/arch|arm64| + |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|r7gd.large| + |node.kubernetes.io/instance-type|r7i.large| #### Resources | Resource | Quantity | |--|--| |cpu|1930m| |ephemeral-storage|17Gi| - |memory|14422Mi| + |memory|14481Mi| |pods|29| - |vpc.amazonaws.com/pod-eni|9| -### `r7gd.xlarge` +### `r7i.xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| - |karpenter.k8s.aws/instance-family|r7gd| + |karpenter.k8s.aws/instance-family|r7i| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| - |karpenter.k8s.aws/instance-local-nvme|237| |karpenter.k8s.aws/instance-memory|32768| - |karpenter.k8s.aws/instance-network-bandwidth|1876| |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| - |kubernetes.io/arch|arm64| + |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|r7gd.xlarge| + |node.kubernetes.io/instance-type|r7i.xlarge| #### Resources | Resource | Quantity | |--|--| |cpu|3920m| |ephemeral-storage|17Gi| - |memory|29258Mi| + |memory|29317Mi| |pods|58| - |vpc.amazonaws.com/pod-eni|18| -### `r7gd.2xlarge` +### `r7i.2xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| - |karpenter.k8s.aws/instance-family|r7gd| + |karpenter.k8s.aws/instance-family|r7i| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| - |karpenter.k8s.aws/instance-local-nvme|474| |karpenter.k8s.aws/instance-memory|65536| - |karpenter.k8s.aws/instance-network-bandwidth|3750| |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| - |kubernetes.io/arch|arm64| + |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|r7gd.2xlarge| + |node.kubernetes.io/instance-type|r7i.2xlarge| #### Resources | Resource | Quantity | |--|--| |cpu|7910m| |ephemeral-storage|17Gi| - |memory|59568Mi| + |memory|59627Mi| |pods|58| - |vpc.amazonaws.com/pod-eni|38| -### `r7gd.4xlarge` +### `r7i.4xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| - |karpenter.k8s.aws/instance-family|r7gd| + |karpenter.k8s.aws/instance-family|r7i| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| - |karpenter.k8s.aws/instance-local-nvme|950| |karpenter.k8s.aws/instance-memory|131072| - |karpenter.k8s.aws/instance-network-bandwidth|7500| |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| - |kubernetes.io/arch|arm64| + |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|r7gd.4xlarge| + |node.kubernetes.io/instance-type|r7i.4xlarge| #### Resources | Resource | Quantity | |--|--| |cpu|15890m| |ephemeral-storage|17Gi| - |memory|118253Mi| + |memory|118312Mi| |pods|234| - |vpc.amazonaws.com/pod-eni|54| -### `r7gd.8xlarge` +### `r7i.8xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| - |karpenter.k8s.aws/instance-family|r7gd| + |karpenter.k8s.aws/instance-family|r7i| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| - |karpenter.k8s.aws/instance-local-nvme|1900| - |karpenter.k8s.aws/instance-memory|196608| - |karpenter.k8s.aws/instance-network-bandwidth|15000| + |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| - |kubernetes.io/arch|arm64| + |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|r7gd.8xlarge| + |node.kubernetes.io/instance-type|r7i.8xlarge| #### Resources | Resource | Quantity | |--|--| |cpu|31850m| |ephemeral-storage|17Gi| - |memory|178874Mi| + |memory|239554Mi| |pods|234| - |vpc.amazonaws.com/pod-eni|54| -### `r7gd.12xlarge` +### `r7i.12xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| - |karpenter.k8s.aws/instance-family|r7gd| + |karpenter.k8s.aws/instance-family|r7i| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| - |karpenter.k8s.aws/instance-local-nvme|2850| - |karpenter.k8s.aws/instance-memory|262144| - |karpenter.k8s.aws/instance-network-bandwidth|22500| + |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| - |kubernetes.io/arch|arm64| + |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|r7gd.12xlarge| + |node.kubernetes.io/instance-type|r7i.12xlarge| #### Resources | Resource | Quantity | |--|--| |cpu|47810m| |ephemeral-storage|17Gi| - |memory|239495Mi| + |memory|360795Mi| |pods|234| - |vpc.amazonaws.com/pod-eni|54| -### `r7gd.16xlarge` +### `r7i.16xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| - |karpenter.k8s.aws/instance-family|r7gd| + |karpenter.k8s.aws/instance-family|r7i| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| - |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|524288| - |karpenter.k8s.aws/instance-network-bandwidth|30000| |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| - |kubernetes.io/arch|arm64| + |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|r7gd.16xlarge| + |node.kubernetes.io/instance-type|r7i.16xlarge| #### Resources | Resource | Quantity | |--|--| |cpu|63770m| |ephemeral-storage|17Gi| - |memory|476445Mi| + |memory|476504Mi| + |pods|737| +### `r7i.24xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7i| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|786432| + |karpenter.k8s.aws/instance-pods|737| + |karpenter.k8s.aws/instance-size|24xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7i.24xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|95690m| + |ephemeral-storage|17Gi| + |memory|718987Mi| + |pods|737| +### `r7i.48xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|192| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7i| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|1572864| + |karpenter.k8s.aws/instance-pods|737| + |karpenter.k8s.aws/instance-size|48xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7i.48xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|191450m| + |ephemeral-storage|17Gi| + |memory|1446437Mi| |pods|737| - |vpc.amazonaws.com/pod-eni|107| ## r7iz Family ### `r7iz.large` #### Labels @@ -16954,6 +17696,36 @@ below are the resources available with some assumptions and after the instance o |memory|29258Mi| |pods|58| ## trn1 Family +### `trn1.2xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-accelerator-count|1| + |karpenter.k8s.aws/instance-accelerator-manufacturer|aws| + |karpenter.k8s.aws/instance-accelerator-name|inferentia| + |karpenter.k8s.aws/instance-category|trn| + |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|trn1| + |karpenter.k8s.aws/instance-generation|1| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-local-nvme|474| + |karpenter.k8s.aws/instance-memory|32768| + |karpenter.k8s.aws/instance-network-bandwidth|3125| + |karpenter.k8s.aws/instance-pods|58| + |karpenter.k8s.aws/instance-size|2xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|trn1.2xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |aws.amazon.com/neuron|1| + |cpu|7910m| + |ephemeral-storage|17Gi| + |memory|29317Mi| + |pods|58| + |vpc.amazonaws.com/pod-eni|17| ### `trn1.32xlarge` #### Labels | Label | Value | @@ -16984,6 +17756,37 @@ below are the resources available with some assumptions and after the instance o |memory|481894Mi| |pods|247| |vpc.amazonaws.com/pod-eni|82| +## trn1n Family +### `trn1n.32xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-accelerator-count|16| + |karpenter.k8s.aws/instance-accelerator-manufacturer|aws| + |karpenter.k8s.aws/instance-accelerator-name|inferentia| + |karpenter.k8s.aws/instance-category|trn| + |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|trn1n| + |karpenter.k8s.aws/instance-generation|1| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-local-nvme|7600| + |karpenter.k8s.aws/instance-memory|524288| + |karpenter.k8s.aws/instance-network-bandwidth|1600000| + |karpenter.k8s.aws/instance-pods|247| + |karpenter.k8s.aws/instance-size|32xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|trn1n.32xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |aws.amazon.com/neuron|16| + |cpu|127610m| + |ephemeral-storage|17Gi| + |memory|481894Mi| + |pods|247| + |vpc.amazonaws.com/pod-eni|120| ## u-12tb1 Family ### `u-12tb1.112xlarge` #### Labels @@ -17159,6 +17962,82 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|8720933Mi| |pods|737| +## vt1 Family +### `vt1.3xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|vt| + |karpenter.k8s.aws/instance-cpu|12| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|vt1| + |karpenter.k8s.aws/instance-generation|1| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|24576| + |karpenter.k8s.aws/instance-network-bandwidth|3120| + |karpenter.k8s.aws/instance-pods|58| + |karpenter.k8s.aws/instance-size|3xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|vt1.3xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|11900m| + |ephemeral-storage|17Gi| + |memory|21739Mi| + |pods|58| + |vpc.amazonaws.com/pod-eni|38| +### `vt1.6xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|vt| + |karpenter.k8s.aws/instance-cpu|24| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|vt1| + |karpenter.k8s.aws/instance-generation|1| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|49152| + |karpenter.k8s.aws/instance-network-bandwidth|6250| + |karpenter.k8s.aws/instance-pods|234| + |karpenter.k8s.aws/instance-size|6xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|vt1.6xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|23870m| + |ephemeral-storage|17Gi| + |memory|42536Mi| + |pods|234| + |vpc.amazonaws.com/pod-eni|54| +### `vt1.24xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|vt| + |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|vt1| + |karpenter.k8s.aws/instance-generation|1| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|196608| + |karpenter.k8s.aws/instance-network-bandwidth|25000| + |karpenter.k8s.aws/instance-pods|737| + |karpenter.k8s.aws/instance-size|24xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|vt1.24xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|95690m| + |ephemeral-storage|17Gi| + |memory|173400Mi| + |pods|737| + |vpc.amazonaws.com/pod-eni|107| ## x1 Family ### `x1.16xlarge` #### Labels diff --git a/website/content/en/preview/reference/metrics.md b/website/content/en/preview/reference/metrics.md index 112f63b0fe9c..374fba82b28c 100644 --- a/website/content/en/preview/reference/metrics.md +++ b/website/content/en/preview/reference/metrics.md @@ -50,6 +50,26 @@ Amount of time required for a replacement machine to become initialized. ### `karpenter_deprovisioning_replacement_machine_launch_failure_counter` The number of times that Karpenter failed to launch a replacement node for deprovisioning. Labeled by deprovisioner. +## Disruption Metrics + +### `karpenter_disruption_actions_performed_total` +Number of disruption actions performed. Labeled by disruption method. + +### `karpenter_disruption_consolidation_timeouts_total` +Number of times the Consolidation algorithm has reached a timeout. Labeled by consolidation type. + +### `karpenter_disruption_eligible_nodes` +Number of nodes eligible for disruption by Karpenter. Labeled by disruption method. + +### `karpenter_disruption_evaluation_duration_seconds` +Duration of the disruption evaluation process in seconds. + +### `karpenter_disruption_replacement_nodeclaim_failures_total` +The number of times that Karpenter failed to launch a replacement node for disruption. Labeled by disruption method. + +### `karpenter_disruption_replacement_nodeclaim_initialized_seconds` +Amount of time required for a replacement nodeclaim to become initialized. + ## Interruption Metrics ### `karpenter_interruption_actions_performed` @@ -87,6 +107,40 @@ Number of machines registered in total by Karpenter. Labeled by the owning provi ### `karpenter_machines_terminated` Number of machines terminated in total by Karpenter. Labeled by reason the machine was terminated and the owning provisioner. +## Nodeclaims Metrics + +### `karpenter_nodeclaims_created` +Number of nodeclaims created in total by Karpenter. Labeled by reason the nodeclaim was created and the owning nodepool. + +### `karpenter_nodeclaims_disrupted` +Number of nodeclaims disrupted in total by Karpenter. Labeled by disruption type of the nodeclaim and the owning nodepool. + +### `karpenter_nodeclaims_drifted` +Number of nodeclaims drifted reasons in total by Karpenter. Labeled by drift type of the nodeclaim and the owning nodepool. + +### `karpenter_nodeclaims_initialized` +Number of nodeclaims initialized in total by Karpenter. Labeled by the owning nodepool. + +### `karpenter_nodeclaims_launched` +Number of nodeclaims launched in total by Karpenter. Labeled by the owning nodepool. + +### `karpenter_nodeclaims_registered` +Number of nodeclaims registered in total by Karpenter. Labeled by the owning nodepool. + +### `karpenter_nodeclaims_terminated` +Number of nodeclaims terminated in total by Karpenter. Labeled by reason the nodeclaim was terminated and the owning nodepool. + +## Nodepools Metrics + +### `karpenter_nodepools_limit` +The nodepool limits are the limits specified on the provisioner that restrict the quantity of resources provisioned. Labeled by nodepool name and resource type. + +### `karpenter_nodepools_usage` +The nodepool usage is the amount of resources that have been provisioned by a particular nodepool. Labeled by nodepool name and resource type. + +### `karpenter_nodepools_usage_pct` +The nodepool usage percentage is the percentage of each resource used based on the resources provisioned and the limits that have been configured. Labeled by nodepool name and resource type. + ## Provisioner Metrics ### `karpenter_provisioner_limit` diff --git a/website/content/en/preview/reference/settings.md b/website/content/en/preview/reference/settings.md index 800cf75c168a..78f959ea8dd6 100644 --- a/website/content/en/preview/reference/settings.md +++ b/website/content/en/preview/reference/settings.md @@ -12,13 +12,16 @@ Karpenter surfaces environment variables and CLI parameters to allow you to conf | Environment Variable | CLI Flag | Description | |--|--|--| -| DISABLE_WEBHOOK | \-\-disable-webhook | Disable the admission and validation webhooks (default = false)| -| ENABLE_PROFILING | \-\-enable-profiling | Enable the profiling on the metric endpoint (default = false)| +| BATCH_IDLE_DURATION | \-\-batch-idle-duration | The maximum amount of time with no new pending pods that if exceeded ends the current batching window. If pods arrive faster than this time, the batching window will be extended up to the maxDuration. If they arrive slower, the pods will be batched separately. (default = 1s)| +| BATCH_MAX_DURATION | \-\-batch-max-duration | The maximum length of a batch window. The longer this is, the more pods we can consider for provisioning at one time which usually results in fewer but larger nodes. (default = 10s)| +| DISABLE_WEBHOOK | \-\-disable-webhook | Disable the admission and validation webhooks| +| ENABLE_PROFILING | \-\-enable-profiling | Enable the profiling on the metric endpoint| +| FEATURE_GATES | \-\-feature-gates | Optional features can be enabled / disabled using feature gates. Current options are: Drift (default = Drift=false)| | HEALTH_PROBE_PORT | \-\-health-probe-port | The port the health probe endpoint binds to for reporting controller health (default = 8081)| | KARPENTER_SERVICE | \-\-karpenter-service | The Karpenter Service name for the dynamic webhook certificate| | KUBE_CLIENT_BURST | \-\-kube-client-burst | The maximum allowed burst of queries to the kube-apiserver (default = 300)| | KUBE_CLIENT_QPS | \-\-kube-client-qps | The smoothed rate of qps to kube-apiserver (default = 200)| -| LEADER_ELECT | \-\-leader-elect | Start leader election client and gain leadership before executing the main loop. Enable this when running replicated components for high availability. (default = true)| +| LEADER_ELECT | \-\-leader-elect | Start leader election client and gain leadership before executing the main loop. Enable this when running replicated components for high availability.| | LOG_LEVEL | \-\-log-level | Log verbosity level. Can be one of 'debug', 'info', or 'error'| | MEMORY_LIMIT | \-\-memory-limit | Memory limit on the container running the controller. The GC soft memory limit is set to 90% of this value. (default = -1)| | METRICS_PORT | \-\-metrics-port | The port the metric endpoint binds to for operating metrics about the controller itself (default = 8000)| diff --git a/website/content/en/preview/troubleshooting.md b/website/content/en/preview/troubleshooting.md index e16c2d97397a..bc50aa8693cc 100644 --- a/website/content/en/preview/troubleshooting.md +++ b/website/content/en/preview/troubleshooting.md @@ -1,7 +1,7 @@ --- title: "Troubleshooting" linkTitle: "Troubleshooting" -weight: 90 +weight: 70 description: > Troubleshoot Karpenter problems --- @@ -255,7 +255,7 @@ spec: When attempting to schedule a large number of pods with PersistentVolumes, it's possible that these pods will co-locate on the same node. Pods will report the following errors in their events using a `kubectl describe pod` call -```console +```bash Warning FailedAttachVolume pod/example-pod AttachVolume.Attach failed for volume "***" : rpc error: code = Internal desc = Could not attach volume "***" to node "***": attachment of disk "***" failed, expected device to be attached but was attaching Warning FailedMount pod/example-pod Unable to attach or mount volumes: unmounted volumes=[***], unattached volumes=[***]: timed out waiting for the condition ``` @@ -266,7 +266,7 @@ In this case, Karpenter may fail to scale-up your nodes due to these pods due to Karpenter does not support [in-tree storage plugins](https://kubernetes.io/blog/2021/12/10/storage-in-tree-to-csi-migration-status-update/) to provision PersistentVolumes, since nearly all of the in-tree plugins have been deprecated in upstream Kubernetes. This means that, if you are using a statically-provisioned PersistentVolume that references a volume source like `AWSElasticBlockStore` or a dynamically-provisioned PersistentVolume that references a StorageClass with a in-tree storage plugin provisioner like `kubernetes.io/aws-ebs`, Karpenter will fail to discover the maxiumum volume attachments for the node. Instead, Karpenter may think the node still has more schedulable space due to memory and cpu constraints when there is really no more schedulable space on the node due to volume limits. When Karpenter sees you are using an in-tree storage plugin on your pod volumes, it will print the following error message into the logs. If you see this message, upgrade your StorageClasses and statically-provisioned PersistentVolumes to use the latest CSI drivers for your cloud provider. -```console +```bash 2023-04-05T23:56:53.363Z ERROR controller.node_state PersistentVolume source 'AWSElasticBlockStore' uses an in-tree storage plugin which is unsupported by Karpenter and is deprecated by Kubernetes. Scale-ups may fail because Karpenter will not discover driver limits. Use a PersistentVolume that references the 'CSI' volume source for Karpenter auto-scaling support. {"commit": "b2af562", "node": "ip-192-168-36-137.us-west-2.compute.internal", "pod": "inflate0-6c4bdb8b75-7qmfd", "volume": "mypd", "persistent-volume": "pvc-11db7489-3c6e-46f3-a958-91f9d5009d41"} 2023-04-05T23:56:53.464Z ERROR controller.node_state StorageClass .spec.provisioner uses an in-tree storage plugin which is unsupported by Karpenter and is deprecated by Kubernetes. Scale-ups may fail because Karpenter will not discover driver limits. Create a new StorageClass with a .spec.provisioner referencing the CSI driver plugin name 'ebs.csi.aws.com'. {"commit": "b2af562", "node": "ip-192-168-36-137.us-west-2.compute.internal", "pod": "inflate0-6c4bdb8b75-7qmfd", "volume": "mypd", "storage-class": "gp2", "provisioner": "kubernetes.io/aws-ebs"} ``` @@ -299,7 +299,7 @@ To avoid this discrepancy between `maxPods` and the supported pod density of the 2. Reduce your `maxPods` value to be under the maximum pod density for the instance types assigned to your Provisioner 3. Remove the `maxPods` value from your [`kubeletConfiguration`]({{}}) if you no longer need it and instead rely on the defaulted values from Karpenter and EKS AMIs. -For more information on pod density, view the [Pod Density Conceptual Documentation]({{}}). +For more information on pod density, view the [Pod Density Section in the NodePools doc]({{}}). #### IP exhaustion in a subnet diff --git a/website/content/en/preview/upgrading/_index.md b/website/content/en/preview/upgrading/_index.md new file mode 100644 index 000000000000..ef1368b22a75 --- /dev/null +++ b/website/content/en/preview/upgrading/_index.md @@ -0,0 +1,10 @@ +--- +title: "Upgrading" +linkTitle: "Upgrading" +weight: 30 +description: > + Upgrading Karpenter guide and reference +cascade: + type: docs +--- + diff --git a/website/content/en/preview/upgrading/compatibility.md b/website/content/en/preview/upgrading/compatibility.md new file mode 100644 index 000000000000..bfde9d049a22 --- /dev/null +++ b/website/content/en/preview/upgrading/compatibility.md @@ -0,0 +1,87 @@ +--- +title: "Compatibility" +linkTitle: "Compatibility" +weight: 20 +description: > + Compatibility issues for Karpenter +--- + +# Compatibility + +To make upgrading easier we aim to minimize the introduction of breaking changes. +Before you begin upgrading Karpenter, consider Karpenter compatibility issues related to Kubernetes and the NodePool API (previously Provisioner). + +## Compatibility Matrix + +[comment]: <> (the content below is generated from hack/docs/compataiblitymetrix_gen_docs.go) + +| KUBERNETES | 1.23 | 1.24 | 1.25 | 1.26 | 1.27 | 1.28 | +|------------|---------|---------|---------|---------|---------|--------| +| karpenter | 0.21.x+ | 0.21.x+ | 0.25.x+ | 0.28.x+ | 0.28.x+ | 0.31.x | + +[comment]: <> (end docs generated content from hack/docs/compataiblitymetrix_gen_docs.go) + +{{% alert title="Note" color="warning" %}} +Karpenter currently does not support the following [new `topologySpreadConstraints` keys](https://kubernetes.io/blog/2023/04/17/fine-grained-pod-topology-spread-features-beta/), promoted to beta in Kubernetes 1.27: +- `matchLabelKeys` +- `nodeAffinityPolicy` +- `nodeTaintsPolicy` + +For more information on Karpenter's support for these keys, view [this tracking issue](https://github.com/aws/karpenter-core/issues/430). +{{% /alert %}} + +## Compatibility issues + +When we introduce a breaking change, we do so only as described in this document. + +Karpenter follows [Semantic Versioning 2.0.0](https://semver.org/) in its stable release versions, while in +major version zero (v0.y.z) [anything may change at any time](https://semver.org/#spec-item-4). +However, to further protect users during this phase we will only introduce breaking changes in minor releases (releases that increment y in x.y.z). +Note this does not mean every minor upgrade has a breaking change as we will also increment the +minor version when we release a new feature. + +Users should therefore check to see if there is a breaking change every time they are upgrading to a new minor version. + +### How Do We Break Incompatibility? + +When there is a breaking change we will: + +* Increment the minor version when in major version 0 +* Add a permanent separate section named `upgrading to vx.y.z+` under [release upgrade notes](#release-upgrade-notes) + clearly explaining the breaking change and what needs to be done on the user side to ensure a safe upgrade +* Add the sentence “This is a breaking change, please refer to the above link for upgrade instructions” to the top of the release notes and in all our announcements + +### How Do We Find Incompatibilities? + +Besides the peer review process for all changes to the code base we also do the followings in order to find +incompatibilities: +* (To be implemented) To check the compatibility of the application, we will automate tests for installing, uninstalling, upgrading from an older version, and downgrading to an older version +* (To be implemented) To check the compatibility of the documentation with the application, we will turn the commands in our documentation into scripts that we can automatically run + +### Security Patches + +While we are in major version 0 we will not release security patches to older versions. +Rather we provide the patches in the latest versions. +When at major version 1 we will have an EOL (end of life) policy where we provide security patches +for a subset of older versions and deprecate the others. + +## Release Types + +Karpenter offers three types of releases. This section explains the purpose of each release type and how the images for each release type are tagged in our [public image repository](https://gallery.ecr.aws/karpenter). + +### Stable Releases + +Stable releases are the most reliable releases that are released with weekly cadence. Stable releases are our only recommended versions for production environments. +Sometimes we skip a stable release because we find instability or problems that need to be fixed before having a stable release. +Stable releases are tagged with Semantic Versioning. For example `v0.13.0`. + +### Release Candidates + +We consider having release candidates for major and important minor versions. Our release candidates are tagged like `vx.y.z-rc.0`, `vx.y.z-rc.1`. The release candidate will then graduate to `vx.y.z` as a normal stable release. +By adopting this practice we allow our users who are early adopters to test out new releases before they are available to the wider community, thereby providing us with early feedback resulting in more stable releases. + +### Snapshot Releases + +We release a snapshot release for every commit that gets merged into the main repository. This enables our users to immediately try a new feature or fix right after it's merged rather than waiting days or weeks for release. +Snapshot releases are suitable for testing, and troubleshooting but users should exercise great care if they decide to use them in production environments. +Snapshot releases are tagged with the git commit hash prefixed by the Karpenter major version. For example `v0-fc17bfc89ebb30a3b102a86012b3e3992ec08adf`. For more detailed examples on how to use snapshot releases look under "Usage" in [Karpenter Helm Chart](https://gallery.ecr.aws/karpenter/karpenter). diff --git a/website/content/en/preview/upgrade-guide.md b/website/content/en/preview/upgrading/upgrade-guide.md similarity index 65% rename from website/content/en/preview/upgrade-guide.md rename to website/content/en/preview/upgrading/upgrade-guide.md index 37e590b54031..dcb24ee1f0e2 100644 --- a/website/content/en/preview/upgrade-guide.md +++ b/website/content/en/preview/upgrading/upgrade-guide.md @@ -8,120 +8,198 @@ description: > Karpenter is a controller that runs in your cluster, but it is not tied to a specific Kubernetes version, as the Cluster Autoscaler is. Use your existing upgrade mechanisms to upgrade your core add-ons in Kubernetes and keep Karpenter up to date on bug fixes and new features. +This guide contains information needed to upgrade to the latest release of Karpenter, along with compatibility issues you need to be aware of when upgrading from earlier Karpenter versions. -To make upgrading easier we aim to minimize introduction of breaking changes with the following: +### CRD Upgrades -## Compatibility Matrix +Karpenter ships with a few Custom Resource Definitions (CRDs). These CRDs are published: +* As an independent helm chart [karpenter-crd](https://gallery.ecr.aws/karpenter/karpenter-crd) - [source](https://github.com/aws/karpenter/blob/main/charts/karpenter-crd) that can be used by Helm to manage the lifecycle of these CRDs. + * To upgrade or install `karpenter-crd` run: + ``` + helm upgrade --install karpenter-crd oci://public.ecr.aws/karpenter/karpenter-crd --version vx.y.z --namespace karpenter --create-namespace + ``` -[comment]: <> (the content below is generated from hack/docs/compataiblitymetrix_gen_docs.go) +{{% alert title="Note" color="warning" %}} +If you get the error `invalid ownership metadata; label validation error:` while installing the `karpenter-crd` chart from an older version of Karpenter, follow the [Troubleshooting Guide]({{}}) for details on how to resolve these errors. +{{% /alert %}} -| KUBERNETES | 1.23 | 1.24 | 1.25 | 1.26 | 1.27 | 1.28 | -|------------|---------|---------|---------|---------|---------|--------| -| karpenter | 0.21.x+ | 0.21.x+ | 0.25.x+ | 0.28.x+ | 0.28.x+ | 0.31.x | +* As part of the helm chart [karpenter](https://gallery.ecr.aws/karpenter/karpenter) - [source](https://github.com/aws/karpenter/blob/main/charts/karpenter/crds). Helm [does not manage the lifecycle of CRDs using this method](https://helm.sh/docs/chart_best_practices/custom_resource_definitions/), the tool will only install the CRD during the first installation of the helm chart. Subsequent chart upgrades will not add or remove CRDs, even if the CRDs have changed. When CRDs are changed, we will make a note in the version's upgrade guide. -[comment]: <> (end docs generated content from hack/docs/compataiblitymetrix_gen_docs.go) +In general, you can reapply the CRDs in the `crds` directory of the Karpenter helm chart: -{{% alert title="Note" color="warning" %}} -Karpenter currently does not support the following [new `topologySpreadConstraints` keys](https://kubernetes.io/blog/2023/04/17/fine-grained-pod-topology-spread-features-beta/), promoted to beta in Kubernetes 1.27: -- `matchLabelKeys` -- `nodeAffinityPolicy` -- `nodeTaintsPolicy` +```shell +kubectl apply -f https://raw.githubusercontent.com/aws/karpenter{{< githubRelRef >}}pkg/apis/crds/karpenter.sh_nodepols.yaml +kubectl apply -f https://raw.githubusercontent.com/aws/karpenter{{< githubRelRef >}}pkg/apis/crds/karpenter.sh_nodeclaims.yaml +kubectl apply -f https://raw.githubusercontent.com/aws/karpenter{{< githubRelRef >}}pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml +``` +g +### Upgrading to v0.32.0+ -For more information on Karpenter's support for these keys, view [this tracking issue](https://github.com/aws/karpenter-core/issues/430). -{{% /alert %}} +#### v1beta1 Migration -## Compatibility issues +Here is some information you should know about upgrading the Karpenter controller to v0.32.x: -To make upgrading easier, we aim to minimize the introduction of breaking changes with the following components: +* **Towards a v1 release**: The latest version of Karpenter sets the stage for Karpenter v1. Karpenter v0.32.x implements the Karpenter v1beta1 API spec. The intention is to have v1beta1 be used as the v1 spec, with only minimal changes needed. +* **Path to upgrading**: This procedure assumes that you are upgrading from Karpenter v0.31.x to v0.32.x. If you are on an earlier version of Karpenter, review the [Release Upgrade Notes]({{< relref "#release-upgrade-notes" >}}) for earlier versions' breaking changes. +* **Enhancing and renaming components**: For v1beta1, APIs have been enhanced to improve and solidify Karpenter APIs. Part of these enhancements includes renaming the Kinds for all Karpenter CustomResources. The following name changes have been made: + * Provisioner -> NodePool + * Machine -> NodeClaim + * AWSNodeTemplate -> EC2NodeClass +* **Running v1alpha1 alongside v1beta1**: Having different Kind names for v1alpha5 and v1beta1 allows them to coexist for the same Karpenter controller for v0.32.x. This gives you time to transition to the new v1beta1 APIs while existing Provisioners and other objects stay in place. Keep in mind that there is no guarantee that the two versions will be able to coexist in future Karpenter versions. -* Provisioner API -* Helm Chart +Some things that will help you with this upgrade include: -We try to maintain compatibility with: +* **[v1beta1 Upgrade Reference]({{< relref "v1beta1-reference" >}})**: Provides a complete reference to help you transition your Provisioner, Machine, and AWSNodeTemplate manifests, as well as other components, to be able to work with the new v1beta1 names, labels, and other elements. +* **[Karpenter conversion tool](https://github.com/aws/karpenter/tree/main/tools/karpenter-convert)**: Simplifies the creation of NodePool and EC2NodeClass manifests. -* The application itself -* The documentation of the application +##### Procedure -When we introduce a breaking change, we do so only as described in this document. +This procedure assumes you are running the Karpenter controller on cluster and want to upgrade that cluster to v0.32.x. -Karpenter follows [Semantic Versioning 2.0.0](https://semver.org/) in its stable release versions, while in -major version zero (v0.y.z) [anything may change at any time](https://semver.org/#spec-item-4). -However, to further protect users during this phase we will only introduce breaking changes in minor releases (releases that increment y in x.y.z). -Note this does not mean every minor upgrade has a breaking change as we will also increment the -minor version when we release a new feature. +**NOTE**: Please read through the entire procedure before beginning the upgrade. There are major changes in this upgrade, so you should carefully evaluate your cluster and workloads before proceeding. -Users should therefore check to see if there is a breaking change every time they are upgrading to a new minor version. -### Custom Resource Definition (CRD) Upgrades +#### Prerequisites -Karpenter ships with a few Custom Resource Definitions (CRDs). These CRDs are published: -* As an independent helm chart [karpenter-crd](https://gallery.ecr.aws/karpenter/karpenter-crd) - [source](https://github.com/aws/karpenter/blob/main/charts/karpenter-crd) that can be used by Helm to manage the lifecycle of these CRDs. - * To upgrade or install `karpenter-crd` run: +To upgrade your provisioner and AWSNodeTemplate YAML files to be compatible with v1beta1, you can either update them manually or use the [karpenter-convert](https://github.com/aws/karpenter/tree/main/tools/karpenter-convert) CLI tool. To install that tool: + +``` +go install github.com/aws/karpenter/tools/karpenter-convert/cmd/karpenter-convert@latest +``` +Add `~/go/bin` to your $PATH, if you have not already done so. + +1. Determine the current cluster version: Run the following to make sure that your Karpenter version is v0.31.x: + ``` + kubectl get pod -A | grep karpenter + kubectl describe pod -n karpenter karpenter-xxxxxxxxxx-xxxxx | grep Image: | grep v0..... + ``` + Sample output: + ``` + Image: public.ecr.aws/karpenter/controller:v0.31.0@sha256:d29767fa9c5c0511a3812397c932f5735234f03a7a875575422b712d15e54a77 + ``` + + {{% alert title="Note" color="primary" %}} + v0.31.2 introduces minor changes to Karpenter so that rollback from v0.32.0 is supported. If you are coming from some other patch version of minor version v0.31.x, note that v0.31.2 is the _only_ patch version that supports rollback. + {{% /alert %}} + +2. Review for breaking changes: If you are already running Karpenter v0.31.x, you can skip this step. If you are running an earlier Karpenter version, you need to review the upgrade notes for each minor release. + +3. Set environment variables for your cluster: + + ```bash + export KARPENTER_VERSION=v0.32.0 + export AWS_PARTITION="aws" # if you are not using standard partitions, you may need to configure to aws-cn / aws-us-gov + export CLUSTER_NAME="${USER}-karpenter-demo" + export AWS_REGION="us-west-2" + export AWS_ACCOUNT_ID="$(aws sts get-caller-identity --query Account --output text)" + export KARPENTER_IAM_ROLE_ARN="arn:${AWS_PARTITION}:iam::${AWS_ACCOUNT_ID}:role/${CLUSTER_NAME}-karpenter" + export CLUSTER_ENDPOINT="$(aws eks describe-cluster --name ${CLUSTER_NAME} --query "cluster.endpoint" --output text)" + ``` + +4. Apply the new Karpenter policy and assign it to the existing Karpenter role: + + ```bash + TEMPOUT=$(mktemp) + curl -fsSL https://raw.githubusercontent.com/aws/karpenter{{< githubRelRef >}}website/content/en/preview/upgrade/v1beta1-controller-policy.json > ${TEMPOUT} + + POLICY_DOCUMENT=$(envsubst < ${TEMPOUT}) + POLICY_NAME="KarpenterControllerPolicy-${CLUSTER_NAME}-v1beta1" + ROLE_NAME="${CLUSTER_NAME}-karpenter" + + POLICY_ARN=$(aws iam create-policy --policy-name "${POLICY_NAME}" --policy-document "${POLICY_DOCUMENT}" | jq -r .Policy.Arn) + aws iam attach-role-policy --role-name "${ROLE_NAME}" --policy-arn "${POLICY_ARN}" + ``` + +5. Apply the v0.32.0 Custom Resource Definitions (CRDs) in the crds directory of the Karpenter helm chart. Here are the ways you can do this: + + * As an independent helm chart [karpenter-crd](https://gallery.ecr.aws/karpenter/karpenter-crd) - [source](https://github.com/aws/karpenter/blob/main/charts/karpenter-crd) that can be used by Helm to manage the lifecycle of these CRDs. To upgrade or install `karpenter-crd` run: ``` helm upgrade --install karpenter-crd oci://public.ecr.aws/karpenter/karpenter-crd --version vx.y.z --namespace karpenter --create-namespace ``` -{{% alert title="Note" color="warning" %}} -If you get the error `invalid ownership metadata; label validation error:` while installing the `karpenter-crd` chart from an older version of Karpenter, follow the [Troubleshooting Guide]({{}}) for details on how to resolve these errors. -{{% /alert %}} + {{% alert title="Note" color="warning" %}} + < If you get the error `invalid ownership metadata; label validation error:` while installing the `karpenter-crd` chart from an older version of Karpenter, follow the [Troubleshooting Guide]({{}}) for details on how to resolve these errors. + {{% /alert %}} -* As part of the helm chart [karpenter](https://gallery.ecr.aws/karpenter/karpenter) - [source](https://github.com/aws/karpenter/blob/main/charts/karpenter/crds). Helm [does not manage the lifecycle of CRDs using this method](https://helm.sh/docs/chart_best_practices/custom_resource_definitions/), the tool will only install the CRD during the first installation of the helm chart. Subsequent chart upgrades will not add or remove CRDs, even if the CRDs have changed. When CRDs are changed, we will make a note in the version's upgrade guide. + * As part of the helm chart [karpenter](https://gallery.ecr.aws/karpenter/karpenter) - [source](https://github.com/aws/karpenter/blob/main/charts/karpenter/crds). Helm [does not manage the lifecycle of CRDs using this method](https://helm.sh/docs/chart_best_practices/custom_resource_definitions/), the tool will only install the CRD during the first installation of the helm chart. Subsequent chart upgrades will not add or remove CRDs, even if the CRDs have changed. When CRDs are changed, we will make a note in the version's upgrade guide. In general, ou can reapply the CRDs in the `crds` directory of the Karpenter helm chart: -In general, you can reapply the CRDs in the `crds` directory of the Karpenter helm chart: + ```bash + kubectl apply -f https://raw.githubusercontent.com/aws/karpenter{{< githubRelRef >}}pkg/apis/crds/karpenter.sh_nodepools.yaml + kubectl apply -f https://raw.githubusercontent.com/aws/karpenter{{< githubRelRef >}}pkg/apis/crds/karpenter.sh_nodeclaims.yaml + kubectl apply -f https://raw.githubusercontent.com/aws/karpenter{{< githubRelRef >}}pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml + ``` +6. Upgrade Karpenter to the new version: + + ```bash + helm registry logout public.ecr.aws + + helm upgrade --install karpenter oci://public.ecr.aws/karpenter/karpenter --version ${KARPENTER_VERSION} --namespace karpenter --create-namespace \ + --set serviceAccount.annotations."eks\.amazonaws\.com/role-arn"=${KARPENTER_IAM_ROLE_ARN} \ + --set settings.aws.defaultInstanceProfile=KarpenterNodeInstanceProfile-${CLUSTER_NAME} \ + --set settings.aws.clusterName=${CLUSTER_NAME} \ + --set settings.aws.interruptionQueueName=${CLUSTER_NAME} \ + --set controller.resources.requests.cpu=1 \ + --set controller.resources.requests.memory=1Gi \ + --set controller.resources.limits.cpu=1 \ + --set controller.resources.limits.memory=1Gi \ + --wait + ``` -```shell -kubectl apply -f https://raw.githubusercontent.com/aws/karpenter{{< githubRelRef >}}pkg/apis/crds/karpenter.sh_provisioners.yaml -kubectl apply -f https://raw.githubusercontent.com/aws/karpenter{{< githubRelRef >}}pkg/apis/crds/karpenter.sh_machines.yaml -kubectl apply -f https://raw.githubusercontent.com/aws/karpenter{{< githubRelRef >}}pkg/apis/crds/karpenter.k8s.aws_awsnodetemplates.yaml -``` +7. Convert each AWSNodeTemplate to an EC2NodeClass. To convert your v1alpha Karpenter manifests to v1beta1, you can either manually apply changes to API components or use the [Karpenter conversion tool](https://github.com/aws/karpenter/tree/main/tools/karpenter-convert). + See the [AWSNodeTemplate to EC2NodeClass]({{< relref "v1beta1-reference#awsnodetemplate-to-ec2nodeclass" >}}) section of the Karpenter Upgrade Reference for details on how to update to Karpenter AWSNodeTemplate objects. Here is an example of how to use the `karpenter-convert` CLI to convert an AWSNodeTemplate file to a EC2NodeClass file: -### How Do We Break Incompatibility? + ```bash + karpenter-convert -f awsnodetemplate.yaml > ec2nodeclass.yaml + ``` -When there is a breaking change we will: +8. Edit the converted EC2NodeClass file manually: -* Increment the minor version when in major version 0 -* Add a permanent separate section named `upgrading to vx.y.z+` under [released upgrade notes](#released-upgrade-notes) - clearly explaining the breaking change and what needs to be done on the user side to ensure a safe upgrade -* Add the sentence “This is a breaking change, please refer to the above link for upgrade instructions” to the top of the release notes and in all our announcements + * Specify your AWS role where there is a `$KARPENTER_NODE_ROLE` placeholder. For example, if you created your cluster using the [Getting Started with Karpenter](https://karpenter.sh/docs/getting-started/getting-started-with-karpenter/) guide, you would use the name `KarpenterNodeRole-$CLUSTER_NAME`, substituting your cluster name for `$CLUSTER_NAME`. + * Otherwise, check the file for accuracy. -### How Do We Find Incompatibilities? +9. When you are satisfied with your EC2NodeClass file, apply it as follows: -Besides the peer review process for all changes to the code base we also do the followings in order to find -incompatibilities: -* (To be implemented) To check the compatibility of the application, we will automate tests for installing, uninstalling, upgrading from an older version, and downgrading to an older version -* (To be implemented) To check the compatibility of the documentation with the application, we will turn the commands in our documentation into scripts that we can automatically run + ```bash + kubectl apply -f ec2nodeclass.yaml + ``` -### Security Patches +10. Convert each Provisioner to a NodePool. Again, either manually update your Provisioner manifests or use the karpenter-convert CLI tool: -While we are in major version 0 we will not release security patches to older versions. -Rather we provide the patches in the latest versions. -When at major version 1 we will have an EOL (end of life) policy where we provide security patches -for a subset of older versions and deprecate the others. + ```bash + karpenter-convert -f provisioner.yaml > nodepool.yaml + ``` -## Release Types +11. When you are satisfied with your NodePool file, apply it as follows: -Karpenter offers three types of releases. This section explains the purpose of each release type and how the images for each release type are tagged in our [public image repository](https://gallery.ecr.aws/karpenter). + ```bash + kubectl apply -f nodepool.yaml + ``` -### Stable Releases +12. Roll over nodes: With the new NodePool yaml in hand, there are several ways you can begin to roll over your nodes to use the new NodePool: -Stable releases are the most reliable releases that are released with weekly cadence. Stable releases are our only recommended versions for production environments. -Sometimes we skip a stable release because we find instability or problems that need to be fixed before having a stable release. -Stable releases are tagged with Semantic Versioning. For example `v0.13.0`. + * Periodic Rolling with [Drift]({{< relref "../concepts/disruption#drift" >}}): Enable [drift]({{< relref "../concepts/disruption#drift" >}}) in your NodePool file, then do the following: + - Add the following taint to the old Provisioner: `karpenter.sh/legacy=true:NoSchedule` + - Wait as Karpenter marks all machines owned by that Provisioner as having drifted. + - Watch as replacement nodes are launched from the new NodePool resource. -### Release Candidates + Because Karpenter will only roll of one node at a time, it may take some time for Karpenter to completely roll all nodes under a Provisioner. -We consider having release candidates for major and important minor versions. Our release candidates are tagged like `vx.y.z-rc.0`, `vx.y.z-rc.1`. The release candidate will then graduate to `vx.y.z` as a normal stable release. -By adopting this practice we allow our users who are early adopters to test out new releases before they are available to the wider community, thereby providing us with early feedback resulting in more stable releases. + * Forced Deletion: For each Provisioner in your cluster: -### Snapshot Releases + - Delete the old Provisioner with: `kubectl delete provisioner --cascade=foreground` + - Wait as Karpenter deletes all the Provisioner's nodes. All nodes will drain simultaneously. New nodes are launched after the old ones have been drained. -We release a snapshot release for every commit that gets merged into the main repository. This enables our users to immediately try a new feature or fix right after it's merged rather than waiting days or weeks for release. -Snapshot releases are suitable for testing, and troubleshooting but users should exercise great care if they decide to use them in production environments. -Snapshot releases are tagged with the git commit hash prefixed by the Karpenter major version. For example `v0-fc17bfc89ebb30a3b102a86012b3e3992ec08adf`. For more detailed examples on how to use snapshot releases look under "Usage" in [Karpenter Helm Chart](https://gallery.ecr.aws/karpenter/karpenter). + * Manual Rolling: For each Provisioner in your cluster: + - Add the following taint to the old Provisioner: `karpenter.sh/legacy=true:NoSchedule` + - For all the nodes owned by the Provisioner, delete one at a time as follows: `kubectl delete node ` -## Released Upgrade Notes +13. Update workload labels: Old v1alpha labels (`karpenter.sh/do-not-consolidate` and `karpenter.sh/do-not-evict`) are deprecated, but will not be dropped until Karpenter v1. However, you can begin updating those labels at any time with `karpenter.sh/do-not-disrupt`. You should check that there are no more Provisioner, AWSNodeTemplate, or Machine resources on your cluster. at which time you can delete the old CRDs. To validate that there are no more machines, type: -### Upgrading to v0.32.0+ + ```bash + kubectl get machines + ``` + +#### Additional Release Notes * Karpenter now serves the webhook prometheus metrics server on port `8001`. If this port is already in-use on the pod or you are running in `hostNetworking` mode, you may need to change this port value. You can configure this port value through the `WEBHOOK_METRICS_PORT` environment variable or the `webhook.metrics.port` value if installing via Helm. * Karpenter now exposes the ability to disable webhooks through the `webhook.enabled=false` value. This value will disable the webhook server and will prevent any permissions, mutating or validating webhook configurations from being deployed to the cluster. @@ -132,7 +210,7 @@ Snapshot releases are tagged with the git commit hash prefixed by the Karpenter ### Upgrading to v0.30.0+ -* Karpenter will now [statically drift]({{}}) on both Provisioner and AWSNodeTemplate Fields. For Provisioner Static Drift, the `karpenter.sh/provisioner-hash` annotation must be present on both the Provisioner and Machine. For AWSNodeTemplate drift, the `karpenter.k8s.aws/nodetemplate-hash` annotation must be present on the AWSNodeTemplate and Machine. Karpenter will not add these annotations to pre-existing nodes, so each of these nodes will need to be recycled one time for the annotations to be added. +* Karpenter will now [statically drift]({{}}) on both Provisioner and AWSNodeTemplate Fields. For Provisioner Static Drift, the `karpenter.sh/provisioner-hash` annotation must be present on both the Provisioner and Machine. For AWSNodeTemplate drift, the `karpenter.k8s.aws/nodetemplate-hash` annotation must be present on the AWSNodeTemplate and Machine. Karpenter will not add these annotations to pre-existing nodes, so each of these nodes will need to be recycled one time for the annotations to be added. * Karpenter will now fail validation on AWSNodeTemplates and Provisioner `spec.provider` that have `amiSelectors`, `subnetSelectors`, or `securityGroupSelectors` set with a combination of id selectors (`aws-ids`, `aws::ids`) and other selectors. * Karpenter now statically sets the `securityContext` at both the pod and container-levels and doesn't allow override values to be passed through the helm chart. This change was made to adhere to [Restricted Pod Security Standard](https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted), which follows pod hardening best practices. @@ -149,7 +227,7 @@ Karpenter `v0.29.1` contains a [file descriptor and memory leak bug](https://git * Karpenter has changed the default metrics service port from 8080 to 8000 and the default webhook service port from 443 to 8443. In `v0.28.0`, the Karpenter pod port was changed to 8000, but referenced the service by name, allowing users to scrape the service at port 8080 for metrics. `v0.29.0` aligns the two ports so that service and pod metrics ports are the same. These ports are set by the `controller.metrics.port` and `webhook.port` helm chart values, so if you have previously set these to non-default values, you may need to update your Prometheus scraper to match these new values. * Karpenter will now reconcile nodes that are drifted due to their Security Groups or their Subnets. If your AWSNodeTemplate's Security Groups differ from the Security Groups used for an instance, Karpenter will consider it drifted. If the Subnet used by an instance is not contained in the allowed list of Subnets for an AWSNodeTemplate, Karpenter will also consider it drifted. - * Since Karpenter uses tags for discovery of Subnets and SecurityGroups, check the [Threat Model]({{}}) to see how to manage this IAM Permission. + * Since Karpenter uses tags for discovery of Subnets and SecurityGroups, check the [Threat Model]({{}}) to see how to manage this IAM Permission. ### Upgrading to v0.28.0+ @@ -158,8 +236,8 @@ Karpenter `v0.28.0` is incompatible with Kubernetes version 1.26+, which can res {{% /alert %}} * The `extraObjects` value is now removed from the Helm chart. Having this value in the chart proved to not work in the majority of Karpenter installs and often led to anti-patterns, where the Karpenter resources installed to manage Karpenter's capacity were directly tied to the install of the Karpenter controller deployments. The Karpenter team recommends that, if you want to install Karpenter manifests alongside the Karpenter helm chart, to do so by creating a separate chart for the manifests, creating a dependency on the controller chart. -* The `aws.nodeNameConvention` setting is now removed from the [`karpenter-global-settings`]({{}}) ConfigMap. Because Karpenter is now driving its orchestration of capacity through Machines, it no longer needs to know the node name, making this setting obsolete. Karpenter ignores configuration that it doesn't recognize in the [`karpenter-global-settings`]({{}}) ConfigMap, so leaving the `aws.nodeNameConvention` in the ConfigMap will simply cause this setting to be ignored. -* Karpenter now defines a set of "restricted tags" which can't be overridden with custom tagging in the AWSNodeTemplate or in the [`karpenter-global-settings`]({{}}) ConfigMap. If you are currently using any of these tag overrides when tagging your instances, webhook validation will now fail. These tags include: +* The `aws.nodeNameConvention` setting is now removed from the [`karpenter-global-settings`]({{}}) ConfigMap. Because Karpenter is now driving its orchestration of capacity through Machines, it no longer needs to know the node name, making this setting obsolete. Karpenter ignores configuration that it doesn't recognize in the [`karpenter-global-settings`]({{}}) ConfigMap, so leaving the `aws.nodeNameConvention` in the ConfigMap will simply cause this setting to be ignored. +* Karpenter now defines a set of "restricted tags" which can't be overridden with custom tagging in the AWSNodeTemplate or in the [`karpenter-global-settings`]({{}}) ConfigMap. If you are currently using any of these tag overrides when tagging your instances, webhook validation will now fail. These tags include: * `karpenter.sh/managed-by` * `karpenter.sh/provisioner-name` @@ -181,7 +259,7 @@ Karpenter creates a mapping between CloudProvider machines and CustomResources i * `karpenter.sh/provisioner-name` * `kubernetes.io/cluster/${CLUSTER_NAME}` -Because Karpenter takes this dependency, any user that has the ability to Create/Delete these tags on CloudProvider machines will have the ability to orchestrate Karpenter to Create/Delete CloudProvider machines as a side effect. Check the [Threat Model]({{}}) to see how this might affect you, and ways to mitigate this. +Because Karpenter takes this dependency, any user that has the ability to Create/Delete these tags on CloudProvider machines will have the ability to orchestrate Karpenter to Create/Delete CloudProvider machines as a side effect. Check the [Threat Model]({{}}) to see how this might affect you, and ways to mitigate this. {{% /alert %}} {{% alert title="Rolling Back" color="warning" %}} @@ -194,7 +272,7 @@ Karpenter marks CloudProvider capacity as "managed by" a Machine using the `karp ### Upgrading to v0.27.3+ * The `defaulting.webhook.karpenter.sh` mutating webhook was removed in `v0.27.3`. If you are coming from an older version of Karpenter where this webhook existed and the webhook was not managed by Helm, you may need to delete the stale webhook. -```console +```bash kubectl delete mutatingwebhookconfigurations defaulting.webhook.karpenter.sh ``` @@ -216,7 +294,7 @@ kubectl delete mutatingwebhookconfigurations defaulting.webhook.karpenter.sh * The `karpenter_allocation_controller_scheduling_duration_seconds` metric name changed to `karpenter_provisioner_scheduling_duration_seconds` ### Upgrading to v0.26.0+ -* The `karpenter.sh/do-not-evict` annotation no longer blocks node termination when running `kubectl delete node`. This annotation on pods will only block automatic deprovisioning that is considered "voluntary," that is, disruptions that can be avoided. Disruptions that Karpenter deems as "involuntary" and will ignore the `karpenter.sh/do-not-evict` annotation include spot interruption and manual deletion of the node. See [Disabling Deprovisioning]({{}}) for more details. +* The `karpenter.sh/do-not-evict` annotation no longer blocks node termination when running `kubectl delete node`. This annotation on pods will only block automatic deprovisioning that is considered "voluntary," that is, disruptions that can be avoided. Disruptions that Karpenter deems as "involuntary" and will ignore the `karpenter.sh/do-not-evict` annotation include spot interruption and manual deletion of the node. See [Disabling Deprovisioning]({{}}) for more details. * Default resources `requests` and `limits` are removed from the Karpenter's controller deployment through the Helm chart. If you have not set custom resource `requests` or `limits` in your helm values and are using Karpenter's defaults, you will now need to set these values in your helm chart deployment. * The `controller.image` value in the helm chart has been broken out to a map consisting of `controller.image.repository`, `controller.image.tag`, and `controller.image.digest`. If manually overriding the `controller.image`, you will need to update your values to the new design. @@ -224,9 +302,9 @@ kubectl delete mutatingwebhookconfigurations defaulting.webhook.karpenter.sh * Cluster Endpoint can now be automatically discovered. If you are using Amazon Elastic Kubernetes Service (EKS), you can now omit the `clusterEndpoint` field in your configuration. In order to allow the resolving, you have to add the permission `eks:DescribeCluster` to the Karpenter Controller IAM role. ### Upgrading to v0.24.0+ -* Settings are no longer updated dynamically while Karpenter is running. If you manually make a change to the [`karpenter-global-settings`]({{}}) ConfigMap, you will need to reload the containers by restarting the deployment with `kubectl rollout restart -n karpenter deploy/karpenter` -* Karpenter no longer filters out instance types internally. Previously, `g2` (not supported by the NVIDIA device plugin) and FPGA instance types were filtered. The only way to filter instance types now is to set requirements on your provisioner or pods using well-known node labels described [here]({{}}). If you are currently using overly broad requirements that allows all of the `g` instance-category, you will want to tighten the requirement, or add an instance-generation requirement. -* `aws.tags` in [`karpenter-global-settings`]({{}}) ConfigMap is now a top-level field and expects the value associated with this key to be a JSON object of string to string. This is change from previous versions where keys were given implicitly by providing the key-value pair `aws.tags.: value` in the ConfigMap. +* Settings are no longer updated dynamically while Karpenter is running. If you manually make a change to the [`karpenter-global-settings`]({{}}) ConfigMap, you will need to reload the containers by restarting the deployment with `kubectl rollout restart -n karpenter deploy/karpenter` +* Karpenter no longer filters out instance types internally. Previously, `g2` (not supported by the NVIDIA device plugin) and FPGA instance types were filtered. The only way to filter instance types now is to set requirements on your provisioner or pods using well-known node labels described [here]({{}}). If you are currently using overly broad requirements that allows all of the `g` instance-category, you will want to tighten the requirement, or add an instance-generation requirement. +* `aws.tags` in [`karpenter-global-settings`]({{}}) ConfigMap is now a top-level field and expects the value associated with this key to be a JSON object of string to string. This is change from previous versions where keys were given implicitly by providing the key-value pair `aws.tags.: value` in the ConfigMap. ### Upgrading to v0.22.0+ * Do not upgrade to this version unless you are on Kubernetes >= v1.21. Karpenter no longer supports Kubernetes v1.20, but now supports Kubernetes v1.25. This change is due to the v1 PDB API, which was introduced in K8s v1.20 and subsequent removal of the v1beta1 API in K8s v1.25. @@ -236,11 +314,11 @@ kubectl delete mutatingwebhookconfigurations defaulting.webhook.karpenter.sh ### Upgrading to v0.19.0+ * The karpenter webhook and controller containers are combined into a single binary, which requires changes to the helm chart. If your Karpenter installation (helm or otherwise) currently customizes the karpenter webhook, your deployment tooling may require minor changes. -* Karpenter now supports native interruption handling. If you were previously using Node Termination Handler for spot interruption handling and health events, you will need to remove the component from your cluster before enabling `aws.interruptionQueueName`. For more details on Karpenter's interruption handling, see the [Interruption Handling Docs]({{< ref "./concepts/disruption/#interruption" >}}). For common questions on the migration process, see the [FAQ]({{< ref "./faq/#interruption-handling" >}}) +* Karpenter now supports native interruption handling. If you were previously using Node Termination Handler for spot interruption handling and health events, you will need to remove the component from your cluster before enabling `aws.interruptionQueueName`. For more details on Karpenter's interruption handling, see the [Interruption Handling Docs]({{< ref "../concepts/disruption/#interruption" >}}). * Instance category defaults are now explicitly persisted in the Provisioner, rather than handled implicitly in memory. By default, Provisioners will limit instance category to c,m,r. If any instance type constraints are applied, it will override this default. If you have created Provisioners in the past with unconstrained instance type, family, or category, Karpenter will now more flexibly use instance types than before. If you would like to apply these constraints, they must be included in the Provisioner CRD. * Karpenter CRD raw YAML URLs have migrated from `https://raw.githubusercontent.com/aws/karpenter{{< githubRelRef >}}charts/karpenter/crds/...` to `https://raw.githubusercontent.com/aws/karpenter{{< githubRelRef >}}pkg/apis/crds/...`. If you reference static Karpenter CRDs or rely on `kubectl replace -f` to apply these CRDs from their remote location, you will need to migrate to the new location. * Pods without an ownerRef (also called "controllerless" or "naked" pods) will now be evicted by default during node termination and consolidation. Users can prevent controllerless pods from being voluntarily disrupted by applying the `karpenter.sh/do-not-evict: "true"` annotation to the pods in question. -* The following CLI options/environment variables are now removed and replaced in favor of pulling settings dynamically from the [`karpenter-global-settings`]({{}}) ConfigMap. See the [Settings docs]({{}}) for more details on configuring the new values in the ConfigMap. +* The following CLI options/environment variables are now removed and replaced in favor of pulling settings dynamically from the [`karpenter-global-settings`]({{}}) ConfigMap. See the [Settings docs]({{}}) for more details on configuring the new values in the ConfigMap. * `CLUSTER_NAME` -> `settings.aws.clusterName` * `CLUSTER_ENDPOINT` -> `settings.aws.clusterEndpoint` @@ -252,11 +330,11 @@ kubectl delete mutatingwebhookconfigurations defaulting.webhook.karpenter.sh * `VM_MEMORY_OVERHEAD` -> `settings.aws.vmMemoryOverheadPercent` ### Upgrading to v0.18.0+ -* v0.18.0 removes the `karpenter_consolidation_nodes_created` and `karpenter_consolidation_nodes_terminated` prometheus metrics in favor of the more generic `karpenter_nodes_created` and `karpenter_nodes_terminated` metrics. You can still see nodes created and terminated by consolidation by checking the `reason` label on the metrics. Check out all the metrics published by Karpenter [here]({{}}). +* v0.18.0 removes the `karpenter_consolidation_nodes_created` and `karpenter_consolidation_nodes_terminated` prometheus metrics in favor of the more generic `karpenter_nodes_created` and `karpenter_nodes_terminated` metrics. You can still see nodes created and terminated by consolidation by checking the `reason` label on the metrics. Check out all the metrics published by Karpenter [here]({{}}). ### Upgrading to v0.17.0+ Karpenter's Helm chart package is now stored in [Karpenter's OCI (Open Container Initiative) registry](https://gallery.ecr.aws/karpenter/karpenter). The Helm CLI supports the new format since [v3.8.0+](https://helm.sh/docs/topics/registries/). -With this change [charts.karpenter.sh](https://charts.karpenter.sh/) is no longer updated but preserved to allow using older Karpenter versions. For examples on working with the Karpenter helm charts look at [Install Karpenter Helm Chart]({{< ref "./getting-started/getting-started-with-karpenter/#install-karpenter-helm-chart" >}}). +With this change [charts.karpenter.sh](https://charts.karpenter.sh/) is no longer updated but preserved to allow using older Karpenter versions. For examples on working with the Karpenter helm charts look at [Install Karpenter Helm Chart]({{< ref "../getting-started/getting-started-with-karpenter/#install-karpenter-helm-chart" >}}). Users who have scripted the installation or upgrading of Karpenter need to adjust their scripts with the following changes: 1. There is no longer a need to add the Karpenter helm repo to helm @@ -306,13 +384,13 @@ aws ec2 delete-launch-template --launch-template-id operator: Exists ``` -* v0.14.0 introduces support for custom AMIs without the need for an entire launch template. You must add the `ec2:DescribeImages` permission to the Karpenter Controller Role for this feature to work. This permission is needed for Karpenter to discover custom images specified. Read the [Custom AMI documentation here]({{}}) to get started +* v0.14.0 introduces support for custom AMIs without the need for an entire launch template. You must add the `ec2:DescribeImages` permission to the Karpenter Controller Role for this feature to work. This permission is needed for Karpenter to discover custom images specified. Read the [Custom AMI documentation here]({{}}) to get started * v0.14.0 adds an an additional default toleration (CriticalAddonOnly=Exists) to the Karpenter helm chart. This may cause Karpenter to run on nodes with that use this Taint which previously would not have been schedulable. This can be overridden by using `--set tolerations[0]=null`. * v0.14.0 deprecates the `AWS_ENI_LIMITED_POD_DENSITY` environment variable in-favor of specifying `spec.kubeletConfiguration.maxPods` on the Provisioner. `AWS_ENI_LIMITED_POD_DENSITY` will continue to work when `maxPods` is not set on the Provisioner. If `maxPods` is set, it will override `AWS_ENI_LIMITED_POD_DENSITY` on that specific Provisioner. ### Upgrading to v0.13.0+ -* v0.13.0 introduces a new CRD named `AWSNodeTemplate` which can be used to specify AWS Cloud Provider parameters. Everything that was previously specified under `spec.provider` in the Provisioner resource, can now be specified in the spec of the new resource. The use of `spec.provider` is deprecated but will continue to function to maintain backwards compatibility for the current API version (v1alpha5) of the Provisioner resource. v0.13.0 also introduces support for custom user data that doesn't require the use of a custom launch template. The user data can be specified in-line in the AWSNodeTemplate resource. Read the [UserData documentation here](../aws/operating-systems) to get started. +* v0.13.0 introduces a new CRD named `AWSNodeTemplate` which can be used to specify AWS Cloud Provider parameters. Everything that was previously specified under `spec.provider` in the Provisioner resource, can now be specified in the spec of the new resource. The use of `spec.provider` is deprecated but will continue to function to maintain backwards compatibility for the current API version (v1alpha5) of the Provisioner resource. v0.13.0 also introduces support for custom user data that doesn't require the use of a custom launch template. The user data can be specified in-line in the AWSNodeTemplate resource. If you are upgrading from v0.10.1 - v0.11.1, a new CRD `awsnodetemplate` was added. In v0.12.0, this crd was renamed to `awsnodetemplates`. Since helm does not manage the lifecycle of CRDs, you will need to perform a few manual steps for this CRD upgrade: 1. Make sure any `awsnodetemplate` manifests are saved somewhere so that they can be reapplied to the cluster. diff --git a/website/content/en/preview/upgrading/v1beta1-controller-policy.json b/website/content/en/preview/upgrading/v1beta1-controller-policy.json new file mode 100644 index 000000000000..e6923be897d2 --- /dev/null +++ b/website/content/en/preview/upgrading/v1beta1-controller-policy.json @@ -0,0 +1,219 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "AllowScopedEC2InstanceActions", + "Effect": "Allow", + "Resource": [ + "arn:${AWS_PARTITION}:ec2:${AWS_REGION}::image/*", + "arn:${AWS_PARTITION}:ec2:${AWS_REGION}::snapshot/*", + "arn:${AWS_PARTITION}:ec2:${AWS_REGION}:*:spot-instances-request/*", + "arn:${AWS_PARTITION}:ec2:${AWS_REGION}:*:security-group/*", + "arn:${AWS_PARTITION}:ec2:${AWS_REGION}:*:subnet/*", + "arn:${AWS_PARTITION}:ec2:${AWS_REGION}:*:launch-template/*" + ], + "Action": [ + "ec2:RunInstances", + "ec2:CreateFleet" + ] + }, + { + "Sid": "AllowScopedEC2InstanceActionsWithTags", + "Effect": "Allow", + "Resource": [ + "arn:${AWS_PARTITION}:ec2:${AWS_REGION}:*:fleet/*", + "arn:${AWS_PARTITION}:ec2:${AWS_REGION}:*:instance/*", + "arn:${AWS_PARTITION}:ec2:${AWS_REGION}:*:volume/*", + "arn:${AWS_PARTITION}:ec2:${AWS_REGION}:*:network-interface/*", + "arn:${AWS_PARTITION}:ec2:${AWS_REGION}:*:launch-template/*" + ], + "Action": [ + "ec2:RunInstances", + "ec2:CreateFleet", + "ec2:CreateLaunchTemplate" + ], + "Condition": { + "StringEquals": { + "aws:RequestTag/kubernetes.io/cluster/${CLUSTER_NAME}": "owned" + }, + "StringLike": { + "aws:RequestTag/karpenter.sh/nodepool": "*" + } + } + }, + { + "Sid": "AllowScopedResourceCreationTagging", + "Effect": "Allow", + "Resource": [ + "arn:${AWS_PARTITION}:ec2:${AWS_REGION}:*:fleet/*", + "arn:${AWS_PARTITION}:ec2:${AWS_REGION}:*:instance/*", + "arn:${AWS_PARTITION}:ec2:${AWS_REGION}:*:volume/*", + "arn:${AWS_PARTITION}:ec2:${AWS_REGION}:*:network-interface/*", + "arn:${AWS_PARTITION}:ec2:${AWS_REGION}:*:launch-template/*" + ], + "Action": "ec2:CreateTags", + "Condition": { + "StringEquals": { + "aws:RequestTag/kubernetes.io/cluster/${CLUSTER_NAME}": "owned", + "ec2:CreateAction": [ + "RunInstances", + "CreateFleet", + "CreateLaunchTemplate" + ] + }, + "StringLike": { + "aws:RequestTag/karpenter.sh/nodepool": "*" + } + } + }, + { + "Sid": "AllowScopedResourceTagging", + "Effect": "Allow", + "Resource": "arn:${AWS_PARTITION}:ec2:${AWS_REGION}:*:instance/*", + "Action": "ec2:CreateTags", + "Condition": { + "StringEquals": { + "aws:ResourceTag/kubernetes.io/cluster/${CLUSTER_NAME}": "owned" + }, + "StringLike": { + "aws:ResourceTag/karpenter.sh/nodepool": "*" + }, + "ForAllValues:StringEquals": { + "aws:TagKeys": [ + "karpenter.sh/nodeclaim", + "Name" + ] + } + } + }, + { + "Sid": "AllowScopedDeletion", + "Effect": "Allow", + "Resource": [ + "arn:${AWS_PARTITION}:ec2:${AWS_REGION}:*:instance/*", + "arn:${AWS_PARTITION}:ec2:${AWS_REGION}:*:launch-template/*" + ], + "Action": [ + "ec2:TerminateInstances", + "ec2:DeleteLaunchTemplate" + ], + "Condition": { + "StringEquals": { + "aws:ResourceTag/kubernetes.io/cluster/${CLUSTER_NAME}": "owned" + }, + "StringLike": { + "aws:ResourceTag/karpenter.sh/nodepool": "*" + } + } + }, + { + "Sid": "AllowRegionalReadActions", + "Effect": "Allow", + "Resource": "*", + "Action": [ + "ec2:DescribeAvailabilityZones", + "ec2:DescribeImages", + "ec2:DescribeInstances", + "ec2:DescribeInstanceTypeOfferings", + "ec2:DescribeInstanceTypes", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSpotPriceHistory", + "ec2:DescribeSubnets" + ], + "Condition": { + "StringEquals": { + "aws:RequestedRegion": "${AWS_REGION}" + } + } + }, + { + "Sid": "AllowSSMReadActions", + "Effect": "Allow", + "Resource": "arn:${AWS_PARTITION}:ssm:${AWS_REGION}::parameter/aws/service/*", + "Action": "ssm:GetParameter" + }, + { + "Sid": "AllowPricingReadActions", + "Effect": "Allow", + "Resource": "*", + "Action": "pricing:GetProducts" + }, + { + "Sid": "AllowInterruptionQueueActions", + "Effect": "Allow", + "Resource": "arn:aws:sqs:${AWS_REGION}:${AWS_ACCOUNT_ID}:${CLUSTER_NAME}", + "Action": [ + "sqs:DeleteMessage", + "sqs:GetQueueAttributes", + "sqs:GetQueueUrl", + "sqs:ReceiveMessage" + ] + }, + { + "Sid": "AllowPassingInstanceRole", + "Effect": "Allow", + "Resource": "arn:${AWS_PARTITION}:iam::${AWS_ACCOUNT_ID}:role/KarpenterNodeRole-${CLUSTER_NAME}", + "Action": "iam:PassRole", + "Condition": { + "StringEquals": { + "iam:PassedToService": "ec2.amazonaws.com" + } + } + }, + { + "Sid": "AllowScopedInstanceProfileCreationActions", + "Effect": "Allow", + "Resource": "*", + "Action": "iam:CreateInstanceProfile", + "Condition": { + "StringEquals": { + "aws:RequestTag/kubernetes.io/cluster/${CLUSTER_NAME}": "owned", + "aws:RequestTag/topology.kubernetes.io/region": "${AWS_REGION}" + } + } + }, + { + "Sid": "AllowScopedInstanceProfileTagActions", + "Effect": "Allow", + "Resource": "*", + "Action": "iam:TagInstanceProfile", + "Condition": { + "StringEquals": { + "aws:ResourceTag/kubernetes.io/cluster/${CLUSTER_NAME}": "owned", + "aws:ResourceTag/topology.kubernetes.io/region": "${AWS_REGION}", + "aws:RequestTag/kubernetes.io/cluster/${CLUSTER_NAME}": "owned", + "aws:RequestTag/topology.kubernetes.io/region": "${AWS_REGION}" + } + } + }, + { + "Sid": "AllowScopedInstanceProfileActions", + "Effect": "Allow", + "Resource": "*", + "Action": [ + "iam:AddRoleToInstanceProfile", + "iam:RemoveRoleFromInstanceProfile", + "iam:DeleteInstanceProfile" + ], + "Condition": { + "StringEquals": { + "aws:ResourceTag/kubernetes.io/cluster/${CLUSTER_NAME}": "owned", + "aws:ResourceTag/topology.kubernetes.io/region": "${AWS_REGION}" + } + } + }, + { + "Sid": "AllowInstanceProfileReadActions", + "Effect": "Allow", + "Resource": "*", + "Action": "iam:GetInstanceProfile" + }, + { + "Sid": "AllowAPIServerEndpointDiscovery", + "Effect": "Allow", + "Resource": "arn:${AWS_PARTITION}:eks:${AWS_REGION}:${AWS_ACCOUNT_ID}:cluster/${CLUSTER_NAME}", + "Action": "eks:DescribeCluster" + } + ] +} \ No newline at end of file diff --git a/website/content/en/preview/upgrading/v1beta1-reference.md b/website/content/en/preview/upgrading/v1beta1-reference.md new file mode 100644 index 000000000000..8b21dcf530e7 --- /dev/null +++ b/website/content/en/preview/upgrading/v1beta1-reference.md @@ -0,0 +1,661 @@ +--- +title: "v1beta1 Upgrade Reference" +linkTitle: "v1beta1 Upgrade Reference" +weight: 30 +description: > + API information for upgrading Karpenter +--- + +Significant changes to the Karpenter APIs have been introduced in Karpenter v0.32.x. +In this release, Karpenter APIs have advanced to v1beta1, in preparation for Karpenter v1 in the near future. +The v1beta1 changes are meant to simplify and improve ease of use of those APIs, as well as solidify the APIs for the v1 release. +Use this document as a reference to the changes that were introduced in the current release and as a guide to how you need to update the manifests and other Karpenter objects you created in previous Karpenter releases. + +The [Upgrade Guide]({{< relref "upgrade-guide" >}}) steps you through the process of upgrading Karpenter for the latest release. +For a more general understanding of Karpenter's compatibility, see the [Compatibility Document]({{< relref "compatibility" >}}). + +## CRD Upgrades + +Karpenter ships with a few Custom Resource Definitions (CRDs). Starting with v0.32.0, CRDs representing Provisioners, Machines, and AWS Node Templates are replaced by those for NodePools, NodeClaims, and EC2Nodeclasses, respectively. +You can find these CRDs by visiting the [Karpenter GitHub repository](https://github.com/aws/karpenter/tree{{< githubRelRef >}}pkg/apis/crds). + +The [Upgrade Guide]({{< relref "upgrade-guide" >}}) describes how to install the new CRDs. + +## Annotations, Labels, and Status Conditions + +Karpenter v1beta1 introduces changes to some common labels, annotations, and status conditions that are present in the project. The tables below lists the v1alpha5 values and their v1beta1 equivalent. + +| Karpenter Labels | | +|---------------------------------|-----------------------------| +| **v1alpha5** | **v1beta1** | +| karpenter.sh/provisioner-name | karpenter.sh/nodepool | + + +| Karpenter Annotations | | +|-------------------------------------|----------------------------------| +| **v1alpha5** | **v1beta1** | +| karpenter.sh/provisioner-hash | karpenter.sh/nodepool-hash | +| karpenter.k8s.aws/nodetemplate-hash | karpenter.k8s.aws/nodeclass-hash | +| karpenter.sh/do-not-consolidate | karpenter.sh/do-not-disrupt | +| karpenter.sh/do-not-evict | karpenter.sh/do-not-disrupt | + +> **Note**: Karpenter dropped the `karpenter.sh/do-not-consolidate` annotation in favor of the `karpenter.sh/do-not-disrupt` annotation on nodes. This annotation specifies that no voluntary disruption should be performed by Karpenter against this node. + +| StatusCondition Types | | +|---------------------------------|----------------| +| **v1alpha5** | **v1beta1** | +| MachineLaunched | Launched | +| MachineRegistered | Registered | +| MachineInitialized | Initialized | +| MachineEmpty | Empty | +| MachineExpired | Expired | +| MachineDrifted | Drifted | + +## Provisioner to NodePool + +Karpenter v1beta1 moves almost all top-level fields under the `NodePool` template field. Similar to Deployments (which template Pods that are orchestrated by the deployment controller), Karpenter NodePool templates NodeClaims (that are orchestrated by the Karpenter controller). Here is an example of a `Provisioner` (v1alpha5) migrated to a `NodePool` (v1beta1): + +Note that: +* The `Limits` and `Weight` fields sit outside of the template section. The `Labels` and `Annotations` fields from the Provisioner are now under the `spec.template.metadata` section. All other fields including requirements, taints, kubelet, and so on, are specified under the `spec.template.spec` section. +* Support for `spec.template.spec.kubelet.containerRuntime` has been dropped. If you are using EKS 1.23 you should upgrade to containerd before using Karpenter v0.32.0, as this field in the kubelet block of the NodePool is not supported. EKS 1.24+ only supports containerd as a supported runtime. + +**Provisioner example (v1alpha)** + +```yaml +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner + ... +spec: + providerRef: + name: default + annotations: + custom-annotation: custom-value + labels: + team: team-a + custom-label: custom-value + requirements: + - key: karpenter.k8s.aws/instance-generation + operator: Gt + values: ["3"] + - key: karpenter.k8s.aws/instance-category + operator: In + values: ["c", "m", "r"] + - key: karpenter.sh/capacity-type + operator: In + values: ["spot"] + taints: + - key: example.com/special-taint + value: "true" + effect: NoSchedule + startupTaints: + - key: example.com/another-taint + value: "true" + effect: NoSchedule + kubelet: + systemReserved: + cpu: 100m + memory: 100Mi + ephemeral-storage: 1Gi + maxPods: 20 + limits: + resources: + cpu: 1000 + memory: 1000Gi + weight: 50 +``` + +**NodePool example (v1beta1)** + +```yaml +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +... +spec: + template: + metadata: + annotations: + custom-annotation: custom-value + labels: + team: team-a + custom-label: custom-value + spec: + requirements: + - key: karpenter.k8s.aws/instance-generation + operator: Gt + values: ["3"] + - key: karpenter.k8s.aws/instance-category + operator: In + values: ["c", "m", "r"] + - key: karpenter.sh/capacity-type + operator: In + values: ["spot"] + taints: + - key: example.com/special-taint + value: "true" + effect: NoSchedule + startupTaints: + - key: example.com/another-taint + value: "true" + effect: NoSchedule + kubelet: + systemReserved: + cpu: 100m + memory: 100Mi + ephemeral-storage: 1Gi + maxPods: 20 + limits: + cpu: 1000 + memory: 1000Gi + weight: 50 +``` + +### Provider + +The Karpenter `spec.provider` field has been deprecated since version v0.7.0 and is now removed in the new `NodePool` resource. Any of the fields that you could specify within the `spec.provider` field are now available in the separate `NodeClass` resource. + +**Provider example (v1alpha)** + +```yaml +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +... +spec: + provider: + amiFamily: Bottlerocket + tags: + test-tag: test-value +``` + +**Nodepool example (v1beta1)** + +```yaml +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +... +nodeClassRef: + name: default +``` + +**EC2NodeClass example (v1beta1)** + +```yaml +apiVersion: karpenter.k8s.aws/v1beta1 +kind: EC2NodeClass +metadata: + name: default +spec: + amiFamily: Bottlerocket + tags: + test-tag: test-value +``` + +### TTLSecondsAfterEmpty + +The Karpenter `spec.ttlSecondsAfterEmpty` field has been removed in favor of a `consolidationPolicy` and `consolidateAfter` field. + +As part of the v1beta1 migration, Karpenter has chosen to collapse the concepts of emptiness and underutilization into a single concept: `consolidation`. +You can now define the types of consolidation that you want to support in your `consolidationPolicy` field. +The current values for this field are `WhenEmpty` or `WhenUnderutilized` (defaulting to `WhenUnderutilized` if not specified). +If specifying `WhenEmpty`, you can define how long you wish to wait for consolidation to act on your empty nodes by tuning the `consolidateAfter` parameter. +This field works the same as the `ttlSecondsAfterEmpty` field except this field now accepts either of the following values: + +* `Never`: This disables empty consolidation. +* Duration String (e.g. “10m”, “1s”): This enables empty consolidation for the time specified. + +**ttlSecondsAfterEmpty example (v1alpha)** + +```yaml +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +... +spec: + ttlSecondsAfterEmpty: 120 +``` + +**consolidationPolicy and consolidateAfter examples (v1beta1)** + +```yaml +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +... +spec: + disruption: + consolidationPolicy: WhenEmpty + consolidateAfter: 2m + +``` + +### Consolidation + +The Karpenter `spec.consolidation` block has also been shifted under `consolidationPolicy`. If you were previously enabling Karpenter’s consolidation feature for underutilized nodes using the `consolidation.enabled` flag, you now enable consolidation through the `consolidationPolicy`. + +**consolidation enabled example (v1alpha)** + +```yaml +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +... +spec: + consolidation: + enabled: true +``` + +**consolidationPolicy WhenUnderutilized example (v1beta1)** + +```yaml +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +... +spec: + disruption: + consolidationPolicy: WhenUnderutilized +``` + +> Note: You currently can’t set the `consolidateAfter` field when specifying `consolidationPolicy: WhenUnderutilized`. Karpenter will use a 15s `consolidateAfter` runtime default. + +### TTLSecondsUntilExpired + +The Karpenter `spec.ttlSecondsUntilExpired` field has been removed in favor of the `expireAfter` field inside of the `disruption` block. This field works the same as it did before except this field now accepts either of the following values: + +* `Never`: This disables expiration. +* Duration String (e.g. “10m”, “1s”): This enables expiration for the time specified. + +**consolidation ttlSecondsUntilExpired example (v1alpha)** + +```yaml +apiVersion: karpenter.sh/v1alpha5 +kind: Provisioner +... +spec: + ttlSecondsUntilExpired: 2592000 # 30 Days = 60 * 60 * 24 * 30 Seconds +``` + +**consolidationPolicy WhenUnderutilized example (v1beta1)** + +```yaml +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +... +spec: + disruption: + expireAfter: 720h # 30 days = 30 * 24 Hours + +``` + +### Defaults + +Karpenter now statically defaults some fields in the v1beta1 if they are not specified when applying the `NodePool` configuration. The following fields are defaulted if unspecified. + +| Field | Default | +|--------------------------------------|------------------------------------------------------------------| +| spec.disruption | {"consolidationPolicy: WhenUnderutilized", expireAfter: "720h"} | +| spec.disruption.consolidationPolicy | WhenUnderutilized | +| spec.disruption.expireAfter | 720h | + + +### spec.template.spec.requirements Defaults Dropped + +Karpenter v1beta1 drops the defaulting logic for the node requirements that were shipped by default with Provisioners in v1alpha5. Previously, Karpenter would create dynamic defaulting in the following cases. If multiple of these cases were satisfied, those default requirements would be combined: + +* If you didn't specify any instance type requirement: + + ```yaml + spec: + requirements: + - key: karpenter.k8s.aws/instance-category + operator: In + values: ["c", "m", "r"] + - key: karpenter.k8s.aws/instance-generation + operator: In + values: ["2"] + ``` + +* If you didn’t specify any capacity type requirement (`karpenter.sh/capacity-type`): + + ```yaml + spec: + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand"] + ``` + +* If you didn’t specify any OS requirement (`kubernetes.io/os`): + ```yaml + spec: + requirements: + - key: kubernetes.io/os + operator: In + values: ["linux"] + ``` + +* If you didn’t specify any architecture requirement (`kubernetes.io/arch`): + ```yaml + spec: + requirements: + - key: kubernetes.io/arch + operator: In + values: ["amd64"] + ``` + +If you were previously relying on this defaulting logic, you will now need to explicitly specify these requirements in your `NodePool`. + +## AWSNodeTemplate to EC2NodeClass + +To configure AWS-specific settings, AWSNodeTemplate (v1alpha) is being changed to EC2NodeClass (v1beta1). Below are ways in which you can update your manifests for the new version. + +{{% alert title="Note" color="warning" %}} +Tag-based [AMI Selection](https://karpenter.sh/v0.31/concepts/node-templates/#ami-selection) is no longer supported for the new v1beta1 `EC2NodeClass` API. That feature allowed users to tag their AMIs using EC2 tags to express “In” requirements on selected. We recommend using different NodePools with different EC2NodeClasses with your various AMI requirement constraints to appropriately constrain your AMIs based on the instance types you’ve selected for a given NodePool. +{{% /alert %}} + +{{% alert title="Note" color="primary" %}} +Karpenter will now tag EC2 instances with their node name instead of with `karpenter.sh/provisioner-name: $PROVISIONER_NAME`. This makes it easier to map nodes to instances if you are not currently using [resource-based naming](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-naming.html). +{{% /alert %}} + + +### InstanceProfile + +The Karpenter `spec.instanceProfile` field has been removed from the EC2NodeClass in favor of the `spec.role` field. Karpenter is also removing support for the `defaultInstanceProfile` specified globally in the karpenter-global-settings, making the `spec.role` field required for all EC2NodeClasses. + +Karpenter will now auto-generate the instance profile in your EC2NodeClass, given the role that you specify. To find the role, type: + +```bash +export INSTANCE_PROFILE_NAME=KarpenterNodeInstanceProfile-bob-karpenter-demo +aws iam get-instance-profile --instance-profile-name $INSTANCE_PROFILE_NAME --query "InstanceProfile.Roles[0].RoleName" +KarpenterNodeRole-bob-karpenter-demo +``` + +**instanceProfile example (v1alpha)** + +```yaml +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +... +spec: + instanceProfile: KarpenterNodeInstanceProfile-karpenter-demo +``` + +**role example (v1beta1)** + +```yaml +apiVersion: karpenter.k8s.aws/v1beta1 +kind: EC2NodeClass +... +spec: + role: KarpenterNodeRole-karpenter-demo +``` + +### SubnetSelector, SecurityGroupSelector, and AMISelector + +Karpenter’s `spec.subnetSelector`, `spec.securityGroupSelector`, and `spec.amiSelector` fields have been modified to support multiple terms and to first-class keys like id and name. If using comma-delimited strings in your `tag`, `id`, or `name` values, you may need to create separate terms for the new fields. + + +**subnetSelector example (v1alpha)** + +```yaml +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +... +spec: + subnetSelector: + karpenter.sh/discovery: karpenter-demo +``` + +**SubnetSelectorTerms.tags example (v1beta1)** + +```yaml +apiVersion: karpenter.k8s.aws/v1beta1 +kind: EC2NodeClass +... +spec: + subnetSelectorTerms: + - tags: + karpenter.sh/discovery: karpenter-demo +``` + +**subnetSelector example (v1alpha)** + +```yaml +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +... +spec: + subnetSelector: + aws::ids: subnet-123,subnet-456 +``` + +**subnetSelectorTerms.id example (v1beta1)** + +```yaml +apiVersion: karpenter.k8s.aws/v1beta1 +kind: EC2NodeClass +... +spec: + subnetSelectorTerms: + - id: subnet-123 + - id: subnet-456 +``` + +**securityGroupSelector example (v1alpha)** + +```yaml +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +... +spec: + securityGroupSelector: + karpenter.sh/discovery: karpenter-demo +``` + +**securityGroupSelectorTerms.tags example (v1beta1)** + +```yaml +apiVersion: compute.k8s.aws/v1beta1 +kind: EC2NodeClass +... +spec: + securityGroupSelectorTerms: + - tags: + karpenter.sh/discovery: karpenter-demo +``` + +**securityGroupSelector example (v1alpha)** + +```yaml +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +... +spec: + securityGroupSelector: + aws::ids: sg-123, sg-456 +``` + + +**securityGroupSelectorTerms.id example (v1beta1)** + +```yaml +apiVersion: compute.k8s.aws/v1beta1 +kind: EC2NodeClass +... +spec: + securityGroupSelectorTerms: + - id: sg-123 + - id: sg-456 +``` + + +**amiSelector example (v1alpha)** + +```yaml +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +... +spec: + amiSelector: + karpenter.sh/discovery: karpenter-demo +``` + + +**amiSelectorTerms.tags example (v1beta1)** + +```yaml +apiVersion: compute.k8s.aws/v1beta1 +kind: EC2NodeClass +... +spec: + amiSelectorTerms: + - tags: + karpenter.sh/discovery: karpenter-demo +``` + +**amiSelector example (v1alpha)** + +```yaml +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +... +spec: + amiSelector: + aws::ids: ami-123,ami-456 +``` + +**amiSelectorTerms example (v1beta1)** + +```yaml +apiVersion: compute.k8s.aws/v1beta1 +kind: EC2NodeClass +... +spec: + amiSelectorTerms: + - id: ami-123 + - id: ami-456 +``` + + +**amiSelector example (v1alpha)** + +```yaml +apiVersion: karpenter.k8s.aws/v1alpha1 +kind: AWSNodeTemplate +... +spec: + amiSelector: + aws::name: my-name1,my-name2 + aws::owners: 123456789,amazon +``` + +**amiSelectorTerms.name example (v1beta1)** + +```yaml +apiVersion: compute.k8s.aws/v1beta1 +kind: EC2NodeClass +... +spec: + amiSelectorTerms: + - name: my-name1 + owner: 123456789 + - name: my-name2 + owner: 123456789 + - name: my-name1 + owner: amazon + - name: my-name2 + owner: amazon +``` + +### LaunchTemplateName + +The `spec.launchTemplateName` field for referencing unmanaged launch templates within Karpenter has been removed. +Find a discussion of the decision to remove `spec.launchTemplateName`, see [RFC: Unmanaged LaunchTemplate Removal](https://github.com/aws/karpenter/blob/main/designs/unmanaged-launch-template-removal.md). + +### AMIFamily + +The AMIFamily field is now required. If you were previously not specifying the AMIFamily field, having Karpenter default the AMIFamily to AL2, you will now have to specify AL2 explicitly. + +## Metrics + +The following table shows v1alpha5 metrics and the v1beta1 version of each metric. All metrics on this table will exist simultaneously, while both v1alpha5 and v1beta1 are supported within the same version. + +| **v1alpha5 Metric Name** | **v1beta1 Metric Name** | +|-----------------------------------------------------------------------|-----------------------------------------------------------------| +| karpenter_machines_created | karpenter_nodeclaims_created | +| karpenter_machines_disrupted | karpenter_nodeclaims_disrupted | +| karpenter_machines_drifted | karpenter_nodeclaims_drifted | +| karpenter_machines_initialized | karpenter_nodeclaims_initialized | +| karpenter_machines_launched | karpenter_nodeclaims_launched | +| karpenter_machines_registered | karpenter_nodeclaims_registered | +| karpenter_machines_terminated | karpenter_nodeclaims_terminated | +| karpenter_provisioners_limit | karpenter_nodepools_limit | +| karpenter_provisioners_usage | karpenter_nodepools_usage | +| karpenter_provisioners_usage_pct | Dropped | +| karpenter_deprovisioning_evaluation_duration_seconds | karpenter_disruption_evaluation_duration_seconds | +| karpenter_deprovisioning_eligible_machines | karpenter_disruption_eligible_nodeclaims | +| karpenter_deprovisioning_replacement_machine_initialized_seconds | karpenter_disruption_replacement_nodeclaims_initialized_seconds | +| karpenter_deprovisioning_replacement_machine_launch_failure_counter | karpenter_disruption_replacement_nodeclaims_launch_failed_total | +| karpenter_deprovisioning_actions_performed | karpenter_disruption_actions_performed_total | +| karpenter_deprovisioning_consolidation_timeouts | karpenter_disruption_consolidation_timeouts_total | +| karpenter_nodes_leases_deleted | karpenter_leases_deleted | + +In addition to these metrics, the MachineNotFound error returned by the `karpenter_cloudprovider_errors_total` values in the error label has been changed to `NodeClaimNotFound`. This is agnostic to the version of the API (Machine or NodeClaim) that actually owns the instance. + +## Global Settings + +The v1beta1 specification removes the `karpenter-global-settings` ConfigMap in favor of setting all Karpenter configuration using environment variables. Along, with this change, Karpenter has chosen to remove certain global variables that can be configured with more specificity in the EC2NodeClass . These values are marked as removed below. + + +| **`karpenter-global-settings` ConfigMap Key** | **Environment Variable** | **CLI Argument** +|---------------------------------------------------|---------------------------------|-------------------------------| +| batchMaxDuration | BATCH_MAX_DURATION | --batch-max-duration | +| batchIdleDuration | BATCH_IDLE_DURATION | --batch-idle-duration | +| assumeRoleARN | ASSUME_ROLE_ARN | --assume-role-arn | +| assumeRoleDuration | ASSUME_ROLE_DURATION | --assume-role-duration | +| clusterCABundle | CLUSTER_CA_BUNDLE | --cluster-ca-bundle | +| clusterName | CLUSTER_NAME | --cluster-name | +| clusterEndpoint | CLUSTER_ENDPOINT | --cluster-endpoint | +| defaultInstanceProfile | Dropped | Dropped | +| enablePodENI | Dropped | Dropped | +| enableENILimitedPodDensity | Dropped | Dropped | +| isolatedVPC | ISOLATED_VPC | --isolated-vpc | +| vmMemoryOverheadPercent | VM_MEMORY_OVERHEAD_PERCENT | --vm-memory-overhead-percent | +| interruptionQueueName | INTERRUPTION_QUEUE_NAME | --interruption-queue-name | +| reservedENIs | RESERVED_ENIS | --reserved-enis | +| featureGates.enableDrift | FEATURE_GATE="Drift=true" | --feature-gates Drift=true | + +## Drift Enabled by Default + +The drift feature will now be enabled by default starting from v0.33.0. If you don’t specify the Drift featureGate, the feature will be assumed to be enabled. You can disable the drift feature by specifying --feature-gates Drift=false. This feature gate is expected to be dropped when core APIs (NodePool, NodeClaim) are bumped to v1. + +## Logging Configuration is No Longer Dynamic + +As part of this deprecation, Karpenter will no longer call out to the APIServer to discover the ConfigMap. Instead, Karpenter will expect the ConfigMap to be mounted on the filesystem at `/etc/karpenter/logging/zap-logger-config`. You can also still choose to override the individual log level of components of the system (webhook and controller) at the paths `/etc/karpenter/logging/loglevel.webhook` and `/etc/karpenter/logging/loglevel.controller`. + +What you do to upgrade this feature depends on how you install Karpenter: + +* If you are using the helm chart to install Karpenter, you won’t need to make any changes for Karpenter to begin using this new mechanism for loading the config. + +* If you are manually configuring the deployment for Karpenter, you will need to add the following sections to your deployment: + + ```yaml + apiVersion: apps/v1 + kind: Deployment + spec: + template: + spec: + ... + containers: + - name: controller + volumeMounts: + - name: config-logging + mountPath: /etc/karpenter/logging + volumes: + - name: config-logging + configMap: + name: config-logging + ``` + +Karpenter will drop support for ConfigMap discovery through the APIServer starting in v0.33.0, meaning that you will need to ensure that you are mounting the config file on the expected filepath by that version. + +## Webhook Support Deprecated in Favor of CEL + +Karpenter v1beta1 APIs now support Common Expression Language (CEL) for validaiton directly through the APIServer. This change means that Karpenter’s validating webhooks are no longer needed to ensure that Karpenter’s NodePools and EC2NodeClasses are configured correctly. + +As a result, Karpenter will now disable webhooks by default by setting the `DISABLE_WEBHOOK` environment variable to `true` starting in v0.33.0. If you are currently on a version of Kubernetes < less than 1.25, CEL validation for Custom Resources is not enabled. We recommend that you enable the webhooks on these versions with `DISABLE_WEBHOOK=false` to get proper validation support for any Karpenter configuration. \ No newline at end of file diff --git a/website/content/en/v0.27/concepts/provisioners.md b/website/content/en/v0.27/concepts/provisioners.md index ef08e92881b1..9c55506c433c 100644 --- a/website/content/en/v0.27/concepts/provisioners.md +++ b/website/content/en/v0.27/concepts/provisioners.md @@ -56,7 +56,7 @@ spec: example.com/owner: "my-team" # Requirements that constrain the parameters of provisioned nodes. - # These requirements are combined with pod.spec.affinity.nodeAffinity rules. + # These requirements are combined with pod.spec.topologySpreadConstraints, pod.spec.affinity.nodeAffinity, pod.spec.affinity.podAffinity, and pod.spec.nodeSelector rules. # Operators { In, NotIn } are supported to enable including or excluding values requirements: - key: "karpenter.k8s.aws/instance-category" diff --git a/website/content/en/v0.28/concepts/provisioners.md b/website/content/en/v0.28/concepts/provisioners.md index 97221c52d2d0..0ae9f0c7f1bf 100644 --- a/website/content/en/v0.28/concepts/provisioners.md +++ b/website/content/en/v0.28/concepts/provisioners.md @@ -56,7 +56,7 @@ spec: example.com/owner: "my-team" # Requirements that constrain the parameters of provisioned nodes. - # These requirements are combined with pod.spec.affinity.nodeAffinity rules. + # These requirements are combined with pod.spec.topologySpreadConstraints, pod.spec.affinity.nodeAffinity, pod.spec.affinity.podAffinity, and pod.spec.nodeSelector rules. # Operators { In, NotIn } are supported to enable including or excluding values requirements: - key: "karpenter.k8s.aws/instance-category" diff --git a/website/content/en/v0.29/concepts/provisioners.md b/website/content/en/v0.29/concepts/provisioners.md index 8c25ea4540b0..186b50594962 100644 --- a/website/content/en/v0.29/concepts/provisioners.md +++ b/website/content/en/v0.29/concepts/provisioners.md @@ -58,7 +58,7 @@ spec: example.com/owner: "my-team" # Requirements that constrain the parameters of provisioned nodes. - # These requirements are combined with pod.spec.affinity.nodeAffinity rules. + # These requirements are combined with pod.spec.topologySpreadConstraints, pod.spec.affinity.nodeAffinity, pod.spec.affinity.podAffinity, and pod.spec.nodeSelector rules. # Operators { In, NotIn } are supported to enable including or excluding values requirements: - key: "karpenter.k8s.aws/instance-category" diff --git a/website/content/en/v0.30/concepts/provisioners.md b/website/content/en/v0.30/concepts/provisioners.md index 8c25ea4540b0..186b50594962 100644 --- a/website/content/en/v0.30/concepts/provisioners.md +++ b/website/content/en/v0.30/concepts/provisioners.md @@ -58,7 +58,7 @@ spec: example.com/owner: "my-team" # Requirements that constrain the parameters of provisioned nodes. - # These requirements are combined with pod.spec.affinity.nodeAffinity rules. + # These requirements are combined with pod.spec.topologySpreadConstraints, pod.spec.affinity.nodeAffinity, pod.spec.affinity.podAffinity, and pod.spec.nodeSelector rules. # Operators { In, NotIn } are supported to enable including or excluding values requirements: - key: "karpenter.k8s.aws/instance-category" diff --git a/website/content/en/v0.31/concepts/provisioners.md b/website/content/en/v0.31/concepts/provisioners.md index a65c84eee24e..5881eb6046f8 100644 --- a/website/content/en/v0.31/concepts/provisioners.md +++ b/website/content/en/v0.31/concepts/provisioners.md @@ -58,7 +58,7 @@ spec: example.com/owner: "my-team" # Requirements that constrain the parameters of provisioned nodes. - # These requirements are combined with pod.spec.affinity.nodeAffinity rules. + # These requirements are combined with pod.spec.topologySpreadConstraints, pod.spec.affinity.nodeAffinity, pod.spec.affinity.podAffinity, and pod.spec.nodeSelector rules. # Operators { In, NotIn, Exists, DoesNotExist, Gt, and Lt } are supported. # https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#operators requirements: diff --git a/website/static/_redirects b/website/static/_redirects index 239a5d444163..5655bc1466e9 100644 --- a/website/static/_redirects +++ b/website/static/_redirects @@ -8,3 +8,8 @@ /preview/concepts/provisioners /preview/concepts/nodepools /preview/concepts/node-templates /preview/concepts/nodeclasses /preview/concepts/deprovisioning /preview/concepts/disruption +/preview/upgrade/* /preview/upgrading/* +/preview/concepts/instance-types /preview/reference/instance-types +/preview/concepts/metrics /preview/reference/metrics +/preview/concepts/settings /preview/reference/settings +/preview/concepts/threat-model /preview/reference/threat-model