From 7014587a05a38750b8d0e1a48216e8684dd515e3 Mon Sep 17 00:00:00 2001 From: Jonathan Innis Date: Mon, 16 Oct 2023 09:42:02 -0700 Subject: [PATCH] docs: Update getting started guide to v1beta1 apis and concepts (#4839) --- .../en/preview/getting-started/_index.md | 2 +- .../getting-started-with-karpenter/_index.md | 23 +++------- .../scripts/step08-apply-helm-chart.sh | 5 +- .../scripts/step12-add-nodepool.sh | 46 +++++++++++++++++++ .../scripts/step12-add-provisioner.sh | 28 ----------- .../migrating-from-cas/_index.md | 33 ++++++------- ...roller-iam.sh => step04-controller-iam.sh} | 0 .../scripts/step04-instance-profile.sh | 6 --- ...6-tag-subnets.sh => step05-tag-subnets.sh} | 0 ...roups.sh => step06-tag-security-groups.sh} | 0 ...it-aws-auth.sh => step07-edit-aws-auth.sh} | 0 ...rate-chart.sh => step08-generate-chart.sh} | 3 +- .../{step10-deploy.sh => step09-deploy.sh} | 6 +-- .../scripts/step10-create-nodepool.sh | 46 +++++++++++++++++++ .../scripts/step11-create-provisioner.sh | 26 ----------- ...tep12-scale-cas.sh => step11-scale-cas.sh} | 0 ...iple-ng.sh => step12-scale-multiple-ng.sh} | 0 ...single-ng.sh => step12-scale-single-ng.sh} | 0 18 files changed, 120 insertions(+), 104 deletions(-) create mode 100755 website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step12-add-nodepool.sh delete mode 100755 website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step12-add-provisioner.sh rename website/content/en/preview/getting-started/migrating-from-cas/scripts/{step05-controller-iam.sh => step04-controller-iam.sh} (100%) delete mode 100644 website/content/en/preview/getting-started/migrating-from-cas/scripts/step04-instance-profile.sh rename website/content/en/preview/getting-started/migrating-from-cas/scripts/{step06-tag-subnets.sh => step05-tag-subnets.sh} (100%) rename website/content/en/preview/getting-started/migrating-from-cas/scripts/{step07-tag-security-groups.sh => step06-tag-security-groups.sh} (100%) rename website/content/en/preview/getting-started/migrating-from-cas/scripts/{step08-edit-aws-auth.sh => step07-edit-aws-auth.sh} (100%) rename website/content/en/preview/getting-started/migrating-from-cas/scripts/{step09-generate-chart.sh => step08-generate-chart.sh} (77%) rename website/content/en/preview/getting-started/migrating-from-cas/scripts/{step10-deploy.sh => step09-deploy.sh} (65%) create mode 100644 website/content/en/preview/getting-started/migrating-from-cas/scripts/step10-create-nodepool.sh delete mode 100644 website/content/en/preview/getting-started/migrating-from-cas/scripts/step11-create-provisioner.sh rename website/content/en/preview/getting-started/migrating-from-cas/scripts/{step12-scale-cas.sh => step11-scale-cas.sh} (100%) rename website/content/en/preview/getting-started/migrating-from-cas/scripts/{step13-scale-multiple-ng.sh => step12-scale-multiple-ng.sh} (100%) rename website/content/en/preview/getting-started/migrating-from-cas/scripts/{step13-scale-single-ng.sh => step12-scale-single-ng.sh} (100%) diff --git a/website/content/en/preview/getting-started/_index.md b/website/content/en/preview/getting-started/_index.md index 77e7a529548f..89a2ba78422c 100644 --- a/website/content/en/preview/getting-started/_index.md +++ b/website/content/en/preview/getting-started/_index.md @@ -12,7 +12,7 @@ cascade: To get started with Karpenter, the [Getting Started with Karpenter]({{< relref "getting-started-with-karpenter" >}}) guide provides an end-to-end procedure for creating a cluster (with `eksctl`) and adding Karpenter. If you prefer, the following instructions use Terraform to create a cluster and add Karpenter: -* [Amazon EKS Blueprints for Terraform](https://aws-ia.github.io/terraform-aws-eks-blueprints): Follow a basic [Getting Started](https://aws-ia.github.io/terraform-aws-eks-blueprints/v4.18.0/getting-started/) guide and also add modules and add-ons. This includes a [Karpenter](https://aws-ia.github.io/terraform-aws-eks-blueprints/v4.18.0/add-ons/karpenter/) add-on that lets you bypass the instructions in this guide for setting up Karpenter. +* [Amazon EKS Blueprints for Terraform](https://aws-ia.github.io/terraform-aws-eks-blueprints): Follow a basic [Getting Started](https://aws-ia.github.io/terraform-aws-eks-blueprints/getting-started/) guide and also add modules and add-ons. This includes a [Karpenter](https://aws-ia.github.io/terraform-aws-eks-blueprints/patterns/karpenter/) add-on that lets you bypass the instructions in this guide for setting up Karpenter. Although not supported, you could also try Karpenter on other Kubernetes distributions running on AWS. For example: diff --git a/website/content/en/preview/getting-started/getting-started-with-karpenter/_index.md b/website/content/en/preview/getting-started/getting-started-with-karpenter/_index.md index efdf7d929b38..924e9a74577d 100644 --- a/website/content/en/preview/getting-started/getting-started-with-karpenter/_index.md +++ b/website/content/en/preview/getting-started/getting-started-with-karpenter/_index.md @@ -92,32 +92,23 @@ See [Enabling Windows support](https://docs.aws.amazon.com/eks/latest/userguide/ Karpenter creates a mapping between CloudProvider machines and CustomResources in the cluster for capacity tracking. To ensure this mapping is consistent, Karpenter utilizes the following tag keys: * `karpenter.sh/managed-by` -* `karpenter.sh/provisioner-name` +* `karpenter.sh/nodepool` * `kubernetes.io/cluster/${CLUSTER_NAME}` Because Karpenter takes this dependency, any user that has the ability to Create/Delete these tags on CloudProvider machines will have the ability to orchestrate Karpenter to Create/Delete CloudProvider machines as a side effect. We recommend that you [enforce tag-based IAM policies](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_tags.html) on these tags against any EC2 instance resource (`i-*`) for any users that might have [CreateTags](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateTags.html)/[DeleteTags](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DeleteTags.html) permissions but should not have [RunInstances](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_RunInstances.html)/[TerminateInstances](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_TerminateInstances.html) permissions. {{% /alert %}} -### 5. Create Provisioner +### 5. Create NodePool -A single Karpenter provisioner is capable of handling many different pod -shapes. Karpenter makes scheduling and provisioning decisions based on pod -attributes such as labels and affinity. In other words, Karpenter eliminates -the need to manage many different node groups. +A single Karpenter NodePool is capable of handling many different pod shapes. Karpenter makes scheduling and provisioning decisions based on pod attributes such as labels and affinity. In other words, Karpenter eliminates the need to manage many different node groups. -Create a default provisioner using the command below. -This provisioner uses `securityGroupSelector` and `subnetSelector` to discover resources used to launch nodes. -We applied the tag `karpenter.sh/discovery` in the `eksctl` command above. -Depending how these resources are shared between clusters, you may need to use different tagging schemes. +Create a default NodePool using the command below. This NodePool uses `securityGroupSelectorTerms` and `subnetSelectorTerms` to discover resources used to launch nodes. We applied the tag `karpenter.sh/discovery` in the `eksctl` command above. Depending on how these resources are shared between clusters, you may need to use different tagging schemes. -The `consolidation` value configures Karpenter to reduce cluster cost by removing and replacing nodes. As a result, consolidation will terminate any empty nodes on the cluster. This behavior can be disabled by leaving the value undefined or setting `consolidation.enabled` to `false`. Review the [provisioner CRD]({{}}) for more information. +The `consolidationPolicy` set to `WhenUnderutilized` in the `disruption` block configures Karpenter to reduce cluster cost by removing and replacing nodes. As a result, consolidation will terminate any empty nodes on the cluster. This behavior can be disabled by setting `consolidateAfter` to `Never`, telling Karpenter that it should never consolidate nodes. Review the [NodePool API docs]({{}}) for more information. -Review the [provisioner CRD]({{}}) for more information. For example, -`ttlSecondsUntilExpired` configures Karpenter to terminate nodes when a maximum age is reached. +Note: This NodePool will create capacity as long as the sum of all created capacity is less than the specified limit. -Note: This provisioner will create capacity as long as the sum of all created capacity is less than the specified limit. - -{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step12-add-provisioner.sh" language="bash"%}} +{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step12-add-nodepool.sh" language="bash"%}} Karpenter is now active and ready to begin provisioning nodes. diff --git a/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step08-apply-helm-chart.sh b/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step08-apply-helm-chart.sh index 99fa927b12b6..b6c86db990e4 100755 --- a/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step08-apply-helm-chart.sh +++ b/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step08-apply-helm-chart.sh @@ -3,9 +3,8 @@ helm registry logout public.ecr.aws helm upgrade --install karpenter oci://public.ecr.aws/karpenter/karpenter --version ${KARPENTER_VERSION} --namespace karpenter --create-namespace \ --set serviceAccount.annotations."eks\.amazonaws\.com/role-arn"=${KARPENTER_IAM_ROLE_ARN} \ - --set settings.aws.clusterName=${CLUSTER_NAME} \ - --set settings.aws.defaultInstanceProfile=KarpenterNodeInstanceProfile-${CLUSTER_NAME} \ - --set settings.aws.interruptionQueueName=${CLUSTER_NAME} \ + --set settings.clusterName=${CLUSTER_NAME} \ + --set settings.interruptionQueueName=${CLUSTER_NAME} \ --set controller.resources.requests.cpu=1 \ --set controller.resources.requests.memory=1Gi \ --set controller.resources.limits.cpu=1 \ diff --git a/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step12-add-nodepool.sh b/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step12-add-nodepool.sh new file mode 100755 index 000000000000..8c518e9e74b8 --- /dev/null +++ b/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step12-add-nodepool.sh @@ -0,0 +1,46 @@ +cat <}} We can now generate a full Karpenter deployment yaml from the helm chart. -{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step09-generate-chart.sh" language="bash" %}} +{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step08-generate-chart.sh" language="bash" %}} Modify the following lines in the karpenter.yaml file. @@ -115,7 +111,7 @@ affinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - - key: karpenter.sh/provisioner-name + - key: karpenter.sh/nodepool operator: DoesNotExist - matchExpressions: - key: eks.amazonaws.com/nodegroup @@ -127,16 +123,15 @@ affinity: - topologyKey: "kubernetes.io/hostname" ``` -Now that our deployment is ready we can create the karpenter namespace, create the provisioner CRD, and then deploy the rest of the karpenter resources. +Now that our deployment is ready we can create the karpenter namespace, create the NodePool CRD, and then deploy the rest of the karpenter resources. -{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step10-deploy.sh" language="bash" %}} +{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step09-deploy.sh" language="bash" %}} -## Create default provisioner +## Create default NodePool -We need to create a default provisioner so Karpenter knows what types of nodes we want for unscheduled workloads. -You can refer to some of the [example provisioners](https://github.com/aws/karpenter/tree{{< githubRelRef >}}examples/provisioner) for specific needs. +We need to create a default NodePool so Karpenter knows what types of nodes we want for unscheduled workloads. You can refer to some of the [example NodePool](https://github.com/aws/karpenter/tree{{< githubRelRef >}}examples/v1beta1) for specific needs. -{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step11-create-provisioner.sh" language="bash" %}} +{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step10-create-nodepool.sh" language="bash" %}} ## Set nodeAffinity for critical workloads (optional) @@ -167,7 +162,7 @@ affinity: Now that karpenter is running we can disable the cluster autoscaler. To do that we will scale the number of replicas to zero. -{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step12-scale-cas.sh" language="bash" %}} +{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step11-scale-cas.sh" language="bash" %}} To get rid of the instances that were added from the node group we can scale our nodegroup down to a minimum size to support Karpenter and other critical services. @@ -175,11 +170,11 @@ To get rid of the instances that were added from the node group we can scale our If you have a single multi-AZ node group, we suggest a minimum of 2 instances. -{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step13-scale-single-ng.sh" language="bash" %}} +{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step12-scale-single-ng.sh" language="bash" %}} Or, if you have multiple single-AZ node groups, we suggest a minimum of 1 instance each. -{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step13-scale-multiple-ng.sh" language="bash" %}} +{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step12-scale-multiple-ng.sh" language="bash" %}} {{% alert title="Note" color="warning" %}} If you have a lot of nodes or workloads you may want to slowly scale down your node groups by a few instances at a time. It is recommended to watch the transition carefully for workloads that may not have enough replicas running or disruption budgets configured. diff --git a/website/content/en/preview/getting-started/migrating-from-cas/scripts/step05-controller-iam.sh b/website/content/en/preview/getting-started/migrating-from-cas/scripts/step04-controller-iam.sh similarity index 100% rename from website/content/en/preview/getting-started/migrating-from-cas/scripts/step05-controller-iam.sh rename to website/content/en/preview/getting-started/migrating-from-cas/scripts/step04-controller-iam.sh diff --git a/website/content/en/preview/getting-started/migrating-from-cas/scripts/step04-instance-profile.sh b/website/content/en/preview/getting-started/migrating-from-cas/scripts/step04-instance-profile.sh deleted file mode 100644 index 3c112c819588..000000000000 --- a/website/content/en/preview/getting-started/migrating-from-cas/scripts/step04-instance-profile.sh +++ /dev/null @@ -1,6 +0,0 @@ -aws iam create-instance-profile \ - --instance-profile-name "KarpenterNodeInstanceProfile-${CLUSTER_NAME}" - -aws iam add-role-to-instance-profile \ - --instance-profile-name "KarpenterNodeInstanceProfile-${CLUSTER_NAME}" \ - --role-name "KarpenterNodeRole-${CLUSTER_NAME}" diff --git a/website/content/en/preview/getting-started/migrating-from-cas/scripts/step06-tag-subnets.sh b/website/content/en/preview/getting-started/migrating-from-cas/scripts/step05-tag-subnets.sh similarity index 100% rename from website/content/en/preview/getting-started/migrating-from-cas/scripts/step06-tag-subnets.sh rename to website/content/en/preview/getting-started/migrating-from-cas/scripts/step05-tag-subnets.sh diff --git a/website/content/en/preview/getting-started/migrating-from-cas/scripts/step07-tag-security-groups.sh b/website/content/en/preview/getting-started/migrating-from-cas/scripts/step06-tag-security-groups.sh similarity index 100% rename from website/content/en/preview/getting-started/migrating-from-cas/scripts/step07-tag-security-groups.sh rename to website/content/en/preview/getting-started/migrating-from-cas/scripts/step06-tag-security-groups.sh diff --git a/website/content/en/preview/getting-started/migrating-from-cas/scripts/step08-edit-aws-auth.sh b/website/content/en/preview/getting-started/migrating-from-cas/scripts/step07-edit-aws-auth.sh similarity index 100% rename from website/content/en/preview/getting-started/migrating-from-cas/scripts/step08-edit-aws-auth.sh rename to website/content/en/preview/getting-started/migrating-from-cas/scripts/step07-edit-aws-auth.sh diff --git a/website/content/en/preview/getting-started/migrating-from-cas/scripts/step09-generate-chart.sh b/website/content/en/preview/getting-started/migrating-from-cas/scripts/step08-generate-chart.sh similarity index 77% rename from website/content/en/preview/getting-started/migrating-from-cas/scripts/step09-generate-chart.sh rename to website/content/en/preview/getting-started/migrating-from-cas/scripts/step08-generate-chart.sh index f2bc603e0eeb..bc66ee53e387 100644 --- a/website/content/en/preview/getting-started/migrating-from-cas/scripts/step09-generate-chart.sh +++ b/website/content/en/preview/getting-started/migrating-from-cas/scripts/step08-generate-chart.sh @@ -1,6 +1,5 @@ helm template karpenter oci://public.ecr.aws/karpenter/karpenter --version ${KARPENTER_VERSION} --namespace karpenter \ - --set settings.aws.defaultInstanceProfile=KarpenterNodeInstanceProfile-${CLUSTER_NAME} \ - --set settings.aws.clusterName=${CLUSTER_NAME} \ + --set settings.clusterName=${CLUSTER_NAME} \ --set serviceAccount.annotations."eks\.amazonaws\.com/role-arn"="arn:${AWS_PARTITION}:iam::${AWS_ACCOUNT_ID}:role/KarpenterControllerRole-${CLUSTER_NAME}" \ --set controller.resources.requests.cpu=1 \ --set controller.resources.requests.memory=1Gi \ diff --git a/website/content/en/preview/getting-started/migrating-from-cas/scripts/step10-deploy.sh b/website/content/en/preview/getting-started/migrating-from-cas/scripts/step09-deploy.sh similarity index 65% rename from website/content/en/preview/getting-started/migrating-from-cas/scripts/step10-deploy.sh rename to website/content/en/preview/getting-started/migrating-from-cas/scripts/step09-deploy.sh index 45322127f591..f35e9513d247 100644 --- a/website/content/en/preview/getting-started/migrating-from-cas/scripts/step10-deploy.sh +++ b/website/content/en/preview/getting-started/migrating-from-cas/scripts/step09-deploy.sh @@ -1,8 +1,8 @@ kubectl create namespace karpenter kubectl create -f \ - https://raw.githubusercontent.com/aws/karpenter/${KARPENTER_VERSION}/pkg/apis/crds/karpenter.sh_provisioners.yaml + https://raw.githubusercontent.com/aws/karpenter/${KARPENTER_VERSION}/pkg/apis/crds/karpenter.sh_nodepools.yaml kubectl create -f \ - https://raw.githubusercontent.com/aws/karpenter/${KARPENTER_VERSION}/pkg/apis/crds/karpenter.k8s.aws_awsnodetemplates.yaml + https://raw.githubusercontent.com/aws/karpenter/${KARPENTER_VERSION}/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml kubectl create -f \ - https://raw.githubusercontent.com/aws/karpenter/${KARPENTER_VERSION}/pkg/apis/crds/karpenter.sh_machines.yaml + https://raw.githubusercontent.com/aws/karpenter/${KARPENTER_VERSION}/pkg/apis/crds/karpenter.sh_nodeclaims.yaml kubectl apply -f karpenter.yaml diff --git a/website/content/en/preview/getting-started/migrating-from-cas/scripts/step10-create-nodepool.sh b/website/content/en/preview/getting-started/migrating-from-cas/scripts/step10-create-nodepool.sh new file mode 100644 index 000000000000..8c518e9e74b8 --- /dev/null +++ b/website/content/en/preview/getting-started/migrating-from-cas/scripts/step10-create-nodepool.sh @@ -0,0 +1,46 @@ +cat <