diff --git a/.github/actions/e2e/cleanup/action.yaml b/.github/actions/e2e/cleanup/action.yaml index 69cbd2471770..7237012066c6 100644 --- a/.github/actions/e2e/cleanup/action.yaml +++ b/.github/actions/e2e/cleanup/action.yaml @@ -17,17 +17,21 @@ inputs: description: "The git commit, tag, or branch to check out. Requires a corresponding Karpenter snapshot release" eksctl_version: description: "Version of eksctl to install" - default: v0.164.0 + default: v0.169.0 + private_cluster: + description: "Whether the cluster that has to be deleted is private or not. Valid values are 'true' or 'false'" + default: 'false' runs: using: "composite" steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: ref: ${{ inputs.git_ref }} - uses: ./.github/actions/e2e/install-eksctl with: version: ${{ inputs.eksctl_version }} - name: delete-cluster + if: ${{ inputs.private_cluster == 'false' }} shell: bash env: CLUSTER_NAME: ${{ inputs.cluster_name }} diff --git a/.github/actions/e2e/dump-logs/action.yaml b/.github/actions/e2e/dump-logs/action.yaml index 5eff21b1567c..c893323d56e4 100644 --- a/.github/actions/e2e/dump-logs/action.yaml +++ b/.github/actions/e2e/dump-logs/action.yaml @@ -17,7 +17,7 @@ runs: using: "composite" steps: - name: configure aws credentials - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 + uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 with: role-to-assume: arn:aws:iam::${{ inputs.account_id }}:role/${{ inputs.role }} aws-region: ${{ inputs.region }} diff --git a/.github/actions/e2e/install-helm/action.yaml b/.github/actions/e2e/install-helm/action.yaml index 43c807b1b9ac..bd141ae413ab 100644 --- a/.github/actions/e2e/install-helm/action.yaml +++ b/.github/actions/e2e/install-helm/action.yaml @@ -10,12 +10,9 @@ runs: - name: install helm shell: bash env: - VERSION: ${{ inputs.version }} + HELM_VERSION: ${{ inputs.version }} run: | - TEMPDIR=$(mktemp -d) - curl -fsSL -o "${TEMPDIR}/get_helm.sh" https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 - chmod 700 "${TEMPDIR}/get_helm.sh" - "${TEMPDIR}/get_helm.sh" --version "$VERSION" + ./test/hack/e2e_scripts/install_helm.sh - name: install helm-diff shell: bash run: | diff --git a/.github/actions/e2e/install-karpenter/action.yaml b/.github/actions/e2e/install-karpenter/action.yaml index 122214994bd8..137f6237bb26 100644 --- a/.github/actions/e2e/install-karpenter/action.yaml +++ b/.github/actions/e2e/install-karpenter/action.yaml @@ -21,19 +21,22 @@ inputs: required: true k8s_version: description: 'Version of Kubernetes to use for the launched cluster' - default: "1.28" + default: "1.29" git_ref: description: "The git commit, tag, or branch to check out. Requires a corresponding Karpenter snapshot release" + private_cluster: + description: "Whether the cluster is private or not. Valid values are 'true' or 'false'" + default: 'false' runs: using: "composite" steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: ref: ${{ inputs.git_ref }} - uses: ./.github/actions/e2e/install-helm with: version: v3.12.3 # Pinned to this version since v3.13.0 has issues with anonymous pulls: https://github.com/helm/helm/issues/12423 - # Label namespace to enforce security stranded and scrape prometheus metrics + # Label namespace to enforce security stranded and scrape prometheus metrics # https://kubernetes.io/docs/concepts/security/pod-security-standards/ - name: add labels to kube-system namespace shell: bash @@ -41,7 +44,7 @@ runs: kubectl label ns kube-system scrape=enabled --overwrite=true kubectl label ns kube-system pod-security.kubernetes.io/warn=restricted --overwrite=true - name: login to ecr via docker - uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0 + uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0 with: registry: ${{ inputs.ecr_account_id }}.dkr.ecr.${{ inputs.ecr_region }}.amazonaws.com logout: true @@ -53,47 +56,13 @@ runs: ACCOUNT_ID: ${{ inputs.account_id }} CLUSTER_NAME: ${{ inputs.cluster_name }} K8S_VERSION: ${{ inputs.k8s_version }} + PRIVATE_CLUSTER: ${{ inputs.private_cluster }} run: | - aws eks update-kubeconfig --name "$CLUSTER_NAME" - - # Parse minor version to determine whether to enable the webhooks - RELEASE_VERSION_MINOR="${K8S_VERSION#*.}" - WEBHOOK_ENABLED=false - if (( RELEASE_VRESION_MINOR < 25 )); then - WEBHOOK_ENABLED=true - fi - - # Remove service account annotation when dropping support for 1.23 - helm upgrade --install karpenter "oci://$ECR_ACCOUNT_ID.dkr.ecr.$ECR_REGION.amazonaws.com/karpenter/snapshot/karpenter" \ - -n kube-system \ - --version "v0-$(git rev-parse HEAD)" \ - --set webhook.enabled=${WEBHOOK_ENABLED} \ - --set serviceAccount.annotations."eks\.amazonaws\.com/role-arn"="arn:aws:iam::$ACCOUNT_ID:role/karpenter-irsa-$CLUSTER_NAME" \ - --set settings.clusterName="$CLUSTER_NAME" \ - --set settings.interruptionQueue="$CLUSTER_NAME" \ - --set settings.featureGates.spotToSpotConsolidation=true \ - --set controller.resources.requests.cpu=3 \ - --set controller.resources.requests.memory=3Gi \ - --set controller.resources.limits.cpu=3 \ - --set controller.resources.limits.memory=3Gi \ - --set serviceMonitor.enabled=true \ - --set serviceMonitor.additionalLabels.scrape=enabled \ - --set "serviceMonitor.endpointConfig.relabelings[0].targetLabel=clusterName" \ - --set "serviceMonitor.endpointConfig.relabelings[0].replacement=$CLUSTER_NAME" \ - --set "serviceMonitor.endpointConfig.relabelings[1].targetLabel=gitRef" \ - --set "serviceMonitor.endpointConfig.relabelings[1].replacement=$(git rev-parse HEAD)" \ - --set "serviceMonitor.endpointConfig.relabelings[2].targetLabel=mostRecentTag" \ - --set "serviceMonitor.endpointConfig.relabelings[2].replacement=$(git describe --abbrev=0 --tags)" \ - --set "serviceMonitor.endpointConfig.relabelings[3].targetLabel=commitsAfterTag" \ - --set "serviceMonitor.endpointConfig.relabelings[3].replacement=\"$(git describe --tags | cut -d '-' -f 2)\"" \ - --wait + ./test/hack/e2e_scripts/install_karpenter.sh - name: diff-karpenter shell: bash env: ECR_ACCOUNT_ID: ${{ inputs.ecr_account_id }} ECR_REGION: ${{ inputs.ecr_region }} run: | - helm diff upgrade --namespace kube-system \ - karpenter oci://$ECR_ACCOUNT_ID.dkr.ecr.$ECR_REGION.amazonaws.com/karpenter/snapshot/karpenter \ - --version v0-$(git rev-parse HEAD) \ - --reuse-values --three-way-merge --detailed-exitcode + ./test/hack/e2e_scripts/diff_karpenter.sh diff --git a/.github/actions/e2e/install-prometheus/action.yaml b/.github/actions/e2e/install-prometheus/action.yaml index 81678c404cf3..f80dd138c8a6 100644 --- a/.github/actions/e2e/install-prometheus/action.yaml +++ b/.github/actions/e2e/install-prometheus/action.yaml @@ -10,6 +10,9 @@ inputs: region: description: "Region to access AWS" required: true + prometheus_region: + description: "Prometheus region" + required: true cluster_name: description: 'Name of the cluster to be launched by eksctl' required: true @@ -18,10 +21,13 @@ inputs: required: true git_ref: description: "The git commit, tag, or branch to check out. Requires a corresponding Karpenter snapshot release" + private_cluster: + description: "Whether the cluster is private or not. Valid values are 'true' or 'false'" + default: 'false' runs: using: "composite" steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: ref: ${{ inputs.git_ref }} - uses: ./.github/actions/e2e/install-helm @@ -39,27 +45,11 @@ runs: - name: install prometheus shell: bash env: + PROMETHEUS_REGION: ${{ inputs.prometheus_region }} REGION: ${{ inputs.region }} WORKSPACE_ID: ${{ inputs.workspace_id }} ACCOUNT_ID: ${{ inputs.account_id }} CLUSTER_NAME: ${{ inputs.cluster_name }} + PRIVATE_CLUSTER: ${{ inputs.private_cluster }} run: | - # Remove service account annotation when dropping support for 1.23 - helm upgrade --install prometheus prometheus-community/kube-prometheus-stack \ - -n prometheus \ - -f ./.github/actions/e2e/install-prometheus/values.yaml \ - --set prometheus.prometheusSpec.remoteWrite[0].url=https://aps-workspaces.$REGION.amazonaws.com/workspaces/$WORKSPACE_ID/api/v1/remote_write \ - --set prometheus.prometheusSpec.remoteWrite[0].sigv4.region=$REGION \ - --set prometheus.serviceAccount.annotations."eks\.amazonaws\.com/role-arn"="arn:aws:iam::$ACCOUNT_ID:role/prometheus-irsa-$CLUSTER_NAME" \ - --set "kubelet.serviceMonitor.cAdvisorRelabelings[0].targetLabel=metrics_path" \ - --set "kubelet.serviceMonitor.cAdvisorRelabelings[0].action=replace" \ - --set "kubelet.serviceMonitor.cAdvisorRelabelings[0].sourceLabels[0]=__metrics_path__" \ - --set "kubelet.serviceMonitor.cAdvisorRelabelings[1].targetLabel=clusterName" \ - --set "kubelet.serviceMonitor.cAdvisorRelabelings[1].replacement=$CLUSTER_NAME" \ - --set "kubelet.serviceMonitor.cAdvisorRelabelings[2].targetLabel=gitRef" \ - --set "kubelet.serviceMonitor.cAdvisorRelabelings[2].replacement=$(git rev-parse HEAD)" \ - --set "kubelet.serviceMonitor.cAdvisorRelabelings[3].targetLabel=mostRecentTag" \ - --set "kubelet.serviceMonitor.cAdvisorRelabelings[3].replacement=$(git describe --abbrev=0 --tags)" \ - --set "kubelet.serviceMonitor.cAdvisorRelabelings[4].targetLabel=commitsAfterTag" \ - --set "kubelet.serviceMonitor.cAdvisorRelabelings[4].replacement=\"$(git describe --tags | cut -d '-' -f 2)\"" \ - --wait + ./test/hack/e2e_scripts/install_prometheus.sh \ No newline at end of file diff --git a/.github/actions/e2e/run-tests-private-cluster/action.yaml b/.github/actions/e2e/run-tests-private-cluster/action.yaml new file mode 100644 index 000000000000..64504b9b6d58 --- /dev/null +++ b/.github/actions/e2e/run-tests-private-cluster/action.yaml @@ -0,0 +1,159 @@ +name: RunTestsPrivateCluster +description: 'Installs Karpenter, Prometheus, runs tests on private cluster and performs clean up' +inputs: + account_id: + description: "Account ID to access AWS" + required: true + suite: + type: string + required: true + ecr_account_id: + description: "Account ID to access ECR Repository" + required: true + prometheus_workspace_id: + description: "Workspace ID for the Prometheus workspace" + required: true + metrics_region: + description: "Metrics region" + required: true + node_role: + description: "Private cluster node role" + required: true + region: + description: "Region to access AWS" + required: true + ecr_region: + description: "Region to access ECR Repository" + required: true + prometheus_region: + description: Region to access Prometheus + required: true + cluster_name: + description: 'Name of the cluster to be launched by eksctl' + required: true + k8s_version: + description: 'Version of Kubernetes to use for the launched cluster' + default: "1.29" + private_cluster: + description: "Whether to create a private cluster which does not add access to the public internet. Valid values are 'true' or 'false'" + default: 'false' + enable_metrics: + description: "Whether to enable metrics for the cluster" + default: 'false' + codebuild_sg: + description: "Codebuild security group to run private cluster tests" + required: true + codebuild_vpc: + description: "Codebuild VPC to run private cluster tests" + required: true + cleanup: + description: "Whether to cleanup resources on failure" + default: 'false' +runs: + using: "composite" + steps: + - name: login to ecr via docker + uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # v3.1.0 + with: + registry: ${{ inputs.account_id }}.dkr.ecr.${{ inputs.region }}.amazonaws.com + logout: true + - name: configure private cluster + if: ${{ inputs.private_cluster }} + shell: bash + env: + REGION: ${{ inputs.region }} + CLUSTER_NAME: ${{ inputs.cluster_name }} + ACCOUNT_ID: ${{ inputs.account_id }} + REPOSITORY: ${{ github.repository }} + RUN_ID: ${{ github.run_id }} + CODEBUILD_SG: ${{ inputs.codebuild_sg }} + CODEBUILD_VPC: ${{ inputs.codebuild_vpc }} + run: | + ./test/hack/e2e_scripts/configure_private_cluster.sh + - name: run private cluster tests on codebuild + env: + SUITE: ${{ inputs.suite }} + CLUSTER_NAME: ${{ inputs.cluster_name }} + INTERRUPTION_QUEUE: ${{ inputs.cluster_name }} + REGION: ${{ inputs.region }} + HELM_VERSION: v3.12.3 # Pinned to this version since v3.13.0 has issues with anonymous pulls: https://github.com/helm/helm/issues/12423 + PROMETHEUS_REGION: ${{ inputs.prometheus_region }} + WORKSPACE_ID: ${{ inputs.prometheus_workspace_id }} + ACCOUNT_ID: ${{ inputs.account_id }} + K8S_VERSION: ${{ inputs.k8s_version }} + ECR_ACCOUNT_ID: ${{ inputs.ecr_account_id }} + ECR_REGION: ${{ inputs.ecr_region }} + PRIVATE_CLUSTER: ${{ inputs.private_cluster }} + ENABLE_METRICS: ${{ inputs.enable_metrics }} + METRICS_REGION: ${{ inputs.metrics_region }} + VPC_PEERING_CONNECTION_ID: ${{ env.VPC_PEERING_CONNECTION_ID }} + NODE_ROLE: ${{ env.NODE_ROLE }} + SG_CB: ${{ inputs.codebuild_sg }} + VPC_CB: ${{ inputs.codebuild_vpc }} + CLUSTER_VPC_ID: ${{ env.CLUSTER_VPC_ID }} + EKS_CLUSTER_SG: ${{ env.EKS_CLUSTER_SG }} + CLEANUP: ${{ inputs.cleanup }} + uses: aws-actions/aws-codebuild-run-build@b0a7ca5730725c01b45af8866100667e32e0a9b1 #v1.0.15 + with: + project-name: E2EPrivateClusterCodeBuildProject-us-east-1 + buildspec-override: | + version: 0.2 + phases: + install: + commands: + # Make sure goenv is up to date + - cd $HOME/.goenv && git pull --ff-only && cd - + # Install Go 1.22 + - goenv install 1.22 && goenv global 1.22 + build: + commands: + - aws eks update-kubeconfig --name $CLUSTER_NAME + - ./test/hack/e2e_scripts/noderole_bootstrap_permission.sh + - ./test/hack/e2e_scripts/install_helm.sh + - helm plugin install https://github.com/databus23/helm-diff || true + - aws ecr get-login-password --region $REGION | docker login --username AWS --password-stdin $ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com + - helm repo add prometheus-community https://prometheus-community.github.io/helm-charts + - helm pull prometheus-community/kube-prometheus-stack + - kubectl create ns prometheus || true + - kubectl label ns prometheus scrape=enabled --overwrite=true + - ./test/hack/e2e_scripts/install_prometheus.sh + - kubectl label ns kube-system scrape=enabled --overwrite=true + - kubectl label ns kube-system pod-security.kubernetes.io/warn=restricted --overwrite=true + - ./test/hack/e2e_scripts/install_karpenter.sh + - ./test/hack/e2e_scripts/diff_karpenter.sh + - kubectl delete nodepool --all + - kubectl delete ec2nodeclass --all + - kubectl delete deployment --all + - PRIVATE_CLUSTER=$CLUSTER_NAME TEST_SUITE=$SUITE ENABLE_METRICS=$ENABLE_METRICS METRICS_REGION=$METRICS_REGION GIT_REF="$(git rev-parse HEAD)" CLUSTER_NAME=$CLUSTER_NAME CLUSTER_ENDPOINT="$(aws eks describe-cluster --name $CLUSTER_NAME --query "cluster.endpoint" --output text)" INTERRUPTION_QUEUE=$CLUSTER_NAME make e2etests + post_build: + commands: + # Describe karpenter pods + - kubectl describe pods -n kube-system -l app.kubernetes.io/name=karpenter + # Describe nodes + - kubectl describe nodes + - | + if [ "${CLEANUP}" = true ]; then + ./test/hack/e2e_scripts/clean_private_cluster.sh + fi + env-vars-for-codebuild: | + SUITE, + CLUSTER_NAME, + INTERRUPTION_QUEUE, + REGION, + HELM_VERSION, + PROMETHEUS_REGION, + WORKSPACE_ID, + ACCOUNT_ID, + K8S_VERSION, + ECR_ACCOUNT_ID, + ECR_REGION, + PRIVATE_CLUSTER, + ENABLE_METRICS, + METRICS_REGION, + VPC_PEERING_CONNECTION_ID, + NODE_ROLE, + SG_CB, + VPC_CB, + CLUSTER_VPC_ID, + EKS_CLUSTER_SG, + CLEANUP \ No newline at end of file diff --git a/.github/actions/e2e/setup-cluster/action.yaml b/.github/actions/e2e/setup-cluster/action.yaml index 9ba803bd31ab..c829662294f0 100644 --- a/.github/actions/e2e/setup-cluster/action.yaml +++ b/.github/actions/e2e/setup-cluster/action.yaml @@ -27,10 +27,10 @@ inputs: required: true k8s_version: description: 'Version of Kubernetes to use for the launched cluster' - default: "1.28" + default: "1.29" eksctl_version: description: "Version of eksctl to install" - default: v0.165.0 + default: v0.169.0 ip_family: description: "IP Family of the cluster. Valid values are IPv4 or IPv6" default: "IPv4" @@ -39,15 +39,18 @@ inputs: default: 'false' git_ref: description: "The git commit, tag, or branch to check out. Requires a corresponding Karpenter snapshot release" - required: false enable_local_zones: description: "Whether to include local zones in the VPC created for the cluster." - required: false default: 'false' + cleanup: + description: "Whether to cleanup resources on failure" + default: 'false' + codebuild_role: + description: "Codebuild Role that must be given an access entry in case of private cluster" runs: using: "composite" steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: ref: ${{ inputs.git_ref }} - uses: ./.github/actions/e2e/install-eksctl @@ -86,6 +89,8 @@ runs: PRIVATE_CLUSTER: ${{ inputs.private_cluster }} GIT_REF: ${{ inputs.git_ref }} ENABLE_LOCAL_ZONES: ${{ inputs.enable_local_zones }} + CLEANUP: ${{ inputs.cleanup }} + CODEBUILD_ROLE: ${{ inputs.codebuild_role }} run: | if [[ "$GIT_REF" == '' ]]; then GIT_REF=$(git rev-parse HEAD) @@ -93,7 +98,7 @@ runs: # Disable Pod Identity for Karpenter on K8s 1.23. Pod Identity is not supported on K8s 1.23 # https://docs.aws.amazon.com/eks/latest/userguide/pod-identities.html#pod-id-considerations - if [[ "$K8S_VERSION" == '1.23' ]]; then + if [[ "$K8S_VERSION" == '1.23' ]] || [[ "$PRIVATE_CLUSTER" == 'true' ]]; then KARPENTER_IAM=""" - metadata: name: karpenter @@ -193,27 +198,18 @@ runs: if [[ $PRIVATE_CLUSTER == 'true' ]]; then yq -i '.privateCluster.enabled=true' clusterconfig.yaml yq -i '.managedNodeGroups[0].privateNetworking=true' clusterconfig.yaml + yq -i '.accessConfig.authenticationMode="API_AND_CONFIG_MAP"' clusterconfig.yaml + CODEBUILD_ROLE_ARN="arn:aws:iam::$ACCOUNT_ID:role/$CODEBUILD_ROLE" + yq -i ".accessConfig.accessEntries[0].principalARN=\"$CODEBUILD_ROLE_ARN\"" clusterconfig.yaml + yq -i '.accessConfig.accessEntries[0].accessPolicies[0].policyARN="arn:aws:eks::aws:cluster-access-policy/AmazonEKSClusterAdminPolicy"' clusterconfig.yaml + yq -i '.accessConfig.accessEntries[0].accessPolicies[0].accessScope.type="cluster"' clusterconfig.yaml fi - eksctl ${cmd} cluster -f clusterconfig.yaml - - # Add the SQS and SSM VPC endpoints if we are creating a private cluster - # We need to grab all of the VPC details for the cluster in order to add the endpoint - if [[ $PRIVATE_CLUSTER == 'true' ]]; then - VPC_CONFIG=$(aws eks describe-cluster --name "$CLUSTER_NAME" --query "cluster.resourcesVpcConfig") - VPC_ID=$(echo $VPC_CONFIG | jq .vpcId -r) - SUBNET_IDS=($(echo $VPC_CONFIG | jq '.subnetIds | join(" ")' -r)) - SECURITY_GROUP_IDS=($(echo $VPC_CONFIG | jq '.securityGroupIds | join(" ")' -r)) - - for SERVICE in "com.amazonaws.$REGION.ssm" "com.amazonaws.$REGION.sqs"; do - aws ec2 create-vpc-endpoint \ - --vpc-id "${VPC_ID}" \ - --vpc-endpoint-type Interface \ - --service-name "${SERVICE}" \ - --subnet-ids ${SUBNET_IDS[@]} \ - --security-group-ids ${SECURITY_GROUP_IDS[@]} \ - --tags "testing/type=e2e" "testing/cluster=$CLUSTER_NAME" "github.com/run-url=https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" "karpenter.sh/discovery=$CLUSTER_NAME" - done + # Disable rollback of the CloudFormation on Create if we aren't cleaning up the run + if [[ $CLEANUP == 'false' ]] && [[ $cmd == 'create' ]]; then + eksctl ${cmd} cluster -f clusterconfig.yaml --cfn-disable-rollback + else + eksctl ${cmd} cluster -f clusterconfig.yaml fi - name: tag oidc provider of the cluster if: always() @@ -231,18 +227,13 @@ runs: aws iam tag-open-id-connect-provider --open-id-connect-provider-arn $arn \ --tags Key=test/git_ref,Value=$GIT_REF Key=testing/type,Value=e2e Key=testing/cluster,Value=$CLUSTER_NAME Key=github.com/run-url,Value=https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} - name: give KarpenterNodeRole permission to bootstrap + if: ${{ inputs.private_cluster == 'false' }} shell: bash env: ACCOUNT_ID: ${{ inputs.account_id }} CLUSTER_NAME: ${{ inputs.cluster_name }} run: | - eksctl create iamidentitymapping \ - --username system:node:{{EC2PrivateDNSName}} \ - --cluster "$CLUSTER_NAME" \ - --arn "arn:aws:iam::$ACCOUNT_ID:role/KarpenterNodeRole-$CLUSTER_NAME" \ - --group system:bootstrappers \ - --group system:nodes \ - --group eks:kube-proxy-windows + ./test/hack/e2e_scripts/noderole_bootstrap_permission.sh - name: cloudformation describe stack events shell: bash if: failure() @@ -255,15 +246,18 @@ runs: aws cloudformation describe-stack-events --stack-name $stack_name done - name: install prometheus + if: ${{ inputs.private_cluster == 'false' }} uses: ./.github/actions/e2e/install-prometheus with: account_id: ${{ inputs.account_id }} role: ${{ inputs.role }} - region: ${{ inputs.prometheus_region }} + prometheus_region: ${{ inputs.prometheus_region }} + region: ${{ inputs.region }} cluster_name: ${{ inputs.cluster_name }} workspace_id: ${{ inputs.prometheus_workspace_id }} git_ref: ${{ inputs.git_ref }} - name: install karpenter + if: ${{ inputs.private_cluster == 'false' }} uses: ./.github/actions/e2e/install-karpenter with: account_id: ${{ inputs.account_id }} @@ -274,3 +268,4 @@ runs: cluster_name: ${{ inputs.cluster_name }} k8s_version: ${{ inputs.k8s_version }} git_ref: ${{ inputs.git_ref }} + private_cluster: ${{ inputs.private_cluster }} diff --git a/.github/actions/e2e/slack/notify/action.yaml b/.github/actions/e2e/slack/notify/action.yaml index 983125610034..3fadcd90954f 100644 --- a/.github/actions/e2e/slack/notify/action.yaml +++ b/.github/actions/e2e/slack/notify/action.yaml @@ -12,21 +12,26 @@ inputs: required: true git_ref: description: "The git commit, tag, or branch to check out. Requires a corresponding Karpenter snapshot release" + workflow_trigger: + description: "Workflow trigger for this run" runs: using: "composite" steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: ref: ${{ inputs.git_ref }} - id: get-run-name shell: bash env: SUITE: ${{ inputs.suite }} + WORKFLOW: ${{ inputs.workflow_trigger }} run: | - if [[ ${{ github.event_name }} == "schedule" && inputs.suite != "soak" ]]; then - RUN_NAME="$SUITE-periodic" - elif [[ ${{ github.event_name }} == "schedule" ]]; then + if [[ ${{ github.event_name }} == "schedule" && "$WORKFLOW" == 'private_cluster' ]]; then + RUN_NAME="privateCluster-periodic" + elif [[ ${{ github.event_name }} == "schedule" && "$WORKFLOW" == 'soak' ]]; then RUN_NAME="soak-periodic" + elif [[ ${{ github.event_name }} == "schedule" ]]; then + RUN_NAME="$SUITE-periodic" else RUN_NAME="$SUITE-${GITHUB_SHA::7}" fi diff --git a/.github/actions/e2e/upgrade-crds/action.yaml b/.github/actions/e2e/upgrade-crds/action.yaml index 168319c49f90..23c28d7fb1c5 100644 --- a/.github/actions/e2e/upgrade-crds/action.yaml +++ b/.github/actions/e2e/upgrade-crds/action.yaml @@ -19,12 +19,12 @@ runs: using: "composite" steps: - name: configure aws credentials - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 + uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 with: role-to-assume: arn:aws:iam::${{ inputs.account_id }}:role/${{ inputs.role }} aws-region: ${{ inputs.region }} role-duration-seconds: 21600 - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: ref: ${{ inputs.git_ref }} - name: install-karpenter diff --git a/.github/actions/install-deps/action.yaml b/.github/actions/install-deps/action.yaml index 04434ecfec41..f0dd71b3b83c 100644 --- a/.github/actions/install-deps/action.yaml +++ b/.github/actions/install-deps/action.yaml @@ -3,11 +3,12 @@ description: 'Installs Go Downloads and installs Karpenter Dependencies' inputs: k8sVersion: description: Kubernetes version to use when installing the toolchain - default: "1.28.x" + default: "1.29.x" runs: using: "composite" steps: - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 + id: setup-go with: go-version-file: go.mod check-latest: true @@ -15,13 +16,15 @@ runs: # Root path permission workaround for caching https://github.com/actions/cache/issues/845#issuecomment-1252594999 - run: sudo chown "$USER" /usr/local shell: bash - - uses: actions/cache@13aacd865c20de90d75de3b17ebe84f7a17d57d2 # v4.0.0 + - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 id: cache-toolchain with: path: | /usr/local/kubebuilder/bin ~/go/bin - key: ${{ runner.os }}-${{ inputs.k8sVersion }}-toolchain-cache-${{ hashFiles('hack/toolchain.sh') }} + # Added go version to compensate for this issue with govulncheck: https://github.com/golang/go/issues/65590. Could re-evaluate if this is necessary once the + # upstream go issue is corrected and if this is causing too many cache misses. + key: ${{ runner.os }}-${{ inputs.k8sVersion }}-${{ steps.setup-go.outputs.go-version }}-toolchain-cache-${{ hashFiles('hack/toolchain.sh') }} - if: ${{ steps.cache-toolchain.outputs.cache-hit != 'true' }} shell: bash env: diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml index 0380b07dfb24..025117413f37 100644 --- a/.github/dependabot.yaml +++ b/.github/dependabot.yaml @@ -122,6 +122,14 @@ updates: action-deps: patterns: - '*' + - package-ecosystem: github-actions + directory: .github/actions/e2e/run-tests-private-cluster + schedule: + interval: weekly + groups: + action-deps: + patterns: + - '*' - package-ecosystem: github-actions directory: .github/actions/e2e/setup-cluster schedule: diff --git a/.github/workflows/approval-comment.yaml b/.github/workflows/approval-comment.yaml index cfa51b069b28..34dfcafe9c05 100644 --- a/.github/workflows/approval-comment.yaml +++ b/.github/workflows/approval-comment.yaml @@ -7,7 +7,7 @@ jobs: if: startsWith(github.event.review.body, '/karpenter snapshot') || startsWith(github.event.review.body, '/karpenter scale') || startsWith(github.event.review.body, '/karpenter versionCompatibility') runs-on: ubuntu-latest steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: fetch-depth: 0 - name: Save info about the review comment as an artifact for other workflows that run on workflow_run to download them @@ -19,7 +19,7 @@ jobs: mkdir -p /tmp/artifacts { echo "$REVIEW_BODY"; echo "$PULL_REQUEST_NUMBER"; echo "$COMMIT_ID"; } >> /tmp/artifacts/metadata.txt cat /tmp/artifacts/metadata.txt - - uses: actions/upload-artifact@694cdabd8bdb0f10b2cea11669e1bf5453eed0a6 # v4.2.0 + - uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 with: name: artifacts path: /tmp/artifacts diff --git a/.github/workflows/ci-test.yaml b/.github/workflows/ci-test.yaml index 8c46f1416c08..58cfdb9b204b 100644 --- a/.github/workflows/ci-test.yaml +++ b/.github/workflows/ci-test.yaml @@ -9,18 +9,19 @@ on: jobs: ci-test: runs-on: ubuntu-latest + if: github.repository == 'aws/karpenter-provider-aws' strategy: matrix: - k8sVersion: ["1.23.x", "1.24.x", "1.25.x", "1.26.x", "1.27.x", "1.28.x"] + k8sVersion: ["1.23.x", "1.24.x", "1.25.x", "1.26.x", "1.27.x", "1.28.x", "1.29.x"] steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - uses: ./.github/actions/install-deps with: k8sVersion: ${{ matrix.k8sVersion }} - run: K8S_VERSION=${{ matrix.k8sVersion }} make ci-test - name: Send coverage # should only send converage once https://docs.coveralls.io/parallel-builds - if: matrix.k8sVersion == '1.28.x' + if: matrix.k8sVersion == '1.29.x' env: COVERALLS_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: goveralls -coverprofile=coverage.out -service=github diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a967a8521f7d..55fcdb828096 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -9,8 +9,9 @@ on: jobs: ci: runs-on: ubuntu-latest + if: github.repository == 'aws/karpenter-provider-aws' steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - uses: ./.github/actions/install-deps - name: Enable the actionlint matcher run: echo "::add-matcher::.github/actionlint-matcher.json" diff --git a/.github/workflows/codegen.yaml b/.github/workflows/codegen.yaml index 326956c3243e..4786e7b3542c 100644 --- a/.github/workflows/codegen.yaml +++ b/.github/workflows/codegen.yaml @@ -12,14 +12,14 @@ jobs: if: github.repository == 'aws/karpenter-provider-aws' runs-on: ubuntu-latest steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - uses: ./.github/actions/install-deps - run: | git config user.name "APICodeGen" git config user.email "APICodeGen@users.noreply.github.com" git remote set-url origin https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }} git config pull.rebase false - - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 + - uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 with: role-to-assume: 'arn:aws:iam::${{ vars.READONLY_ACCOUNT_ID }}:role/${{ vars.READONLY_ROLE_NAME }}' aws-region: ${{ vars.READONLY_REGION }} diff --git a/.github/workflows/codeql-analysis.yaml b/.github/workflows/codeql-analysis.yaml index e6f06b545401..c78fc63f866d 100644 --- a/.github/workflows/codeql-analysis.yaml +++ b/.github/workflows/codeql-analysis.yaml @@ -10,12 +10,13 @@ on: jobs: analyze-go: name: Analyze Go + if: github.repository == 'aws/karpenter-provider-aws' runs-on: ubuntu-latest permissions: actions: read # github/codeql-action/init@v2 security-events: write # github/codeql-action/init@v2 steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - uses: ./.github/actions/install-deps - run: make vulncheck - uses: github/codeql-action/init@df32e399139a3050671466d7d9b3cbacc1cfd034 # v2.22.8 @@ -27,12 +28,13 @@ jobs: # https://github.blog/2023-08-09-four-tips-to-keep-your-github-actions-workflows-secure/#2-enable-code-scanning-for-workflows analyze-github-actions: name: Analyze Github Actions + if: github.repository == 'aws/karpenter-provider-aws' runs-on: ubuntu-latest permissions: actions: read # github/codeql-action/init@v2 security-events: write # github/codeql-action/init@v2 steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - uses: github/codeql-action/init@df32e399139a3050671466d7d9b3cbacc1cfd034 # v2.22.8 with: languages: javascript diff --git a/.github/workflows/deflake.yaml b/.github/workflows/deflake.yaml deleted file mode 100644 index 1fba63ef89d1..000000000000 --- a/.github/workflows/deflake.yaml +++ /dev/null @@ -1,27 +0,0 @@ -name: Deflake -on: - schedule: - - cron: '0 12 * * *' - workflow_dispatch: -jobs: - deflake: - if: github.repository == 'aws/karpenter-provider-aws' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - uses: ./.github/actions/install-deps - - name: Running tests 5 times to find flaky tests - id: run-deflake - run: make deflake - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - if: always() - name: Post commit status - run: | - curl \ - -X POST \ - -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}"\ - -H "X-GitHub-Api-Version: 2022-11-28" \ - https://api.github.com/repos/${{ github.repository }}/statuses/${{ github.sha }} \ - -d '{"state":"${{ steps.run-deflake.outcome }}","target_url":"${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}","context":"Deflake Tests (cron)"}' diff --git a/.github/workflows/dryrun-gen-pr.yaml b/.github/workflows/dryrun-gen-pr.yaml new file mode 100644 index 000000000000..13e2832ad888 --- /dev/null +++ b/.github/workflows/dryrun-gen-pr.yaml @@ -0,0 +1,13 @@ +name: DryRunGenPR +on: + pull_request: + workflow_dispatch: +jobs: + dryrun-gen: + if: github.repository == 'aws/karpenter-provider-aws' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + - run: make prepare-website + env: + GIT_TAG: v0.10000.0 # Mock version for testing website generation diff --git a/.github/workflows/docgen.yaml b/.github/workflows/dryrun-gen.yaml similarity index 62% rename from .github/workflows/docgen.yaml rename to .github/workflows/dryrun-gen.yaml index bcb6145f6221..b18c62b3697e 100644 --- a/.github/workflows/docgen.yaml +++ b/.github/workflows/dryrun-gen.yaml @@ -1,19 +1,19 @@ -name: DocGenCI +name: DryRunGen on: push: branches: - 'main' - 'release-v*' jobs: - docgen-ci: + dryrun-gen: permissions: id-token: write # aws-actions/configure-aws-credentials@v4.0.1 if: github.repository == 'aws/karpenter-provider-aws' runs-on: ubuntu-latest steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - uses: ./.github/actions/install-deps - - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 + - uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 with: role-to-assume: 'arn:aws:iam::${{ vars.READONLY_ACCOUNT_ID }}:role/${{ vars.READONLY_ROLE_NAME }}' aws-region: ${{ vars.READONLY_REGION }} @@ -21,3 +21,6 @@ jobs: - run: make codegen env: ENABLE_GIT_PUSH: false + - run: make prepare-website + env: + GIT_TAG: v0.10000.0 # Mock version for testing website generation diff --git a/.github/workflows/e2e-cleanup.yaml b/.github/workflows/e2e-cleanup.yaml index d2f2be1e43c6..a5844c6331bf 100644 --- a/.github/workflows/e2e-cleanup.yaml +++ b/.github/workflows/e2e-cleanup.yaml @@ -21,11 +21,11 @@ jobs: name: cleanup-${{ inputs.cluster_name }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: ref: ${{ inputs.git_ref }} - name: configure aws credentials - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 + uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 with: role-to-assume: arn:aws:iam::${{ vars.CI_ACCOUNT_ID }}:role/${{ vars.CI_ROLE_NAME }} aws-region: ${{ inputs.region }} @@ -38,4 +38,4 @@ jobs: region: ${{ inputs.region }} cluster_name: ${{ inputs.cluster_name }} git_ref: ${{ inputs.git_ref }} - eksctl_version: v0.165.0 + eksctl_version: v0.169.0 diff --git a/.github/workflows/e2e-matrix.yaml b/.github/workflows/e2e-matrix.yaml index 60c37b9f6f7c..d0cb69569e10 100644 --- a/.github/workflows/e2e-matrix.yaml +++ b/.github/workflows/e2e-matrix.yaml @@ -7,7 +7,7 @@ on: default: "us-east-2" k8s_version: type: string - default: "1.28" + default: "1.29" cleanup: type: boolean required: true @@ -40,7 +40,8 @@ on: - "1.26" - "1.27" - "1.28" - default: "1.28" + - "1.29" + default: "1.29" cleanup: type: boolean required: true diff --git a/.github/workflows/e2e-private-cluster-trigger.yaml b/.github/workflows/e2e-private-cluster-trigger.yaml new file mode 100644 index 000000000000..970539aad33b --- /dev/null +++ b/.github/workflows/e2e-private-cluster-trigger.yaml @@ -0,0 +1,19 @@ +name: E2EPrivateClusterTrigger +on: + schedule: + - cron: '7 0 * * 4' +jobs: + private-cluster-trigger: + if: github.repository == 'aws/karpenter-provider-aws' + permissions: + id-token: write # aws-actions/configure-aws-credentials@v4.0.1 + statuses: write # ./.github/actions/commit-status/start + uses: ./.github/workflows/e2e.yaml + with: + suite: Integration + region: us-east-1 + workflow_trigger: "private_cluster" + cleanup: true + codebuild_region: US_EAST_1 + secrets: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} diff --git a/.github/workflows/e2e-soak-trigger.yaml b/.github/workflows/e2e-soak-trigger.yaml index f334db8a8cf2..862473a2c2dd 100644 --- a/.github/workflows/e2e-soak-trigger.yaml +++ b/.github/workflows/e2e-soak-trigger.yaml @@ -11,9 +11,9 @@ jobs: outputs: PREEXISTING_CLUSTERS: ${{ steps.list_clusters.outputs.PREEXISTING_CLUSTERS }} steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - name: configure aws credentials - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a + uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 with: role-to-assume: arn:aws:iam::${{ vars.CI_ACCOUNT_ID }}:role/${{ vars.CI_ROLE_NAME }} aws-region: eu-north-1 diff --git a/.github/workflows/e2e-upgrade.yaml b/.github/workflows/e2e-upgrade.yaml index bef22611e612..0ad5a6e3f3e0 100644 --- a/.github/workflows/e2e-upgrade.yaml +++ b/.github/workflows/e2e-upgrade.yaml @@ -22,7 +22,8 @@ on: - "1.26" - "1.27" - "1.28" - default: "1.28" + - "1.29" + default: "1.29" cleanup: required: true default: true @@ -39,7 +40,7 @@ on: default: "us-east-2" k8s_version: type: string - default: "1.28" + default: "1.29" cleanup: required: true type: boolean @@ -57,7 +58,7 @@ jobs: runs-on: ubuntu-latest steps: # This additional checkout can be removed when the commit status action is added to the from_git_ref version of Karpenter - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: ref: ${{ inputs.to_git_ref }} - if: always() && github.event_name == 'workflow_run' @@ -66,11 +67,11 @@ jobs: name: ${{ github.workflow }} (${{ inputs.k8s_version }}) / e2e (Upgrade) git_ref: ${{ inputs.to_git_ref }} - uses: ./.github/actions/install-deps - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: ref: ${{ inputs.from_git_ref }} - name: configure aws credentials - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 + uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 with: role-to-assume: arn:aws:iam::${{ vars.CI_ACCOUNT_ID }}:role/${{ vars.CI_ROLE_NAME }} aws-region: ${{ inputs.region }} @@ -89,14 +90,14 @@ jobs: region: ${{ inputs.region }} cluster_name: ${{ steps.generate-cluster-name.outputs.CLUSTER_NAME }} k8s_version: ${{ inputs.k8s_version }} - eksctl_version: v0.165.0 + eksctl_version: v0.169.0 ip_family: IPv4 # Set the value to IPv6 if IPv6 suite, else IPv4 git_ref: ${{ inputs.from_git_ref }} ecr_account_id: ${{ vars.SNAPSHOT_ACCOUNT_ID }} ecr_region: ${{ vars.SNAPSHOT_REGION }} prometheus_workspace_id: ${{ vars.WORKSPACE_ID }} prometheus_region: ${{ vars.PROMETHEUS_REGION }} - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: ref: ${{ inputs.to_git_ref }} - name: upgrade eks cluster '${{ steps.generate-cluster-name.outputs.CLUSTER_NAME }}' @@ -107,7 +108,7 @@ jobs: region: ${{ inputs.region }} cluster_name: ${{ steps.generate-cluster-name.outputs.CLUSTER_NAME }} k8s_version: ${{ inputs.k8s_version }} - eksctl_version: v0.165.0 + eksctl_version: v0.169.0 ip_family: IPv4 # Set the value to IPv6 if IPv6 suite, else IPv4 git_ref: ${{ inputs.to_git_ref }} ecr_account_id: ${{ vars.SNAPSHOT_ACCOUNT_ID }} @@ -151,7 +152,7 @@ jobs: region: ${{ inputs.region }} cluster_name: ${{ steps.generate-cluster-name.outputs.CLUSTER_NAME }} git_ref: ${{ inputs.to_git_ref }} - eksctl_version: v0.165.0 + eksctl_version: v0.169.0 - if: always() && github.event_name == 'workflow_run' uses: ./.github/actions/commit-status/end with: diff --git a/.github/workflows/e2e-version-compatibility-trigger.yaml b/.github/workflows/e2e-version-compatibility-trigger.yaml index 9a5f82899640..e4b9f59ddab8 100644 --- a/.github/workflows/e2e-version-compatibility-trigger.yaml +++ b/.github/workflows/e2e-version-compatibility-trigger.yaml @@ -34,7 +34,7 @@ jobs: strategy: fail-fast: false matrix: - k8s_version: [ "1.23", "1.24", "1.25", "1.26", "1.27", "1.28" ] + k8s_version: [ "1.23", "1.24", "1.25", "1.26", "1.27", "1.28", "1.29"] uses: ./.github/workflows/e2e-matrix.yaml with: region: ${{ inputs.region || 'eu-west-1' }} diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 130f37ce2c32..dc120754be4a 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -36,7 +36,8 @@ on: - "1.26" - "1.27" - "1.28" - default: "1.28" + - "1.29" + default: "1.29" cluster_name: type: string cleanup: @@ -46,6 +47,8 @@ on: enable_metrics: type: boolean default: false + codebuild_region: + type: string workflow_call: inputs: git_ref: @@ -58,7 +61,7 @@ on: required: true k8s_version: type: string - default: "1.28" + default: "1.29" enable_metrics: type: boolean default: false @@ -67,6 +70,8 @@ on: required: true workflow_trigger: type: string + codebuild_region: + type: string cluster_name: type: string description: If cluster_name is empty, a new cluster will be created. Otherwise, tests will run on an existing cluster @@ -83,7 +88,7 @@ jobs: name: suite-${{ inputs.suite }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: ref: ${{ inputs.git_ref }} - if: always() && github.event_name == 'workflow_run' @@ -93,7 +98,7 @@ jobs: git_ref: ${{ inputs.git_ref }} - uses: ./.github/actions/install-deps - name: configure aws credentials - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 + uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 with: role-to-assume: arn:aws:iam::${{ vars.CI_ACCOUNT_ID }}:role/${{ vars.CI_ROLE_NAME }} aws-region: ${{ inputs.region }} @@ -127,24 +132,43 @@ jobs: region: ${{ inputs.region }} cluster_name: ${{ steps.generate-cluster-name.outputs.CLUSTER_NAME }} k8s_version: ${{ inputs.k8s_version }} - eksctl_version: v0.165.0 + eksctl_version: v0.169.0 ip_family: ${{ contains(inputs.suite, 'IPv6') && 'IPv6' || 'IPv4' }} # Set the value to IPv6 if IPv6 suite, else IPv4 - private_cluster: ${{ inputs.suite == 'PrivateCluster' }} + private_cluster: ${{ inputs.workflow_trigger == 'private_cluster' }} git_ref: ${{ inputs.git_ref }} ecr_account_id: ${{ vars.SNAPSHOT_ACCOUNT_ID }} ecr_region: ${{ vars.SNAPSHOT_REGION }} prometheus_workspace_id: ${{ vars.WORKSPACE_ID }} prometheus_region: ${{ vars.PROMETHEUS_REGION }} enable_local_zones: ${{ inputs.suite == 'LocalZone' }} + cleanup: ${{ inputs.cleanup }} + codebuild_role: ${{ vars[format('{0}_CODEBUILD_ROLE', inputs.codebuild_region)] }} + - name: run tests for private cluster + if: ${{ inputs.workflow_trigger == 'private_cluster' }} + uses: ./.github/actions/e2e/run-tests-private-cluster + with: + cluster_name: ${{ steps.generate-cluster-name.outputs.CLUSTER_NAME }} + suite: ${{ inputs.suite }} + prometheus_region: ${{ vars.PROMETHEUS_REGION }} + prometheus_workspace_id: ${{ vars.WORKSPACE_ID }} + region: ${{ inputs.region }} + account_id: ${{ vars.CI_ACCOUNT_ID }} + k8s_version: ${{ inputs.k8s_version }} + ecr_account_id: ${{ vars.SNAPSHOT_ACCOUNT_ID }} + ecr_region: ${{ vars.SNAPSHOT_REGION }} + private_cluster: ${{ inputs.workflow_trigger == 'private_cluster' }} + enable_metrics: ${{ inputs.enable_metrics }} + metrics_region: ${{ vars.TIMESTREAM_REGION }} + node_role: ${{ env.NODE_ROLE }} + cleanup: ${{ inputs.cleanup }} + codebuild_sg: ${{ vars[format('{0}_CODEBUILD_SG', inputs.codebuild_region)] }} + codebuild_vpc: ${{ vars[format('{0}_CODEBUILD_VPC', inputs.codebuild_region)] }} - name: run the ${{ inputs.suite }} test suite + if: ${{ inputs.workflow_trigger != 'private_cluster' }} env: SUITE: ${{ inputs.suite }} ENABLE_METRICS: ${{ inputs.enable_metrics }} run: | - # If we are performing the PrivateCluster test suite, then we should just run the 'Integration' test suite - if [[ $SUITE == 'PrivateCluster' ]]; then - SUITE="Integration" - fi aws eks update-kubeconfig --name ${{ steps.generate-cluster-name.outputs.CLUSTER_NAME }} # Clean up the cluster before running all tests kubectl delete nodepool --all @@ -160,11 +184,12 @@ jobs: with: cluster_name: ${{ steps.generate-cluster-name.outputs.CLUSTER_NAME }} url: ${{ inputs.workflow_trigger == 'soak' && secrets.SLACK_WEBHOOK_SOAK_URL || secrets.SLACK_WEBHOOK_URL }} - suite: ${{ inputs.workflow_trigger == 'soak' && 'soak' || inputs.suite }} + suite: ${{ inputs.suite }} git_ref: ${{ inputs.git_ref }} + workflow_trigger: ${{ inputs.workflow_trigger }} - name: dump logs on failure uses: ./.github/actions/e2e/dump-logs - if: failure() || cancelled() + if: (failure() || cancelled()) && inputs.workflow_trigger != 'private_cluster' with: account_id: ${{ vars.CI_ACCOUNT_ID }} role: ${{ vars.CI_ROLE_NAME }} @@ -179,7 +204,8 @@ jobs: region: ${{ inputs.region }} cluster_name: ${{ steps.generate-cluster-name.outputs.CLUSTER_NAME }} git_ref: ${{ inputs.git_ref }} - eksctl_version: v0.165.0 + eksctl_version: v0.169.0 + private_cluster: ${{ inputs.workflow_trigger == 'private_cluster' }} - if: always() && github.event_name == 'workflow_run' uses: ./.github/actions/commit-status/end with: diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 1ae5f0b7fb3a..7b8495483349 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -16,7 +16,7 @@ jobs: if: github.repository == 'aws/karpenter-provider-aws' runs-on: ubuntu-latest steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: fetch-depth: 0 - name: Create GitHub Release @@ -28,12 +28,12 @@ jobs: - uses: ./.github/actions/e2e/install-helm with: version: v3.12.3 # Pinned to this version since v3.13.0 has issues with pushing to public ECR: https://github.com/helm/helm/issues/12442 - - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 + - uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 with: role-to-assume: 'arn:aws:iam::${{ vars.RELEASE_ACCOUNT_ID }}:role/${{ vars.RELEASE_ROLE_NAME }}' aws-region: ${{ vars.RELEASE_REGION }} - run: make release - - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 + - uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 with: role-to-assume: 'arn:aws:iam::${{ vars.READONLY_ACCOUNT_ID }}:role/${{ vars.READONLY_ROLE_NAME }}' aws-region: ${{ vars.READONLY_REGION }} diff --git a/.github/workflows/resolve-args.yaml b/.github/workflows/resolve-args.yaml index 4856c5728e00..2ffbd51ac1c0 100644 --- a/.github/workflows/resolve-args.yaml +++ b/.github/workflows/resolve-args.yaml @@ -19,7 +19,7 @@ jobs: steps: # Download the artifact and resolve the commit if initiated by PR snapshot # Otherwise, use the currently checked-out branch to run the E2E tests against - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - if: github.event_name == 'workflow_run' uses: ./.github/actions/download-artifact - id: resolve-step diff --git a/.github/workflows/resource-count.yaml b/.github/workflows/resource-count.yaml index 0b1cf0d94e4e..fa54688ae03c 100644 --- a/.github/workflows/resource-count.yaml +++ b/.github/workflows/resource-count.yaml @@ -14,9 +14,9 @@ jobs: region: [us-east-2, us-west-2, eu-west-1, eu-north-1] runs-on: ubuntu-latest steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - name: configure aws credentials - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 + uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 with: role-to-assume: arn:aws:iam::${{ vars.CI_ACCOUNT_ID }}:role/${{ vars.CI_ROLE_NAME }} aws-region: ${{ matrix.region }} diff --git a/.github/workflows/pr-snapshot.yaml b/.github/workflows/snapshot-pr.yaml similarity index 68% rename from .github/workflows/pr-snapshot.yaml rename to .github/workflows/snapshot-pr.yaml index 67fb2e16e0b7..76d3c4729b4e 100644 --- a/.github/workflows/pr-snapshot.yaml +++ b/.github/workflows/snapshot-pr.yaml @@ -1,4 +1,4 @@ -name: PullRequestSnapshot +name: SnapshotPR on: workflow_run: workflows: @@ -6,7 +6,7 @@ on: types: - completed jobs: - release: + snapshot: permissions: id-token: write pull-requests: write @@ -14,7 +14,7 @@ jobs: if: github.event.workflow_run.conclusion == 'success' runs-on: ubuntu-latest steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - uses: ./.github/actions/download-artifact - id: metadata run: | @@ -22,7 +22,7 @@ jobs: pr_commit="$(tail -n 1 /tmp/artifacts/metadata.txt)" echo PR_COMMIT="$pr_commit" >> "$GITHUB_ENV" echo PR_NUMBER="$pr_number" >> "$GITHUB_ENV" - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: ref: ${{ env.PR_COMMIT }} - uses: ./.github/actions/commit-status/start @@ -30,7 +30,7 @@ jobs: name: "${{ github.workflow }} / ${{ github.job }} (pull_request_review)" git_ref: ${{ env.PR_COMMIT }} - uses: ./.github/actions/install-deps - - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 + - uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 with: role-to-assume: 'arn:aws:iam::${{ vars.SNAPSHOT_ACCOUNT_ID }}:role/${{ vars.SNAPSHOT_ROLE_NAME }}' aws-region: ${{ vars.SNAPSHOT_REGION }} @@ -45,10 +45,19 @@ jobs: issue_number: process.env.PR_NUMBER, owner: context.repo.owner, repo: context.repo.repo, - body: `Snapshot successfully published to \`oci://${process.env.SNAPSHOT_ACCOUNT_ID}.dkr.ecr.${process.env.SNAPSHOT_REGION}.amazonaws.com/karpenter/snapshot/karpenter:v0-${process.env.PR_COMMIT}\`. + body: `Snapshot successfully published to \`oci://${process.env.SNAPSHOT_ACCOUNT_ID}.dkr.ecr.${process.env.SNAPSHOT_REGION}.amazonaws.com/karpenter/snapshot/karpenter:0-${process.env.PR_COMMIT}\`. To install you must login to the ECR repo with an AWS account: ~~~ aws ecr get-login-password --region ${process.env.SNAPSHOT_REGION} | docker login --username AWS --password-stdin ${process.env.SNAPSHOT_ACCOUNT_ID}.dkr.ecr.${process.env.SNAPSHOT_REGION}.amazonaws.com + + helm upgrade --install karpenter oci://${process.env.SNAPSHOT_ACCOUNT_ID}.dkr.ecr.${process.env.SNAPSHOT_REGION}.amazonaws.com/karpenter/snapshot/karpenter --version "0-${process.env.PR_COMMIT}" --namespace "kube-system" --create-namespace \\ + --set "settings.clusterName=\${CLUSTER_NAME}" \\ + --set "settings.interruptionQueue=\${CLUSTER_NAME}" \\ + --set controller.resources.requests.cpu=1 \\ + --set controller.resources.requests.memory=1Gi \\ + --set controller.resources.limits.cpu=1 \\ + --set controller.resources.limits.memory=1Gi \\ + --wait ~~~ ` }) diff --git a/.github/workflows/snapshot.yaml b/.github/workflows/snapshot.yaml index 4a9437e28d47..17dc8c0ccd61 100644 --- a/.github/workflows/snapshot.yaml +++ b/.github/workflows/snapshot.yaml @@ -5,16 +5,16 @@ on: - 'main' - 'release-v*' jobs: - release: + snapshot: permissions: id-token: write # aws-actions/configure-aws-credentials@v4.0.1 runs-on: ubuntu-latest steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 with: fetch-depth: 0 - uses: ./.github/actions/install-deps - - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 + - uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 with: role-to-assume: 'arn:aws:iam::${{ vars.SNAPSHOT_ACCOUNT_ID }}:role/${{ vars.SNAPSHOT_ROLE_NAME }}' aws-region: ${{ vars.SNAPSHOT_REGION }} diff --git a/.github/workflows/sweeper.yaml b/.github/workflows/sweeper.yaml index cd2a1446ecac..b6b4dfd1db2c 100644 --- a/.github/workflows/sweeper.yaml +++ b/.github/workflows/sweeper.yaml @@ -14,9 +14,9 @@ jobs: region: [us-east-2, us-west-2, eu-west-1, eu-north-1] runs-on: ubuntu-latest steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 - name: configure aws credentials - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 + uses: aws-actions/configure-aws-credentials@e3dd6a429d7300a6a4c196c26e071d42e0343502 # v4.0.2 with: role-to-assume: arn:aws:iam::${{ vars.CI_ACCOUNT_ID }}:role/${{ vars.CI_ROLE_NAME }} aws-region: ${{ matrix.region }} diff --git a/.ko.yaml b/.ko.yaml index aafa60d0e357..7a663228cd39 100644 --- a/.ko.yaml +++ b/.ko.yaml @@ -1 +1,4 @@ defaultBaseImage: public.ecr.aws/eks-distro-build-tooling/eks-distro-minimal-base +defaultPlatforms: + - linux/arm64 + - linux/amd64 diff --git a/ADOPTERS.md b/ADOPTERS.md index 7ab9dd062e0b..56a37a46ca7b 100644 --- a/ADOPTERS.md +++ b/ADOPTERS.md @@ -49,6 +49,7 @@ If you are open to others contacting you about your use of Karpenter on Slack, a | Target Australia | Using Karpenter to manage (scale, auto update etc) the containerised (EKS) compute that powers much of the Target Online platform | `@gazal-k` | [Target Australia Online Shopping](https://www.target.com.au) | | The Scale Factory | Using Karpenter (controllers on EC2/Fargate) to efficiently scale K8s workloads and empowering our customer teams to do the same | `@marko` | [Homepage](https://www.scalefactory.com) | | Tyk Cloud | Scaling workloads for the Cloud Free plan | `@Artem Hluvchynskyi`, `@gowtham` | [Tyk Cloud](https://tyk.io/cloud/) | +| VictoriaMetrics Cloud | Managed Time Series Database Services from VictoriaMetrics | `@Max G` | [VictoriaMetrics Cloud](https://victoriametrics.com/products/managed/) | | Wehkamp | Using Karpenter to scale the EKS clusters for our e-commerce platforms | `@ChrisV` | [Wehkamp](https://www.wehkamp.nl) & [Wehkamp Techblog](https://medium.com/wehkamp-techblog)| | Whoosh | Using Karpenter to scale the EKS clusters for many purposes | `@vainkop` | [Whoosh](https://whoosh.bike) | | Next Insurance | Using Karpenter to manage the nodes in all our EKS clusters, including dev and prod, on demand and spots | `@moshebs` | [Homepage](https://www.nextinsurance.com)| diff --git a/Makefile b/Makefile index ad33b12e5422..1aa29f5e69c4 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,7 @@ -export K8S_VERSION ?= 1.27.x CLUSTER_NAME ?= $(shell kubectl config view --minify -o jsonpath='{.clusters[].name}' | rev | cut -d"/" -f1 | rev | cut -d"." -f1) ## Inject the app version into operator.Version -LDFLAGS ?= -ldflags=-X=sigs.k8s.io/karpenter/pkg/operator.Version=$(shell git describe --tags --always) +LDFLAGS ?= -ldflags=-X=sigs.k8s.io/karpenter/pkg/operator.Version=$(shell git describe --tags --always | cut -d"v" -f2) GOFLAGS ?= $(LDFLAGS) WITH_GOFLAGS = GOFLAGS="$(GOFLAGS)" @@ -23,17 +22,16 @@ HELM_OPTS ?= --set serviceAccount.annotations.eks\\.amazonaws\\.com/role-arn=${K # CR for local builds of Karpenter KARPENTER_NAMESPACE ?= kube-system -KARPENTER_VERSION ?= $(shell git tag --sort=committerdate | tail -1) +KARPENTER_VERSION ?= $(shell git tag --sort=committerdate | tail -1 | cut -d"v" -f2) KO_DOCKER_REPO ?= ${AWS_ACCOUNT_ID}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/dev -GETTING_STARTED_SCRIPT_DIR = website/content/en/preview/getting-started/getting-started-with-karpenter/scripts +KOCACHE ?= ~/.ko # Common Directories MOD_DIRS = $(shell find . -path "./website" -prune -o -name go.mod -type f -print | xargs dirname) KARPENTER_CORE_DIR = $(shell go list -m -f '{{ .Dir }}' sigs.k8s.io/karpenter) -# TEST_SUITE enables you to select a specific test suite directory to run "make e2etests" or "make test" against +# TEST_SUITE enables you to select a specific test suite directory to run "make e2etests" against TEST_SUITE ?= "..." -TEST_TIMEOUT ?= "3h" help: ## Display help @awk 'BEGIN {FS = ":.*##"; printf "Usage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) @@ -55,12 +53,21 @@ run: ## Run Karpenter controller binary against your local cluster go run ./cmd/controller/main.go test: ## Run tests - go test -v ./pkg/$(shell echo $(TEST_SUITE) | tr A-Z a-z)/... \ + go test -v ./pkg/... \ -cover -coverprofile=coverage.out -outputdir=. -coverpkg=./... \ --ginkgo.focus="${FOCUS}" \ --ginkgo.randomize-all \ --ginkgo.vv +deflake: ## Run randomized, racing tests until the test fails to catch flakes + ginkgo \ + --race \ + --focus="${FOCUS}" \ + --randomize-all \ + --until-it-fails \ + -v \ + ./pkg/... + e2etests: ## Run the e2e suite against your local cluster cd test && CLUSTER_ENDPOINT=${CLUSTER_ENDPOINT} \ CLUSTER_NAME=${CLUSTER_NAME} \ @@ -68,18 +75,18 @@ e2etests: ## Run the e2e suite against your local cluster go test \ -p 1 \ -count 1 \ - -timeout ${TEST_TIMEOUT} \ + -timeout 3h \ -v \ ./suites/$(shell echo $(TEST_SUITE) | tr A-Z a-z)/... \ --ginkgo.focus="${FOCUS}" \ - --ginkgo.timeout=${TEST_TIMEOUT} \ + --ginkgo.timeout=3h \ --ginkgo.grace-period=3m \ --ginkgo.vv e2etests-deflake: ## Run the e2e suite against your local cluster cd test && CLUSTER_NAME=${CLUSTER_NAME} ginkgo \ --focus="${FOCUS}" \ - --timeout=${TEST_TIMEOUT} \ + --timeout=3h \ --grace-period=3m \ --until-it-fails \ --vv \ @@ -88,18 +95,6 @@ e2etests-deflake: ## Run the e2e suite against your local cluster benchmark: go test -tags=test_performance -run=NoTests -bench=. ./... -deflake: ## Run randomized, racing, code-covered tests to deflake failures - for i in $(shell seq 1 5); do make test || exit 1; done - -deflake-until-it-fails: ## Run randomized, racing tests until the test fails to catch flakes - ginkgo \ - --race \ - --focus="${FOCUS}" \ - --randomize-all \ - --until-it-fails \ - -v \ - ./pkg/... - coverage: go tool cover -html coverage.out -o coverage.html @@ -124,13 +119,11 @@ vulncheck: ## Verify code vulnerabilities @govulncheck ./pkg/... licenses: download ## Verifies dependency licenses - ! go-licenses csv ./... | grep -v -e 'MIT' -e 'Apache-2.0' -e 'BSD-3-Clause' -e 'BSD-2-Clause' -e 'ISC' -e 'MPL-2.0' - -setup: ## Sets up the IAM roles needed prior to deploying the karpenter-controller. This command only needs to be run once - CLUSTER_NAME=${CLUSTER_NAME} ./$(GETTING_STARTED_SCRIPT_DIR)/add-roles.sh $(KARPENTER_VERSION) + # TODO: remove nodeadm check once license is updated + ! go-licenses csv ./... | grep -v -e 'MIT' -e 'Apache-2.0' -e 'BSD-3-Clause' -e 'BSD-2-Clause' -e 'ISC' -e 'MPL-2.0' -e 'github.com/awslabs/amazon-eks-ami/nodeadm' image: ## Build the Karpenter controller images using ko build - $(eval CONTROLLER_IMG=$(shell $(WITH_GOFLAGS) KO_DOCKER_REPO="$(KO_DOCKER_REPO)" ko build --bare github.com/aws/karpenter-provider-aws/cmd/controller)) + $(eval CONTROLLER_IMG=$(shell $(WITH_GOFLAGS) KOCACHE=$(KOCACHE) KO_DOCKER_REPO="$(KO_DOCKER_REPO)" ko build --bare github.com/aws/karpenter-provider-aws/cmd/controller)) $(eval IMG_REPOSITORY=$(shell echo $(CONTROLLER_IMG) | cut -d "@" -f 1 | cut -d ":" -f 1)) $(eval IMG_TAG=$(shell echo $(CONTROLLER_IMG) | cut -d "@" -f 1 | cut -d ":" -f 2 -s)) $(eval IMG_DIGEST=$(shell echo $(CONTROLLER_IMG) | cut -d "@" -f 2)) @@ -166,9 +159,6 @@ snapshot: ## Builds and publishes snapshot release release: ## Builds and publishes stable release $(WITH_GOFLAGS) ./hack/release/release.sh -release-crd: ## Packages and publishes a karpenter-crd helm chart - $(WITH_GOFLAGS) ./hack/release/release-crd.sh - prepare-website: ## prepare the website for release ./hack/release/prepare-website.sh @@ -194,7 +184,7 @@ update-karpenter: ## Update kubernetes-sigs/karpenter to latest go get -u sigs.k8s.io/karpenter@HEAD go mod tidy -.PHONY: help dev ci release test e2etests verify tidy download docgen codegen apply delete toolchain licenses vulncheck issues website nightly snapshot +.PHONY: help presubmit ci-test ci-non-test run test deflake e2etests e2etests-deflake benchmark coverage verify vulncheck licenses image apply install delete docgen codegen stable-release-pr snapshot release prepare-website toolchain issues website tidy download update-karpenter define newline diff --git a/charts/karpenter-crd/Chart.yaml b/charts/karpenter-crd/Chart.yaml index b6bccbee45b5..766c31ea6be8 100644 --- a/charts/karpenter-crd/Chart.yaml +++ b/charts/karpenter-crd/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: karpenter-crd description: A Helm chart for Karpenter Custom Resource Definitions (CRDs) type: application -version: 0.33.0 -appVersion: 0.33.0 +version: 0.35.4 +appVersion: 0.35.4 keywords: - cluster - node diff --git a/charts/karpenter-crd/artifacthub-repo.yaml b/charts/karpenter-crd/artifacthub-repo.yaml new file mode 100644 index 000000000000..194c8d2496ea --- /dev/null +++ b/charts/karpenter-crd/artifacthub-repo.yaml @@ -0,0 +1,7 @@ +repositoryID: fda7ffc4-4672-4218-8264-321ec3b4e3cc +owners: [] +# - name: awsadmin1 +# email: artifacthub1@aws.com +ignore: + - name: karpenter-crd + version: (?:^\d+$)|(?:^v?0\.0\.0)|(?:^v?\d+\-) diff --git a/charts/karpenter/Chart.yaml b/charts/karpenter/Chart.yaml index 2e09fdedb2cd..eb480286be99 100644 --- a/charts/karpenter/Chart.yaml +++ b/charts/karpenter/Chart.yaml @@ -2,8 +2,8 @@ apiVersion: v2 name: karpenter description: A Helm chart for Karpenter, an open-source node provisioning project built for Kubernetes. type: application -version: 0.33.0 -appVersion: 0.33.0 +version: 0.35.4 +appVersion: 0.35.4 keywords: - cluster - node @@ -13,4 +13,22 @@ keywords: home: https://karpenter.sh/ icon: https://repository-images.githubusercontent.com/278480393/dab059c8-caa1-4b55-aaa7-3d30e47a5616 sources: - - https://github.com/aws/karpenter/ + - https://github.com/aws/karpenter-provider-aws/ +annotations: + artifacthub.io/alternativeName: karpenter-provider-aws + artifacthub.io/crds: | + - kind: EC2NodeClass + version: v1beta1 + name: ec2nodeclasses.karpenter.k8s.aws + displayName: EC2NodeClass + description: EC2NodeClass is the Schema for the EC2NodeClass API. + - kind: NodeClaim + version: v1beta1 + name: nodeclaims.karpenter.sh + displayName: NodeClaim + description: NodeClaim is the Schema for the NodeClaims API. + - kind: NodePool + version: v1beta1 + name: nodepools.karpenter.sh + displayName: NodePool + description: NodePool is the Schema for the NodePools API. diff --git a/charts/karpenter/README.md b/charts/karpenter/README.md index 29db446c547f..8eeab2785604 100644 --- a/charts/karpenter/README.md +++ b/charts/karpenter/README.md @@ -2,7 +2,7 @@ A Helm chart for Karpenter, an open-source node provisioning project built for Kubernetes. -![Version: 0.33.0](https://img.shields.io/badge/Version-0.33.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.33.0](https://img.shields.io/badge/AppVersion-0.33.0-informational?style=flat-square) +![Version: 0.35.4](https://img.shields.io/badge/Version-0.35.4-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.35.4](https://img.shields.io/badge/AppVersion-0.35.4-informational?style=flat-square) ## Documentation @@ -15,13 +15,27 @@ You can follow the detailed installation instruction in the [documentation](http ```bash helm upgrade --install --namespace karpenter --create-namespace \ karpenter oci://public.ecr.aws/karpenter/karpenter \ - --version v0.33.0 \ + --version 0.35.4 \ --set "serviceAccount.annotations.eks\.amazonaws\.com/role-arn=${KARPENTER_IAM_ROLE_ARN}" \ --set settings.clusterName=${CLUSTER_NAME} \ --set settings.interruptionQueue=${CLUSTER_NAME} \ --wait ``` +### Verification + +As the OCI Helm chart is signed by [Cosign](https://github.com/sigstore/cosign) as part of the release process you can verify the chart before installing it by running the following command. + +```shell +cosign verify public.ecr.aws/karpenter/karpenter:0.35.4 \ + --certificate-oidc-issuer=https://token.actions.githubusercontent.com \ + --certificate-identity-regexp='https://github\.com/aws/karpenter-provider-aws/\.github/workflows/release\.yaml@.+' \ + --certificate-github-workflow-repository=aws/karpenter-provider-aws \ + --certificate-github-workflow-name=Release \ + --certificate-github-workflow-ref=refs/tags/v0.35.4 \ + --annotations version=0.35.4 +``` + ## Values | Key | Type | Default | Description | @@ -34,15 +48,15 @@ helm upgrade --install --namespace karpenter --create-namespace \ | controller.envFrom | list | `[]` | | | controller.extraVolumeMounts | list | `[]` | Additional volumeMounts for the controller pod. | | controller.healthProbe.port | int | `8081` | The container port to use for http health probe. | -| controller.image.digest | string | `"sha256:5e5f59f74d86ff7f13d7d80b89afff8c661cb4e3265f2fdda95b76dd9c838cc1"` | SHA256 digest of the controller image. | +| controller.image.digest | string | `"sha256:27a73db80b78e523370bcca77418f6d2136eea10a99fc87d02d2df059fcf5fb7"` | SHA256 digest of the controller image. | | controller.image.repository | string | `"public.ecr.aws/karpenter/controller"` | Repository path to the controller image. | -| controller.image.tag | string | `"v0.33.0"` | Tag of the controller image. | +| controller.image.tag | string | `"0.35.4"` | Tag of the controller image. | | controller.metrics.port | int | `8000` | The container port to use for metrics. | | controller.resources | object | `{}` | Resources for the controller pod. | | controller.sidecarContainer | list | `[]` | Additional sidecarContainer config | | controller.sidecarVolumeMounts | list | `[]` | Additional volumeMounts for the sidecar - this will be added to the volume mounts on top of extraVolumeMounts | | dnsConfig | object | `{}` | Configure DNS Config for the pod | -| dnsPolicy | string | `"Default"` | Configure the DNS Policy for the pod | +| dnsPolicy | string | `"ClusterFirst"` | Configure the DNS Policy for the pod | | extraVolumes | list | `[]` | Additional volumes for the pod. | | fullnameOverride | string | `""` | Overrides the chart's computed fullname. | | hostNetwork | bool | `false` | Bind the pod to the host network. This is required when using a custom CNI. | @@ -64,7 +78,7 @@ helm upgrade --install --namespace karpenter --create-namespace \ | podDisruptionBudget.maxUnavailable | int | `1` | | | podDisruptionBudget.name | string | `"karpenter"` | | | podLabels | object | `{}` | Additional labels for the pod. | -| podSecurityContext | object | `{"fsGroup":65536}` | SecurityContext for the pod. | +| podSecurityContext | object | `{"fsGroup":65532}` | SecurityContext for the pod. | | priorityClassName | string | `"system-cluster-critical"` | PriorityClass name for the pod. | | replicas | int | `2` | Number of replicas. | | revisionHistoryLimit | int | `10` | The number of old ReplicaSets to retain to allow rollback. | @@ -84,7 +98,7 @@ helm upgrade --install --namespace karpenter --create-namespace \ | settings.clusterName | string | `""` | Cluster name. | | settings.featureGates | object | `{"drift":true,"spotToSpotConsolidation":false}` | Feature Gate configuration values. Feature Gates will follow the same graduation process and requirements as feature gates in Kubernetes. More information here https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/#feature-gates-for-alpha-or-beta-features | | settings.featureGates.drift | bool | `true` | drift is in BETA and is enabled by default. Setting drift to false disables the drift disruption method to watch for drift between currently deployed nodes and the desired state of nodes set in nodepools and nodeclasses | -| settings.featureGates.spotToSpotConsolidation | bool | `false` | spotToSpotConsolidation is disabled by default. Setting this to true will enable spot replacement consolidation for both single and multi-node consolidation. | +| settings.featureGates.spotToSpotConsolidation | bool | `false` | spotToSpotConsolidation is ALPHA and is disabled by default. Setting this to true will enable spot replacement consolidation for both single and multi-node consolidation. | | settings.interruptionQueue | string | `""` | interruptionQueue is disabled if not specified. Enabling interruption handling may require additional permissions on the controller service account. Additional permissions are outlined in the docs. | | settings.isolatedVPC | bool | `false` | If true then assume we can't reach AWS services which don't have a VPC endpoint This also has the effect of disabling look-ups to the AWS pricing endpoint | | settings.reservedENIs | string | `"0"` | Reserved ENIs are not included in the calculations for max-pods or kube-reserved This is most often used in the VPC CNI custom networking setup https://docs.aws.amazon.com/eks/latest/userguide/cni-custom-network.html | diff --git a/charts/karpenter/README.md.gotmpl b/charts/karpenter/README.md.gotmpl index ed86390ec4df..1e641e1ee586 100644 --- a/charts/karpenter/README.md.gotmpl +++ b/charts/karpenter/README.md.gotmpl @@ -14,13 +14,27 @@ You can follow the detailed installation instruction in the [documentation](http ```bash helm upgrade --install --namespace karpenter --create-namespace \ karpenter oci://public.ecr.aws/karpenter/{{ template "chart.name" . }} \ - --version v{{ template "chart.version" . }} \ + --version {{ template "chart.version" . }} \ --set "serviceAccount.annotations.eks\.amazonaws\.com/role-arn=${KARPENTER_IAM_ROLE_ARN}" \ --set settings.clusterName=${CLUSTER_NAME} \ --set settings.interruptionQueue=${CLUSTER_NAME} \ --wait ``` +### Verification + +As the OCI Helm chart is signed by [Cosign](https://github.com/sigstore/cosign) as part of the release process you can verify the chart before installing it by running the following command. + +```shell +cosign verify public.ecr.aws/karpenter/karpenter:{{ template "chart.version" . }} \ + --certificate-oidc-issuer=https://token.actions.githubusercontent.com \ + --certificate-identity-regexp='https://github\.com/aws/karpenter-provider-aws/\.github/workflows/release\.yaml@.+' \ + --certificate-github-workflow-repository=aws/karpenter-provider-aws \ + --certificate-github-workflow-name=Release \ + --certificate-github-workflow-ref=refs/tags/v{{ template "chart.version" . }} \ + --annotations version={{ template "chart.version" . }} +``` + {{ template "chart.requirementsSection" . }} {{ template "chart.valuesSection" . }} diff --git a/charts/karpenter/artifacthub-repo.yaml b/charts/karpenter/artifacthub-repo.yaml new file mode 100644 index 000000000000..48d9ca7ce581 --- /dev/null +++ b/charts/karpenter/artifacthub-repo.yaml @@ -0,0 +1,7 @@ +repositoryID: 356cb63f-9ee3-4956-9c20-003e416715c7 +owners: [] +# - name: awsadmin1 +# email: artifacthub1@aws.com +ignore: + - name: karpenter + version: (?:^\d+$)|(?:^v?0\.0\.0)|(?:^v?\d+\-) diff --git a/charts/karpenter/templates/deployment.yaml b/charts/karpenter/templates/deployment.yaml index 4b4f9450a569..d1994b3485b9 100644 --- a/charts/karpenter/templates/deployment.yaml +++ b/charts/karpenter/templates/deployment.yaml @@ -59,8 +59,8 @@ spec: containers: - name: controller securityContext: - runAsUser: 65536 - runAsGroup: 65536 + runAsUser: 65532 + runAsGroup: 65532 runAsNonRoot: true seccompProfile: type: RuntimeDefault diff --git a/charts/karpenter/templates/servicemonitor.yaml b/charts/karpenter/templates/servicemonitor.yaml index 70f07608fb2c..3ab5f9288650 100644 --- a/charts/karpenter/templates/servicemonitor.yaml +++ b/charts/karpenter/templates/servicemonitor.yaml @@ -21,9 +21,9 @@ spec: matchLabels: {{- include "karpenter.selectorLabels" . | nindent 6 }} endpoints: - - port: http-metrics - path: /metrics + - port: http-metrics + path: /metrics {{- with .Values.serviceMonitor.endpointConfig }} - {{- toYaml . | nindent 6 }} + {{- toYaml . | nindent 4 }} {{- end }} {{- end -}} diff --git a/charts/karpenter/values.yaml b/charts/karpenter/values.yaml index ff90cde71016..8dfc528cd085 100644 --- a/charts/karpenter/values.yaml +++ b/charts/karpenter/values.yaml @@ -45,8 +45,8 @@ podDisruptionBudget: name: karpenter maxUnavailable: 1 # -- SecurityContext for the pod. -podSecurityContext: - fsGroup: 65536 +podSecurityContext: + fsGroup: 65532 # -- PriorityClass name for the pod. priorityClassName: system-cluster-critical # -- Override the default termination grace period for the pod. @@ -99,9 +99,9 @@ controller: # -- Repository path to the controller image. repository: public.ecr.aws/karpenter/controller # -- Tag of the controller image. - tag: v0.33.0 + tag: 0.35.4 # -- SHA256 digest of the controller image. - digest: sha256:5e5f59f74d86ff7f13d7d80b89afff8c661cb4e3265f2fdda95b76dd9c838cc1 + digest: sha256:27a73db80b78e523370bcca77418f6d2136eea10a99fc87d02d2df059fcf5fb7 # -- Additional environment variables for the controller pod. env: [] # - name: AWS_REGION @@ -202,6 +202,6 @@ settings: # Setting drift to false disables the drift disruption method to watch for drift between currently deployed nodes # and the desired state of nodes set in nodepools and nodeclasses drift: true - # -- spotToSpotConsolidation is disabled by default. + # -- spotToSpotConsolidation is ALPHA and is disabled by default. # Setting this to true will enable spot replacement consolidation for both single and multi-node consolidation. spotToSpotConsolidation: false diff --git a/cmd/controller/main.go b/cmd/controller/main.go index 94dc0950929e..b1871bd91f1e 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -47,7 +47,6 @@ func main() { WithControllers(ctx, corecontrollers.NewControllers( op.Clock, op.GetClient(), - op.KubernetesInterface, state.NewCluster(op.Clock, op.GetClient(), cloudProvider), op.EventRecorder, cloudProvider, @@ -60,7 +59,7 @@ func main() { op.GetClient(), op.EventRecorder, op.UnavailableOfferingsCache, - awsCloudProvider, + cloudProvider, op.SubnetProvider, op.SecurityGroupProvider, op.InstanceProfileProvider, diff --git a/designs/v1beta1-api.md b/designs/v1beta1-api.md index 13faad7ece38..409558a71a96 100644 --- a/designs/v1beta1-api.md +++ b/designs/v1beta1-api.md @@ -344,12 +344,13 @@ status: 6. `karpenter.k8s.aws/instance-local-nvme` 7. `karpenter.k8s.aws/instance-size` 8. `karpenter.k8s.aws/instance-cpu` -9. `karpenter.k8s.aws/instance-memory` -10. `karpenter.k8s.aws/instance-network-bandwidth` -11. `karpenter.k8s.aws/instance-gpu-name` -12. `karpenter.k8s.aws/instance-gpu-manufacturer` -13. `karpenter.k8s.aws/instance-gpu-count` -14. `karpenter.k8s.aws/instance-gpu-memory` -15. `karpenter.k8s.aws/instance-accelerator-name` -16. `karpenter.k8s.aws/instance-accelerator-manufacturer` -17. `karpenter.k8s.aws/instance-accelerator-count` +9. `karpenter.k8s.aws/instance-cpu-manufacturer` +10. `karpenter.k8s.aws/instance-memory` +11. `karpenter.k8s.aws/instance-network-bandwidth` +12. `karpenter.k8s.aws/instance-gpu-name` +13. `karpenter.k8s.aws/instance-gpu-manufacturer` +14. `karpenter.k8s.aws/instance-gpu-count` +15. `karpenter.k8s.aws/instance-gpu-memory` +16. `karpenter.k8s.aws/instance-accelerator-name` +17. `karpenter.k8s.aws/instance-accelerator-manufacturer` +18. `karpenter.k8s.aws/instance-accelerator-count` diff --git a/examples/v1beta1/100-cpu-limit.yaml b/examples/v1beta1/100-cpu-limit.yaml index b0fb8151366f..c027aaac713e 100644 --- a/examples/v1beta1/100-cpu-limit.yaml +++ b/examples/v1beta1/100-cpu-limit.yaml @@ -28,6 +28,8 @@ spec: operator: Gt values: ["2"] nodeClassRef: + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass name: default limits: cpu: 100 diff --git a/examples/v1beta1/al2-custom-ami.yaml b/examples/v1beta1/al2-custom-ami.yaml index c1b808598b65..5aadf58c31b8 100644 --- a/examples/v1beta1/al2-custom-ami.yaml +++ b/examples/v1beta1/al2-custom-ami.yaml @@ -28,6 +28,8 @@ spec: operator: Gt values: ["2"] nodeClassRef: + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass name: al2 --- apiVersion: karpenter.k8s.aws/v1beta1 diff --git a/examples/v1beta1/al2-custom-userdata.yaml b/examples/v1beta1/al2-custom-userdata.yaml index e9b5873884a2..e5bb04c20071 100644 --- a/examples/v1beta1/al2-custom-userdata.yaml +++ b/examples/v1beta1/al2-custom-userdata.yaml @@ -28,6 +28,8 @@ spec: operator: Gt values: ["2"] nodeClassRef: + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass name: al2 --- apiVersion: karpenter.k8s.aws/v1beta1 diff --git a/examples/v1beta1/al2-kubelet-log-query.yaml b/examples/v1beta1/al2-kubelet-log-query.yaml index 013469976c5b..ecfc907937e9 100644 --- a/examples/v1beta1/al2-kubelet-log-query.yaml +++ b/examples/v1beta1/al2-kubelet-log-query.yaml @@ -27,6 +27,8 @@ spec: operator: Gt values: ["2"] nodeClassRef: + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass name: al2 --- apiVersion: karpenter.k8s.aws/v1beta1 diff --git a/examples/v1beta1/al2023-custom-userdata.yaml b/examples/v1beta1/al2023-custom-userdata.yaml new file mode 100644 index 000000000000..409792c56bee --- /dev/null +++ b/examples/v1beta1/al2023-custom-userdata.yaml @@ -0,0 +1,68 @@ +# This example NodePool will provision instances using the AL2023 EKS-Optimized AMI. +--- +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: default + annotations: + kubernetes.io/description: "General purpose NodePool for generic workloads" +spec: + template: + spec: + requirements: + - key: kubernetes.io/arch + operator: In + values: ["amd64"] + - key: kubernetes.io/os + operator: In + values: ["linux"] + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand"] + - key: karpenter.k8s.aws/instance-category + operator: In + values: ["c", "m", "r"] + - key: karpenter.k8s.aws/instance-generation + operator: Gt + values: ["2"] + nodeClassRef: + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass + name: al2023 +--- +apiVersion: karpenter.k8s.aws/v1beta1 +kind: EC2NodeClass +metadata: + name: al2023 + annotations: + kubernetes.io/description: "EC2NodeClass for running Amazon Linux 2023 nodes with custom user data" +spec: + amiFamily: AL2023 # Amazon Linux 2023 + role: "KarpenterNodeRole-${CLUSTER_NAME}" # replace with your cluster name + subnetSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + securityGroupSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + userData: | + MIME-Version: 1.0 + Content-Type: multipart/mixed; boundary="//" + + --// + Content-Type: application/node.eks.aws + + # Example custom nodeconfig which mounts individual drives on an instance + apiVersion: node.eks.aws/v1alpha1 + kind: NodeConfig + spec: + instance: + localStorage: + strategy: Mount + + --// + Content-Type: text/x-shellscript; charset="us-ascii" + + #!/bin/bash + echo "Hello, AL2023!" + --// diff --git a/examples/v1beta1/bottlerocket.yaml b/examples/v1beta1/bottlerocket.yaml index 0c2574a4e35b..3330a876fd18 100644 --- a/examples/v1beta1/bottlerocket.yaml +++ b/examples/v1beta1/bottlerocket.yaml @@ -27,6 +27,8 @@ spec: operator: Gt values: ["2"] nodeClassRef: + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass name: bottlerocket --- apiVersion: karpenter.k8s.aws/v1beta1 diff --git a/examples/v1beta1/br-custom-userdata.yaml b/examples/v1beta1/br-custom-userdata.yaml index fa18decf4c3b..99f3efb09301 100644 --- a/examples/v1beta1/br-custom-userdata.yaml +++ b/examples/v1beta1/br-custom-userdata.yaml @@ -28,6 +28,8 @@ spec: operator: Gt values: ["2"] nodeClassRef: + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass name: bottlerocket --- apiVersion: karpenter.k8s.aws/v1beta1 diff --git a/examples/v1beta1/custom-family.yaml b/examples/v1beta1/custom-family.yaml index daad19810935..92aa2725266b 100644 --- a/examples/v1beta1/custom-family.yaml +++ b/examples/v1beta1/custom-family.yaml @@ -27,6 +27,8 @@ spec: operator: Gt values: ["2"] nodeClassRef: + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass name: custom-family --- apiVersion: karpenter.k8s.aws/v1beta1 diff --git a/examples/v1beta1/general-purpose.yaml b/examples/v1beta1/general-purpose.yaml index 43e32d8c436d..d801ae8c498d 100644 --- a/examples/v1beta1/general-purpose.yaml +++ b/examples/v1beta1/general-purpose.yaml @@ -26,6 +26,8 @@ spec: operator: Gt values: ["2"] nodeClassRef: + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass name: default --- apiVersion: karpenter.k8s.aws/v1beta1 diff --git a/examples/v1beta1/instance-store-ephemeral-storage.yaml b/examples/v1beta1/instance-store-ephemeral-storage.yaml index ea7f49a28819..7ace15c928c2 100644 --- a/examples/v1beta1/instance-store-ephemeral-storage.yaml +++ b/examples/v1beta1/instance-store-ephemeral-storage.yaml @@ -30,6 +30,8 @@ spec: operator: Gt values: ["300"] nodeClassRef: + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass name: instance-store-ephemeral-storage --- apiVersion: karpenter.k8s.aws/v1beta1 diff --git a/examples/v1beta1/large-instances.yaml b/examples/v1beta1/large-instances.yaml index da6fac3ba4ba..162262b76c43 100644 --- a/examples/v1beta1/large-instances.yaml +++ b/examples/v1beta1/large-instances.yaml @@ -18,6 +18,8 @@ spec: operator: Gt values: ["8191"] nodeClassRef: + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass name: default --- apiVersion: karpenter.k8s.aws/v1beta1 diff --git a/examples/v1beta1/minValues-family.yaml b/examples/v1beta1/minValues-family.yaml new file mode 100644 index 000000000000..30a54e737f89 --- /dev/null +++ b/examples/v1beta1/minValues-family.yaml @@ -0,0 +1,52 @@ +# This example will use spot instance type for all provisioned instances +# and enforces minValues to instance families which means at least that number of unique instance +# families is required by the scheduler for the NodeClaim creation. +--- +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: spot + annotations: + kubernetes.io/description: "NodePool for provisioning spot capacity" +spec: + template: + spec: + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["spot"] + - key: kubernetes.io/arch + operator: In + values: ["amd64"] + - key: kubernetes.io/os + operator: In + values: ["linux"] + - key: karpenter.k8s.aws/instance-category + operator: In + values: ["c", "m", "r"] + - key: karpenter.k8s.aws/instance-family + operator: Exists + minValues: 5 + - key: karpenter.k8s.aws/instance-generation + operator: Gt + values: ["2"] + nodeClassRef: + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass + name: default +--- +apiVersion: karpenter.k8s.aws/v1beta1 +kind: EC2NodeClass +metadata: + name: default + annotations: + kubernetes.io/description: "General purpose EC2NodeClass for running Amazon Linux 2 nodes" +spec: + amiFamily: AL2 # Amazon Linux 2 + role: "KarpenterNodeRole-${CLUSTER_NAME}" # replace with your cluster name + subnetSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + securityGroupSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name \ No newline at end of file diff --git a/examples/v1beta1/minValues-multiple-keys.yaml b/examples/v1beta1/minValues-multiple-keys.yaml new file mode 100644 index 000000000000..1fa780147d8b --- /dev/null +++ b/examples/v1beta1/minValues-multiple-keys.yaml @@ -0,0 +1,56 @@ +# This example will use spot instance type for all provisioned instances and enforces minValues to various keys where it is defined i.e +# at least 2 unique instance families from [c,m,r], 5 unique instance families [eg: "m5","m5d","m5dn","c5","c5d","c4" etc], 2 unique instance types [eg: "c5.2xlarge","c4.xlarge" etc] are required by the scheduler for the NodeClaim creation. +# This ensures minimum flexiblity required to schedule pods into spot nodes. +--- +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: spot + annotations: + kubernetes.io/description: "NodePool for provisioning spot capacity" +spec: + template: + spec: + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["spot"] + - key: kubernetes.io/arch + operator: In + values: ["amd64"] + - key: kubernetes.io/os + operator: In + values: ["linux"] + - key: karpenter.k8s.aws/instance-category + operator: In + values: ["c", "m", "r"] + minValues: 2 + - key: karpenter.k8s.aws/instance-family + operator: Exists + minValues: 5 + - key: node.kubernetes.io/instance-type + operator: Exists + minValues: 2 + - key: karpenter.k8s.aws/instance-generation + operator: Gt + values: ["2"] + nodeClassRef: + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass + name: default +--- +apiVersion: karpenter.k8s.aws/v1beta1 +kind: EC2NodeClass +metadata: + name: default + annotations: + kubernetes.io/description: "General purpose EC2NodeClass for running Amazon Linux 2 nodes" +spec: + amiFamily: AL2 # Amazon Linux 2 + role: "KarpenterNodeRole-${CLUSTER_NAME}" # replace with your cluster name + subnetSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + securityGroupSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name \ No newline at end of file diff --git a/examples/v1beta1/multiple-arch.yaml b/examples/v1beta1/multiple-arch.yaml index 91529c628908..b7c4c8a853fa 100644 --- a/examples/v1beta1/multiple-arch.yaml +++ b/examples/v1beta1/multiple-arch.yaml @@ -28,6 +28,8 @@ spec: operator: Gt values: ["2"] nodeClassRef: + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass name: default --- apiVersion: karpenter.sh/v1beta1 @@ -56,6 +58,8 @@ spec: operator: Gt values: ["2"] nodeClassRef: + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass name: default --- apiVersion: karpenter.k8s.aws/v1beta1 diff --git a/examples/v1beta1/multiple-ebs.yaml b/examples/v1beta1/multiple-ebs.yaml index 5400e110bdcc..24335cacae5f 100644 --- a/examples/v1beta1/multiple-ebs.yaml +++ b/examples/v1beta1/multiple-ebs.yaml @@ -27,6 +27,8 @@ spec: operator: Gt values: ["2"] nodeClassRef: + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass name: multiple-ebs --- apiVersion: karpenter.k8s.aws/v1beta1 diff --git a/examples/v1beta1/node-ttls.yaml b/examples/v1beta1/node-ttls.yaml index 84c00f34f716..6f0218d12018 100644 --- a/examples/v1beta1/node-ttls.yaml +++ b/examples/v1beta1/node-ttls.yaml @@ -28,6 +28,8 @@ spec: operator: Gt values: ["2"] nodeClassRef: + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass name: default disruption: consolidationPolicy: WhenEmpty diff --git a/examples/v1beta1/spot.yaml b/examples/v1beta1/spot.yaml index 7c8903748658..774887a3a682 100644 --- a/examples/v1beta1/spot.yaml +++ b/examples/v1beta1/spot.yaml @@ -27,6 +27,8 @@ spec: operator: Gt values: ["2"] nodeClassRef: + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass name: default --- apiVersion: karpenter.k8s.aws/v1beta1 diff --git a/examples/v1beta1/ubuntu-kubelet-log-query.yaml b/examples/v1beta1/ubuntu-kubelet-log-query.yaml index 9c6a4c27570c..1f5893071d73 100644 --- a/examples/v1beta1/ubuntu-kubelet-log-query.yaml +++ b/examples/v1beta1/ubuntu-kubelet-log-query.yaml @@ -27,6 +27,8 @@ spec: operator: Gt values: ["2"] nodeClassRef: + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass name: ubuntu --- apiVersion: karpenter.k8s.aws/v1beta1 diff --git a/examples/v1beta1/windows-custom-userdata.yaml b/examples/v1beta1/windows-custom-userdata.yaml index 412519503eba..dd638dc40b6f 100644 --- a/examples/v1beta1/windows-custom-userdata.yaml +++ b/examples/v1beta1/windows-custom-userdata.yaml @@ -29,6 +29,8 @@ spec: operator: Gt values: ["2"] nodeClassRef: + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass name: windows2022 --- apiVersion: karpenter.k8s.aws/v1beta1 diff --git a/examples/v1beta1/windows2019.yaml b/examples/v1beta1/windows2019.yaml index 8b3791ac8c22..edabbee534d6 100644 --- a/examples/v1beta1/windows2019.yaml +++ b/examples/v1beta1/windows2019.yaml @@ -26,6 +26,8 @@ spec: operator: Gt values: ["2"] nodeClassRef: + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass name: windows2019 --- apiVersion: karpenter.k8s.aws/v1beta1 diff --git a/examples/v1beta1/windows2022.yaml b/examples/v1beta1/windows2022.yaml index 899512fbc805..24fd6ccb6857 100644 --- a/examples/v1beta1/windows2022.yaml +++ b/examples/v1beta1/windows2022.yaml @@ -26,6 +26,8 @@ spec: operator: Gt values: ["2"] nodeClassRef: + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass name: windows2022 --- apiVersion: karpenter.k8s.aws/v1beta1 diff --git a/go.mod b/go.mod index 4c1aefd504ec..a41cd6f7cf20 100644 --- a/go.mod +++ b/go.mod @@ -1,33 +1,35 @@ module github.com/aws/karpenter-provider-aws -go 1.21 +go 1.22 require ( github.com/Pallinder/go-randomdata v1.2.0 - github.com/PuerkitoBio/goquery v1.8.1 - github.com/aws/aws-sdk-go v1.50.0 + github.com/PuerkitoBio/goquery v1.9.1 + github.com/aws/aws-sdk-go v1.51.12 github.com/aws/karpenter-provider-aws/tools/kompat v0.0.0-20231207011214-752356948623 + github.com/awslabs/amazon-eks-ami/nodeadm v0.0.0-20240229193347-cfab22a10647 github.com/go-logr/zapr v1.3.0 github.com/imdario/mergo v0.3.16 github.com/mitchellh/hashstructure/v2 v2.0.2 - github.com/onsi/ginkgo/v2 v2.15.0 - github.com/onsi/gomega v1.31.1 + github.com/onsi/ginkgo/v2 v2.17.1 + github.com/onsi/gomega v1.32.0 github.com/patrickmn/go-cache v2.1.0+incompatible - github.com/pelletier/go-toml/v2 v2.1.1 - github.com/prometheus/client_golang v1.18.0 + github.com/pelletier/go-toml/v2 v2.2.0 + github.com/prometheus/client_golang v1.19.0 github.com/samber/lo v1.39.0 go.uber.org/multierr v1.11.0 - go.uber.org/zap v1.26.0 + go.uber.org/zap v1.27.0 golang.org/x/sync v0.6.0 golang.org/x/time v0.5.0 - k8s.io/api v0.29.1 - k8s.io/apiextensions-apiserver v0.29.1 - k8s.io/apimachinery v0.29.1 - k8s.io/client-go v0.29.1 - k8s.io/utils v0.0.0-20230726121419-3b25d923346b + k8s.io/api v0.29.3 + k8s.io/apiextensions-apiserver v0.29.3 + k8s.io/apimachinery v0.29.3 + k8s.io/client-go v0.29.3 + k8s.io/utils v0.0.0-20240102154912-e7106e64919e knative.dev/pkg v0.0.0-20231010144348-ca8c009405dd - sigs.k8s.io/controller-runtime v0.17.0 - sigs.k8s.io/karpenter v0.33.1-0.20240116233859-f19e1d8dfbfa + sigs.k8s.io/controller-runtime v0.17.2 + sigs.k8s.io/karpenter v0.35.1-0.20240404200702-545d88a836d4 + sigs.k8s.io/yaml v1.4.0 ) require ( @@ -42,7 +44,6 @@ require ( github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/deckarep/golang-set v1.8.0 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch v5.7.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.8.0 // indirect @@ -57,7 +58,7 @@ require ( github.com/gobuffalo/flect v1.0.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.3 // indirect + github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect @@ -72,30 +73,29 @@ require ( github.com/kelseyhightower/envconfig v1.4.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-runewidth v0.0.15 // indirect - github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/client_model v0.5.0 // indirect - github.com/prometheus/common v0.45.0 // indirect + github.com/prometheus/client_model v0.6.0 // indirect + github.com/prometheus/common v0.48.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect github.com/prometheus/statsd_exporter v0.24.0 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect - github.com/spf13/cobra v1.7.0 // indirect + github.com/spf13/cobra v1.8.0 // indirect github.com/spf13/pflag v1.0.5 // indirect go.opencensus.io v0.24.0 // indirect go.uber.org/automaxprocs v1.5.3 // indirect golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect - golang.org/x/net v0.19.0 // indirect - golang.org/x/oauth2 v0.13.0 // indirect - golang.org/x/sys v0.16.0 // indirect - golang.org/x/term v0.15.0 // indirect + golang.org/x/net v0.23.0 // indirect + golang.org/x/oauth2 v0.16.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/term v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect - golang.org/x/tools v0.16.1 // indirect + golang.org/x/tools v0.17.0 // indirect gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/api v0.146.0 // indirect google.golang.org/appengine v1.6.8 // indirect @@ -103,16 +103,15 @@ require ( google.golang.org/genproto/googleapis/api v0.0.0-20231009173412-8bfb1ae86b6c // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20231009173412-8bfb1ae86b6c // indirect google.golang.org/grpc v1.58.3 // indirect - google.golang.org/protobuf v1.31.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/cloud-provider v0.29.0 // indirect - k8s.io/component-base v0.29.1 // indirect - k8s.io/csi-translation-lib v0.29.0 // indirect - k8s.io/klog/v2 v2.120.0 // indirect + k8s.io/cloud-provider v0.29.3 // indirect + k8s.io/component-base v0.29.3 // indirect + k8s.io/csi-translation-lib v0.29.3 // indirect + k8s.io/klog/v2 v2.120.1 // indirect k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect - sigs.k8s.io/yaml v1.4.0 // indirect ) diff --git a/go.sum b/go.sum index b57cefa12dd7..81643e3546c4 100644 --- a/go.sum +++ b/go.sum @@ -41,24 +41,25 @@ github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0 github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Pallinder/go-randomdata v1.2.0 h1:DZ41wBchNRb/0GfsePLiSwb0PHZmT67XY00lCDlaYPg= github.com/Pallinder/go-randomdata v1.2.0/go.mod h1:yHmJgulpD2Nfrm0cR9tI/+oAgRqCQQixsA8HyRZfV9Y= -github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= -github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= +github.com/PuerkitoBio/goquery v1.9.1 h1:mTL6XjbJTZdpfL+Gwl5U2h1l9yEkJjhmlTeV9VPW7UI= +github.com/PuerkitoBio/goquery v1.9.1/go.mod h1:cW1n6TmIMDoORQU5IU/P1T3tGFunOeXEpGP2WHRwkbY= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= -github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= github.com/andybalholm/cascadia v1.3.2 h1:3Xi6Dw5lHF15JtdcmAHD3i1+T8plmv7BQ/nsViSLyss= github.com/andybalholm/cascadia v1.3.2/go.mod h1:7gtRlve5FxPPgIgX36uWBX58OdBsSS6lUvCFb+h7KvU= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= -github.com/aws/aws-sdk-go v1.50.0 h1:HBtrLeO+QyDKnc3t1+5DR1RxodOHCGr8ZcrHudpv7jI= -github.com/aws/aws-sdk-go v1.50.0/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= +github.com/aws/aws-sdk-go v1.51.12 h1:DvuhIHZXwnjaR1/Gu19gUe1EGPw4J0qSJw4Qs/5PA8g= +github.com/aws/aws-sdk-go v1.51.12/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/aws/karpenter-provider-aws/tools/kompat v0.0.0-20231207011214-752356948623 h1:DQEFtmPyyMVHOyqva+DaWR6iAQG4h0KJbpSJAYlsnEo= github.com/aws/karpenter-provider-aws/tools/kompat v0.0.0-20231207011214-752356948623/go.mod h1:fpKKbSoh7nKrbAw8V44Ov1sgosfUvR1ZtyN9k44zHfY= +github.com/awslabs/amazon-eks-ami/nodeadm v0.0.0-20240229193347-cfab22a10647 h1:8yRBVsjGmI7qQsPWtIrbWP+XfwHO9Wq7gdLVzjqiZFs= +github.com/awslabs/amazon-eks-ami/nodeadm v0.0.0-20240229193347-cfab22a10647/go.mod h1:9NafTAUHL0FlMeL6Cu5PXnMZ1q/LnC9X2emLXHsVbM8= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -79,13 +80,11 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/deckarep/golang-set v1.8.0 h1:sk9/l/KqpunDwP7pSjUg0keiOOLEnOBHzykLrsPppp4= -github.com/deckarep/golang-set v1.8.0/go.mod h1:5nI87KwE7wgsBU1F4GKAw2Qod7p5kyS383rP6+o6qqo= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -165,8 +164,8 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= @@ -254,8 +253,6 @@ github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= -github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/hashstructure/v2 v2.0.2 h1:vGKWl0YJqUNxE8d+h8f6NJLcCJrgbhC4NcD46KavDd4= @@ -273,14 +270,14 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= -github.com/onsi/ginkgo/v2 v2.15.0 h1:79HwNRBAZHOEwrczrgSOPy+eFTTlIGELKy5as+ClttY= -github.com/onsi/ginkgo/v2 v2.15.0/go.mod h1:HlxMHtYF57y6Dpf+mc5529KKmSq9h2FpCF+/ZkwUxKM= -github.com/onsi/gomega v1.31.1 h1:KYppCUK+bUgAZwHOu7EXVBKyQA6ILvOESHkn/tgoqvo= -github.com/onsi/gomega v1.31.1/go.mod h1:y40C95dwAD1Nz36SsEnxvfFe8FFfNxzI5eJ0EYGyAy0= +github.com/onsi/ginkgo/v2 v2.17.1 h1:V++EzdbhI4ZV4ev0UTIj0PzhzOcReJFyJaLjtSF55M8= +github.com/onsi/ginkgo/v2 v2.17.1/go.mod h1:llBI3WDLL9Z6taip6f33H76YcWtJv+7R3HigUjbIBOs= +github.com/onsi/gomega v1.32.0 h1:JRYU78fJ1LPxlckP6Txi/EYqJvjtMrDC04/MM5XRHPk= +github.com/onsi/gomega v1.32.0/go.mod h1:a4x4gW6Pz2yK1MAmvluYme5lvYTn61afQ2ETw/8n4Lg= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= -github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI= -github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/pelletier/go-toml/v2 v2.2.0 h1:QLgLl2yMN7N+ruc31VynXs1vhMZa7CeHHejIeBAsoHo= +github.com/pelletier/go-toml/v2 v2.2.0/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -296,22 +293,22 @@ github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqr github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= -github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= -github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= +github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU= +github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= -github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/client_model v0.6.0 h1:k1v3CzpSRUTrKMppY35TLwPvxHqBu0bYgxZzqGIgaos= +github.com/prometheus/client_model v0.6.0/go.mod h1:NTQHnmxFpouOD0DpvP4XujX3CdOAGQPoaGhyTchlyt8= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.35.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= -github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= -github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= +github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= +github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= @@ -338,14 +335,15 @@ github.com/samber/lo v1.39.0/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXn github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= -github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= +github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -353,8 +351,9 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stvp/go-udp-testing v0.0.0-20201019212854-469649b16807/go.mod h1:7jxmlfBCDBXRzr0eAQJ48XC1hBu1np4CS5+cHEYfwpc= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -378,8 +377,8 @@ go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= -go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -456,15 +455,13 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= -golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= -golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -472,8 +469,8 @@ golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= -golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY= -golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0= +golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= +golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -533,14 +530,14 @@ golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= -golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= -golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= -golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -602,8 +599,8 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= -golang.org/x/tools v0.16.1 h1:TLyB3WofjdOEepBHAU20JdNC1Zbg87elYofWYAY5oZA= -golang.org/x/tools v0.16.1/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= +golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc= +golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -703,8 +700,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -733,37 +730,37 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -k8s.io/api v0.29.1 h1:DAjwWX/9YT7NQD4INu49ROJuZAAAP/Ijki48GUPzxqw= -k8s.io/api v0.29.1/go.mod h1:7Kl10vBRUXhnQQI8YR/R327zXC8eJ7887/+Ybta+RoQ= -k8s.io/apiextensions-apiserver v0.29.1 h1:S9xOtyk9M3Sk1tIpQMu9wXHm5O2MX6Y1kIpPMimZBZw= -k8s.io/apiextensions-apiserver v0.29.1/go.mod h1:zZECpujY5yTW58co8V2EQR4BD6A9pktVgHhvc0uLfeU= -k8s.io/apimachinery v0.29.1 h1:KY4/E6km/wLBguvCZv8cKTeOwwOBqFNjwJIdMkMbbRc= -k8s.io/apimachinery v0.29.1/go.mod h1:6HVkd1FwxIagpYrHSwJlQqZI3G9LfYWRPAkUvLnXTKU= -k8s.io/client-go v0.29.1 h1:19B/+2NGEwnFLzt0uB5kNJnfTsbV8w6TgQRz9l7ti7A= -k8s.io/client-go v0.29.1/go.mod h1:TDG/psL9hdet0TI9mGyHJSgRkW3H9JZk2dNEUS7bRks= -k8s.io/cloud-provider v0.29.0 h1:Qgk/jHsSKGRk/ltTlN6e7eaNuuamLROOzVBd0RPp94M= -k8s.io/cloud-provider v0.29.0/go.mod h1:gBCt7YYKFV4oUcJ/0xF9lS/9il4MxKunJ+ZKvh39WGo= -k8s.io/component-base v0.29.1 h1:MUimqJPCRnnHsskTTjKD+IC1EHBbRCVyi37IoFBrkYw= -k8s.io/component-base v0.29.1/go.mod h1:fP9GFjxYrLERq1GcWWZAE3bqbNcDKDytn2srWuHTtKc= -k8s.io/csi-translation-lib v0.29.0 h1:we4X1yUlDikvm5Rv0dwMuPHNw6KwjwsQiAuOPWXha8M= -k8s.io/csi-translation-lib v0.29.0/go.mod h1:Cp6t3CNBSm1dXS17V8IImUjkqfIB6KCj8Fs8wf6uyTA= -k8s.io/klog/v2 v2.120.0 h1:z+q5mfovBj1fKFxiRzsa2DsJLPIVMk/KFL81LMOfK+8= -k8s.io/klog/v2 v2.120.0/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= +k8s.io/api v0.29.3 h1:2ORfZ7+bGC3YJqGpV0KSDDEVf8hdGQ6A03/50vj8pmw= +k8s.io/api v0.29.3/go.mod h1:y2yg2NTyHUUkIoTC+phinTnEa3KFM6RZ3szxt014a80= +k8s.io/apiextensions-apiserver v0.29.3 h1:9HF+EtZaVpFjStakF4yVufnXGPRppWFEQ87qnO91YeI= +k8s.io/apiextensions-apiserver v0.29.3/go.mod h1:po0XiY5scnpJfFizNGo6puNU6Fq6D70UJY2Cb2KwAVc= +k8s.io/apimachinery v0.29.3 h1:2tbx+5L7RNvqJjn7RIuIKu9XTsIZ9Z5wX2G22XAa5EU= +k8s.io/apimachinery v0.29.3/go.mod h1:hx/S4V2PNW4OMg3WizRrHutyB5la0iCUbZym+W0EQIU= +k8s.io/client-go v0.29.3 h1:R/zaZbEAxqComZ9FHeQwOh3Y1ZUs7FaHKZdQtIc2WZg= +k8s.io/client-go v0.29.3/go.mod h1:tkDisCvgPfiRpxGnOORfkljmS+UrW+WtXAy2fTvXJB0= +k8s.io/cloud-provider v0.29.3 h1:y39hNq0lrPD1qmqQ2ykwMJGeWF9LsepVkR2a4wskwLc= +k8s.io/cloud-provider v0.29.3/go.mod h1:daDV1WkAO6pTrdsn7v8TpN/q9n75ExUC4RJDl7vlPKk= +k8s.io/component-base v0.29.3 h1:Oq9/nddUxlnrCuuR2K/jp6aflVvc0uDvxMzAWxnGzAo= +k8s.io/component-base v0.29.3/go.mod h1:Yuj33XXjuOk2BAaHsIGHhCKZQAgYKhqIxIjIr2UXYio= +k8s.io/csi-translation-lib v0.29.3 h1:GNYCE0f86K3Xkyrk7WKKwQZkJrum6QQapbOzYxZv6Mg= +k8s.io/csi-translation-lib v0.29.3/go.mod h1:snAzieA58/oiQXQZr27b0+b6/3+ZzitwI+57cUsMKKQ= +k8s.io/klog/v2 v2.120.1 h1:QXU6cPEOIslTGvZaXvFWiP9VKyeet3sawzTOvdXb4Vw= +k8s.io/klog/v2 v2.120.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= -k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= -k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20240102154912-e7106e64919e h1:eQ/4ljkx21sObifjzXwlPKpdGLrCfRziVtos3ofG/sQ= +k8s.io/utils v0.0.0-20240102154912-e7106e64919e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= knative.dev/pkg v0.0.0-20231010144348-ca8c009405dd h1:KJXBX9dOmRTUWduHg1gnWtPGIEl+GMh8UHdrBEZgOXE= knative.dev/pkg v0.0.0-20231010144348-ca8c009405dd/go.mod h1:36cYnaOVHkzmhgybmYX6zDaTl3PakFeJQJl7wi6/RLE= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/controller-runtime v0.17.0 h1:fjJQf8Ukya+VjogLO6/bNX9HE6Y2xpsO5+fyS26ur/s= -sigs.k8s.io/controller-runtime v0.17.0/go.mod h1:+MngTvIQQQhfXtwfdGw/UOQ/aIaqsYywfCINOtwMO/s= +sigs.k8s.io/controller-runtime v0.17.2 h1:FwHwD1CTUemg0pW2otk7/U5/i5m2ymzvOXdbeGOUvw0= +sigs.k8s.io/controller-runtime v0.17.2/go.mod h1:+MngTvIQQQhfXtwfdGw/UOQ/aIaqsYywfCINOtwMO/s= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/karpenter v0.33.1-0.20240116233859-f19e1d8dfbfa h1:XW2HP9imNJu99aRP1ZCSwZLrgVENCdF6sZ0h4L1jLmI= -sigs.k8s.io/karpenter v0.33.1-0.20240116233859-f19e1d8dfbfa/go.mod h1:ExeDTBVknsbC9x9K4/9gwrTt1Mo9HgMaV1NHPELHQPw= +sigs.k8s.io/karpenter v0.35.1-0.20240404200702-545d88a836d4 h1:eb3ubw9tFiV62Gq1Ci5r5BCJ8OlKu2Mqf7h3tJIRe0U= +sigs.k8s.io/karpenter v0.35.1-0.20240404200702-545d88a836d4/go.mod h1:fieFojxOec/l0tDmFT7R+g/Y+SGQbL9VlcYO8xb3sLo= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= diff --git a/hack/code/instancetype_testdata_gen/main.go b/hack/code/instancetype_testdata_gen/main.go index 35b684036226..9069cb63cd60 100644 --- a/hack/code/instancetype_testdata_gen/main.go +++ b/hack/code/instancetype_testdata_gen/main.go @@ -119,6 +119,7 @@ func getInstanceTypeInfo(info *ec2.InstanceTypeInfo) string { fmt.Fprintf(src, "BareMetal: aws.Bool(%t),\n", lo.FromPtr(info.BareMetal)) fmt.Fprintf(src, "Hypervisor: aws.String(\"%s\"),\n", lo.FromPtr(info.Hypervisor)) fmt.Fprintf(src, "ProcessorInfo: &ec2.ProcessorInfo{\n") + fmt.Fprintf(src, "Manufacturer: aws.String(\"%s\"),\n", lo.FromPtr(info.ProcessorInfo.Manufacturer)) fmt.Fprintf(src, "SupportedArchitectures: aws.StringSlice([]string{%s}),\n", getStringSliceData(info.ProcessorInfo.SupportedArchitectures)) fmt.Fprintf(src, "},\n") fmt.Fprintf(src, "VCpuInfo: &ec2.VCpuInfo{\n") diff --git a/hack/code/prices_gen/main.go b/hack/code/prices_gen/main.go index 1a2cfa6a998b..62518c9adee0 100644 --- a/hack/code/prices_gen/main.go +++ b/hack/code/prices_gen/main.go @@ -34,6 +34,7 @@ import ( "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/reconcile" + controllerspricing "github.com/aws/karpenter-provider-aws/pkg/controllers/pricing" "github.com/aws/karpenter-provider-aws/pkg/operator/options" "github.com/aws/karpenter-provider-aws/pkg/providers/pricing" "github.com/aws/karpenter-provider-aws/pkg/test" @@ -108,7 +109,7 @@ func main() { for _, region := range getAWSRegions(opts.partition) { log.Println("fetching for", region) pricingProvider := pricing.NewProvider(ctx, pricing.NewAPI(sess, region), ec2, region) - controller := pricing.NewController(pricingProvider) + controller := controllerspricing.NewController(pricingProvider) _, err := controller.Reconcile(ctx, reconcile.Request{NamespacedName: types.NamespacedName{}}) if err != nil { log.Fatalf("failed to initialize pricing provider %s", err) diff --git a/hack/docgen.sh b/hack/docgen.sh index 58655b38b2c5..037114698363 100755 --- a/hack/docgen.sh +++ b/hack/docgen.sh @@ -2,8 +2,10 @@ set -euo pipefail compatibilitymatrix() { + # versionCount is the number of K8s versions to display in the compatibility matrix + versionCount=7 go run hack/docs/version_compatibility.go hack/docs/compatibility-karpenter.yaml "$(git describe --exact-match --tags || echo "no tag")" - go run hack/docs/compatibilitymetrix_gen_docs.go website/content/en/preview/upgrading/compatibility.md hack/docs/compatibility-karpenter.yaml 6 + go run hack/docs/compatibilitymetrix_gen_docs.go website/content/en/preview/upgrading/compatibility.md hack/docs/compatibility-karpenter.yaml $versionCount } @@ -11,4 +13,4 @@ compatibilitymatrix go run hack/docs/metrics_gen_docs.go pkg/ ${KARPENTER_CORE_DIR}/pkg website/content/en/preview/reference/metrics.md go run hack/docs/instancetypes_gen_docs.go website/content/en/preview/reference/instance-types.md go run hack/docs/configuration_gen_docs.go website/content/en/preview/reference/settings.md -cd charts/karpenter && helm-docs \ No newline at end of file +cd charts/karpenter && helm-docs diff --git a/hack/docs/compatibility-karpenter.yaml b/hack/docs/compatibility-karpenter.yaml index bb5f4ef1fe9e..0a35c0a6a884 100644 --- a/hack/docs/compatibility-karpenter.yaml +++ b/hack/docs/compatibility-karpenter.yaml @@ -35,4 +35,13 @@ compatibility: maxK8sVersion: 1.28 - appVersion: 0.32.x minK8sVersion: 1.23 - maxK8sVersion: 1.28 \ No newline at end of file + maxK8sVersion: 1.28 + - appVersion: 0.33.x + minK8sVersion: 1.23 + maxK8sVersion: 1.28 + - appVersion: 0.34.x + minK8sVersion: 1.23 + maxK8sVersion: 1.29 + - appVersion: 0.35.x + minK8sVersion: 1.23 + maxK8sVersion: 1.29 \ No newline at end of file diff --git a/hack/docs/metrics_gen_docs.go b/hack/docs/metrics_gen_docs.go index 5522034f0001..ffc7c07ed852 100644 --- a/hack/docs/metrics_gen_docs.go +++ b/hack/docs/metrics_gen_docs.go @@ -284,6 +284,7 @@ func getIdentMapping(identName string) (string, error) { "consistencySubsystem": "consistency", "batcherSubsystem": "cloudprovider_batcher", "cloudProviderSubsystem": "cloudprovider", + "stateSubsystem": "cluster_state", } if v, ok := identMapping[identName]; ok { return v, nil diff --git a/hack/github/community-contributors.sh b/hack/github/community-contributors.sh index 0404a56186db..1466a6d2b523 100755 --- a/hack/github/community-contributors.sh +++ b/hack/github/community-contributors.sh @@ -77,6 +77,9 @@ COMMUNITY_CONTRIBUTIONS=$( .author != "njtran" and .author != "dewjam" and .author != "suket22" and + .author != "Jigisha Patil" and + .author != "jigisha620" and + .author != "nikmohan123" and .author != "StableRelease" and .author != "dependabot[bot]" and .author != "github-actions[bot]" and diff --git a/hack/release/common.sh b/hack/release/common.sh index ae7ea45b537a..1edf2091beb1 100644 --- a/hack/release/common.sh +++ b/hack/release/common.sh @@ -1,62 +1,46 @@ #!/usr/bin/env bash set -euo pipefail -config(){ - GITHUB_ACCOUNT="aws" - AWS_ACCOUNT_ID="605559265918" - ECR_GALLERY_NAME="karpenter" - RELEASE_REPO_ECR=${RELEASE_REPO_ECR:-public.ecr.aws/${ECR_GALLERY_NAME}/} - RELEASE_REPO_GH=${RELEASE_REPO_GH:-ghcr.io/${GITHUB_ACCOUNT}/karpenter} - - PRIVATE_HOST="${AWS_ACCOUNT_ID}.dkr.ecr.us-west-2.amazonaws.com" - SNAPSHOT_REPO_ECR=${SNAPSHOT_REPO_ECR:-${PRIVATE_HOST}/karpenter/snapshot/} - - CURRENT_MAJOR_VERSION="0" - RELEASE_PLATFORM="--platform=linux/amd64,linux/arm64" - - MAIN_GITHUB_ACCOUNT="aws" - RELEASE_TYPE_STABLE="stable" - RELEASE_TYPE_SNAPSHOT="snapshot" -} +GITHUB_ACCOUNT="aws" +AWS_ACCOUNT_ID="605559265918" +ECR_GALLERY_NAME="karpenter" +RELEASE_REPO_ECR=${RELEASE_REPO_ECR:-public.ecr.aws/${ECR_GALLERY_NAME}/} +RELEASE_REPO_GH=${RELEASE_REPO_GH:-ghcr.io/${GITHUB_ACCOUNT}/karpenter} -# versionData sets all the version properties for the passed release version. It sets the values -# RELEASE_VERSION_MAJOR, RELEASE_VERSION_MINOR, and RELEASE_VERSION_PATCH to be used by other scripts -versionData(){ - local VERSION="$1" - local VERSION="${VERSION#[vV]}" - RELEASE_VERSION_MAJOR="${VERSION%%\.*}" - RELEASE_VERSION_MINOR="${VERSION#*.}" - RELEASE_VERSION_MINOR="${RELEASE_VERSION_MINOR%.*}" - RELEASE_VERSION_PATCH="${VERSION##*.}" - RELEASE_MINOR_VERSION="v${RELEASE_VERSION_MAJOR}.${RELEASE_VERSION_MINOR}" -} +SNAPSHOT_ECR="${AWS_ACCOUNT_ID}.dkr.ecr.us-west-2.amazonaws.com" +SNAPSHOT_REPO_ECR=${SNAPSHOT_REPO_ECR:-${SNAPSHOT_ECR}/karpenter/snapshot/} +CURRENT_MAJOR_VERSION="0" snapshot() { - RELEASE_VERSION=$1 + local commit_sha version helm_chart_version + + commit_sha="${1}" + version="${commit_sha}" + helm_chart_version="${CURRENT_MAJOR_VERSION}-${commit_sha}" + echo "Release Type: snapshot -Release Version: ${RELEASE_VERSION} -Commit: $(git rev-parse HEAD) -Helm Chart Version $(helmChartVersion $RELEASE_VERSION)" +Release Version: ${version} +Commit: ${commit_sha} +Helm Chart Version ${helm_chart_version}" authenticatePrivateRepo - buildImages "${SNAPSHOT_REPO_ECR}" - cosignImages - publishHelmChart "karpenter" "${RELEASE_VERSION}" "${SNAPSHOT_REPO_ECR}" - publishHelmChart "karpenter-crd" "${RELEASE_VERSION}" "${SNAPSHOT_REPO_ECR}" + build "${SNAPSHOT_REPO_ECR}" "${version}" "${helm_chart_version}" "${commit_sha}" } release() { - RELEASE_VERSION=$1 + local commit_sha version helm_chart_version + + commit_sha="${1}" + version="${2}" + helm_chart_version="${version}" + echo "Release Type: stable -Release Version: ${RELEASE_VERSION} -Commit: $(git rev-parse HEAD) -Helm Chart Version $(helmChartVersion $RELEASE_VERSION)" +Release Version: ${version} +Commit: ${commit_sha} +Helm Chart Version ${helm_chart_version}" authenticate - buildImages "${RELEASE_REPO_ECR}" - cosignImages - publishHelmChart "karpenter" "${RELEASE_VERSION}" "${RELEASE_REPO_ECR}" - publishHelmChart "karpenter-crd" "${RELEASE_VERSION}" "${RELEASE_REPO_ECR}" + build "${RELEASE_REPO_ECR}" "${version}" "${helm_chart_version}" "${commit_sha}" } authenticate() { @@ -64,125 +48,168 @@ authenticate() { } authenticatePrivateRepo() { - aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin ${PRIVATE_HOST} + aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin ${SNAPSHOT_ECR} } -buildImages() { - RELEASE_REPO=$1 - # Set the SOURCE_DATE_EPOCH and KO_DATA_DATE_EPOCH values for reproducable builds with timestamps - # https://ko.build/advanced/faq/ - CONTROLLER_IMG=$(GOFLAGS=${GOFLAGS} SOURCE_DATE_EPOCH=$(git log -1 --format='%ct') KO_DATA_DATE_EPOCH=$(git log -1 --format='%ct') KO_DOCKER_REPO=${RELEASE_REPO} ko publish -B -t "${RELEASE_VERSION}" "${RELEASE_PLATFORM}" ./cmd/controller) - HELM_CHART_VERSION=$(helmChartVersion "$RELEASE_VERSION") - IMG_REPOSITORY=$(echo "$CONTROLLER_IMG" | cut -d "@" -f 1 | cut -d ":" -f 1) - IMG_TAG=$(echo "$CONTROLLER_IMG" | cut -d "@" -f 1 | cut -d ":" -f 2 -s) - IMG_DIGEST=$(echo "$CONTROLLER_IMG" | cut -d "@" -f 2) - yq e -i ".controller.image.repository = \"${IMG_REPOSITORY}\"" charts/karpenter/values.yaml - yq e -i ".controller.image.tag = \"${IMG_TAG}\"" charts/karpenter/values.yaml - yq e -i ".controller.image.digest = \"${IMG_DIGEST}\"" charts/karpenter/values.yaml - yq e -i ".appVersion = \"${RELEASE_VERSION#v}\"" charts/karpenter/Chart.yaml - yq e -i ".version = \"${HELM_CHART_VERSION#v}\"" charts/karpenter/Chart.yaml - yq e -i ".appVersion = \"${RELEASE_VERSION#v}\"" charts/karpenter-crd/Chart.yaml - yq e -i ".version = \"${HELM_CHART_VERSION#v}\"" charts/karpenter-crd/Chart.yaml -} +build() { + local oci_repo version helm_chart_version commit_sha date_epoch build_date img img_repo img_tag img_digest + + oci_repo="${1}" + version="${2}" + helm_chart_version="${3}" + commit_sha="${4}" -releaseType(){ - RELEASE_VERSION=$1 + date_epoch="$(dateEpoch)" + build_date="$(buildDate "${date_epoch}")" - if [[ "${RELEASE_VERSION}" == v* ]]; then - echo $RELEASE_TYPE_STABLE - else - echo $RELEASE_TYPE_SNAPSHOT + img="$(GOFLAGS=${GOFLAGS:-} SOURCE_DATE_EPOCH="${date_epoch}" KO_DATA_DATE_EPOCH="${date_epoch}" KO_DOCKER_REPO="${oci_repo}" ko publish -B -t "${version}" ./cmd/controller)" + img_repo="$(echo "${img}" | cut -d "@" -f 1 | cut -d ":" -f 1)" + img_tag="$(echo "${img}" | cut -d "@" -f 1 | cut -d ":" -f 2 -s)" + img_digest="$(echo "${img}" | cut -d "@" -f 2)" + + cosignOciArtifact "${version}" "${commit_sha}" "${build_date}" "${img}" + + yq e -i ".controller.image.repository = \"${img_repo}\"" charts/karpenter/values.yaml + yq e -i ".controller.image.tag = \"${img_tag}\"" charts/karpenter/values.yaml + yq e -i ".controller.image.digest = \"${img_digest}\"" charts/karpenter/values.yaml + + publishHelmChart "${oci_repo}" "karpenter" "${helm_chart_version}" "${commit_sha}" "${build_date}" + publishHelmChart "${oci_repo}" "karpenter-crd" "${helm_chart_version}" "${commit_sha}" "${build_date}" +} + +publishHelmChart() { + local oci_repo helm_chart version commit_sha build_date ah_config_file_name helm_chart_artifact helm_chart_digest + + oci_repo="${1}" + helm_chart="${2}" + version="${3}" + commit_sha="${4}" + build_date="${5}" + + ah_config_file_name="${helm_chart}/artifacthub-repo.yaml" + helm_chart_artifact="${helm_chart}-${version}.tgz" + + yq e -i ".appVersion = \"${version}\"" "charts/${helm_chart}/Chart.yaml" + yq e -i ".version = \"${version}\"" "charts/${helm_chart}/Chart.yaml" + + cd charts + if [[ -s "${ah_config_file_name}" ]] && [[ "$oci_repo" == "${RELEASE_REPO_ECR}" ]]; then + # ECR requires us to create an empty config file for an alternative + # media type artifact push rather than /dev/null + # https://github.com/aws/containers-roadmap/issues/1074 + temp=$(mktemp) + echo {} > "${temp}" + oras push "${oci_repo}${helm_chart}:artifacthub.io" --config "${temp}:application/vnd.cncf.artifacthub.config.v1+yaml" "${ah_config_file_name}:application/vnd.cncf.artifacthub.repository-metadata.layer.v1.yaml" fi + helm dependency update "${helm_chart}" + helm lint "${helm_chart}" + helm package "${helm_chart}" --version "${version}" + helm push "${helm_chart_artifact}" "oci://${oci_repo}" + rm "${helm_chart_artifact}" + cd .. + + helm_chart_digest="$(crane digest "${oci_repo}/${helm_chart}:${version}")" + cosignOciArtifact "${version}" "${commit_sha}" "${build_date}" "${oci_repo}${helm_chart}:${version}@${helm_chart_digest}" } -helmChartVersion(){ - RELEASE_VERSION=$1 - if [[ $(releaseType "$RELEASE_VERSION") == "$RELEASE_TYPE_STABLE" ]]; then - echo "$RELEASE_VERSION" - fi +cosignOciArtifact() { + local version commit_sha build_date artifact - if [[ $(releaseType "$RELEASE_VERSION") == "$RELEASE_TYPE_SNAPSHOT" ]]; then - echo "v${CURRENT_MAJOR_VERSION}-${RELEASE_VERSION}" - fi + version="${1}" + commit_sha="${2}" + build_date="${3}" + artifact="${4}" + + cosign sign --yes -a version="${version}" -a commitSha="${commit_sha}" -a buildDate="${build_date}" "${artifact}" } -buildDate(){ - # Set the SOURCE_DATE_EPOCH and KO_DATA_DATE_EPOCH values for reproducable builds with timestamps - # https://ko.build/advanced/faq/ - DATE_FMT="+%Y-%m-%dT%H:%M:%SZ" - SOURCE_DATE_EPOCH=$(git log -1 --format='%ct') - echo "$(date -u -r "${SOURCE_DATE_EPOCH}" $DATE_FMT 2>/dev/null)" +dateEpoch() { + git log -1 --format='%ct' } -cosignImages() { - COSIGN_EXPERIMENTAL=1 cosign sign \ - -a GIT_HASH="$(git rev-parse HEAD)" \ - -a GIT_VERSION="${RELEASE_VERSION}" \ - -a BUILD_DATE="$(buildDate)" \ - "${CONTROLLER_IMG}" +buildDate() { + local date_epoch + + date_epoch="${1}" + + date -u --date="@${date_epoch}" "+%Y-%m-%dT%H:%M:%SZ" 2>/dev/null } -publishHelmChart() { - CHART_NAME=$1 - RELEASE_VERSION=$2 - RELEASE_REPO=$3 - HELM_CHART_VERSION=$(helmChartVersion "$RELEASE_VERSION") - HELM_CHART_FILE_NAME="${CHART_NAME}-${HELM_CHART_VERSION}.tgz" - - cd charts - helm dependency update "${CHART_NAME}" - helm lint "${CHART_NAME}" - helm package "${CHART_NAME}" --version "$HELM_CHART_VERSION" - helm push "${HELM_CHART_FILE_NAME}" "oci://${RELEASE_REPO}" - rm "${HELM_CHART_FILE_NAME}" - cd .. +prepareWebsite() { + local version version_parts short_version + + version="${1}" + # shellcheck disable=SC2206 + version_parts=(${version//./ }) + short_version="${version_parts[0]}.${version_parts[1]}" + + createNewWebsiteDirectory "${short_version}" "${version}" + removeOldWebsiteDirectories + editWebsiteConfig "${version}" + editWebsiteVersionsMenu } createNewWebsiteDirectory() { - RELEASE_VERSION=$1 - versionData "${RELEASE_VERSION}" - - mkdir -p "website/content/en/${RELEASE_MINOR_VERSION}" - cp -r website/content/en/preview/* "website/content/en/${RELEASE_MINOR_VERSION}/" - find "website/content/en/${RELEASE_MINOR_VERSION}/" -type f | xargs perl -i -p -e "s/{{< param \"latest_release_version\" >}}/${RELEASE_VERSION}/g;" - find website/content/en/${RELEASE_MINOR_VERSION}/*/*/*.yaml -type f | xargs perl -i -p -e "s/preview/${RELEASE_MINOR_VERSION}/g;" - find "website/content/en/${RELEASE_MINOR_VERSION}/" -type f | xargs perl -i -p -e "s/{{< githubRelRef >}}/\/${RELEASE_VERSION}\//g;" - - rm -rf website/content/en/docs - mkdir -p website/content/en/docs - cp -r website/content/en/${RELEASE_MINOR_VERSION}/* website/content/en/docs/ + local short_version version + + short_version="${1}" + version="${2}" + + mkdir -p "website/content/en/v${short_version}" + cp -r website/content/en/preview/* "website/content/en/v${short_version}/" + + # Update parameterized variables in the preview documentation to be statically set in the versioned documentation + # shellcheck disable=SC2038 + find "website/content/en/v${short_version}/" -type f -print | xargs perl -i -p -e "s/{{< param \"latest_release_version\" >}}/${version}/g;" + # shellcheck disable=SC2038 + find "website/content/en/v${short_version}/" -type f | xargs perl -i -p -e "s/{{< param \"latest_k8s_version\" >}}/$(yq .params.latest_k8s_version website/hugo.yaml)/g;" + # shellcheck disable=SC2038 + find "website/content/en/v${short_version}/"*/*/*.yaml -type f | xargs perl -i -p -e "s/preview/v${short_version}/g;" + # shellcheck disable=SC2038 + find "website/content/en/v${short_version}/" -type f | xargs perl -i -p -e "s/{{< githubRelRef >}}/\/v${version}\//g;" + + rm -rf website/content/en/docs + mkdir -p website/content/en/docs + cp -r "website/content/en/v${short_version}/"* website/content/en/docs/ } removeOldWebsiteDirectories() { - local n=3 + local n=3 last_n_versions all + # Get all the directories except the last n directories sorted from earliest to latest version # preview, docs, and v0.32 are special directories that we always propagate into the set of directory options # Keep the v0.32 version around while we are supporting v1beta1 migration # Drop it once we no longer want to maintain the v0.32 version in the docs - last_n_versions=$(find website/content/en/* -type d -name "*" -maxdepth 0 | grep -v "preview\|docs\|v0.32" | sort | tail -n "$n") + last_n_versions=$(find website/content/en/* -maxdepth 0 -type d -name "*" | grep -v "preview\|docs\|v0.32" | sort | tail -n "${n}") last_n_versions+=$(echo -e "\nwebsite/content/en/preview") last_n_versions+=$(echo -e "\nwebsite/content/en/docs") last_n_versions+=$(echo -e "\nwebsite/content/en/v0.32") - all=$(find website/content/en/* -type d -name "*" -maxdepth 0) + all=$(find website/content/en/* -maxdepth 0 -type d -name "*") + ## symmetric difference - comm -3 <(sort <<< $last_n_versions) <(sort <<< $all) | tr -d '\t' | xargs -r -n 1 rm -r + # shellcheck disable=SC2086 + comm -3 <(sort <<< ${last_n_versions}) <(sort <<< ${all}) | tr -d '\t' | xargs -r -n 1 rm -r } editWebsiteConfig() { - RELEASE_VERSION=$1 - yq -i ".params.latest_release_version = \"${RELEASE_VERSION}\"" website/hugo.yaml + local version="${1}" + + yq -i ".params.latest_release_version = \"${version}\"" website/hugo.yaml } # editWebsiteVersionsMenu sets relevant releases in the version dropdown menu of the website # without increasing the size of the set. # It uses the current version directories (ignoring the docs directory) to generate this list editWebsiteVersionsMenu() { - VERSIONS=($(find website/content/en/* -type d -name "*" -maxdepth 0 | xargs basename | grep -v "docs\|preview")) - VERSIONS+=('preview') + local versions version + + # shellcheck disable=SC2207 + versions=($(find website/content/en/* -maxdepth 0 -type d -name "*" -print0 | xargs -0 -r -n 1 basename | grep -v "docs\|preview")) + versions+=('preview') yq -i '.params.versions = []' website/hugo.yaml - for VERSION in "${VERSIONS[@]}"; do - yq -i ".params.versions += \"${VERSION}\"" website/hugo.yaml + for version in "${versions[@]}"; do + yq -i ".params.versions += \"${version}\"" website/hugo.yaml done } diff --git a/hack/release/prepare-website.sh b/hack/release/prepare-website.sh index 96b396b2bc84..80bfb8e41672 100755 --- a/hack/release/prepare-website.sh +++ b/hack/release/prepare-website.sh @@ -1,19 +1,15 @@ #!/usr/bin/env bash set -euo pipefail -SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" +# shellcheck source=hack/release/common.sh source "${SCRIPT_DIR}/common.sh" -config - -GIT_TAG=${GIT_TAG:-$(git describe --exact-match --tags || echo "none")} -if [[ $(releaseType "$GIT_TAG") != $RELEASE_TYPE_STABLE ]]; then +git_tag="${GIT_TAG:-$(git describe --exact-match --tags || echo "none")}" +if [[ "${git_tag}" != v* ]]; then echo "Not a stable release. Missing required git tag." exit 1 fi -echo "RenderingPrep website files for ${GIT_TAG}" +echo "RenderingPrep website files for ${git_tag}" -createNewWebsiteDirectory "$GIT_TAG" -removeOldWebsiteDirectories -editWebsiteConfig "$GIT_TAG" -editWebsiteVersionsMenu +prepareWebsite "${git_tag#v}" diff --git a/hack/release/release-crd.sh b/hack/release/release-crd.sh deleted file mode 100755 index fb3ab0a074ef..000000000000 --- a/hack/release/release-crd.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -HEAD_HASH=$(git rev-parse HEAD) -GIT_TAG=$(git describe --exact-match --tags || echo "no tag") - -SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) -source "${SCRIPT_DIR}/common.sh" - -config -publishHelmChart "karpenter-crd" "${HEAD_HASH}" "${RELEASE_REPO_GH}" - -if [[ $(releaseType $GIT_TAG) == $RELEASE_TYPE_STABLE ]]; then - publishHelmChart "karpenter-crd" "${GIT_TAG}" "${RELEASE_REPO_GH}" -fi diff --git a/hack/release/release.sh b/hack/release/release.sh index 2775e026fb4c..d058460eef9e 100755 --- a/hack/release/release.sh +++ b/hack/release/release.sh @@ -1,20 +1,20 @@ #!/usr/bin/env bash set -euo pipefail -GIT_TAG=$(git describe --exact-match --tags || echo "no tag") -if [[ "$GIT_TAG" == "no tag" ]]; then - echo "Failed to release: commit is untagged" - exit 1 -fi -HEAD_HASH=$(git rev-parse HEAD) - -SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" +# shellcheck source=hack/release/common.sh source "${SCRIPT_DIR}/common.sh" -config + +git_tag="$(git describe --exact-match --tags || echo "no tag")" +if [[ "${git_tag}" == "no tag" ]]; then + echo "Failed to release: commit is untagged" + exit 1 +fi +commit_sha="$(git rev-parse HEAD)" # Don't release with a dirty commit! if [[ "$(git status --porcelain)" != "" ]]; then - exit 1 + exit 1 fi -release "$GIT_TAG" +release "${commit_sha}" "${git_tag#v}" diff --git a/hack/release/snapshot.sh b/hack/release/snapshot.sh index 3213f5bf3bb2..14626271950b 100755 --- a/hack/release/snapshot.sh +++ b/hack/release/snapshot.sh @@ -1,15 +1,15 @@ #!/usr/bin/env bash set -euo pipefail -HEAD_HASH=$(git rev-parse HEAD) - -SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" +# shellcheck source=hack/release/common.sh source "${SCRIPT_DIR}/common.sh" -config + +commit_sha="$(git rev-parse HEAD)" # Don't release with a dirty commit! if [[ "$(git status --porcelain)" != "" ]]; then - exit 1 + exit 1 fi -snapshot "$HEAD_HASH" +snapshot "${commit_sha}" diff --git a/hack/release/stable-pr.sh b/hack/release/stable-pr.sh index 22dfa2858e30..eee4a5d69282 100755 --- a/hack/release/stable-pr.sh +++ b/hack/release/stable-pr.sh @@ -1,49 +1,31 @@ #!/usr/bin/env bash set -euo pipefail -# updateKarpenterCoreGoMod bumps the karpenter-core go.mod to the release version so that the -# karpenter and karpenter-core release versions match -updateKarpenterCoreGoMod(){ - RELEASE_VERSION=$1 - if [[ $GITHUB_ACCOUNT != $MAIN_GITHUB_ACCOUNT ]]; then - echo "not updating go mod for a repo other than the main repo" - return - fi - go get -u "sigs.k8s.io/karpenter@${RELEASE_VERSION}" - cd test - go get -u "sigs.k8s.io/karpenter@${RELEASE_VERSION}" - cd .. - make tidy -} - -SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) +SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" +# shellcheck source=hack/release/common.sh source "${SCRIPT_DIR}/common.sh" -config - -GIT_TAG=$(git describe --exact-match --tags || echo "none") -if [[ $(releaseType $GIT_TAG) != $RELEASE_TYPE_STABLE ]]; then +git_tag="$(git describe --exact-match --tags || echo "none")" +if [[ "${git_tag}" != v* ]]; then echo "Not a stable release. Missing required git tag." exit 1 fi -versionData "$GIT_TAG" -updateKarpenterCoreGoMod "$GIT_TAG" - git config user.name "StableRelease" git config user.email "StableRelease@users.noreply.github.com" -git remote set-url origin https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPO} +git remote set-url origin "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPO}" git config pull.rebase false -BRANCH_NAME="release-${GIT_TAG}" -git checkout -b "${BRANCH_NAME}" +branch_name="release-${git_tag}" +git checkout -b "${branch_name}" git add go.mod git add go.sum +git add hack/docs git add website git add charts/karpenter-crd/Chart.yaml git add charts/karpenter/Chart.yaml git add charts/karpenter/Chart.lock git add charts/karpenter/values.yaml git add charts/karpenter/README.md -git commit -m "Stable Release updates Release ${GIT_TAG}." -git push --set-upstream origin "${BRANCH_NAME}" +git commit -m "Stable Release updates Release ${git_tag}." +git push --set-upstream origin "${branch_name}" diff --git a/hack/toolchain.sh b/hack/toolchain.sh index 55f01e175d72..9bf6dbc47c96 100755 --- a/hack/toolchain.sh +++ b/hack/toolchain.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail -K8S_VERSION="${K8S_VERSION:="1.28.x"}" +K8S_VERSION="${K8S_VERSION:="1.29.x"}" KUBEBUILDER_ASSETS="/usr/local/kubebuilder/bin" main() { @@ -23,6 +23,8 @@ tools() { go install github.com/onsi/ginkgo/v2/ginkgo@latest go install github.com/rhysd/actionlint/cmd/actionlint@latest go install github.com/mattn/goveralls@latest + go install github.com/google/go-containerregistry/cmd/crane@latest + go install oras.land/oras/cmd/oras@latest if ! echo "$PATH" | grep -q "${GOPATH:-undefined}/bin\|$HOME/go/bin"; then echo "Go workspace's \"bin\" directory is not in PATH. Run 'export PATH=\"\$PATH:\${GOPATH:-\$HOME/go}/bin\"'." @@ -33,8 +35,17 @@ kubebuilder() { sudo mkdir -p ${KUBEBUILDER_ASSETS} sudo chown "${USER}" ${KUBEBUILDER_ASSETS} arch=$(go env GOARCH) - ln -sf $(setup-envtest use -p path "${K8S_VERSION}" --arch="${arch}" --bin-dir="${KUBEBUILDER_ASSETS}")/* ${KUBEBUILDER_ASSETS} + ln -sf "$(setup-envtest use -p path "${K8S_VERSION}" --arch="${arch}" --bin-dir="${KUBEBUILDER_ASSETS}")"/* ${KUBEBUILDER_ASSETS} find $KUBEBUILDER_ASSETS + + # Install latest binaries for 1.25.x (contains CEL fix) + if [[ "${K8S_VERSION}" = "1.25.x" ]] && [[ "$OSTYPE" == "linux"* ]]; then + for binary in 'kube-apiserver' 'kubectl'; do + rm $KUBEBUILDER_ASSETS/$binary + wget -P $KUBEBUILDER_ASSETS dl.k8s.io/v1.25.16/bin/linux/${arch}/${binary} + chmod +x $KUBEBUILDER_ASSETS/$binary + done + fi } main "$@" diff --git a/hack/validation/labels.sh b/hack/validation/labels.sh index 144b66c1e053..1fb9c6293b32 100755 --- a/hack/validation/labels.sh +++ b/hack/validation/labels.sh @@ -4,4 +4,4 @@ # ## checking for restricted labels while filtering out well known labels yq eval '.spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.template.properties.metadata.properties.labels.x-kubernetes-validations += [ - {"message": "label domain \"karpenter.k8s.aws\" is restricted", "rule": "self.all(x, x in [\"karpenter.k8s.aws/instance-encryption-in-transit-supported\", \"karpenter.k8s.aws/instance-category\", \"karpenter.k8s.aws/instance-hypervisor\", \"karpenter.k8s.aws/instance-family\", \"karpenter.k8s.aws/instance-generation\", \"karpenter.k8s.aws/instance-local-nvme\", \"karpenter.k8s.aws/instance-size\", \"karpenter.k8s.aws/instance-cpu\",\"karpenter.k8s.aws/instance-memory\", \"karpenter.k8s.aws/instance-network-bandwidth\", \"karpenter.k8s.aws/instance-gpu-name\", \"karpenter.k8s.aws/instance-gpu-manufacturer\", \"karpenter.k8s.aws/instance-gpu-count\", \"karpenter.k8s.aws/instance-gpu-memory\", \"karpenter.k8s.aws/instance-accelerator-name\", \"karpenter.k8s.aws/instance-accelerator-manufacturer\", \"karpenter.k8s.aws/instance-accelerator-count\"] || !x.find(\"^([^/]+)\").endsWith(\"karpenter.k8s.aws\"))"}]' -i pkg/apis/crds/karpenter.sh_nodepools.yaml \ No newline at end of file + {"message": "label domain \"karpenter.k8s.aws\" is restricted", "rule": "self.all(x, x in [\"karpenter.k8s.aws/instance-encryption-in-transit-supported\", \"karpenter.k8s.aws/instance-category\", \"karpenter.k8s.aws/instance-hypervisor\", \"karpenter.k8s.aws/instance-family\", \"karpenter.k8s.aws/instance-generation\", \"karpenter.k8s.aws/instance-local-nvme\", \"karpenter.k8s.aws/instance-size\", \"karpenter.k8s.aws/instance-cpu\",\"karpenter.k8s.aws/instance-cpu-manufacturer\",\"karpenter.k8s.aws/instance-memory\", \"karpenter.k8s.aws/instance-network-bandwidth\", \"karpenter.k8s.aws/instance-gpu-name\", \"karpenter.k8s.aws/instance-gpu-manufacturer\", \"karpenter.k8s.aws/instance-gpu-count\", \"karpenter.k8s.aws/instance-gpu-memory\", \"karpenter.k8s.aws/instance-accelerator-name\", \"karpenter.k8s.aws/instance-accelerator-manufacturer\", \"karpenter.k8s.aws/instance-accelerator-count\"] || !x.find(\"^([^/]+)\").endsWith(\"karpenter.k8s.aws\"))"}]' -i pkg/apis/crds/karpenter.sh_nodepools.yaml \ No newline at end of file diff --git a/hack/validation/requirements.sh b/hack/validation/requirements.sh index b33f9774163b..763d359fab16 100755 --- a/hack/validation/requirements.sh +++ b/hack/validation/requirements.sh @@ -4,9 +4,9 @@ ## checking for restricted labels while filtering out well known labels yq eval '.spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.requirements.items.properties.key.x-kubernetes-validations += [ - {"message": "label domain \"karpenter.k8s.aws\" is restricted", "rule": "self in [\"karpenter.k8s.aws/instance-encryption-in-transit-supported\", \"karpenter.k8s.aws/instance-category\", \"karpenter.k8s.aws/instance-hypervisor\", \"karpenter.k8s.aws/instance-family\", \"karpenter.k8s.aws/instance-generation\", \"karpenter.k8s.aws/instance-local-nvme\", \"karpenter.k8s.aws/instance-size\", \"karpenter.k8s.aws/instance-cpu\",\"karpenter.k8s.aws/instance-memory\", \"karpenter.k8s.aws/instance-network-bandwidth\", \"karpenter.k8s.aws/instance-gpu-name\", \"karpenter.k8s.aws/instance-gpu-manufacturer\", \"karpenter.k8s.aws/instance-gpu-count\", \"karpenter.k8s.aws/instance-gpu-memory\", \"karpenter.k8s.aws/instance-accelerator-name\", \"karpenter.k8s.aws/instance-accelerator-manufacturer\", \"karpenter.k8s.aws/instance-accelerator-count\"] || !self.find(\"^([^/]+)\").endsWith(\"karpenter.k8s.aws\")"}]' -i pkg/apis/crds/karpenter.sh_nodeclaims.yaml + {"message": "label domain \"karpenter.k8s.aws\" is restricted", "rule": "self in [\"karpenter.k8s.aws/instance-encryption-in-transit-supported\", \"karpenter.k8s.aws/instance-category\", \"karpenter.k8s.aws/instance-hypervisor\", \"karpenter.k8s.aws/instance-family\", \"karpenter.k8s.aws/instance-generation\", \"karpenter.k8s.aws/instance-local-nvme\", \"karpenter.k8s.aws/instance-size\", \"karpenter.k8s.aws/instance-cpu\",\"karpenter.k8s.aws/instance-cpu-manufacturer\",\"karpenter.k8s.aws/instance-memory\", \"karpenter.k8s.aws/instance-network-bandwidth\", \"karpenter.k8s.aws/instance-gpu-name\", \"karpenter.k8s.aws/instance-gpu-manufacturer\", \"karpenter.k8s.aws/instance-gpu-count\", \"karpenter.k8s.aws/instance-gpu-memory\", \"karpenter.k8s.aws/instance-accelerator-name\", \"karpenter.k8s.aws/instance-accelerator-manufacturer\", \"karpenter.k8s.aws/instance-accelerator-count\"] || !self.find(\"^([^/]+)\").endsWith(\"karpenter.k8s.aws\")"}]' -i pkg/apis/crds/karpenter.sh_nodeclaims.yaml # # Adding validation for nodepool # ## checking for restricted labels while filtering out well known labels yq eval '.spec.versions[0].schema.openAPIV3Schema.properties.spec.properties.template.properties.spec.properties.requirements.items.properties.key.x-kubernetes-validations += [ - {"message": "label domain \"karpenter.k8s.aws\" is restricted", "rule": "self in [\"karpenter.k8s.aws/instance-encryption-in-transit-supported\", \"karpenter.k8s.aws/instance-category\", \"karpenter.k8s.aws/instance-hypervisor\", \"karpenter.k8s.aws/instance-family\", \"karpenter.k8s.aws/instance-generation\", \"karpenter.k8s.aws/instance-local-nvme\", \"karpenter.k8s.aws/instance-size\", \"karpenter.k8s.aws/instance-cpu\",\"karpenter.k8s.aws/instance-memory\", \"karpenter.k8s.aws/instance-network-bandwidth\", \"karpenter.k8s.aws/instance-gpu-name\", \"karpenter.k8s.aws/instance-gpu-manufacturer\", \"karpenter.k8s.aws/instance-gpu-count\", \"karpenter.k8s.aws/instance-gpu-memory\", \"karpenter.k8s.aws/instance-accelerator-name\", \"karpenter.k8s.aws/instance-accelerator-manufacturer\", \"karpenter.k8s.aws/instance-accelerator-count\"] || !self.find(\"^([^/]+)\").endsWith(\"karpenter.k8s.aws\")"}]' -i pkg/apis/crds/karpenter.sh_nodepools.yaml + {"message": "label domain \"karpenter.k8s.aws\" is restricted", "rule": "self in [\"karpenter.k8s.aws/instance-encryption-in-transit-supported\", \"karpenter.k8s.aws/instance-category\", \"karpenter.k8s.aws/instance-hypervisor\", \"karpenter.k8s.aws/instance-family\", \"karpenter.k8s.aws/instance-generation\", \"karpenter.k8s.aws/instance-local-nvme\", \"karpenter.k8s.aws/instance-size\", \"karpenter.k8s.aws/instance-cpu\",\"karpenter.k8s.aws/instance-cpu-manufacturer\",\"karpenter.k8s.aws/instance-memory\", \"karpenter.k8s.aws/instance-network-bandwidth\", \"karpenter.k8s.aws/instance-gpu-name\", \"karpenter.k8s.aws/instance-gpu-manufacturer\", \"karpenter.k8s.aws/instance-gpu-count\", \"karpenter.k8s.aws/instance-gpu-memory\", \"karpenter.k8s.aws/instance-accelerator-name\", \"karpenter.k8s.aws/instance-accelerator-manufacturer\", \"karpenter.k8s.aws/instance-accelerator-count\"] || !self.find(\"^([^/]+)\").endsWith(\"karpenter.k8s.aws\")"}]' -i pkg/apis/crds/karpenter.sh_nodepools.yaml diff --git a/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml b/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml index 2beec0384c5d..fe04920f8dad 100644 --- a/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml +++ b/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.13.0 + controller-gen.kubebuilder.io/version: v0.14.0 name: ec2nodeclasses.karpenter.k8s.aws spec: group: karpenter.k8s.aws @@ -25,26 +25,32 @@ spec: description: EC2NodeClass is the Schema for the EC2NodeClass API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: - description: EC2NodeClassSpec is the top level specification for the AWS - Karpenter Provider. This will contain configuration necessary to launch - instances in AWS. + description: |- + EC2NodeClassSpec is the top level specification for the AWS Karpenter Provider. + This will contain configuration necessary to launch instances in AWS. properties: amiFamily: description: AMIFamily is the AMI family that instances use. enum: - AL2 + - AL2023 - Bottlerocket - Ubuntu - Custom @@ -55,28 +61,30 @@ spec: description: AMISelectorTerms is a list of or ami selector terms. The terms are ORed. items: - description: AMISelectorTerm defines selection logic for an ami - used by Karpenter to launch nodes. If multiple fields are used - for selection, the requirements are ANDed. + description: |- + AMISelectorTerm defines selection logic for an ami used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. properties: id: description: ID is the ami id in EC2 pattern: ami-[0-9a-z]+ type: string name: - description: Name is the ami name in EC2. This value is the - name field, which is different from the name tag. + description: |- + Name is the ami name in EC2. + This value is the name field, which is different from the name tag. type: string owner: - description: Owner is the owner for the ami. You can specify - a combination of AWS account IDs, "self", "amazon", and "aws-marketplace" + description: |- + Owner is the owner for the ami. + You can specify a combination of AWS account IDs, "self", "amazon", and "aws-marketplace" type: string tags: additionalProperties: type: string - description: Tags is a map of key/value tags used to select - subnets Specifying '*' for a value selects all values for - a given tag key. + description: |- + Tags is a map of key/value tags used to select subnets + Specifying '*' for a value selects all values for a given tag key. maxProperties: 20 type: object x-kubernetes-validations: @@ -92,6 +100,10 @@ spec: of other fields in amiSelectorTerms' rule: '!self.all(x, has(x.id) && (has(x.tags) || has(x.name) || has(x.owner)))' + associatePublicIPAddress: + description: AssociatePublicIPAddress controls if public IP addresses + are assigned to instances that are launched with the nodeclass. + type: boolean blockDeviceMappings: description: BlockDeviceMappings to be applied to provisioned nodes. items: @@ -108,27 +120,38 @@ spec: volume is deleted on instance termination. type: boolean encrypted: - description: Encrypted indicates whether the EBS volume - is encrypted. Encrypted volumes can only be attached to - instances that support Amazon EBS encryption. If you are - creating a volume from a snapshot, you can't specify an - encryption value. + description: |- + Encrypted indicates whether the EBS volume is encrypted. Encrypted volumes can only + be attached to instances that support Amazon EBS encryption. If you are creating + a volume from a snapshot, you can't specify an encryption value. type: boolean iops: - description: "IOPS is the number of I/O operations per second - (IOPS). For gp3, io1, and io2 volumes, this represents - the number of IOPS that are provisioned for the volume. - For gp2 volumes, this represents the baseline performance - of the volume and the rate at which the volume accumulates - I/O credits for bursting. \n The following are the supported - values for each volume type: \n * gp3: 3,000-16,000 IOPS - \n * io1: 100-64,000 IOPS \n * io2: 100-64,000 IOPS \n - For io1 and io2 volumes, we guarantee 64,000 IOPS only - for Instances built on the Nitro System (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances). - Other instance families guarantee performance up to 32,000 - IOPS. \n This parameter is supported for io1, io2, and - gp3 volumes only. This parameter is not supported for - gp2, st1, sc1, or standard volumes." + description: |- + IOPS is the number of I/O operations per second (IOPS). For gp3, io1, and io2 volumes, + this represents the number of IOPS that are provisioned for the volume. For + gp2 volumes, this represents the baseline performance of the volume and the + rate at which the volume accumulates I/O credits for bursting. + + + The following are the supported values for each volume type: + + + * gp3: 3,000-16,000 IOPS + + + * io1: 100-64,000 IOPS + + + * io2: 100-64,000 IOPS + + + For io1 and io2 volumes, we guarantee 64,000 IOPS only for Instances built + on the Nitro System (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-types.html#ec2-nitro-instances). + Other instance families guarantee performance up to 32,000 IOPS. + + + This parameter is supported for io1, io2, and gp3 volumes only. This parameter + is not supported for gp2, st1, sc1, or standard volumes. format: int64 type: integer kmsKeyID: @@ -139,9 +162,9 @@ spec: description: SnapshotID is the ID of an EBS snapshot type: string throughput: - description: 'Throughput to provision for a gp3 volume, - with a maximum of 1,000 MiB/s. Valid Range: Minimum value - of 125. Maximum value of 1000.' + description: |- + Throughput to provision for a gp3 volume, with a maximum of 1,000 MiB/s. + Valid Range: Minimum value of 125. Maximum value of 1000. format: int64 type: integer volumeSize: @@ -151,15 +174,27 @@ spec: anyOf: - type: integer - type: string - description: "VolumeSize in `Gi`, `G`, `Ti`, or `T`. You - must specify either a snapshot ID or a volume size. The - following are the supported volumes sizes for each volume - type: \n * gp2 and gp3: 1-16,384 \n * io1 and io2: 4-16,384 - \n * st1 and sc1: 125-16,384 \n * standard: 1-1,024" + description: |- + VolumeSize in `Gi`, `G`, `Ti`, or `T`. You must specify either a snapshot ID or + a volume size. The following are the supported volumes sizes for each volume + type: + + + * gp2 and gp3: 1-16,384 + + + * io1 and io2: 4-16,384 + + + * st1 and sc1: 125-16,384 + + + * standard: 1-1,024 x-kubernetes-int-or-string: true volumeType: - description: VolumeType of the block device. For more information, - see Amazon EBS volume types (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html) + description: |- + VolumeType of the block device. + For more information, see Amazon EBS volume types (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html) in the Amazon Elastic Compute Cloud User Guide. enum: - standard @@ -175,9 +210,9 @@ spec: - message: snapshotID or volumeSize must be defined rule: has(self.snapshotID) || has(self.volumeSize) rootVolume: - description: RootVolume is a flag indicating if this device - is mounted as kubelet root dir. You can configure at most - one root volume in BlockDeviceMappings. + description: |- + RootVolume is a flag indicating if this device is mounted as kubelet root dir. You can + configure at most one root volume in BlockDeviceMappings. type: boolean type: object maxItems: 50 @@ -187,18 +222,20 @@ spec: rule: self.filter(x, has(x.rootVolume)?x.rootVolume==true:false).size() <= 1 context: - description: Context is a Reserved field in EC2 APIs https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html + description: |- + Context is a Reserved field in EC2 APIs + https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html type: string detailedMonitoring: description: DetailedMonitoring controls if detailed monitoring is enabled for instances that are launched type: boolean instanceProfile: - description: InstanceProfile is the AWS entity that instances use. - This field is mutually exclusive from role. The instance profile - should already have a role assigned to it that Karpenter has PassRole - permission on for instance launch using this instanceProfile to - succeed. + description: |- + InstanceProfile is the AWS entity that instances use. + This field is mutually exclusive from role. + The instance profile should already have a role assigned to it that Karpenter + has PassRole permission on for instance launch using this instanceProfile to succeed. type: string x-kubernetes-validations: - message: instanceProfile cannot be empty @@ -215,76 +252,91 @@ spec: httpProtocolIPv6: disabled httpPutResponseHopLimit: 2 httpTokens: required - description: "MetadataOptions for the generated launch template of - provisioned nodes. \n This specifies the exposure of the Instance - Metadata Service to provisioned EC2 nodes. For more information, - see Instance Metadata and User Data (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) - in the Amazon Elastic Compute Cloud User Guide. \n Refer to recommended, - security best practices (https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node) + description: |- + MetadataOptions for the generated launch template of provisioned nodes. + + + This specifies the exposure of the Instance Metadata Service to + provisioned EC2 nodes. For more information, + see Instance Metadata and User Data + (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) + in the Amazon Elastic Compute Cloud User Guide. + + + Refer to recommended, security best practices + (https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node) for limiting exposure of Instance Metadata and User Data to pods. If omitted, defaults to httpEndpoint enabled, with httpProtocolIPv6 - disabled, with httpPutResponseLimit of 2, and with httpTokens required." + disabled, with httpPutResponseLimit of 2, and with httpTokens + required. properties: httpEndpoint: default: enabled - description: "HTTPEndpoint enables or disables the HTTP metadata - endpoint on provisioned nodes. If metadata options is non-nil, - but this parameter is not specified, the default state is \"enabled\". - \n If you specify a value of \"disabled\", instance metadata - will not be accessible on the node." + description: |- + HTTPEndpoint enables or disables the HTTP metadata endpoint on provisioned + nodes. If metadata options is non-nil, but this parameter is not specified, + the default state is "enabled". + + + If you specify a value of "disabled", instance metadata will not be accessible + on the node. enum: - enabled - disabled type: string httpProtocolIPv6: default: disabled - description: HTTPProtocolIPv6 enables or disables the IPv6 endpoint - for the instance metadata service on provisioned nodes. If metadata - options is non-nil, but this parameter is not specified, the - default state is "disabled". + description: |- + HTTPProtocolIPv6 enables or disables the IPv6 endpoint for the instance metadata + service on provisioned nodes. If metadata options is non-nil, but this parameter + is not specified, the default state is "disabled". enum: - enabled - disabled type: string httpPutResponseHopLimit: default: 2 - description: HTTPPutResponseHopLimit is the desired HTTP PUT response - hop limit for instance metadata requests. The larger the number, - the further instance metadata requests can travel. Possible - values are integers from 1 to 64. If metadata options is non-nil, - but this parameter is not specified, the default value is 2. + description: |- + HTTPPutResponseHopLimit is the desired HTTP PUT response hop limit for + instance metadata requests. The larger the number, the further instance + metadata requests can travel. Possible values are integers from 1 to 64. + If metadata options is non-nil, but this parameter is not specified, the + default value is 2. format: int64 maximum: 64 minimum: 1 type: integer httpTokens: default: required - description: "HTTPTokens determines the state of token usage for - instance metadata requests. If metadata options is non-nil, - but this parameter is not specified, the default state is \"required\". - \n If the state is optional, one can choose to retrieve instance - metadata with or without a signed token header on the request. - If one retrieves the IAM role credentials without a token, the - version 1.0 role credentials are returned. If one retrieves - the IAM role credentials using a valid signed token, the version - 2.0 role credentials are returned. \n If the state is \"required\", - one must send a signed token header with any instance metadata - retrieval requests. In this state, retrieving the IAM role credentials - always returns the version 2.0 credentials; the version 1.0 - credentials are not available." + description: |- + HTTPTokens determines the state of token usage for instance metadata + requests. If metadata options is non-nil, but this parameter is not + specified, the default state is "required". + + + If the state is optional, one can choose to retrieve instance metadata with + or without a signed token header on the request. If one retrieves the IAM + role credentials without a token, the version 1.0 role credentials are + returned. If one retrieves the IAM role credentials using a valid signed + token, the version 2.0 role credentials are returned. + + + If the state is "required", one must send a signed token header with any + instance metadata retrieval requests. In this state, retrieving the IAM + role credentials always returns the version 2.0 credentials; the version + 1.0 credentials are not available. enum: - required - optional type: string type: object role: - description: Role is the AWS identity that nodes use. This field is - immutable. This field is mutually exclusive from instanceProfile. - Marking this field as immutable avoids concerns around terminating - managed instance profiles from running instances. This field may - be made mutable in the future, assuming the correct garbage collection - and drift handling is implemented for the old instance profiles - on an update. + description: |- + Role is the AWS identity that nodes use. This field is immutable. + This field is mutually exclusive from instanceProfile. + Marking this field as immutable avoids concerns around terminating managed instance profiles from running instances. + This field may be made mutable in the future, assuming the correct garbage collection and drift handling is implemented + for the old instance profiles on an update. type: string x-kubernetes-validations: - message: role cannot be empty @@ -295,24 +347,25 @@ spec: description: SecurityGroupSelectorTerms is a list of or security group selector terms. The terms are ORed. items: - description: SecurityGroupSelectorTerm defines selection logic for - a security group used by Karpenter to launch nodes. If multiple - fields are used for selection, the requirements are ANDed. + description: |- + SecurityGroupSelectorTerm defines selection logic for a security group used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. properties: id: description: ID is the security group id in EC2 pattern: sg-[0-9a-z]+ type: string name: - description: Name is the security group name in EC2. This value - is the name field, which is different from the name tag. + description: |- + Name is the security group name in EC2. + This value is the name field, which is different from the name tag. type: string tags: additionalProperties: type: string - description: Tags is a map of key/value tags used to select - subnets Specifying '*' for a value selects all values for - a given tag key. + description: |- + Tags is a map of key/value tags used to select subnets + Specifying '*' for a value selects all values for a given tag key. maxProperties: 20 type: object x-kubernetes-validations: @@ -336,9 +389,9 @@ spec: description: SubnetSelectorTerms is a list of or subnet selector terms. The terms are ORed. items: - description: SubnetSelectorTerm defines selection logic for a subnet - used by Karpenter to launch nodes. If multiple fields are used - for selection, the requirements are ANDed. + description: |- + SubnetSelectorTerm defines selection logic for a subnet used by Karpenter to launch nodes. + If multiple fields are used for selection, the requirements are ANDed. properties: id: description: ID is the subnet id in EC2 @@ -347,9 +400,9 @@ spec: tags: additionalProperties: type: string - description: Tags is a map of key/value tags used to select - subnets Specifying '*' for a value selects all values for - a given tag key. + description: |- + Tags is a map of key/value tags used to select subnets + Specifying '*' for a value selects all values for a given tag key. maxProperties: 20 type: object x-kubernetes-validations: @@ -377,17 +430,19 @@ spec: rule: self.all(k, k != '') - message: tag contains a restricted tag matching kubernetes.io/cluster/ rule: self.all(k, !k.startsWith('kubernetes.io/cluster') ) - - message: tag contains a restricted tag matching karpenter.sh/provisioner-name - rule: self.all(k, k != 'karpenter.sh/provisioner-name') - message: tag contains a restricted tag matching karpenter.sh/nodepool rule: self.all(k, k != 'karpenter.sh/nodepool') - message: tag contains a restricted tag matching karpenter.sh/managed-by rule: self.all(k, k !='karpenter.sh/managed-by') + - message: tag contains a restricted tag matching karpenter.sh/nodeclaim + rule: self.all(k, k !='karpenter.sh/nodeclaim') + - message: tag contains a restricted tag matching karpenter.k8s.aws/ec2nodeclass + rule: self.all(k, k !='karpenter.k8s.aws/ec2nodeclass') userData: - description: UserData to be applied to the provisioned nodes. It must - be in the appropriate format based on the AMIFamily in use. Karpenter - will merge certain fields into this UserData to ensure nodes are - being provisioned with the correct configuration. + description: |- + UserData to be applied to the provisioned nodes. + It must be in the appropriate format based on the AMIFamily in use. Karpenter will merge certain fields into + this UserData to ensure nodes are being provisioned with the correct configuration. type: string required: - amiFamily @@ -410,8 +465,9 @@ spec: description: EC2NodeClassStatus contains the resolved state of the EC2NodeClass properties: amis: - description: AMI contains the current AMI values that are available - to the cluster under the AMI selectors. + description: |- + AMI contains the current AMI values that are available to the + cluster under the AMI selectors. items: description: AMI contains resolved AMI selector values utilized for node launch @@ -426,26 +482,32 @@ spec: description: Requirements of the AMI to be utilized on an instance type items: - description: A node selector requirement is a selector that - contains values, a key, and an operator that relates the - key and values. + description: |- + A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values + and minValues that represent the requirement to have at least that many values. properties: key: description: The label key that the selector applies to. type: string + minValues: + description: |- + This field is ALPHA and can be dropped or replaced at any time + MinValues is the minimum number of unique values required to define the flexibility of the specific requirement. + maximum: 50 + minimum: 1 + type: integer operator: - description: Represents a key's relationship to a set - of values. Valid operators are In, NotIn, Exists, DoesNotExist. - Gt, and Lt. + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string values: - description: An array of string values. If the operator - is In or NotIn, the values array must be non-empty. - If the operator is Exists or DoesNotExist, the values - array must be empty. If the operator is Gt or Lt, the - values array must have a single element, which will - be interpreted as an integer. This array is replaced - during a strategic merge patch. + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. items: type: string type: array @@ -464,8 +526,9 @@ spec: for the role type: string securityGroups: - description: SecurityGroups contains the current Security Groups values - that are available to the cluster under the SecurityGroups selectors. + description: |- + SecurityGroups contains the current Security Groups values that are available to the + cluster under the SecurityGroups selectors. items: description: SecurityGroup contains resolved SecurityGroup selector values utilized for node launch @@ -481,8 +544,9 @@ spec: type: object type: array subnets: - description: Subnets contains the current Subnet values that are available - to the cluster under the subnet selectors. + description: |- + Subnets contains the current Subnet values that are available to the + cluster under the subnet selectors. items: description: Subnet contains resolved Subnet selector values utilized for node launch diff --git a/pkg/apis/crds/karpenter.sh_nodeclaims.yaml b/pkg/apis/crds/karpenter.sh_nodeclaims.yaml index dfc62e0a7e54..37abdeca8e80 100644 --- a/pkg/apis/crds/karpenter.sh_nodeclaims.yaml +++ b/pkg/apis/crds/karpenter.sh_nodeclaims.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.13.0 + controller-gen.kubebuilder.io/version: v0.14.0 name: nodeclaims.karpenter.sh spec: group: karpenter.sh @@ -50,10 +50,19 @@ spec: description: NodeClaim is the Schema for the NodeClaims API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object @@ -61,10 +70,15 @@ spec: description: NodeClaimSpec describes the desired state of the NodeClaim properties: kubelet: - description: Kubelet defines args to be used when configuring kubelet on provisioned nodes. They are a subset of the upstream types, recognizing not all options may be supported. Wherever possible, the types and names should reflect the upstream kubelet types. + description: |- + Kubelet defines args to be used when configuring kubelet on provisioned nodes. + They are a subset of the upstream types, recognizing not all options may be supported. + Wherever possible, the types and names should reflect the upstream kubelet types. properties: clusterDNS: - description: clusterDNS is a list of IP addresses for the cluster DNS server. Note that not all providers may use all addresses. + description: |- + clusterDNS is a list of IP addresses for the cluster DNS server. + Note that not all providers may use all addresses. items: type: string type: array @@ -81,7 +95,9 @@ spec: - message: valid keys for evictionHard are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) evictionMaxPodGracePeriod: - description: EvictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use when terminating pods in response to soft eviction thresholds being met. + description: |- + EvictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use when terminating pods in + response to soft eviction thresholds being met. format: int32 type: integer evictionSoft: @@ -102,24 +118,30 @@ spec: - message: valid keys for evictionSoftGracePeriod are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) imageGCHighThresholdPercent: - description: ImageGCHighThresholdPercent is the percent of disk usage after which image garbage collection is always run. The percent is calculated by dividing this field value by 100, so this field must be between 0 and 100, inclusive. When specified, the value must be greater than ImageGCLowThresholdPercent. + description: |- + ImageGCHighThresholdPercent is the percent of disk usage after which image + garbage collection is always run. The percent is calculated by dividing this + field value by 100, so this field must be between 0 and 100, inclusive. + When specified, the value must be greater than ImageGCLowThresholdPercent. format: int32 maximum: 100 minimum: 0 type: integer imageGCLowThresholdPercent: - description: ImageGCLowThresholdPercent is the percent of disk usage before which image garbage collection is never run. Lowest disk usage to garbage collect to. The percent is calculated by dividing this field value by 100, so the field value must be between 0 and 100, inclusive. When specified, the value must be less than imageGCHighThresholdPercent + description: |- + ImageGCLowThresholdPercent is the percent of disk usage before which image + garbage collection is never run. Lowest disk usage to garbage collect to. + The percent is calculated by dividing this field value by 100, + so the field value must be between 0 and 100, inclusive. + When specified, the value must be less than imageGCHighThresholdPercent format: int32 maximum: 100 minimum: 0 type: integer kubeReserved: additionalProperties: - anyOf: - - type: integer - - type: string + type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true description: KubeReserved contains resources reserved for Kubernetes system components. type: object x-kubernetes-validations: @@ -128,22 +150,24 @@ spec: - message: kubeReserved value cannot be a negative resource quantity rule: self.all(x, !self[x].startsWith('-')) maxPods: - description: MaxPods is an override for the maximum number of pods that can run on a worker node instance. + description: |- + MaxPods is an override for the maximum number of pods that can run on + a worker node instance. format: int32 minimum: 0 type: integer podsPerCore: - description: PodsPerCore is an override for the number of pods that can run on a worker node instance based on the number of cpu cores. This value cannot exceed MaxPods, so, if MaxPods is a lower value, that value will be used. + description: |- + PodsPerCore is an override for the number of pods that can run on a worker node + instance based on the number of cpu cores. This value cannot exceed MaxPods, so, if + MaxPods is a lower value, that value will be used. format: int32 minimum: 0 type: integer systemReserved: additionalProperties: - anyOf: - - type: integer - - type: string + type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true description: SystemReserved contains resources reserved for OS system daemons and kernel memory. type: object x-kubernetes-validations: @@ -177,7 +201,9 @@ spec: requirements: description: Requirements are layered with GetLabels and applied to every node. items: - description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: |- + A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values + and minValues that represent the requirement to have at least that many values. properties: key: description: The label key that the selector applies to. @@ -194,9 +220,18 @@ spec: - message: label "kubernetes.io/hostname" is restricted rule: self != "kubernetes.io/hostname" - message: label domain "karpenter.k8s.aws" is restricted - rule: self in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") + rule: self in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") + minValues: + description: |- + This field is ALPHA and can be dropped or replaced at any time + MinValues is the minimum number of unique values required to define the flexibility of the specific requirement. + maximum: 50 + minimum: 1 + type: integer operator: - description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string enum: - In @@ -206,7 +241,12 @@ spec: - Gt - Lt values: - description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. items: type: string type: array @@ -223,6 +263,8 @@ spec: rule: 'self.all(x, x.operator == ''In'' ? x.values.size() != 0 : true)' - message: requirements operator 'Gt' or 'Lt' must have a single positive integer value rule: 'self.all(x, (x.operator == ''Gt'' || x.operator == ''Lt'') ? (x.values.size() == 1 && int(x.values[0]) >= 0) : true)' + - message: requirements with 'minValues' must have at least that many values specified in the 'values' field + rule: 'self.all(x, (x.operator == ''In'' && has(x.minValues)) ? x.values.size() >= x.minValues : true)' resources: description: Resources models the resource requirements for the NodeClaim to launch properties: @@ -237,12 +279,21 @@ spec: type: object type: object startupTaints: - description: StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. + description: |- + StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically + within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by + daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning + purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. items: - description: The node this Taint is attached to has the "effect" on any pod that does not tolerate the Taint. + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. properties: effect: - description: Required. The effect of the taint on pods that do not tolerate the taint. Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. type: string enum: - NoSchedule @@ -254,7 +305,9 @@ spec: minLength: 1 pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ timeAdded: - description: TimeAdded represents the time at which the taint was added. It is only written for NoExecute taints. + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. format: date-time type: string value: @@ -269,10 +322,15 @@ spec: taints: description: Taints will be applied to the NodeClaim's node. items: - description: The node this Taint is attached to has the "effect" on any pod that does not tolerate the Taint. + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. properties: effect: - description: Required. The effect of the taint on pods that do not tolerate the taint. Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. type: string enum: - NoSchedule @@ -284,7 +342,9 @@ spec: minLength: 1 pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ timeAdded: - description: TimeAdded represents the time at which the taint was added. It is only written for NoExecute taints. + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. format: date-time type: string value: @@ -324,10 +384,15 @@ spec: conditions: description: Conditions contains signals for health and readiness items: - description: 'Condition defines a readiness condition for a Knative resource. See: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#typical-status-properties' + description: |- + Condition defines a readiness condition for a Knative resource. + See: https://github.com/kubernetes/community/blob/master/contributors/devel/sig-architecture/api-conventions.md#typical-status-properties properties: lastTransitionTime: - description: LastTransitionTime is the last time the condition transitioned from one status to another. We use VolatileTime in place of metav1.Time to exclude this from creating equality.Semantic differences (all other things held constant). + description: |- + LastTransitionTime is the last time the condition transitioned from one status to another. + We use VolatileTime in place of metav1.Time to exclude this from creating equality.Semantic + differences (all other things held constant). type: string message: description: A human readable message indicating details about the transition. @@ -336,7 +401,9 @@ spec: description: The reason for the condition's last transition. type: string severity: - description: Severity with which to treat failures of this type of condition. When this is not specified, it defaults to Error. + description: |- + Severity with which to treat failures of this type of condition. + When this is not specified, it defaults to Error. type: string status: description: Status of the condition, one of True, False, Unknown. @@ -359,6 +426,8 @@ spec: description: ProviderID of the corresponding node object type: string type: object + required: + - spec type: object served: true storage: true diff --git a/pkg/apis/crds/karpenter.sh_nodepools.yaml b/pkg/apis/crds/karpenter.sh_nodepools.yaml index bcea499e74c0..44a38267e3e9 100644 --- a/pkg/apis/crds/karpenter.sh_nodepools.yaml +++ b/pkg/apis/crds/karpenter.sh_nodepools.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.13.0 + controller-gen.kubebuilder.io/version: v0.14.0 name: nodepools.karpenter.sh spec: group: karpenter.sh @@ -30,15 +30,28 @@ spec: description: NodePool is the Schema for the NodePools API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object spec: - description: NodePoolSpec is the top level nodepool specification. Nodepools launch nodes in response to pods that are unschedulable. A single nodepool is capable of managing a diverse set of nodes. Node properties are determined from a combination of nodepool and pod scheduling constraints. + description: |- + NodePoolSpec is the top level nodepool specification. Nodepools + launch nodes in response to pods that are unschedulable. A single nodepool + is capable of managing a diverse set of nodes. Node properties are determined + from a combination of nodepool and pod scheduling constraints. properties: disruption: default: @@ -49,21 +62,44 @@ spec: budgets: default: - nodes: 10% - description: Budgets is a list of Budgets. If there are multiple active budgets, Karpenter uses the most restrictive value. If left undefined, this will default to one budget with a value to 10%. + description: |- + Budgets is a list of Budgets. + If there are multiple active budgets, Karpenter uses + the most restrictive value. If left undefined, + this will default to one budget with a value to 10%. items: - description: Budget defines when Karpenter will restrict the number of Node Claims that can be terminating simultaneously. + description: |- + Budget defines when Karpenter will restrict the + number of Node Claims that can be terminating simultaneously. properties: duration: - description: Duration determines how long a Budget is active since each Schedule hit. Only minutes and hours are accepted, as cron does not work in seconds. If omitted, the budget is always active. This is required if Schedule is set. This regex has an optional 0s at the end since the duration.String() always adds a 0s at the end. - pattern: ^([0-9]+(m|h)+(0s)?)$ + description: |- + Duration determines how long a Budget is active since each Schedule hit. + Only minutes and hours are accepted, as cron does not work in seconds. + If omitted, the budget is always active. + This is required if Schedule is set. + This regex has an optional 0s at the end since the duration.String() always adds + a 0s at the end. + pattern: ^((([0-9]+(h|m))|([0-9]+h[0-9]+m))(0s)?)$ type: string nodes: default: 10% - description: 'Nodes dictates the maximum number of NodeClaims owned by this NodePool that can be terminating at once. This is calculated by counting nodes that have a deletion timestamp set, or are actively being deleted by Karpenter. This field is required when specifying a budget. This cannot be of type intstr.IntOrString since kubebuilder doesn''t support pattern checking for int nodes for IntOrString nodes. Ref: https://github.com/kubernetes-sigs/controller-tools/blob/55efe4be40394a288216dab63156b0a64fb82929/pkg/crd/markers/validation.go#L379-L388' + description: |- + Nodes dictates the maximum number of NodeClaims owned by this NodePool + that can be terminating at once. This is calculated by counting nodes that + have a deletion timestamp set, or are actively being deleted by Karpenter. + This field is required when specifying a budget. + This cannot be of type intstr.IntOrString since kubebuilder doesn't support pattern + checking for int nodes for IntOrString nodes. + Ref: https://github.com/kubernetes-sigs/controller-tools/blob/55efe4be40394a288216dab63156b0a64fb82929/pkg/crd/markers/validation.go#L379-L388 pattern: ^((100|[0-9]{1,2})%|[0-9]+)$ type: string schedule: - description: Schedule specifies when a budget begins being active, following the upstream cronjob syntax. If omitted, the budget is always active. Timezones are not supported. This field is required if Duration is set. + description: |- + Schedule specifies when a budget begins being active, following + the upstream cronjob syntax. If omitted, the budget is always active. + Timezones are not supported. + This field is required if Duration is set. pattern: ^(@(annually|yearly|monthly|weekly|daily|midnight|hourly))|((.+)\s(.+)\s(.+)\s(.+)\s(.+))$ type: string required: @@ -73,21 +109,30 @@ spec: type: array x-kubernetes-validations: - message: '''schedule'' must be set with ''duration''' - rule: '!self.all(x, (has(x.schedule) && !has(x.duration)) || (!has(x.schedule) && has(x.duration)))' + rule: self.all(x, has(x.schedule) == has(x.duration)) consolidateAfter: - description: ConsolidateAfter is the duration the controller will wait before attempting to terminate nodes that are underutilized. Refer to ConsolidationPolicy for how underutilization is considered. + description: |- + ConsolidateAfter is the duration the controller will wait + before attempting to terminate nodes that are underutilized. + Refer to ConsolidationPolicy for how underutilization is considered. pattern: ^(([0-9]+(s|m|h))+)|(Never)$ type: string consolidationPolicy: default: WhenUnderutilized - description: ConsolidationPolicy describes which nodes Karpenter can disrupt through its consolidation algorithm. This policy defaults to "WhenUnderutilized" if not specified + description: |- + ConsolidationPolicy describes which nodes Karpenter can disrupt through its consolidation + algorithm. This policy defaults to "WhenUnderutilized" if not specified enum: - WhenEmpty - WhenUnderutilized type: string expireAfter: default: 720h - description: ExpireAfter is the duration the controller will wait before terminating a node, measured from when the node is created. This is useful to implement features like eventually consistent node upgrade, memory leak protection, and disruption testing. + description: |- + ExpireAfter is the duration the controller will wait + before terminating a node, measured from when the node is created. This + is useful to implement features like eventually consistent node upgrade, + memory leak protection, and disruption testing. pattern: ^(([0-9]+(s|m|h))+)|(Never)$ type: string type: object @@ -106,21 +151,31 @@ spec: description: Limits define a set of bounds for provisioning capacity. type: object template: - description: Template contains the template of possibilities for the provisioning logic to launch a NodeClaim with. NodeClaims launched from this NodePool will often be further constrained than the template specifies. + description: |- + Template contains the template of possibilities for the provisioning logic to launch a NodeClaim with. + NodeClaims launched from this NodePool will often be further constrained than the template specifies. properties: metadata: properties: annotations: additionalProperties: type: string - description: 'Annotations is an unstructured key value map stored with a resource that may be set by external tools to store and retrieve arbitrary metadata. They are not queryable and should be preserved when modifying objects. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations' + description: |- + Annotations is an unstructured key value map stored with a resource that may be + set by external tools to store and retrieve arbitrary metadata. They are not + queryable and should be preserved when modifying objects. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations type: object labels: additionalProperties: type: string maxLength: 63 pattern: ^(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])?$ - description: 'Map of string keys and values that can be used to organize and categorize (scope and select) objects. May match selectors of replication controllers and services. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels' + description: |- + Map of string keys and values that can be used to organize and categorize + (scope and select) objects. May match selectors of replication controllers + and services. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels type: object maxProperties: 100 x-kubernetes-validations: @@ -135,16 +190,21 @@ spec: - message: label "kubernetes.io/hostname" is restricted rule: self.all(x, x != "kubernetes.io/hostname") - message: label domain "karpenter.k8s.aws" is restricted - rule: self.all(x, x in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !x.find("^([^/]+)").endsWith("karpenter.k8s.aws")) + rule: self.all(x, x in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !x.find("^([^/]+)").endsWith("karpenter.k8s.aws")) type: object spec: description: NodeClaimSpec describes the desired state of the NodeClaim properties: kubelet: - description: Kubelet defines args to be used when configuring kubelet on provisioned nodes. They are a subset of the upstream types, recognizing not all options may be supported. Wherever possible, the types and names should reflect the upstream kubelet types. + description: |- + Kubelet defines args to be used when configuring kubelet on provisioned nodes. + They are a subset of the upstream types, recognizing not all options may be supported. + Wherever possible, the types and names should reflect the upstream kubelet types. properties: clusterDNS: - description: clusterDNS is a list of IP addresses for the cluster DNS server. Note that not all providers may use all addresses. + description: |- + clusterDNS is a list of IP addresses for the cluster DNS server. + Note that not all providers may use all addresses. items: type: string type: array @@ -161,7 +221,9 @@ spec: - message: valid keys for evictionHard are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) evictionMaxPodGracePeriod: - description: EvictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use when terminating pods in response to soft eviction thresholds being met. + description: |- + EvictionMaxPodGracePeriod is the maximum allowed grace period (in seconds) to use when terminating pods in + response to soft eviction thresholds being met. format: int32 type: integer evictionSoft: @@ -182,24 +244,30 @@ spec: - message: valid keys for evictionSoftGracePeriod are ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available'] rule: self.all(x, x in ['memory.available','nodefs.available','nodefs.inodesFree','imagefs.available','imagefs.inodesFree','pid.available']) imageGCHighThresholdPercent: - description: ImageGCHighThresholdPercent is the percent of disk usage after which image garbage collection is always run. The percent is calculated by dividing this field value by 100, so this field must be between 0 and 100, inclusive. When specified, the value must be greater than ImageGCLowThresholdPercent. + description: |- + ImageGCHighThresholdPercent is the percent of disk usage after which image + garbage collection is always run. The percent is calculated by dividing this + field value by 100, so this field must be between 0 and 100, inclusive. + When specified, the value must be greater than ImageGCLowThresholdPercent. format: int32 maximum: 100 minimum: 0 type: integer imageGCLowThresholdPercent: - description: ImageGCLowThresholdPercent is the percent of disk usage before which image garbage collection is never run. Lowest disk usage to garbage collect to. The percent is calculated by dividing this field value by 100, so the field value must be between 0 and 100, inclusive. When specified, the value must be less than imageGCHighThresholdPercent + description: |- + ImageGCLowThresholdPercent is the percent of disk usage before which image + garbage collection is never run. Lowest disk usage to garbage collect to. + The percent is calculated by dividing this field value by 100, + so the field value must be between 0 and 100, inclusive. + When specified, the value must be less than imageGCHighThresholdPercent format: int32 maximum: 100 minimum: 0 type: integer kubeReserved: additionalProperties: - anyOf: - - type: integer - - type: string + type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true description: KubeReserved contains resources reserved for Kubernetes system components. type: object x-kubernetes-validations: @@ -208,22 +276,24 @@ spec: - message: kubeReserved value cannot be a negative resource quantity rule: self.all(x, !self[x].startsWith('-')) maxPods: - description: MaxPods is an override for the maximum number of pods that can run on a worker node instance. + description: |- + MaxPods is an override for the maximum number of pods that can run on + a worker node instance. format: int32 minimum: 0 type: integer podsPerCore: - description: PodsPerCore is an override for the number of pods that can run on a worker node instance based on the number of cpu cores. This value cannot exceed MaxPods, so, if MaxPods is a lower value, that value will be used. + description: |- + PodsPerCore is an override for the number of pods that can run on a worker node + instance based on the number of cpu cores. This value cannot exceed MaxPods, so, if + MaxPods is a lower value, that value will be used. format: int32 minimum: 0 type: integer systemReserved: additionalProperties: - anyOf: - - type: integer - - type: string + type: string pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true description: SystemReserved contains resources reserved for OS system daemons and kernel memory. type: object x-kubernetes-validations: @@ -257,7 +327,9 @@ spec: requirements: description: Requirements are layered with GetLabels and applied to every node. items: - description: A node selector requirement is a selector that contains values, a key, and an operator that relates the key and values. + description: |- + A node selector requirement with min values is a selector that contains values, a key, an operator that relates the key and values + and minValues that represent the requirement to have at least that many values. properties: key: description: The label key that the selector applies to. @@ -276,9 +348,18 @@ spec: - message: label "kubernetes.io/hostname" is restricted rule: self != "kubernetes.io/hostname" - message: label domain "karpenter.k8s.aws" is restricted - rule: self in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") + rule: self in ["karpenter.k8s.aws/instance-encryption-in-transit-supported", "karpenter.k8s.aws/instance-category", "karpenter.k8s.aws/instance-hypervisor", "karpenter.k8s.aws/instance-family", "karpenter.k8s.aws/instance-generation", "karpenter.k8s.aws/instance-local-nvme", "karpenter.k8s.aws/instance-size", "karpenter.k8s.aws/instance-cpu","karpenter.k8s.aws/instance-cpu-manufacturer","karpenter.k8s.aws/instance-memory", "karpenter.k8s.aws/instance-network-bandwidth", "karpenter.k8s.aws/instance-gpu-name", "karpenter.k8s.aws/instance-gpu-manufacturer", "karpenter.k8s.aws/instance-gpu-count", "karpenter.k8s.aws/instance-gpu-memory", "karpenter.k8s.aws/instance-accelerator-name", "karpenter.k8s.aws/instance-accelerator-manufacturer", "karpenter.k8s.aws/instance-accelerator-count"] || !self.find("^([^/]+)").endsWith("karpenter.k8s.aws") + minValues: + description: |- + This field is ALPHA and can be dropped or replaced at any time + MinValues is the minimum number of unique values required to define the flexibility of the specific requirement. + maximum: 50 + minimum: 1 + type: integer operator: - description: Represents a key's relationship to a set of values. Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. + description: |- + Represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt. type: string enum: - In @@ -288,7 +369,12 @@ spec: - Gt - Lt values: - description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. If the operator is Exists or DoesNotExist, the values array must be empty. If the operator is Gt or Lt, the values array must have a single element, which will be interpreted as an integer. This array is replaced during a strategic merge patch. + description: |- + An array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. If the operator is Gt or Lt, the values + array must have a single element, which will be interpreted as an integer. + This array is replaced during a strategic merge patch. items: type: string type: array @@ -305,6 +391,8 @@ spec: rule: 'self.all(x, x.operator == ''In'' ? x.values.size() != 0 : true)' - message: requirements operator 'Gt' or 'Lt' must have a single positive integer value rule: 'self.all(x, (x.operator == ''Gt'' || x.operator == ''Lt'') ? (x.values.size() == 1 && int(x.values[0]) >= 0) : true)' + - message: requirements with 'minValues' must have at least that many values specified in the 'values' field + rule: 'self.all(x, (x.operator == ''In'' && has(x.minValues)) ? x.values.size() >= x.minValues : true)' resources: description: Resources models the resource requirements for the NodeClaim to launch properties: @@ -318,13 +406,23 @@ spec: description: Requests describes the minimum required resources for the NodeClaim to launch type: object type: object + maxProperties: 0 startupTaints: - description: StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. + description: |- + StartupTaints are taints that are applied to nodes upon startup which are expected to be removed automatically + within a short period of time, typically by a DaemonSet that tolerates the taint. These are commonly used by + daemonsets to allow initialization and enforce startup ordering. StartupTaints are ignored for provisioning + purposes in that pods are not required to tolerate a StartupTaint in order to have nodes provisioned for them. items: - description: The node this Taint is attached to has the "effect" on any pod that does not tolerate the Taint. + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. properties: effect: - description: Required. The effect of the taint on pods that do not tolerate the taint. Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. type: string enum: - NoSchedule @@ -336,7 +434,9 @@ spec: minLength: 1 pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ timeAdded: - description: TimeAdded represents the time at which the taint was added. It is only written for NoExecute taints. + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. format: date-time type: string value: @@ -351,10 +451,15 @@ spec: taints: description: Taints will be applied to the NodeClaim's node. items: - description: The node this Taint is attached to has the "effect" on any pod that does not tolerate the Taint. + description: |- + The node this Taint is attached to has the "effect" on + any pod that does not tolerate the Taint. properties: effect: - description: Required. The effect of the taint on pods that do not tolerate the taint. Valid effects are NoSchedule, PreferNoSchedule and NoExecute. + description: |- + Required. The effect of the taint on pods + that do not tolerate the taint. + Valid effects are NoSchedule, PreferNoSchedule and NoExecute. type: string enum: - NoSchedule @@ -366,7 +471,9 @@ spec: minLength: 1 pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*(\/))?([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9]$ timeAdded: - description: TimeAdded represents the time at which the taint was added. It is only written for NoExecute taints. + description: |- + TimeAdded represents the time at which the taint was added. + It is only written for NoExecute taints. format: date-time type: string value: @@ -386,7 +493,11 @@ spec: - spec type: object weight: - description: Weight is the priority given to the nodepool during scheduling. A higher numerical weight indicates that this nodepool will be ordered ahead of other nodepools with lower weights. A nodepool with no weight will be treated as if it is a nodepool with a weight of 0. + description: |- + Weight is the priority given to the nodepool during scheduling. A higher + numerical weight indicates that this nodepool will be ordered + ahead of other nodepools with lower weights. A nodepool with no weight + will be treated as if it is a nodepool with a weight of 0. format: int32 maximum: 100 minimum: 1 @@ -407,6 +518,8 @@ spec: description: Resources is the list of resources that have been provisioned. type: object type: object + required: + - spec type: object served: true storage: true diff --git a/pkg/apis/v1beta1/ec2nodeclass.go b/pkg/apis/v1beta1/ec2nodeclass.go index 17b607278581..b5e96ac1dc6b 100644 --- a/pkg/apis/v1beta1/ec2nodeclass.go +++ b/pkg/apis/v1beta1/ec2nodeclass.go @@ -41,6 +41,9 @@ type EC2NodeClassSpec struct { // +kubebuilder:validation:MaxItems:=30 // +required SecurityGroupSelectorTerms []SecurityGroupSelectorTerm `json:"securityGroupSelectorTerms" hash:"ignore"` + // AssociatePublicIPAddress controls if public IP addresses are assigned to instances that are launched with the nodeclass. + // +optional + AssociatePublicIPAddress *bool `json:"associatePublicIPAddress,omitempty"` // AMISelectorTerms is a list of or ami selector terms. The terms are ORed. // +kubebuilder:validation:XValidation:message="expected at least one, got none, ['tags', 'id', 'name']",rule="self.all(x, has(x.tags) || has(x.id) || has(x.name))" // +kubebuilder:validation:XValidation:message="'id' is mutually exclusive, cannot be set with a combination of other fields in amiSelectorTerms",rule="!self.all(x, has(x.id) && (has(x.tags) || has(x.name) || has(x.owner)))" @@ -48,7 +51,7 @@ type EC2NodeClassSpec struct { // +optional AMISelectorTerms []AMISelectorTerm `json:"amiSelectorTerms,omitempty" hash:"ignore"` // AMIFamily is the AMI family that instances use. - // +kubebuilder:validation:Enum:={AL2,Bottlerocket,Ubuntu,Custom,Windows2019,Windows2022} + // +kubebuilder:validation:Enum:={AL2,AL2023,Bottlerocket,Ubuntu,Custom,Windows2019,Windows2022} // +required AMIFamily *string `json:"amiFamily"` // UserData to be applied to the provisioned nodes. @@ -75,9 +78,10 @@ type EC2NodeClassSpec struct { // Tags to be applied on ec2 resources like instances and launch templates. // +kubebuilder:validation:XValidation:message="empty tag keys aren't supported",rule="self.all(k, k != '')" // +kubebuilder:validation:XValidation:message="tag contains a restricted tag matching kubernetes.io/cluster/",rule="self.all(k, !k.startsWith('kubernetes.io/cluster') )" - // +kubebuilder:validation:XValidation:message="tag contains a restricted tag matching karpenter.sh/provisioner-name",rule="self.all(k, k != 'karpenter.sh/provisioner-name')" // +kubebuilder:validation:XValidation:message="tag contains a restricted tag matching karpenter.sh/nodepool",rule="self.all(k, k != 'karpenter.sh/nodepool')" // +kubebuilder:validation:XValidation:message="tag contains a restricted tag matching karpenter.sh/managed-by",rule="self.all(k, k !='karpenter.sh/managed-by')" + // +kubebuilder:validation:XValidation:message="tag contains a restricted tag matching karpenter.sh/nodeclaim",rule="self.all(k, k !='karpenter.sh/nodeclaim')" + // +kubebuilder:validation:XValidation:message="tag contains a restricted tag matching karpenter.k8s.aws/ec2nodeclass",rule="self.all(k, k !='karpenter.k8s.aws/ec2nodeclass')" // +optional Tags map[string]string `json:"tags,omitempty"` // BlockDeviceMappings to be applied to provisioned nodes. @@ -289,7 +293,7 @@ type BlockDevice struct { // +kubebuilder:validation:Pattern:="^((?:[1-9][0-9]{0,3}|[1-4][0-9]{4}|[5][0-8][0-9]{3}|59000)Gi|(?:[1-9][0-9]{0,3}|[1-5][0-9]{4}|[6][0-3][0-9]{3}|64000)G|([1-9]||[1-5][0-7]|58)Ti|([1-9]||[1-5][0-9]|6[0-3]|64)T)$" // +kubebuilder:validation:XIntOrString // +optional - VolumeSize *resource.Quantity `json:"volumeSize,omitempty"` + VolumeSize *resource.Quantity `json:"volumeSize,omitempty" hash:"string"` // VolumeType of the block device. // For more information, see Amazon EBS volume types (https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/EBSVolumeTypes.html) // in the Amazon Elastic Compute Cloud User Guide. @@ -325,6 +329,12 @@ type EC2NodeClass struct { Status EC2NodeClassStatus `json:"status,omitempty"` } +// We need to bump the EC2NodeClassHashVersion when we make an update to the EC2NodeClass CRD under these conditions: +// 1. A field changes its default value for an existing field that is already hashed +// 2. A field is added to the hash calculation with an already-set value +// 3. A field is removed from the hash calculations +const EC2NodeClassHashVersion = "v2" + func (in *EC2NodeClass) Hash() string { return fmt.Sprint(lo.Must(hashstructure.Hash(in.Spec, hashstructure.FormatV2, &hashstructure.HashOptions{ SlicesAsSets: true, diff --git a/pkg/apis/v1beta1/ec2nodeclass_hash_test.go b/pkg/apis/v1beta1/ec2nodeclass_hash_test.go index 9581d09940e9..ccdde5a389ec 100644 --- a/pkg/apis/v1beta1/ec2nodeclass_hash_test.go +++ b/pkg/apis/v1beta1/ec2nodeclass_hash_test.go @@ -18,6 +18,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/imdario/mergo" "github.com/samber/lo" + "k8s.io/apimachinery/pkg/api/resource" "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" "github.com/aws/karpenter-provider-aws/pkg/test" @@ -27,76 +28,137 @@ import ( ) var _ = Describe("Hash", func() { - const staticHash = "16608948681250225098" + const staticHash = "10790156025840984195" var nodeClass *v1beta1.EC2NodeClass BeforeEach(func() { nodeClass = test.EC2NodeClass(v1beta1.EC2NodeClass{ Spec: v1beta1.EC2NodeClassSpec{ - AMIFamily: aws.String(v1beta1.AMIFamilyAL2), - Context: aws.String("context-1"), + AMIFamily: lo.ToPtr(v1beta1.AMIFamilyAL2023), Role: "role-1", Tags: map[string]string{ "keyTag-1": "valueTag-1", "keyTag-2": "valueTag-2", }, + Context: lo.ToPtr("fake-context"), + DetailedMonitoring: lo.ToPtr(false), + AssociatePublicIPAddress: lo.ToPtr(false), MetadataOptions: &v1beta1.MetadataOptions{ - HTTPEndpoint: aws.String("test-metadata-1"), + HTTPEndpoint: lo.ToPtr("disabled"), + HTTPProtocolIPv6: lo.ToPtr("disabled"), + HTTPPutResponseHopLimit: lo.ToPtr(int64(1)), + HTTPTokens: lo.ToPtr("optional"), }, BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{ { - DeviceName: aws.String("map-device-1"), + DeviceName: lo.ToPtr("map-device-1"), + RootVolume: false, + EBS: &v1beta1.BlockDevice{ + DeleteOnTermination: lo.ToPtr(false), + Encrypted: lo.ToPtr(false), + IOPS: lo.ToPtr(int64(0)), + KMSKeyID: lo.ToPtr("fakeKMSKeyID"), + SnapshotID: lo.ToPtr("fakeSnapshot"), + Throughput: lo.ToPtr(int64(0)), + VolumeSize: resource.NewScaledQuantity(2, resource.Giga), + VolumeType: lo.ToPtr("standard"), + }, }, { - DeviceName: aws.String("map-device-2"), + DeviceName: lo.ToPtr("map-device-2"), }, }, - UserData: aws.String("userdata-test-1"), - DetailedMonitoring: aws.Bool(false), + UserData: aws.String("userdata-test-1"), }, }) }) DescribeTable( - "should match static hash", - func(hash string, changes ...v1beta1.EC2NodeClass) { - modifiedNodeClass := test.EC2NodeClass(append([]v1beta1.EC2NodeClass{*nodeClass}, changes...)...) - Expect(modifiedNodeClass.Hash()).To(Equal(hash)) + "should match static hash on field value change", + func(hash string, changes v1beta1.EC2NodeClass) { + Expect(mergo.Merge(nodeClass, changes, mergo.WithOverride, mergo.WithSliceDeepCopy)).To(Succeed()) + Expect(nodeClass.Hash()).To(Equal(hash)) }, - Entry("Base EC2NodeClass", staticHash), + Entry("Base EC2NodeClass", staticHash, v1beta1.EC2NodeClass{}), // Static fields, expect changed hash from base - Entry("UserData Drift", "588756456110800812", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{UserData: aws.String("userdata-test-2")}}), - Entry("Tags Drift", "2471764681523766508", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{Tags: map[string]string{"keyTag-test-3": "valueTag-test-3"}}}), - Entry("MetadataOptions Drift", "11030161632375731908", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{MetadataOptions: &v1beta1.MetadataOptions{HTTPEndpoint: aws.String("test-metadata-2")}}}), - Entry("BlockDeviceMappings Drift", "436753305915039702", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{DeviceName: aws.String("map-device-test-3")}}}}), - Entry("Context Drift", "3729470655588343019", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{Context: aws.String("context-2")}}), - Entry("DetailedMonitoring Drift", "17892305444040067573", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{DetailedMonitoring: aws.Bool(true)}}), - Entry("AMIFamily Drift", "9493798894326942407", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{AMIFamily: aws.String(v1beta1.AMIFamilyBottlerocket)}}), - Entry("Reorder Tags", staticHash, v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{Tags: map[string]string{"keyTag-2": "valueTag-2", "keyTag-1": "valueTag-1"}}}), - Entry("Reorder BlockDeviceMapping", staticHash, v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{DeviceName: aws.String("map-device-2")}, {DeviceName: aws.String("map-device-1")}}}}), + Entry("UserData", "18317182711135792962", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{UserData: aws.String("userdata-test-2")}}), + Entry("Tags", "7254882043893135054", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{Tags: map[string]string{"keyTag-test-3": "valueTag-test-3"}}}), + Entry("Context", "17271601354348855032", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{Context: aws.String("context-2")}}), + Entry("DetailedMonitoring", "3320998103335094348", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{DetailedMonitoring: aws.Bool(true)}}), + Entry("AMIFamily", "11029247967399146065", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{AMIFamily: aws.String(v1beta1.AMIFamilyBottlerocket)}}), + Entry("InstanceStorePolicy", "15591048753403695860", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{InstanceStorePolicy: lo.ToPtr(v1beta1.InstanceStorePolicyRAID0)}}), + Entry("AssociatePublicIPAddress", "8788624850560996180", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{AssociatePublicIPAddress: lo.ToPtr(true)}}), + Entry("MetadataOptions HTTPEndpoint", "12130088184516131939", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{MetadataOptions: &v1beta1.MetadataOptions{HTTPEndpoint: lo.ToPtr("enabled")}}}), + Entry("MetadataOptions HTTPProtocolIPv6", "9851778617676567202", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{MetadataOptions: &v1beta1.MetadataOptions{HTTPProtocolIPv6: lo.ToPtr("enabled")}}}), + Entry("MetadataOptions HTTPPutResponseHopLimit", "10114972825726256442", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{MetadataOptions: &v1beta1.MetadataOptions{HTTPPutResponseHopLimit: lo.ToPtr(int64(10))}}}), + Entry("MetadataOptions HTTPTokens", "15328515228245883488", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{MetadataOptions: &v1beta1.MetadataOptions{HTTPTokens: lo.ToPtr("required")}}}), + Entry("BlockDeviceMapping DeviceName", "14855383487702710824", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{DeviceName: lo.ToPtr("map-device-test-3")}}}}), + Entry("BlockDeviceMapping RootVolume", "9591488558660758449", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{RootVolume: true}}}}), + Entry("BlockDeviceMapping DeleteOnTermination", "2802222466202766732", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{EBS: &v1beta1.BlockDevice{DeleteOnTermination: lo.ToPtr(true)}}}}}), + Entry("BlockDeviceMapping Encrypted", "16743053872042184219", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{EBS: &v1beta1.BlockDevice{Encrypted: lo.ToPtr(true)}}}}}), + Entry("BlockDeviceMapping IOPS", "17284705682110195253", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{EBS: &v1beta1.BlockDevice{IOPS: lo.ToPtr(int64(10))}}}}}), + Entry("BlockDeviceMapping KMSKeyID", "9151019926310241707", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{EBS: &v1beta1.BlockDevice{KMSKeyID: lo.ToPtr("test")}}}}}), + Entry("BlockDeviceMapping SnapshotID", "5250341140179985875", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{EBS: &v1beta1.BlockDevice{SnapshotID: lo.ToPtr("test")}}}}}), + Entry("BlockDeviceMapping Throughput", "16711481758638864953", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{EBS: &v1beta1.BlockDevice{Throughput: lo.ToPtr(int64(10))}}}}}), + Entry("BlockDeviceMapping VolumeType", "488614640133725370", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{EBS: &v1beta1.BlockDevice{VolumeType: lo.ToPtr("io1")}}}}}), // Behavior / Dynamic fields, expect same hash as base Entry("Modified AMISelector", staticHash, v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{AMISelectorTerms: []v1beta1.AMISelectorTerm{{Tags: map[string]string{"ami-test-key": "ami-test-value"}}}}}), Entry("Modified SubnetSelector", staticHash, v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{SubnetSelectorTerms: []v1beta1.SubnetSelectorTerm{{Tags: map[string]string{"subnet-test-key": "subnet-test-value"}}}}}), Entry("Modified SecurityGroupSelector", staticHash, v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{SecurityGroupSelectorTerms: []v1beta1.SecurityGroupSelectorTerm{{Tags: map[string]string{"security-group-test-key": "security-group-test-value"}}}}}), ) + // We create a separate test for updating blockDeviceMapping volumeSize, since resource.Quantity is a struct, and mergo.WithSliceDeepCopy + // doesn't work well with unexported fields, like the ones that are present in resource.Quantity + It("should match static hash when updating blockDeviceMapping volumeSize", func() { + nodeClass.Spec.BlockDeviceMappings[0].EBS.VolumeSize = resource.NewScaledQuantity(10, resource.Giga) + Expect(nodeClass.Hash()).To(Equal("4802236799448001710")) + }) It("should match static hash for instanceProfile", func() { nodeClass.Spec.Role = "" nodeClass.Spec.InstanceProfile = lo.ToPtr("test-instance-profile") - Expect(nodeClass.Hash()).To(Equal("15756064858220068103")) + Expect(nodeClass.Hash()).To(Equal("7914642030762404205")) + }) + It("should match static hash when reordering tags", func() { + nodeClass.Spec.Tags = map[string]string{"keyTag-2": "valueTag-2", "keyTag-1": "valueTag-1"} + Expect(nodeClass.Hash()).To(Equal(staticHash)) + }) + It("should match static hash when reordering blockDeviceMappings", func() { + nodeClass.Spec.BlockDeviceMappings[0], nodeClass.Spec.BlockDeviceMappings[1] = nodeClass.Spec.BlockDeviceMappings[1], nodeClass.Spec.BlockDeviceMappings[0] + Expect(nodeClass.Hash()).To(Equal(staticHash)) }) DescribeTable("should change hash when static fields are updated", func(changes v1beta1.EC2NodeClass) { hash := nodeClass.Hash() - Expect(mergo.Merge(nodeClass, changes, mergo.WithOverride)).To(Succeed()) + Expect(mergo.Merge(nodeClass, changes, mergo.WithOverride, mergo.WithSliceDeepCopy)).To(Succeed()) updatedHash := nodeClass.Hash() Expect(hash).ToNot(Equal(updatedHash)) }, - Entry("UserData Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{UserData: aws.String("userdata-test-2")}}), - Entry("Tags Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{Tags: map[string]string{"keyTag-test-3": "valueTag-test-3"}}}), - Entry("MetadataOptions Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{MetadataOptions: &v1beta1.MetadataOptions{HTTPEndpoint: aws.String("test-metadata-2")}}}), - Entry("BlockDeviceMappings Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{DeviceName: aws.String("map-device-test-3")}}}}), - Entry("Context Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{Context: aws.String("context-2")}}), - Entry("DetailedMonitoring Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{DetailedMonitoring: aws.Bool(true)}}), - Entry("AMIFamily Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{AMIFamily: aws.String(v1beta1.AMIFamilyBottlerocket)}}), + Entry("UserData", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{UserData: aws.String("userdata-test-2")}}), + Entry("Tags", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{Tags: map[string]string{"keyTag-test-3": "valueTag-test-3"}}}), + Entry("Context", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{Context: aws.String("context-2")}}), + Entry("DetailedMonitoring", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{DetailedMonitoring: aws.Bool(true)}}), + Entry("AMIFamily", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{AMIFamily: aws.String(v1beta1.AMIFamilyBottlerocket)}}), + Entry("InstanceStorePolicy", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{InstanceStorePolicy: lo.ToPtr(v1beta1.InstanceStorePolicyRAID0)}}), + Entry("AssociatePublicIPAddress", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{AssociatePublicIPAddress: lo.ToPtr(true)}}), + Entry("MetadataOptions HTTPEndpoint", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{MetadataOptions: &v1beta1.MetadataOptions{HTTPEndpoint: lo.ToPtr("enabled")}}}), + Entry("MetadataOptions HTTPProtocolIPv6", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{MetadataOptions: &v1beta1.MetadataOptions{HTTPProtocolIPv6: lo.ToPtr("enabled")}}}), + Entry("MetadataOptions HTTPPutResponseHopLimit", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{MetadataOptions: &v1beta1.MetadataOptions{HTTPPutResponseHopLimit: lo.ToPtr(int64(10))}}}), + Entry("MetadataOptions HTTPTokens", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{MetadataOptions: &v1beta1.MetadataOptions{HTTPTokens: lo.ToPtr("required")}}}), + Entry("BlockDeviceMapping DeviceName", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{DeviceName: lo.ToPtr("map-device-test-3")}}}}), + Entry("BlockDeviceMapping RootVolume", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{RootVolume: true}}}}), + Entry("BlockDeviceMapping DeleteOnTermination", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{EBS: &v1beta1.BlockDevice{DeleteOnTermination: lo.ToPtr(true)}}}}}), + Entry("BlockDeviceMapping Encrypted", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{EBS: &v1beta1.BlockDevice{Encrypted: lo.ToPtr(true)}}}}}), + Entry("BlockDeviceMapping IOPS", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{EBS: &v1beta1.BlockDevice{IOPS: lo.ToPtr(int64(10))}}}}}), + Entry("BlockDeviceMapping KMSKeyID", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{EBS: &v1beta1.BlockDevice{KMSKeyID: lo.ToPtr("test")}}}}}), + Entry("BlockDeviceMapping SnapshotID", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{EBS: &v1beta1.BlockDevice{SnapshotID: lo.ToPtr("test")}}}}}), + Entry("BlockDeviceMapping Throughput", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{EBS: &v1beta1.BlockDevice{Throughput: lo.ToPtr(int64(10))}}}}}), + Entry("BlockDeviceMapping VolumeType", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{EBS: &v1beta1.BlockDevice{VolumeType: lo.ToPtr("io1")}}}}}), ) + // We create a separate test for updating blockDeviceMapping volumeSize, since resource.Quantity is a struct, and mergo.WithSliceDeepCopy + // doesn't work well with unexported fields, like the ones that are present in resource.Quantity + It("should change hash blockDeviceMapping volumeSize is updated", func() { + hash := nodeClass.Hash() + nodeClass.Spec.BlockDeviceMappings[0].EBS.VolumeSize = resource.NewScaledQuantity(10, resource.Giga) + updatedHash := nodeClass.Hash() + Expect(hash).ToNot(Equal(updatedHash)) + }) It("should change hash when instanceProfile is updated", func() { nodeClass.Spec.Role = "" nodeClass.Spec.InstanceProfile = lo.ToPtr("test-instance-profile") @@ -105,15 +167,18 @@ var _ = Describe("Hash", func() { updatedHash := nodeClass.Hash() Expect(hash).ToNot(Equal(updatedHash)) }) - DescribeTable("should not change hash when slices are re-ordered", func(changes v1beta1.EC2NodeClass) { + It("should not change hash when tags are re-ordered", func() { hash := nodeClass.Hash() - Expect(mergo.Merge(nodeClass, changes, mergo.WithOverride)).To(Succeed()) + nodeClass.Spec.Tags = map[string]string{"keyTag-2": "valueTag-2", "keyTag-1": "valueTag-1"} updatedHash := nodeClass.Hash() Expect(hash).To(Equal(updatedHash)) - }, - Entry("Reorder Tags", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{Tags: map[string]string{"keyTag-2": "valueTag-2", "keyTag-1": "valueTag-1"}}}), - Entry("Reorder BlockDeviceMapping", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{DeviceName: aws.String("map-device-2")}, {DeviceName: aws.String("map-device-1")}}}}), - ) + }) + It("should not change hash when blockDeviceMappings are re-ordered", func() { + hash := nodeClass.Hash() + nodeClass.Spec.BlockDeviceMappings[0], nodeClass.Spec.BlockDeviceMappings[1] = nodeClass.Spec.BlockDeviceMappings[1], nodeClass.Spec.BlockDeviceMappings[0] + updatedHash := nodeClass.Hash() + Expect(hash).To(Equal(updatedHash)) + }) It("should not change hash when behavior/dynamic fields are updated", func() { hash := nodeClass.Hash() diff --git a/pkg/apis/v1beta1/ec2nodeclass_status.go b/pkg/apis/v1beta1/ec2nodeclass_status.go index c98cab494a29..611e94d62117 100644 --- a/pkg/apis/v1beta1/ec2nodeclass_status.go +++ b/pkg/apis/v1beta1/ec2nodeclass_status.go @@ -14,7 +14,9 @@ limitations under the License. package v1beta1 -import v1 "k8s.io/api/core/v1" +import ( + corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" +) // Subnet contains resolved Subnet selector values utilized for node launch type Subnet struct { @@ -46,7 +48,7 @@ type AMI struct { Name string `json:"name,omitempty"` // Requirements of the AMI to be utilized on an instance type // +required - Requirements []v1.NodeSelectorRequirement `json:"requirements"` + Requirements []corev1beta1.NodeSelectorRequirementWithMinValues `json:"requirements"` } // EC2NodeClassStatus contains the resolved state of the EC2NodeClass diff --git a/pkg/apis/v1beta1/ec2nodeclass_validation_cel_test.go b/pkg/apis/v1beta1/ec2nodeclass_validation_cel_test.go index 5e5bbda783d9..b0104c9c506a 100644 --- a/pkg/apis/v1beta1/ec2nodeclass_validation_cel_test.go +++ b/pkg/apis/v1beta1/ec2nodeclass_validation_cel_test.go @@ -83,6 +83,14 @@ var _ = Describe("CEL/Validation", func() { corev1beta1.ManagedByAnnotationKey: "test", } Expect(env.Client.Create(ctx, nc)).To(Not(Succeed())) + nc.Spec.Tags = map[string]string{ + v1beta1.LabelNodeClass: "test", + } + Expect(env.Client.Create(ctx, nc)).To(Not(Succeed())) + nc.Spec.Tags = map[string]string{ + "karpenter.sh/nodeclaim": "test", + } + Expect(env.Client.Create(ctx, nc)).To(Not(Succeed())) }) }) Context("SubnetSelectorTerms", func() { diff --git a/pkg/apis/v1beta1/ec2nodeclass_validation_webhook_test.go b/pkg/apis/v1beta1/ec2nodeclass_validation_webhook_test.go index 91a64160d4ca..18035649e235 100644 --- a/pkg/apis/v1beta1/ec2nodeclass_validation_webhook_test.go +++ b/pkg/apis/v1beta1/ec2nodeclass_validation_webhook_test.go @@ -95,6 +95,14 @@ var _ = Describe("Webhook/Validation", func() { "karpenter.sh/managed-by": "test", } Expect(nc.Validate(ctx)).To(Not(Succeed())) + nc.Spec.Tags = map[string]string{ + v1beta1.LabelNodeClass: "test", + } + Expect(nc.Validate(ctx)).To(Not(Succeed())) + nc.Spec.Tags = map[string]string{ + "karpenter.sh/nodeclaim": "test", + } + Expect(nc.Validate(ctx)).To(Not(Succeed())) }) }) Context("SubnetSelectorTerms", func() { diff --git a/pkg/apis/v1beta1/labels.go b/pkg/apis/v1beta1/labels.go index 1b86eb70726f..cadf8dd902c6 100644 --- a/pkg/apis/v1beta1/labels.go +++ b/pkg/apis/v1beta1/labels.go @@ -35,6 +35,7 @@ func init() { LabelInstanceSize, LabelInstanceLocalNVME, LabelInstanceCPU, + LabelInstanceCPUManufacturer, LabelInstanceMemory, LabelInstanceNetworkBandwidth, LabelInstanceGPUName, @@ -70,9 +71,12 @@ var ( regexp.MustCompile(`^kubernetes\.io/cluster/[0-9A-Za-z][A-Za-z0-9\-_]*$`), regexp.MustCompile(fmt.Sprintf("^%s$", regexp.QuoteMeta(v1beta1.NodePoolLabelKey))), regexp.MustCompile(fmt.Sprintf("^%s$", regexp.QuoteMeta(v1beta1.ManagedByAnnotationKey))), + regexp.MustCompile(fmt.Sprintf("^%s$", regexp.QuoteMeta(LabelNodeClass))), + regexp.MustCompile(fmt.Sprintf("^%s$", regexp.QuoteMeta(TagNodeClaim))), } AMIFamilyBottlerocket = "Bottlerocket" AMIFamilyAL2 = "AL2" + AMIFamilyAL2023 = "AL2023" AMIFamilyUbuntu = "Ubuntu" AMIFamilyWindows2019 = "Windows2019" AMIFamilyWindows2022 = "Windows2022" @@ -100,6 +104,7 @@ var ( LabelInstanceLocalNVME = Group + "/instance-local-nvme" LabelInstanceSize = Group + "/instance-size" LabelInstanceCPU = Group + "/instance-cpu" + LabelInstanceCPUManufacturer = Group + "/instance-cpu-manufacturer" LabelInstanceMemory = Group + "/instance-memory" LabelInstanceNetworkBandwidth = Group + "/instance-network-bandwidth" LabelInstanceGPUName = Group + "/instance-gpu-name" @@ -110,5 +115,9 @@ var ( LabelInstanceAcceleratorManufacturer = Group + "/instance-accelerator-manufacturer" LabelInstanceAcceleratorCount = Group + "/instance-accelerator-count" AnnotationEC2NodeClassHash = Group + "/ec2nodeclass-hash" + AnnotationEC2NodeClassHashVersion = Group + "/ec2nodeclass-hash-version" AnnotationInstanceTagged = Group + "/tagged" + + TagNodeClaim = v1beta1.Group + "/nodeclaim" + TagName = "Name" ) diff --git a/pkg/apis/v1beta1/nodepool_validation_cel_test.go b/pkg/apis/v1beta1/nodepool_validation_cel_test.go index 3acbd0b8d868..574e8a0779f9 100644 --- a/pkg/apis/v1beta1/nodepool_validation_cel_test.go +++ b/pkg/apis/v1beta1/nodepool_validation_cel_test.go @@ -43,10 +43,12 @@ var _ = Describe("CEL/Validation", func() { Kind: "NodeClaim", Name: "default", }, - Requirements: []v1.NodeSelectorRequirement{ + Requirements: []v1beta1.NodeSelectorRequirementWithMinValues{ { - Key: v1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpExists, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.CapacityTypeLabelKey, + Operator: v1.NodeSelectorOpExists, + }, }, }, }, @@ -58,8 +60,8 @@ var _ = Describe("CEL/Validation", func() { It("should allow restricted domains exceptions", func() { oldNodePool := nodePool.DeepCopy() for label := range v1beta1.LabelDomainExceptions { - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: label + "/test", Operator: v1.NodeSelectorOpIn, Values: []string{"test"}}, + nodePool.Spec.Template.Spec.Requirements = []v1beta1.NodeSelectorRequirementWithMinValues{ + {NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: label + "/test", Operator: v1.NodeSelectorOpIn, Values: []string{"test"}}}, } Expect(env.Client.Create(ctx, nodePool)).To(Succeed()) Expect(nodePool.RuntimeValidate()).To(Succeed()) @@ -70,8 +72,8 @@ var _ = Describe("CEL/Validation", func() { It("should allow well known label exceptions", func() { oldNodePool := nodePool.DeepCopy() for label := range v1beta1.WellKnownLabels.Difference(sets.New(v1beta1.NodePoolLabelKey)) { - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: label, Operator: v1.NodeSelectorOpIn, Values: []string{"test"}}, + nodePool.Spec.Template.Spec.Requirements = []v1beta1.NodeSelectorRequirementWithMinValues{ + {NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: label, Operator: v1.NodeSelectorOpIn, Values: []string{"test"}}}, } Expect(env.Client.Create(ctx, nodePool)).To(Succeed()) Expect(nodePool.RuntimeValidate()).To(Succeed()) diff --git a/pkg/apis/v1beta1/zz_generated.deepcopy.go b/pkg/apis/v1beta1/zz_generated.deepcopy.go index 430fb6bb0840..781d88c876c8 100644 --- a/pkg/apis/v1beta1/zz_generated.deepcopy.go +++ b/pkg/apis/v1beta1/zz_generated.deepcopy.go @@ -19,8 +19,8 @@ limitations under the License. package v1beta1 import ( - "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" + apisv1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" ) // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. @@ -28,7 +28,7 @@ func (in *AMI) DeepCopyInto(out *AMI) { *out = *in if in.Requirements != nil { in, out := &in.Requirements, &out.Requirements - *out = make([]v1.NodeSelectorRequirement, len(*in)) + *out = make([]apisv1beta1.NodeSelectorRequirementWithMinValues, len(*in)) for i := range *in { (*in)[i].DeepCopyInto(&(*out)[i]) } @@ -223,6 +223,11 @@ func (in *EC2NodeClassSpec) DeepCopyInto(out *EC2NodeClassSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.AssociatePublicIPAddress != nil { + in, out := &in.AssociatePublicIPAddress, &out.AssociatePublicIPAddress + *out = new(bool) + **out = **in + } if in.AMISelectorTerms != nil { in, out := &in.AMISelectorTerms, &out.AMISelectorTerms *out = make([]AMISelectorTerm, len(*in)) diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index 7f0ba1d97510..7bb586693f6d 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -36,4 +36,8 @@ const ( const ( // DefaultCleanupInterval triggers cache cleanup (lazy eviction) at this interval. DefaultCleanupInterval = time.Minute + // UnavailableOfferingsCleanupInterval triggers cache cleanup (lazy eviction) at this interval. + // We drop the cleanup interval down for the ICE cache to get quicker reactivity to offerings + // that become available after they get evicted from the cache + UnavailableOfferingsCleanupInterval = time.Second * 10 ) diff --git a/pkg/cache/unavailableofferings.go b/pkg/cache/unavailableofferings.go index 5051ad58300c..bbc2c16b3fb5 100644 --- a/pkg/cache/unavailableofferings.go +++ b/pkg/cache/unavailableofferings.go @@ -35,10 +35,14 @@ type UnavailableOfferings struct { } func NewUnavailableOfferings() *UnavailableOfferings { - return &UnavailableOfferings{ - cache: cache.New(UnavailableOfferingsTTL, DefaultCleanupInterval), + uo := &UnavailableOfferings{ + cache: cache.New(UnavailableOfferingsTTL, UnavailableOfferingsCleanupInterval), SeqNum: 0, } + uo.cache.OnEvicted(func(_ string, _ interface{}) { + atomic.AddUint64(&uo.SeqNum, 1) + }) + return uo } // IsUnavailable returns true if the offering appears in the cache diff --git a/pkg/cloudprovider/cloudprovider.go b/pkg/cloudprovider/cloudprovider.go index d1c5de336d87..a2d8b579d53a 100644 --- a/pkg/cloudprovider/cloudprovider.go +++ b/pkg/cloudprovider/cloudprovider.go @@ -101,7 +101,10 @@ func (c *CloudProvider) Create(ctx context.Context, nodeClaim *corev1beta1.NodeC return i.Name == instance.Type }) nc := c.instanceToNodeClaim(instance, instanceType) - nc.Annotations = lo.Assign(nc.Annotations, map[string]string{v1beta1.AnnotationEC2NodeClassHash: nodeClass.Hash()}) + nc.Annotations = lo.Assign(nodeClass.Annotations, map[string]string{ + v1beta1.AnnotationEC2NodeClassHash: nodeClass.Hash(), + v1beta1.AnnotationEC2NodeClassHashVersion: v1beta1.EC2NodeClassHashVersion, + }) return nc, nil } @@ -208,6 +211,16 @@ func (c *CloudProvider) Name() string { return "aws" } +func (c *CloudProvider) GetSupportedNodeClasses() []schema.GroupVersionKind { + return []schema.GroupVersionKind{ + { + Group: v1beta1.SchemeGroupVersion.Group, + Version: v1beta1.SchemeGroupVersion.Version, + Kind: "EC2NodeClass", + }, + } +} + func (c *CloudProvider) resolveNodeClassFromNodeClaim(ctx context.Context, nodeClaim *corev1beta1.NodeClaim) (*v1beta1.EC2NodeClass, error) { nodeClass := &v1beta1.EC2NodeClass{} if err := c.kubeClient.Get(ctx, types.NamespacedName{Name: nodeClaim.Spec.NodeClassRef.Name}, nodeClass); err != nil { @@ -215,7 +228,9 @@ func (c *CloudProvider) resolveNodeClassFromNodeClaim(ctx context.Context, nodeC } // For the purposes of NodeClass CloudProvider resolution, we treat deleting NodeClasses as NotFound if !nodeClass.DeletionTimestamp.IsZero() { - return nil, errors.NewNotFound(v1beta1.SchemeGroupVersion.WithResource("ec2nodeclasses").GroupResource(), nodeClass.Name) + // For the purposes of NodeClass CloudProvider resolution, we treat deleting NodeClasses as NotFound, + // but we return a different error message to be clearer to users + return nil, newTerminatingNodeClassError(nodeClass.Name) } return nodeClass, nil } @@ -225,9 +240,10 @@ func (c *CloudProvider) resolveNodeClassFromNodePool(ctx context.Context, nodePo if err := c.kubeClient.Get(ctx, types.NamespacedName{Name: nodePool.Spec.Template.Spec.NodeClassRef.Name}, nodeClass); err != nil { return nil, err } - // For the purposes of NodeClass CloudProvider resolution, we treat deleting NodeClasses as NotFound if !nodeClass.DeletionTimestamp.IsZero() { - return nil, errors.NewNotFound(v1beta1.SchemeGroupVersion.WithResource("ec2nodeclasses").GroupResource(), nodeClass.Name) + // For the purposes of NodeClass CloudProvider resolution, we treat deleting NodeClasses as NotFound, + // but we return a different error message to be clearer to users + return nil, newTerminatingNodeClassError(nodeClass.Name) } return nodeClass, nil } @@ -237,7 +253,7 @@ func (c *CloudProvider) resolveInstanceTypes(ctx context.Context, nodeClaim *cor if err != nil { return nil, fmt.Errorf("getting instance types, %w", err) } - reqs := scheduling.NewNodeSelectorRequirements(nodeClaim.Spec.Requirements...) + reqs := scheduling.NewNodeSelectorRequirementsWithMinValues(nodeClaim.Spec.Requirements...) return lo.Filter(instanceTypes, func(i *cloudprovider.InstanceType, _ int) bool { return reqs.Compatible(i.Requirements, scheduling.AllowUndefinedWellKnownLabels) == nil && len(i.Offerings.Compatible(reqs).Available()) > 0 && @@ -270,7 +286,7 @@ func (c *CloudProvider) resolveNodePoolFromInstance(ctx context.Context, instanc } return nodePool, nil } - return nil, errors.NewNotFound(schema.GroupResource{Group: corev1beta1.Group, Resource: "NodePool"}, "") + return nil, errors.NewNotFound(schema.GroupResource{Group: corev1beta1.Group, Resource: "nodepools"}, "") } func (c *CloudProvider) instanceToNodeClaim(i *instance.Instance, instanceType *cloudprovider.InstanceType) *corev1beta1.NodeClaim { @@ -317,3 +333,11 @@ func (c *CloudProvider) instanceToNodeClaim(i *instance.Instance, instanceType * nodeClaim.Status.ImageID = i.ImageID return nodeClaim } + +// newTerminatingNodeClassError returns a NotFound error for handling by +func newTerminatingNodeClassError(name string) *errors.StatusError { + qualifiedResource := schema.GroupResource{Group: corev1beta1.Group, Resource: "ec2nodeclasses"} + err := errors.NewNotFound(qualifiedResource, name) + err.ErrStatus.Message = fmt.Sprintf("%s %q is terminating, treating as not found", qualifiedResource.String(), name) + return err +} diff --git a/pkg/cloudprovider/drift.go b/pkg/cloudprovider/drift.go index fe259ef397d1..69dc5aad1783 100644 --- a/pkg/cloudprovider/drift.go +++ b/pkg/cloudprovider/drift.go @@ -135,9 +135,16 @@ func (c *CloudProvider) areSecurityGroupsDrifted(ctx context.Context, ec2Instanc } func (c *CloudProvider) areStaticFieldsDrifted(nodeClaim *corev1beta1.NodeClaim, nodeClass *v1beta1.EC2NodeClass) cloudprovider.DriftReason { - nodeClassHash, foundHashNodeClass := nodeClass.Annotations[v1beta1.AnnotationEC2NodeClassHash] - nodeClaimHash, foundHashNodeClaim := nodeClaim.Annotations[v1beta1.AnnotationEC2NodeClassHash] - if !foundHashNodeClass || !foundHashNodeClaim { + nodeClassHash, foundNodeClassHash := nodeClass.Annotations[v1beta1.AnnotationEC2NodeClassHash] + nodeClassHashVersion, foundNodeClassHashVersion := nodeClass.Annotations[v1beta1.AnnotationEC2NodeClassHashVersion] + nodeClaimHash, foundNodeClaimHash := nodeClaim.Annotations[v1beta1.AnnotationEC2NodeClassHash] + nodeClaimHashVersion, foundNodeClaimHashVersion := nodeClaim.Annotations[v1beta1.AnnotationEC2NodeClassHashVersion] + + if !foundNodeClassHash || !foundNodeClaimHash || !foundNodeClassHashVersion || !foundNodeClaimHashVersion { + return "" + } + // validate that the hash version for the EC2NodeClass is the same as the NodeClaim before evaluating for static drift + if nodeClassHashVersion != nodeClaimHashVersion { return "" } return lo.Ternary(nodeClassHash != nodeClaimHash, NodeClassDrift, "") diff --git a/pkg/cloudprovider/suite_test.go b/pkg/cloudprovider/suite_test.go index 9cc73de9837e..4e980174a4ba 100644 --- a/pkg/cloudprovider/suite_test.go +++ b/pkg/cloudprovider/suite_test.go @@ -18,12 +18,14 @@ import ( "context" "fmt" "net" + "strings" "testing" "time" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/tools/record" clock "k8s.io/utils/clock/testing" @@ -36,12 +38,11 @@ import ( "github.com/aws/karpenter-provider-aws/pkg/apis" "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + "github.com/aws/karpenter-provider-aws/pkg/cloudprovider" "github.com/aws/karpenter-provider-aws/pkg/fake" "github.com/aws/karpenter-provider-aws/pkg/operator/options" "github.com/aws/karpenter-provider-aws/pkg/test" - "github.com/aws/karpenter-provider-aws/pkg/cloudprovider" - corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" corecloudproivder "sigs.k8s.io/karpenter/pkg/cloudprovider" "sigs.k8s.io/karpenter/pkg/controllers/provisioning" @@ -84,7 +85,7 @@ var _ = BeforeSuite(func() { cloudProvider = cloudprovider.New(awsEnv.InstanceTypesProvider, awsEnv.InstanceProvider, recorder, env.Client, awsEnv.AMIProvider, awsEnv.SecurityGroupProvider, awsEnv.SubnetProvider) cluster = state.NewCluster(fakeClock, env.Client, cloudProvider) - prov = provisioning.NewProvisioner(env.Client, env.KubernetesInterface.CoreV1(), recorder, cloudProvider, cluster) + prov = provisioning.NewProvisioner(env.Client, recorder, cloudProvider, cluster) }) var _ = AfterSuite(func() { @@ -120,8 +121,8 @@ var _ = Describe("CloudProvider", func() { NodeClassRef: &corev1beta1.NodeClassReference{ Name: nodeClass.Name, }, - Requirements: []v1.NodeSelectorRequirement{ - {Key: corev1beta1.CapacityTypeLabelKey, Operator: v1.NodeSelectorOpIn, Values: []string{corev1beta1.CapacityTypeOnDemand}}, + Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ + {NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: corev1beta1.CapacityTypeLabelKey, Operator: v1.NodeSelectorOpIn, Values: []string{corev1beta1.CapacityTypeOnDemand}}}, }, }, }, @@ -140,11 +141,13 @@ var _ = Describe("CloudProvider", func() { }) It("should return an ICE error when there are no instance types to launch", func() { // Specify no instance types and expect to receive a capacity error - nodeClaim.Spec.Requirements = []v1.NodeSelectorRequirement{ + nodeClaim.Spec.Requirements = []corev1beta1.NodeSelectorRequirementWithMinValues{ { - Key: v1.LabelInstanceTypeStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{"test-instance-type"}, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1.LabelInstanceTypeStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{"test-instance-type"}, + }, }, } ExpectApplied(ctx, env.Client, nodePool, nodeClass, nodeClaim) @@ -168,15 +171,16 @@ var _ = Describe("CloudProvider", func() { Expect(ok).To(BeTrue()) }) Context("EC2 Context", func() { + contextID := "context-1234" It("should set context on the CreateFleet request if specified on the NodePool", func() { - nodeClass.Spec.Context = aws.String("context-1234") + nodeClass.Spec.Context = aws.String(contextID) ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) ExpectScheduled(ctx, env.Client, pod) Expect(awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) createFleetInput := awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Pop() - Expect(aws.StringValue(createFleetInput.Context)).To(Equal("context-1234")) + Expect(aws.StringValue(createFleetInput.Context)).To(Equal(contextID)) }) It("should default to no EC2 Context", func() { ExpectApplied(ctx, env.Client, nodePool, nodeClass) @@ -188,6 +192,302 @@ var _ = Describe("CloudProvider", func() { Expect(createFleetInput.Context).To(BeNil()) }) }) + Context("MinValues", func() { + It("CreateFleet input should respect minValues for In operator requirement from NodePool", func() { + // Create fake InstanceTypes where one instances can fit 2 pods and another one can fit only 1 pod. + // This specific type of inputs will help us differentiate the scenario we are trying to test where ideally + // 1 instance launch would have been sufficient to fit the pods and was cheaper but we would launch 2 separate + // instances to meet the minimum requirement. + instances := fake.MakeInstances() + instances, _ = fake.MakeUniqueInstancesAndFamilies(instances, 2) + instances[0].VCpuInfo = &ec2.VCpuInfo{DefaultVCpus: aws.Int64(1)} + instances[1].VCpuInfo = &ec2.VCpuInfo{DefaultVCpus: aws.Int64(8)} + awsEnv.EC2API.DescribeInstanceTypesOutput.Set(&ec2.DescribeInstanceTypesOutput{InstanceTypes: instances}) + awsEnv.EC2API.DescribeInstanceTypeOfferingsOutput.Set(&ec2.DescribeInstanceTypeOfferingsOutput{InstanceTypeOfferings: fake.MakeInstanceOfferings(instances)}) + now := time.Now() + awsEnv.EC2API.DescribeSpotPriceHistoryOutput.Set(&ec2.DescribeSpotPriceHistoryOutput{ + SpotPriceHistory: []*ec2.SpotPrice{ + { + AvailabilityZone: aws.String("test-zone-1a"), + InstanceType: instances[0].InstanceType, + SpotPrice: aws.String("0.002"), + Timestamp: &now, + }, + { + AvailabilityZone: aws.String("test-zone-1a"), + InstanceType: instances[1].InstanceType, + SpotPrice: aws.String("0.003"), + Timestamp: &now, + }, + }, + }) + Expect(awsEnv.PricingProvider.UpdateSpotPricing(ctx)).To(Succeed()) + instanceNames := lo.Map(instances, func(info *ec2.InstanceTypeInfo, _ int) string { return *info.InstanceType }) + + // Define NodePool that has minValues on instance-type requirement. + nodePool = coretest.NodePool(corev1beta1.NodePool{ + Spec: corev1beta1.NodePoolSpec{ + Template: corev1beta1.NodeClaimTemplate{ + Spec: corev1beta1.NodeClaimSpec{ + NodeClassRef: &corev1beta1.NodeClassReference{ + Name: nodeClass.Name, + }, + Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ + { + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: corev1beta1.CapacityTypeLabelKey, + Operator: v1.NodeSelectorOpIn, + Values: []string{corev1beta1.CapacityTypeSpot}, + }, + }, + { + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1.LabelInstanceTypeStable, + Operator: v1.NodeSelectorOpIn, + Values: instanceNames, + }, + MinValues: lo.ToPtr(2), + }, + }, + }, + }, + }, + }) + + ExpectApplied(ctx, env.Client, nodePool, nodeClass) + + // 2 pods are created with resources such that both fit together only in one of the 2 InstanceTypes created above. + pod1 := coretest.UnschedulablePod( + coretest.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("0.9")}, + }, + }) + pod2 := coretest.UnschedulablePod( + coretest.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("0.9")}, + }, + }) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod1, pod2) + + // Under normal circumstances 1 node would have been created that fits both the pods but + // here minValue enforces to include both the instances. And since one of the instances can + // only fit 1 pod, only 1 pod is scheduled to run in the node to be launched by CreateFleet. + node1 := ExpectScheduled(ctx, env.Client, pod1) + node2 := ExpectScheduled(ctx, env.Client, pod2) + + // This ensures that the pods are scheduled in 2 different nodes. + Expect(node1.Name).ToNot(Equal(node2.Name)) + Expect(awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(2)) + createFleetInput := awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Pop() + uniqueInstanceTypes := sets.Set[string]{} + for _, launchTemplateConfig := range createFleetInput.LaunchTemplateConfigs { + for _, override := range launchTemplateConfig.Overrides { + uniqueInstanceTypes.Insert(*override.InstanceType) + } + } + // This ensures that we have sent the minimum number of requirements defined in the NodePool. + Expect(len(uniqueInstanceTypes)).To(BeNumerically(">=", 2)) + }) + It("CreateFleet input should respect minValues for Exists Operator in requirement from NodePool", func() { + // Create fake InstanceTypes where one instances can fit 2 pods and another one can fit only 1 pod. + instances := fake.MakeInstances() + instances, _ = fake.MakeUniqueInstancesAndFamilies(instances, 2) + instances[0].VCpuInfo = &ec2.VCpuInfo{DefaultVCpus: aws.Int64(1)} + instances[1].VCpuInfo = &ec2.VCpuInfo{DefaultVCpus: aws.Int64(8)} + awsEnv.EC2API.DescribeInstanceTypesOutput.Set(&ec2.DescribeInstanceTypesOutput{InstanceTypes: instances}) + awsEnv.EC2API.DescribeInstanceTypeOfferingsOutput.Set(&ec2.DescribeInstanceTypeOfferingsOutput{InstanceTypeOfferings: fake.MakeInstanceOfferings(instances)}) + now := time.Now() + awsEnv.EC2API.DescribeSpotPriceHistoryOutput.Set(&ec2.DescribeSpotPriceHistoryOutput{ + SpotPriceHistory: []*ec2.SpotPrice{ + { + AvailabilityZone: aws.String("test-zone-1a"), + InstanceType: instances[0].InstanceType, + SpotPrice: aws.String("0.002"), + Timestamp: &now, + }, + { + AvailabilityZone: aws.String("test-zone-1a"), + InstanceType: instances[1].InstanceType, + SpotPrice: aws.String("0.003"), + Timestamp: &now, + }, + }, + }) + Expect(awsEnv.PricingProvider.UpdateSpotPricing(ctx)).To(Succeed()) + instanceNames := lo.Map(instances, func(info *ec2.InstanceTypeInfo, _ int) string { return *info.InstanceType }) + + // Define NodePool that has minValues on instance-type requirement. + nodePool = coretest.NodePool(corev1beta1.NodePool{ + Spec: corev1beta1.NodePoolSpec{ + Template: corev1beta1.NodeClaimTemplate{ + Spec: corev1beta1.NodeClaimSpec{ + NodeClassRef: &corev1beta1.NodeClassReference{ + Name: nodeClass.Name, + }, + Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ + { + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1.LabelInstanceTypeStable, + Operator: v1.NodeSelectorOpExists, + }, + MinValues: lo.ToPtr(2), + }, + { + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1.LabelInstanceTypeStable, + Operator: v1.NodeSelectorOpIn, + Values: instanceNames, + }, + MinValues: lo.ToPtr(1), + }, + }, + }, + }, + }, + }) + + ExpectApplied(ctx, env.Client, nodePool, nodeClass) + + // 2 pods are created with resources such that both fit together only in one of the 2 InstanceTypes created above. + pod1 := coretest.UnschedulablePod( + coretest.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("0.9")}, + }, + }) + pod2 := coretest.UnschedulablePod( + coretest.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("0.9")}, + }, + }) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod1, pod2) + + // Under normal circumstances 1 node would have been created that fits both the pods but + // here minValue enforces to include both the instances. And since one of the instances can + // only fit 1 pod, only 1 pod is scheduled to run in the node to be launched by CreateFleet. + node1 := ExpectScheduled(ctx, env.Client, pod1) + node2 := ExpectScheduled(ctx, env.Client, pod2) + + // This ensures that the pods are scheduled in 2 different nodes. + Expect(node1.Name).ToNot(Equal(node2.Name)) + Expect(awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(2)) + createFleetInput := awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Pop() + uniqueInstanceTypes := sets.Set[string]{} + for _, launchTemplateConfig := range createFleetInput.LaunchTemplateConfigs { + for _, override := range launchTemplateConfig.Overrides { + uniqueInstanceTypes.Insert(*override.InstanceType) + } + } + // This ensures that we have sent the minimum number of requirements defined in the NodePool. + Expect(len(uniqueInstanceTypes)).To(BeNumerically(">=", 2)) + }) + It("CreateFleet input should respect minValues from multiple keys in NodePool", func() { + // Create fake InstanceTypes where 2 instances can fit 2 pods individually and one can fit only 1 pod. + instances := fake.MakeInstances() + uniqInstanceTypes, instanceFamilies := fake.MakeUniqueInstancesAndFamilies(instances, 3) + uniqInstanceTypes[0].VCpuInfo = &ec2.VCpuInfo{DefaultVCpus: aws.Int64(1)} + uniqInstanceTypes[1].VCpuInfo = &ec2.VCpuInfo{DefaultVCpus: aws.Int64(4)} + uniqInstanceTypes[2].VCpuInfo = &ec2.VCpuInfo{DefaultVCpus: aws.Int64(8)} + awsEnv.EC2API.DescribeInstanceTypesOutput.Set(&ec2.DescribeInstanceTypesOutput{InstanceTypes: uniqInstanceTypes}) + awsEnv.EC2API.DescribeInstanceTypeOfferingsOutput.Set(&ec2.DescribeInstanceTypeOfferingsOutput{InstanceTypeOfferings: fake.MakeInstanceOfferings(uniqInstanceTypes)}) + now := time.Now() + awsEnv.EC2API.DescribeSpotPriceHistoryOutput.Set(&ec2.DescribeSpotPriceHistoryOutput{ + SpotPriceHistory: []*ec2.SpotPrice{ + { + AvailabilityZone: aws.String("test-zone-1a"), + InstanceType: uniqInstanceTypes[0].InstanceType, + SpotPrice: aws.String("0.002"), + Timestamp: &now, + }, + { + AvailabilityZone: aws.String("test-zone-1a"), + InstanceType: uniqInstanceTypes[1].InstanceType, + SpotPrice: aws.String("0.003"), + Timestamp: &now, + }, + { + AvailabilityZone: aws.String("test-zone-1a"), + InstanceType: uniqInstanceTypes[2].InstanceType, + SpotPrice: aws.String("0.004"), + Timestamp: &now, + }, + }, + }) + Expect(awsEnv.PricingProvider.UpdateSpotPricing(ctx)).To(Succeed()) + instanceNames := lo.Map(uniqInstanceTypes, func(info *ec2.InstanceTypeInfo, _ int) string { return *info.InstanceType }) + + // Define NodePool that has minValues in multiple requirements. + nodePool = coretest.NodePool(corev1beta1.NodePool{ + Spec: corev1beta1.NodePoolSpec{ + Template: corev1beta1.NodeClaimTemplate{ + Spec: corev1beta1.NodeClaimSpec{ + NodeClassRef: &corev1beta1.NodeClassReference{ + Name: nodeClass.Name, + }, + Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ + { + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1.LabelInstanceTypeStable, + Operator: v1.NodeSelectorOpIn, + Values: instanceNames, + }, + // consider at least 2 unique instance types + MinValues: lo.ToPtr(2), + }, + { + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceFamily, + Operator: v1.NodeSelectorOpIn, + Values: instanceFamilies.UnsortedList(), + }, + // consider at least 3 unique instance families + MinValues: lo.ToPtr(3), + }, + }, + }, + }, + }, + }) + + ExpectApplied(ctx, env.Client, nodePool, nodeClass) + pod1 := coretest.UnschedulablePod( + coretest.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("0.9")}, + }, + }) + pod2 := coretest.UnschedulablePod( + coretest.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("0.9")}, + }, + }) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod1, pod2) + + // Under normal circumstances 1 node would have been created that fits both the pods but + // here minValue enforces to include all the 3 instances to satisfy both the instance-type and instance-family requirements. + // And since one of the instances can only fit 1 pod, only 1 pod is scheduled to run in the node to be launched by CreateFleet. + node1 := ExpectScheduled(ctx, env.Client, pod1) + node2 := ExpectScheduled(ctx, env.Client, pod2) + Expect(node1.Name).ToNot(Equal(node2.Name)) + Expect(awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(2)) + createFleetInput := awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Pop() + uniqueInstanceTypes, uniqueInstanceFamilies := sets.Set[string]{}, sets.Set[string]{} + for _, launchTemplateConfig := range createFleetInput.LaunchTemplateConfigs { + for _, override := range launchTemplateConfig.Overrides { + uniqueInstanceTypes.Insert(*override.InstanceType) + uniqueInstanceFamilies.Insert(strings.Split(*override.InstanceType, ".")[0]) + } + } + // Ensure that there are at least minimum number of unique instance types as per the requirement in the CreateFleet request. + Expect(len(uniqueInstanceTypes)).To(BeNumerically("==", 3)) + // Ensure that there are at least minimum number of unique instance families as per the requirement in the CreateFleet request. + Expect(len(uniqueInstanceFamilies)).To(BeNumerically("==", 3)) + }) + }) Context("NodeClaim Drift", func() { var armAMIID, amdAMIID string var validSecurityGroup string @@ -293,10 +593,14 @@ var _ = Describe("CloudProvider", func() { Reservations: []*ec2.Reservation{{Instances: []*ec2.Instance{instance}}}, }) nodeClass.Annotations = lo.Assign(nodeClass.Annotations, map[string]string{ - v1beta1.AnnotationEC2NodeClassHash: nodeClass.Hash(), + v1beta1.AnnotationEC2NodeClassHash: nodeClass.Hash(), + v1beta1.AnnotationEC2NodeClassHashVersion: v1beta1.EC2NodeClassHashVersion, }) nodeClaim.Status.ProviderID = fake.ProviderID(lo.FromPtr(instance.InstanceId)) - nodeClaim.Annotations = lo.Assign(nodeClaim.Annotations, map[string]string{v1beta1.AnnotationEC2NodeClassHash: nodeClass.Hash()}) + nodeClaim.Annotations = lo.Assign(nodeClaim.Annotations, map[string]string{ + v1beta1.AnnotationEC2NodeClassHash: nodeClass.Hash(), + v1beta1.AnnotationEC2NodeClassHashVersion: v1beta1.EC2NodeClassHashVersion, + }) nodeClaim.Labels = lo.Assign(nodeClaim.Labels, map[string]string{v1.LabelInstanceTypeStable: selectedInstanceType.Name}) }) It("should not fail if NodeClass does not exist", func() { @@ -418,14 +722,56 @@ var _ = Describe("CloudProvider", func() { Expect(isDrifted).To(Equal(cloudprovider.AMIDrift)) }) Context("Static Drift Detection", func() { - DescribeTable("should return drifted if the spec is updated", + BeforeEach(func() { + nodeClass = &v1beta1.EC2NodeClass{ + ObjectMeta: nodeClass.ObjectMeta, + Spec: v1beta1.EC2NodeClassSpec{ + SubnetSelectorTerms: nodeClass.Spec.SubnetSelectorTerms, + SecurityGroupSelectorTerms: nodeClass.Spec.SecurityGroupSelectorTerms, + Role: nodeClass.Spec.Role, + UserData: lo.ToPtr("Fake Userdata"), + Tags: map[string]string{ + "fakeKey": "fakeValue", + }, + Context: lo.ToPtr("fake-context"), + DetailedMonitoring: lo.ToPtr(false), + AMIFamily: lo.ToPtr(v1beta1.AMIFamilyAL2023), + AssociatePublicIPAddress: lo.ToPtr(false), + MetadataOptions: &v1beta1.MetadataOptions{ + HTTPEndpoint: lo.ToPtr("disabled"), + HTTPProtocolIPv6: lo.ToPtr("disabled"), + HTTPPutResponseHopLimit: lo.ToPtr(int64(1)), + HTTPTokens: lo.ToPtr("optional"), + }, + BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{ + { + DeviceName: lo.ToPtr("fakeName"), + RootVolume: false, + EBS: &v1beta1.BlockDevice{ + DeleteOnTermination: lo.ToPtr(false), + Encrypted: lo.ToPtr(false), + IOPS: lo.ToPtr(int64(0)), + KMSKeyID: lo.ToPtr("fakeKMSKeyID"), + SnapshotID: lo.ToPtr("fakeSnapshot"), + Throughput: lo.ToPtr(int64(0)), + VolumeSize: resource.NewScaledQuantity(2, resource.Giga), + VolumeType: lo.ToPtr("standard"), + }, + }, + }, + }, + } + nodeClass.Annotations = lo.Assign(nodeClass.Annotations, map[string]string{v1beta1.AnnotationEC2NodeClassHash: nodeClass.Hash()}) + nodeClaim.Annotations = lo.Assign(nodeClaim.Annotations, map[string]string{v1beta1.AnnotationEC2NodeClassHash: nodeClass.Hash()}) + }) + DescribeTable("should return drifted if a statically drifted EC2NodeClass.Spec field is updated", func(changes v1beta1.EC2NodeClass) { ExpectApplied(ctx, env.Client, nodePool, nodeClass) isDrifted, err := cloudProvider.IsDrifted(ctx, nodeClaim) Expect(err).NotTo(HaveOccurred()) Expect(isDrifted).To(BeEmpty()) - Expect(mergo.Merge(nodeClass, changes, mergo.WithOverride)) + Expect(mergo.Merge(nodeClass, changes, mergo.WithOverride, mergo.WithSliceDeepCopy)).To(Succeed()) nodeClass.Annotations = lo.Assign(nodeClass.Annotations, map[string]string{v1beta1.AnnotationEC2NodeClassHash: nodeClass.Hash()}) ExpectApplied(ctx, env.Client, nodeClass) @@ -433,14 +779,38 @@ var _ = Describe("CloudProvider", func() { Expect(err).NotTo(HaveOccurred()) Expect(isDrifted).To(Equal(cloudprovider.NodeClassDrift)) }, - Entry("UserData Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{UserData: aws.String("userdata-test-2")}}), - Entry("Tags Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{Tags: map[string]string{"keyTag-test-3": "valueTag-test-3"}}}), - Entry("MetadataOptions Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{MetadataOptions: &v1beta1.MetadataOptions{HTTPEndpoint: aws.String("disabled")}}}), - Entry("BlockDeviceMappings Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{DeviceName: aws.String("map-device-test-3")}}}}), - Entry("Context Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{Context: aws.String("context-2")}}), - Entry("DetailedMonitoring Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{DetailedMonitoring: aws.Bool(true)}}), - Entry("AMIFamily Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{AMIFamily: aws.String(v1beta1.AMIFamilyBottlerocket)}}), + Entry("UserData", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{UserData: lo.ToPtr("userdata-test-2")}}), + Entry("Tags", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{Tags: map[string]string{"keyTag-test-3": "valueTag-test-3"}}}), + Entry("Context", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{Context: lo.ToPtr("context-2")}}), + Entry("DetailedMonitoring", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{DetailedMonitoring: aws.Bool(true)}}), + Entry("AMIFamily", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{AMIFamily: lo.ToPtr(v1beta1.AMIFamilyBottlerocket)}}), + Entry("InstanceStorePolicy", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{InstanceStorePolicy: lo.ToPtr(v1beta1.InstanceStorePolicyRAID0)}}), + Entry("AssociatePublicIPAddress", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{AssociatePublicIPAddress: lo.ToPtr(true)}}), + Entry("MetadataOptions HTTPEndpoint", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{MetadataOptions: &v1beta1.MetadataOptions{HTTPEndpoint: lo.ToPtr("enabled")}}}), + Entry("MetadataOptions HTTPProtocolIPv6", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{MetadataOptions: &v1beta1.MetadataOptions{HTTPProtocolIPv6: lo.ToPtr("enabled")}}}), + Entry("MetadataOptions HTTPPutResponseHopLimit", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{MetadataOptions: &v1beta1.MetadataOptions{HTTPPutResponseHopLimit: lo.ToPtr(int64(10))}}}), + Entry("MetadataOptions HTTPTokens", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{MetadataOptions: &v1beta1.MetadataOptions{HTTPTokens: lo.ToPtr("required")}}}), + Entry("BlockDeviceMapping DeviceName", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{DeviceName: lo.ToPtr("map-device-test-3")}}}}), + Entry("BlockDeviceMapping RootVolume", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{RootVolume: true}}}}), + Entry("BlockDeviceMapping DeleteOnTermination", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{EBS: &v1beta1.BlockDevice{DeleteOnTermination: lo.ToPtr(true)}}}}}), + Entry("BlockDeviceMapping Encrypted", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{EBS: &v1beta1.BlockDevice{Encrypted: lo.ToPtr(true)}}}}}), + Entry("BlockDeviceMapping IOPS", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{EBS: &v1beta1.BlockDevice{IOPS: lo.ToPtr(int64(10))}}}}}), + Entry("BlockDeviceMapping KMSKeyID", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{EBS: &v1beta1.BlockDevice{KMSKeyID: lo.ToPtr("test")}}}}}), + Entry("BlockDeviceMapping SnapshotID", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{EBS: &v1beta1.BlockDevice{SnapshotID: lo.ToPtr("test")}}}}}), + Entry("BlockDeviceMapping Throughput", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{EBS: &v1beta1.BlockDevice{Throughput: lo.ToPtr(int64(10))}}}}}), + Entry("BlockDeviceMapping VolumeType", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{{EBS: &v1beta1.BlockDevice{VolumeType: lo.ToPtr("io1")}}}}}), ) + // We create a separate test for updating blockDeviceMapping volumeSize, since resource.Quantity is a struct, and mergo.WithSliceDeepCopy + // doesn't work well with unexported fields, like the ones that are present in resource.Quantity + It("should return drifted when updating blockDeviceMapping volumeSize", func() { + nodeClass.Spec.BlockDeviceMappings[0].EBS.VolumeSize = resource.NewScaledQuantity(10, resource.Giga) + nodeClass.Annotations = lo.Assign(nodeClass.Annotations, map[string]string{v1beta1.AnnotationEC2NodeClassHash: nodeClass.Hash()}) + + ExpectApplied(ctx, env.Client, nodeClass) + isDrifted, err := cloudProvider.IsDrifted(ctx, nodeClaim) + Expect(err).NotTo(HaveOccurred()) + Expect(isDrifted).To(Equal(cloudprovider.NodeClassDrift)) + }) DescribeTable("should not return drifted if dynamic fields are updated", func(changes v1beta1.EC2NodeClass) { ExpectApplied(ctx, env.Client, nodePool, nodeClass) @@ -460,8 +830,58 @@ var _ = Describe("CloudProvider", func() { Entry("Subnet Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{SubnetSelectorTerms: []v1beta1.SubnetSelectorTerm{{Tags: map[string]string{"sn-key-1": "sn-value-1"}}}}}), Entry("SecurityGroup Drift", v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{SecurityGroupSelectorTerms: []v1beta1.SecurityGroupSelectorTerm{{Tags: map[string]string{"sg-key": "sg-value"}}}}}), ) - It("should not return drifted if karpenter.k8s.aws/nodeclass-hash annotation is not present on the NodeClaim", func() { - nodeClaim.Annotations = map[string]string{} + It("should not return drifted if karpenter.k8s.aws/ec2nodeclass-hash annotation is not present on the NodeClaim", func() { + nodeClaim.Annotations = map[string]string{ + v1beta1.AnnotationEC2NodeClassHashVersion: v1beta1.EC2NodeClassHashVersion, + } + nodeClass.Spec.Tags = map[string]string{ + "Test Key": "Test Value", + } + ExpectApplied(ctx, env.Client, nodePool, nodeClass) + isDrifted, err := cloudProvider.IsDrifted(ctx, nodeClaim) + Expect(err).NotTo(HaveOccurred()) + Expect(isDrifted).To(BeEmpty()) + }) + It("should not return drifted if the NodeClaim's karpenter.k8s.aws/ec2nodeclass-hash-version annotation does not match the EC2NodeClass's", func() { + nodeClass.ObjectMeta.Annotations = map[string]string{ + v1beta1.AnnotationEC2NodeClassHash: "test-hash-111111", + v1beta1.AnnotationEC2NodeClassHashVersion: "test-hash-version-1", + } + nodeClaim.ObjectMeta.Annotations = map[string]string{ + v1beta1.AnnotationEC2NodeClassHash: "test-hash-222222", + v1beta1.AnnotationEC2NodeClassHashVersion: "test-hash-version-2", + } + ExpectApplied(ctx, env.Client, nodePool, nodeClass) + isDrifted, err := cloudProvider.IsDrifted(ctx, nodeClaim) + Expect(err).NotTo(HaveOccurred()) + Expect(isDrifted).To(BeEmpty()) + }) + It("should not return drifted if karpenter.k8s.aws/ec2nodeclass-hash-version annotation is not present on the NodeClass", func() { + nodeClass.ObjectMeta.Annotations = map[string]string{ + v1beta1.AnnotationEC2NodeClassHash: "test-hash-111111", + } + nodeClaim.ObjectMeta.Annotations = map[string]string{ + v1beta1.AnnotationEC2NodeClassHash: "test-hash-222222", + v1beta1.AnnotationEC2NodeClassHashVersion: "test-hash-version-2", + } + // should trigger drift + nodeClass.Spec.Tags = map[string]string{ + "Test Key": "Test Value", + } + ExpectApplied(ctx, env.Client, nodePool, nodeClass) + isDrifted, err := cloudProvider.IsDrifted(ctx, nodeClaim) + Expect(err).NotTo(HaveOccurred()) + Expect(isDrifted).To(BeEmpty()) + }) + It("should not return drifted if karpenter.k8s.aws/ec2nodeclass-hash-version annotation is not present on the NodeClaim", func() { + nodeClass.ObjectMeta.Annotations = map[string]string{ + v1beta1.AnnotationEC2NodeClassHash: "test-hash-111111", + v1beta1.AnnotationEC2NodeClassHashVersion: "test-hash-version-1", + } + nodeClaim.ObjectMeta.Annotations = map[string]string{ + v1beta1.AnnotationEC2NodeClassHash: "test-hash-222222", + } + // should trigger drift nodeClass.Spec.Tags = map[string]string{ "Test Key": "Test Value", } @@ -637,11 +1057,13 @@ var _ = Describe("CloudProvider", func() { }) Context("EFA", func() { It("should include vpc.amazonaws.com/efa on a nodeclaim if it requests it", func() { - nodeClaim.Spec.Requirements = []v1.NodeSelectorRequirement{ + nodeClaim.Spec.Requirements = []corev1beta1.NodeSelectorRequirementWithMinValues{ { - Key: v1.LabelInstanceTypeStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{"dl1.24xlarge"}, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1.LabelInstanceTypeStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{"dl1.24xlarge"}, + }, }, } nodeClaim.Spec.Resources.Requests = v1.ResourceList{v1beta1.ResourceEFA: resource.MustParse("1")} @@ -651,11 +1073,13 @@ var _ = Describe("CloudProvider", func() { Expect(lo.Keys(cloudProviderNodeClaim.Status.Allocatable)).To(ContainElement(v1beta1.ResourceEFA)) }) It("shouldn't include vpc.amazonaws.com/efa on a nodeclaim if it doesn't request it", func() { - nodeClaim.Spec.Requirements = []v1.NodeSelectorRequirement{ + nodeClaim.Spec.Requirements = []corev1beta1.NodeSelectorRequirementWithMinValues{ { - Key: v1.LabelInstanceTypeStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{"dl1.24xlarge"}, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1.LabelInstanceTypeStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{"dl1.24xlarge"}, + }, }, } ExpectApplied(ctx, env.Client, nodePool, nodeClass, nodeClaim) diff --git a/pkg/controllers/controllers.go b/pkg/controllers/controllers.go index 6a5d7fe7dd36..6f4d2fab39e2 100644 --- a/pkg/controllers/controllers.go +++ b/pkg/controllers/controllers.go @@ -17,20 +17,21 @@ package controllers import ( "context" + "sigs.k8s.io/karpenter/pkg/cloudprovider" + + controllerspricing "github.com/aws/karpenter-provider-aws/pkg/controllers/pricing" "github.com/aws/karpenter-provider-aws/pkg/providers/launchtemplate" "github.com/aws/aws-sdk-go/aws/session" servicesqs "github.com/aws/aws-sdk-go/service/sqs" "github.com/samber/lo" "k8s.io/utils/clock" - "knative.dev/pkg/logging" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/karpenter/pkg/events" "sigs.k8s.io/karpenter/pkg/operator/controller" "github.com/aws/karpenter-provider-aws/pkg/cache" - "github.com/aws/karpenter-provider-aws/pkg/cloudprovider" "github.com/aws/karpenter-provider-aws/pkg/controllers/interruption" nodeclaimgarbagecollection "github.com/aws/karpenter-provider-aws/pkg/controllers/nodeclaim/garbagecollection" nodeclaimtagging "github.com/aws/karpenter-provider-aws/pkg/controllers/nodeclaim/tagging" @@ -46,22 +47,18 @@ import ( ) func NewControllers(ctx context.Context, sess *session.Session, clk clock.Clock, kubeClient client.Client, recorder events.Recorder, - unavailableOfferings *cache.UnavailableOfferings, cloudProvider *cloudprovider.CloudProvider, subnetProvider *subnet.Provider, + unavailableOfferings *cache.UnavailableOfferings, cloudProvider cloudprovider.CloudProvider, subnetProvider *subnet.Provider, securityGroupProvider *securitygroup.Provider, instanceProfileProvider *instanceprofile.Provider, instanceProvider *instance.Provider, pricingProvider *pricing.Provider, amiProvider *amifamily.Provider, launchTemplateProvider *launchtemplate.Provider) []controller.Controller { controllers := []controller.Controller{ - nodeclass.NewNodeClassController(kubeClient, recorder, subnetProvider, securityGroupProvider, amiProvider, instanceProfileProvider, launchTemplateProvider), + nodeclass.NewController(kubeClient, recorder, subnetProvider, securityGroupProvider, amiProvider, instanceProfileProvider, launchTemplateProvider), nodeclaimgarbagecollection.NewController(kubeClient, cloudProvider), nodeclaimtagging.NewController(kubeClient, instanceProvider), + controllerspricing.NewController(pricingProvider), } if options.FromContext(ctx).InterruptionQueue != "" { controllers = append(controllers, interruption.NewController(kubeClient, clk, recorder, lo.Must(sqs.NewProvider(ctx, servicesqs.New(sess), options.FromContext(ctx).InterruptionQueue)), unavailableOfferings)) } - if options.FromContext(ctx).IsolatedVPC { - logging.FromContext(ctx).Infof("assuming isolated VPC, pricing information will not be updated") - } else { - controllers = append(controllers, pricing.NewController(pricingProvider)) - } return controllers } diff --git a/pkg/controllers/interruption/controller.go b/pkg/controllers/interruption/controller.go index e2f3cfc4798e..cd92e1441093 100644 --- a/pkg/controllers/interruption/controller.go +++ b/pkg/controllers/interruption/controller.go @@ -215,8 +215,9 @@ func (c *Controller) deleteNodeClaim(ctx context.Context, nodeClaim *v1beta1.Nod logging.FromContext(ctx).Infof("initiating delete from interruption message") c.recorder.Publish(interruptionevents.TerminatingOnInterruption(node, nodeClaim)...) metrics.NodeClaimsTerminatedCounter.With(prometheus.Labels{ - metrics.ReasonLabel: terminationReasonLabel, - metrics.NodePoolLabel: nodeClaim.Labels[v1beta1.NodePoolLabelKey], + metrics.ReasonLabel: terminationReasonLabel, + metrics.NodePoolLabel: nodeClaim.Labels[v1beta1.NodePoolLabelKey], + metrics.CapacityTypeLabel: nodeClaim.Labels[v1beta1.CapacityTypeLabelKey], }).Inc() return nil } diff --git a/pkg/controllers/nodeclaim/garbagecollection/controller.go b/pkg/controllers/nodeclaim/garbagecollection/controller.go index 6e06163bb28b..dae66f7f6f1e 100644 --- a/pkg/controllers/nodeclaim/garbagecollection/controller.go +++ b/pkg/controllers/nodeclaim/garbagecollection/controller.go @@ -28,21 +28,19 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/karpenter/pkg/cloudprovider" "sigs.k8s.io/karpenter/pkg/apis/v1beta1" - corecloudprovider "sigs.k8s.io/karpenter/pkg/cloudprovider" "sigs.k8s.io/karpenter/pkg/operator/controller" - - "github.com/aws/karpenter-provider-aws/pkg/cloudprovider" ) type Controller struct { kubeClient client.Client - cloudProvider *cloudprovider.CloudProvider + cloudProvider cloudprovider.CloudProvider successfulCount uint64 // keeps track of successful reconciles for more aggressive requeueing near the start of the controller } -func NewController(kubeClient client.Client, cloudProvider *cloudprovider.CloudProvider) *Controller { +func NewController(kubeClient client.Client, cloudProvider cloudprovider.CloudProvider) *Controller { return &Controller{ kubeClient: kubeClient, cloudProvider: cloudProvider, @@ -93,7 +91,7 @@ func (c *Controller) Reconcile(ctx context.Context, _ reconcile.Request) (reconc func (c *Controller) garbageCollect(ctx context.Context, nodeClaim *v1beta1.NodeClaim, nodeList *v1.NodeList) error { ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("provider-id", nodeClaim.Status.ProviderID)) if err := c.cloudProvider.Delete(ctx, nodeClaim); err != nil { - return corecloudprovider.IgnoreNodeClaimNotFoundError(err) + return cloudprovider.IgnoreNodeClaimNotFoundError(err) } logging.FromContext(ctx).Debugf("garbage collected cloudprovider instance") diff --git a/pkg/controllers/nodeclaim/garbagecollection/suite_test.go b/pkg/controllers/nodeclaim/garbagecollection/suite_test.go index 71ecc6f9faee..6527c20a2a4a 100644 --- a/pkg/controllers/nodeclaim/garbagecollection/suite_test.go +++ b/pkg/controllers/nodeclaim/garbagecollection/suite_test.go @@ -111,6 +111,10 @@ var _ = Describe("GarbageCollection", func() { Key: aws.String(corev1beta1.NodePoolLabelKey), Value: aws.String(nodePool.Name), }, + { + Key: aws.String(v1beta1.LabelNodeClass), + Value: aws.String(nodeClass.Name), + }, { Key: aws.String(corev1beta1.ManagedByAnnotationKey), Value: aws.String(options.FromContext(ctx).ClusterName), @@ -175,6 +179,10 @@ var _ = Describe("GarbageCollection", func() { Key: aws.String(corev1beta1.NodePoolLabelKey), Value: aws.String("default"), }, + { + Key: aws.String(v1beta1.LabelNodeClass), + Value: aws.String("default"), + }, { Key: aws.String(corev1beta1.ManagedByAnnotationKey), Value: aws.String(options.FromContext(ctx).ClusterName), diff --git a/pkg/controllers/nodeclaim/tagging/controller.go b/pkg/controllers/nodeclaim/tagging/controller.go index c9246dc4960f..5e73fe76ad92 100644 --- a/pkg/controllers/nodeclaim/tagging/controller.go +++ b/pkg/controllers/nodeclaim/tagging/controller.go @@ -38,11 +38,6 @@ import ( corecontroller "sigs.k8s.io/karpenter/pkg/operator/controller" ) -const ( - TagNodeClaim = corev1beta1.Group + "/nodeclaim" - TagName = "Name" -) - type Controller struct { kubeClient client.Client instanceProvider *instance.Provider @@ -96,8 +91,8 @@ func (c *Controller) Builder(_ context.Context, m manager.Manager) corecontrolle func (c *Controller) tagInstance(ctx context.Context, nc *corev1beta1.NodeClaim, id string) error { tags := map[string]string{ - TagName: nc.Status.NodeName, - TagNodeClaim: nc.Name, + v1beta1.TagName: nc.Status.NodeName, + v1beta1.TagNodeClaim: nc.Name, } // Remove tags which have been already populated diff --git a/pkg/controllers/nodeclaim/tagging/suite_test.go b/pkg/controllers/nodeclaim/tagging/suite_test.go index ac5f15d47354..6addad95b22b 100644 --- a/pkg/controllers/nodeclaim/tagging/suite_test.go +++ b/pkg/controllers/nodeclaim/tagging/suite_test.go @@ -120,7 +120,7 @@ var _ = Describe("TaggingController", func() { ExpectReconcileSucceeded(ctx, taggingController, client.ObjectKeyFromObject(nodeClaim)) Expect(nodeClaim.Annotations).To(Not(HaveKey(v1beta1.AnnotationInstanceTagged))) Expect(lo.ContainsBy(ec2Instance.Tags, func(tag *ec2.Tag) bool { - return *tag.Key == tagging.TagName + return *tag.Key == v1beta1.TagName })).To(BeFalse()) }) @@ -136,7 +136,7 @@ var _ = Describe("TaggingController", func() { ExpectReconcileSucceeded(ctx, taggingController, client.ObjectKeyFromObject(nodeClaim)) Expect(nodeClaim.Annotations).To(Not(HaveKey(v1beta1.AnnotationInstanceTagged))) Expect(lo.ContainsBy(ec2Instance.Tags, func(tag *ec2.Tag) bool { - return *tag.Key == tagging.TagName + return *tag.Key == v1beta1.TagName })).To(BeFalse()) }) @@ -183,7 +183,7 @@ var _ = Describe("TaggingController", func() { ExpectReconcileSucceeded(ctx, taggingController, client.ObjectKeyFromObject(nodeClaim)) Expect(nodeClaim.Annotations).To(Not(HaveKey(v1beta1.AnnotationInstanceTagged))) Expect(lo.ContainsBy(ec2Instance.Tags, func(tag *ec2.Tag) bool { - return *tag.Key == tagging.TagName + return *tag.Key == v1beta1.TagName })).To(BeFalse()) }) @@ -211,8 +211,8 @@ var _ = Describe("TaggingController", func() { Expect(nodeClaim.Annotations).To(HaveKey(v1beta1.AnnotationInstanceTagged)) expectedTags := map[string]string{ - tagging.TagName: nodeClaim.Status.NodeName, - tagging.TagNodeClaim: nodeClaim.Name, + v1beta1.TagName: nodeClaim.Status.NodeName, + v1beta1.TagNodeClaim: nodeClaim.Name, } instanceTags := instance.NewInstance(ec2Instance).Tags for tag, value := range expectedTags { @@ -222,9 +222,9 @@ var _ = Describe("TaggingController", func() { Expect(instanceTags).To(HaveKeyWithValue(tag, value)) } }, - Entry("with only karpenter.k8s.aws/nodeclaim tag", tagging.TagName), - Entry("with only Name tag", tagging.TagNodeClaim), + Entry("with only karpenter.k8s.aws/nodeclaim tag", v1beta1.TagName), + Entry("with only Name tag", v1beta1.TagNodeClaim), Entry("with both Name and karpenter.k8s.aws/nodeclaim tags"), - Entry("with nothing to tag", tagging.TagName, tagging.TagNodeClaim), + Entry("with nothing to tag", v1beta1.TagName, v1beta1.TagNodeClaim), ) }) diff --git a/pkg/controllers/nodeclass/controller.go b/pkg/controllers/nodeclass/controller.go index 874c2e3c95a4..51ba027c9ef1 100644 --- a/pkg/controllers/nodeclass/controller.go +++ b/pkg/controllers/nodeclass/controller.go @@ -20,6 +20,8 @@ import ( "sort" "time" + "k8s.io/apimachinery/pkg/api/errors" + "github.com/aws/karpenter-provider-aws/pkg/providers/launchtemplate" "github.com/aws/aws-sdk-go/service/ec2" @@ -51,6 +53,8 @@ import ( "github.com/aws/karpenter-provider-aws/pkg/providers/subnet" ) +var _ corecontroller.FinalizingTypedController[*v1beta1.EC2NodeClass] = (*Controller)(nil) + type Controller struct { kubeClient client.Client recorder events.Recorder @@ -62,8 +66,9 @@ type Controller struct { } func NewController(kubeClient client.Client, recorder events.Recorder, subnetProvider *subnet.Provider, securityGroupProvider *securitygroup.Provider, - amiProvider *amifamily.Provider, instanceProfileProvider *instanceprofile.Provider, launchTemplateProvider *launchtemplate.Provider) *Controller { - return &Controller{ + amiProvider *amifamily.Provider, instanceProfileProvider *instanceprofile.Provider, launchTemplateProvider *launchtemplate.Provider) corecontroller.Controller { + + return corecontroller.Typed[*v1beta1.EC2NodeClass](kubeClient, &Controller{ kubeClient: kubeClient, recorder: recorder, subnetProvider: subnetProvider, @@ -71,30 +76,45 @@ func NewController(kubeClient client.Client, recorder events.Recorder, subnetPro amiProvider: amiProvider, instanceProfileProvider: instanceProfileProvider, launchTemplateProvider: launchTemplateProvider, - } + }) } func (c *Controller) Reconcile(ctx context.Context, nodeClass *v1beta1.EC2NodeClass) (reconcile.Result, error) { stored := nodeClass.DeepCopy() controllerutil.AddFinalizer(nodeClass, v1beta1.TerminationFinalizer) - nodeClass.Annotations = lo.Assign(nodeClass.Annotations, map[string]string{v1beta1.AnnotationEC2NodeClassHash: nodeClass.Hash()}) - err := multierr.Combine( + + if nodeClass.Annotations[v1beta1.AnnotationEC2NodeClassHashVersion] != v1beta1.EC2NodeClassHashVersion { + if err := c.updateNodeClaimHash(ctx, nodeClass); err != nil { + return reconcile.Result{}, err + } + } + nodeClass.Annotations = lo.Assign(nodeClass.Annotations, map[string]string{ + v1beta1.AnnotationEC2NodeClassHash: nodeClass.Hash(), + v1beta1.AnnotationEC2NodeClassHashVersion: v1beta1.EC2NodeClassHashVersion, + }) + + errs := multierr.Combine( c.resolveSubnets(ctx, nodeClass), c.resolveSecurityGroups(ctx, nodeClass), c.resolveAMIs(ctx, nodeClass), c.resolveInstanceProfile(ctx, nodeClass), ) + if lo.FromPtr(nodeClass.Spec.AMIFamily) == v1beta1.AMIFamilyAL2023 { + if err := c.launchTemplateProvider.ResolveClusterCIDR(ctx); err != nil { + errs = multierr.Append(errs, fmt.Errorf("resolving cluster CIDR, %w", err)) + } + } if !equality.Semantic.DeepEqual(stored, nodeClass) { statusCopy := nodeClass.DeepCopy() - if patchErr := c.kubeClient.Patch(ctx, nodeClass, client.MergeFrom(stored)); err != nil { - err = multierr.Append(err, client.IgnoreNotFound(patchErr)) + if err := c.kubeClient.Patch(ctx, nodeClass, client.MergeFrom(stored)); err != nil { + errs = multierr.Append(errs, client.IgnoreNotFound(err)) } - if patchErr := c.kubeClient.Status().Patch(ctx, statusCopy, client.MergeFrom(stored)); err != nil { - err = multierr.Append(err, client.IgnoreNotFound(patchErr)) + if err := c.kubeClient.Status().Patch(ctx, statusCopy, client.MergeFrom(stored)); err != nil { + errs = multierr.Append(errs, client.IgnoreNotFound(err)) } } - if err != nil { - return reconcile.Result{}, err + if errs != nil { + return reconcile.Result{}, errs } return reconcile.Result{RequeueAfter: 5 * time.Minute}, nil } @@ -122,7 +142,13 @@ func (c *Controller) Finalize(ctx context.Context, nodeClass *v1beta1.EC2NodeCla } controllerutil.RemoveFinalizer(nodeClass, v1beta1.TerminationFinalizer) if !equality.Semantic.DeepEqual(stored, nodeClass) { - if err := c.kubeClient.Patch(ctx, nodeClass, client.MergeFrom(stored)); err != nil { + // We call Update() here rather than Patch() because patching a list with a JSON merge patch + // can cause races due to the fact that it fully replaces the list on a change + // https://github.com/kubernetes/kubernetes/issues/111643#issuecomment-2016489732 + if err := c.kubeClient.Update(ctx, nodeClass); err != nil { + if errors.IsConflict(err) { + return reconcile.Result{Requeue: true}, nil + } return reconcile.Result{}, client.IgnoreNotFound(fmt.Errorf("removing termination finalizer, %w", err)) } } @@ -214,25 +240,50 @@ func (c *Controller) resolveInstanceProfile(ctx context.Context, nodeClass *v1be return nil } -var _ corecontroller.FinalizingTypedController[*v1beta1.EC2NodeClass] = (*NodeClassController)(nil) +// Updating `ec2nodeclass-hash-version` annotation inside the karpenter controller means a breaking change has been made to the hash calculation. +// `ec2nodeclass-hash` annotation on the EC2NodeClass will be updated, due to the breaking change, making the `ec2nodeclass-hash` on the NodeClaim different from +// EC2NodeClass. Since, we cannot rely on the `ec2nodeclass-hash` on the NodeClaims, due to the breaking change, we will need to re-calculate the hash and update the annotation. +// For more information on the Drift Hash Versioning: https://github.com/kubernetes-sigs/karpenter/blob/main/designs/drift-hash-versioning.md +func (c *Controller) updateNodeClaimHash(ctx context.Context, nodeClass *v1beta1.EC2NodeClass) error { + ncList := &corev1beta1.NodeClaimList{} + if err := c.kubeClient.List(ctx, ncList, client.MatchingFields{"spec.nodeClassRef.name": nodeClass.Name}); err != nil { + return err + } + + errs := make([]error, len(ncList.Items)) + for i := range ncList.Items { + nc := ncList.Items[i] + stored := nc.DeepCopy() -//nolint:revive -type NodeClassController struct { - *Controller -} + if nc.Annotations[v1beta1.AnnotationEC2NodeClassHashVersion] != v1beta1.EC2NodeClassHashVersion { + nc.Annotations = lo.Assign(nc.Annotations, map[string]string{ + v1beta1.AnnotationEC2NodeClassHashVersion: v1beta1.EC2NodeClassHashVersion, + }) -func NewNodeClassController(kubeClient client.Client, recorder events.Recorder, subnetProvider *subnet.Provider, securityGroupProvider *securitygroup.Provider, - amiProvider *amifamily.Provider, instanceProfileProvider *instanceprofile.Provider, launchTemplateProvider *launchtemplate.Provider) corecontroller.Controller { - return corecontroller.Typed[*v1beta1.EC2NodeClass](kubeClient, &NodeClassController{ - Controller: NewController(kubeClient, recorder, subnetProvider, securityGroupProvider, amiProvider, instanceProfileProvider, launchTemplateProvider), - }) + // Any NodeClaim that is already drifted will remain drifted if the karpenter.k8s.aws/nodepool-hash-version doesn't match + // Since the hashing mechanism has changed we will not be able to determine if the drifted status of the NodeClaim has changed + if nc.StatusConditions().GetCondition(corev1beta1.Drifted) == nil { + nc.Annotations = lo.Assign(nc.Annotations, map[string]string{ + v1beta1.AnnotationEC2NodeClassHash: nodeClass.Hash(), + }) + } + + if !equality.Semantic.DeepEqual(stored, nc) { + if err := c.kubeClient.Patch(ctx, &nc, client.MergeFrom(stored)); err != nil { + errs[i] = client.IgnoreNotFound(err) + } + } + } + } + + return multierr.Combine(errs...) } -func (c *NodeClassController) Name() string { +func (c *Controller) Name() string { return "nodeclass" } -func (c *NodeClassController) Builder(_ context.Context, m manager.Manager) corecontroller.Builder { +func (c *Controller) Builder(_ context.Context, m manager.Manager) corecontroller.Builder { return corecontroller.Adapt(controllerruntime. NewControllerManagedBy(m). For(&v1beta1.EC2NodeClass{}). diff --git a/pkg/controllers/nodeclass/suite_test.go b/pkg/controllers/nodeclass/suite_test.go index 7a69ebe288d0..0c0a436697c3 100644 --- a/pkg/controllers/nodeclass/suite_test.go +++ b/pkg/controllers/nodeclass/suite_test.go @@ -23,12 +23,14 @@ import ( "github.com/imdario/mergo" "github.com/samber/lo" v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/record" _ "knative.dev/pkg/system/testing" "sigs.k8s.io/controller-runtime/pkg/client" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" + "github.com/aws/aws-sdk-go/service/eks" "github.com/aws/aws-sdk-go/service/iam" corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" "sigs.k8s.io/karpenter/pkg/events" @@ -68,7 +70,7 @@ var _ = BeforeSuite(func() { ctx = options.ToContext(ctx, test.Options()) awsEnv = test.NewEnvironment(ctx, env) - nodeClassController = nodeclass.NewNodeClassController(env.Client, events.NewRecorder(&record.FakeRecorder{}), awsEnv.SubnetProvider, awsEnv.SecurityGroupProvider, awsEnv.AMIProvider, awsEnv.InstanceProfileProvider, awsEnv.LaunchTemplateProvider) + nodeClassController = nodeclass.NewController(env.Client, events.NewRecorder(&record.FakeRecorder{}), awsEnv.SubnetProvider, awsEnv.SecurityGroupProvider, awsEnv.AMIProvider, awsEnv.InstanceProfileProvider, awsEnv.LaunchTemplateProvider) }) var _ = AfterSuite(func() { @@ -107,6 +109,46 @@ var _ = Describe("NodeClassController", func() { }, }) }) + Context("Cluster CIDR Resolution", func() { + BeforeEach(func() { + // Cluster CIDR will only be resolved once per lifetime of the launch template provider, reset to nil between tests + awsEnv.LaunchTemplateProvider.ClusterCIDR.Store(nil) + }) + It("shouldn't resolve cluster CIDR for non-AL2023 NodeClasses", func() { + for _, family := range []string{ + v1beta1.AMIFamilyAL2, + v1beta1.AMIFamilyBottlerocket, + v1beta1.AMIFamilyUbuntu, + v1beta1.AMIFamilyWindows2019, + v1beta1.AMIFamilyWindows2022, + v1beta1.AMIFamilyCustom, + } { + nodeClass.Spec.AMIFamily = lo.ToPtr(family) + ExpectApplied(ctx, env.Client, nodeClass) + ExpectReconcileSucceeded(ctx, nodeClassController, client.ObjectKeyFromObject(nodeClass)) + Expect(awsEnv.LaunchTemplateProvider.ClusterCIDR.Load()).To(BeNil()) + } + }) + It("should resolve cluster CIDR for IPv4 clusters", func() { + nodeClass.Spec.AMIFamily = lo.ToPtr(v1beta1.AMIFamilyAL2023) + ExpectApplied(ctx, env.Client, nodeClass) + ExpectReconcileSucceeded(ctx, nodeClassController, client.ObjectKeyFromObject(nodeClass)) + Expect(lo.FromPtr(awsEnv.LaunchTemplateProvider.ClusterCIDR.Load())).To(Equal("10.100.0.0/16")) + }) + It("should resolve cluster CIDR for IPv6 clusters", func() { + awsEnv.EKSAPI.DescribeClusterBehavior.Output.Set(&eks.DescribeClusterOutput{ + Cluster: &eks.Cluster{ + KubernetesNetworkConfig: &eks.KubernetesNetworkConfigResponse{ + ServiceIpv6Cidr: lo.ToPtr("2001:db8::/64"), + }, + }, + }) + nodeClass.Spec.AMIFamily = lo.ToPtr(v1beta1.AMIFamilyAL2023) + ExpectApplied(ctx, env.Client, nodeClass) + ExpectReconcileSucceeded(ctx, nodeClassController, client.ObjectKeyFromObject(nodeClass)) + Expect(lo.FromPtr(awsEnv.LaunchTemplateProvider.ClusterCIDR.Load())).To(Equal("2001:db8::/64")) + }) + }) Context("Subnet Status", func() { It("Should update EC2NodeClass status for Subnets", func() { ExpectApplied(ctx, env.Client, nodeClass) @@ -589,68 +631,88 @@ var _ = Describe("NodeClassController", func() { { Name: "test-ami-3", ID: "ami-id-789", - Requirements: []v1.NodeSelectorRequirement{ + Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ { - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.ArchitectureArm64}, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{corev1beta1.ArchitectureArm64}, + }, }, { - Key: v1beta1.LabelInstanceGPUCount, - Operator: v1.NodeSelectorOpDoesNotExist, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceGPUCount, + Operator: v1.NodeSelectorOpDoesNotExist, + }, }, { - Key: v1beta1.LabelInstanceAcceleratorCount, - Operator: v1.NodeSelectorOpDoesNotExist, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceAcceleratorCount, + Operator: v1.NodeSelectorOpDoesNotExist, + }, }, }, }, { Name: "test-ami-2", ID: "ami-id-456", - Requirements: []v1.NodeSelectorRequirement{ + Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ { - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.ArchitectureAmd64}, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{corev1beta1.ArchitectureAmd64}, + }, }, { - Key: v1beta1.LabelInstanceGPUCount, - Operator: v1.NodeSelectorOpExists, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceGPUCount, + Operator: v1.NodeSelectorOpExists, + }, }, }, }, { Name: "test-ami-2", ID: "ami-id-456", - Requirements: []v1.NodeSelectorRequirement{ + Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ { - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.ArchitectureAmd64}, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{corev1beta1.ArchitectureAmd64}, + }, }, { - Key: v1beta1.LabelInstanceAcceleratorCount, - Operator: v1.NodeSelectorOpExists, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceAcceleratorCount, + Operator: v1.NodeSelectorOpExists, + }, }, }, }, { Name: "test-ami-1", ID: "ami-id-123", - Requirements: []v1.NodeSelectorRequirement{ + Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ { - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.ArchitectureAmd64}, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{corev1beta1.ArchitectureAmd64}, + }, }, { - Key: v1beta1.LabelInstanceGPUCount, - Operator: v1.NodeSelectorOpDoesNotExist, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceGPUCount, + Operator: v1.NodeSelectorOpDoesNotExist, + }, }, { - Key: v1beta1.LabelInstanceAcceleratorCount, - Operator: v1.NodeSelectorOpDoesNotExist, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceAcceleratorCount, + Operator: v1.NodeSelectorOpDoesNotExist, + }, }, }, }, @@ -697,38 +759,50 @@ var _ = Describe("NodeClassController", func() { { Name: "test-ami-2", ID: "ami-id-456", - Requirements: []v1.NodeSelectorRequirement{ + Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ { - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.ArchitectureArm64}, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{corev1beta1.ArchitectureArm64}, + }, }, { - Key: v1beta1.LabelInstanceGPUCount, - Operator: v1.NodeSelectorOpDoesNotExist, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceGPUCount, + Operator: v1.NodeSelectorOpDoesNotExist, + }, }, { - Key: v1beta1.LabelInstanceAcceleratorCount, - Operator: v1.NodeSelectorOpDoesNotExist, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceAcceleratorCount, + Operator: v1.NodeSelectorOpDoesNotExist, + }, }, }, }, { Name: "test-ami-1", ID: "ami-id-123", - Requirements: []v1.NodeSelectorRequirement{ + Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ { - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.ArchitectureAmd64}, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{corev1beta1.ArchitectureAmd64}, + }, }, { - Key: v1beta1.LabelInstanceGPUCount, - Operator: v1.NodeSelectorOpDoesNotExist, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceGPUCount, + Operator: v1.NodeSelectorOpDoesNotExist, + }, }, { - Key: v1beta1.LabelInstanceAcceleratorCount, - Operator: v1.NodeSelectorOpDoesNotExist, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceAcceleratorCount, + Operator: v1.NodeSelectorOpDoesNotExist, + }, }, }, }, @@ -743,12 +817,14 @@ var _ = Describe("NodeClassController", func() { { Name: "test-ami-3", ID: "ami-test3", - Requirements: []v1.NodeSelectorRequirement{ + Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ { - Key: "kubernetes.io/arch", - Operator: "In", - Values: []string{ - "amd64", + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: "kubernetes.io/arch", + Operator: "In", + Values: []string{ + "amd64", + }, }, }, }, @@ -758,7 +834,7 @@ var _ = Describe("NodeClassController", func() { }) }) Context("Static Drift Hash", func() { - DescribeTable("should update the static drift hash when static field is updated", func(changes *v1beta1.EC2NodeClass) { + DescribeTable("should update the drift hash when static field is updated", func(changes *v1beta1.EC2NodeClass) { ExpectApplied(ctx, env.Client, nodeClass) ExpectReconcileSucceeded(ctx, nodeClassController, client.ObjectKeyFromObject(nodeClass)) nodeClass = ExpectExists(ctx, env.Client, nodeClass) @@ -785,7 +861,7 @@ var _ = Describe("NodeClassController", func() { Entry("MetadataOptions Drift", &v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{MetadataOptions: &v1beta1.MetadataOptions{HTTPEndpoint: aws.String("disabled")}}}), Entry("Context Drift", &v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{Context: aws.String("context-2")}}), ) - It("should not update the static drift hash when dynamic field is updated", func() { + It("should not update the drift hash when dynamic field is updated", func() { ExpectApplied(ctx, env.Client, nodeClass) ExpectReconcileSucceeded(ctx, nodeClassController, client.ObjectKeyFromObject(nodeClass)) nodeClass = ExpectExists(ctx, env.Client, nodeClass) @@ -814,6 +890,128 @@ var _ = Describe("NodeClassController", func() { nodeClass = ExpectExists(ctx, env.Client, nodeClass) Expect(nodeClass.Annotations[v1beta1.AnnotationEC2NodeClassHash]).To(Equal(expectedHash)) }) + It("should update ec2nodeclass-hash-version annotation when the ec2nodeclass-hash-version on the NodeClass does not match with the controller hash version", func() { + nodeClass.Annotations = map[string]string{ + v1beta1.AnnotationEC2NodeClassHash: "abceduefed", + v1beta1.AnnotationEC2NodeClassHashVersion: "test", + } + ExpectApplied(ctx, env.Client, nodeClass) + + ExpectReconcileSucceeded(ctx, nodeClassController, client.ObjectKeyFromObject(nodeClass)) + nodeClass = ExpectExists(ctx, env.Client, nodeClass) + + expectedHash := nodeClass.Hash() + // Expect ec2nodeclass-hash on the NodeClass to be updated + Expect(nodeClass.Annotations).To(HaveKeyWithValue(v1beta1.AnnotationEC2NodeClassHash, expectedHash)) + Expect(nodeClass.Annotations).To(HaveKeyWithValue(v1beta1.AnnotationEC2NodeClassHashVersion, v1beta1.EC2NodeClassHashVersion)) + }) + It("should update ec2nodeclass-hash-versions on all NodeClaims when the ec2nodeclass-hash-version does not match with the controller hash version", func() { + nodeClass.Annotations = map[string]string{ + v1beta1.AnnotationEC2NodeClassHash: "abceduefed", + v1beta1.AnnotationEC2NodeClassHashVersion: "test", + } + nodeClaimOne := coretest.NodeClaim(corev1beta1.NodeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + v1beta1.AnnotationEC2NodeClassHash: "123456", + v1beta1.AnnotationEC2NodeClassHashVersion: "test", + }, + }, + Spec: corev1beta1.NodeClaimSpec{ + NodeClassRef: &corev1beta1.NodeClassReference{ + Name: nodeClass.Name, + }, + }, + }) + nodeClaimTwo := coretest.NodeClaim(corev1beta1.NodeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + v1beta1.AnnotationEC2NodeClassHash: "123456", + v1beta1.AnnotationEC2NodeClassHashVersion: "test", + }, + }, + Spec: corev1beta1.NodeClaimSpec{ + NodeClassRef: &corev1beta1.NodeClassReference{ + Name: nodeClass.Name, + }, + }, + }) + + ExpectApplied(ctx, env.Client, nodeClass, nodeClaimOne, nodeClaimTwo) + + ExpectReconcileSucceeded(ctx, nodeClassController, client.ObjectKeyFromObject(nodeClass)) + nodeClass = ExpectExists(ctx, env.Client, nodeClass) + nodeClaimOne = ExpectExists(ctx, env.Client, nodeClaimOne) + nodeClaimTwo = ExpectExists(ctx, env.Client, nodeClaimTwo) + + expectedHash := nodeClass.Hash() + // Expect ec2nodeclass-hash on the NodeClaims to be updated + Expect(nodeClaimOne.Annotations).To(HaveKeyWithValue(v1beta1.AnnotationEC2NodeClassHash, expectedHash)) + Expect(nodeClaimOne.Annotations).To(HaveKeyWithValue(v1beta1.AnnotationEC2NodeClassHashVersion, v1beta1.EC2NodeClassHashVersion)) + Expect(nodeClaimTwo.Annotations).To(HaveKeyWithValue(v1beta1.AnnotationEC2NodeClassHash, expectedHash)) + Expect(nodeClaimTwo.Annotations).To(HaveKeyWithValue(v1beta1.AnnotationEC2NodeClassHashVersion, v1beta1.EC2NodeClassHashVersion)) + }) + It("should not update ec2nodeclass-hash on all NodeClaims when the ec2nodeclass-hash-version matches the controller hash version", func() { + nodeClass.Annotations = map[string]string{ + v1beta1.AnnotationEC2NodeClassHash: "abceduefed", + v1beta1.AnnotationEC2NodeClassHashVersion: "test-version", + } + nodeClaim := coretest.NodeClaim(corev1beta1.NodeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + v1beta1.AnnotationEC2NodeClassHash: "1234564654", + v1beta1.AnnotationEC2NodeClassHashVersion: v1beta1.EC2NodeClassHashVersion, + }, + }, + Spec: corev1beta1.NodeClaimSpec{ + NodeClassRef: &corev1beta1.NodeClassReference{ + Name: nodeClass.Name, + }, + }, + }) + ExpectApplied(ctx, env.Client, nodeClass, nodeClaim) + + ExpectReconcileSucceeded(ctx, nodeClassController, client.ObjectKeyFromObject(nodeClass)) + nodeClass = ExpectExists(ctx, env.Client, nodeClass) + nodeClaim = ExpectExists(ctx, env.Client, nodeClaim) + + expectedHash := nodeClass.Hash() + + // Expect ec2nodeclass-hash on the NodeClass to be updated + Expect(nodeClass.Annotations).To(HaveKeyWithValue(v1beta1.AnnotationEC2NodeClassHash, expectedHash)) + Expect(nodeClass.Annotations).To(HaveKeyWithValue(v1beta1.AnnotationEC2NodeClassHashVersion, v1beta1.EC2NodeClassHashVersion)) + // Expect ec2nodeclass-hash on the NodeClaims to stay the same + Expect(nodeClaim.Annotations).To(HaveKeyWithValue(v1beta1.AnnotationEC2NodeClassHash, "1234564654")) + Expect(nodeClaim.Annotations).To(HaveKeyWithValue(v1beta1.AnnotationEC2NodeClassHashVersion, v1beta1.EC2NodeClassHashVersion)) + }) + It("should not update ec2nodeclass-hash on the NodeClaim if it's drifted and the ec2nodeclass-hash-version does not match the controller hash version", func() { + nodeClass.Annotations = map[string]string{ + v1beta1.AnnotationEC2NodeClassHash: "abceduefed", + v1beta1.AnnotationEC2NodeClassHashVersion: "test", + } + nodeClaim := coretest.NodeClaim(corev1beta1.NodeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + v1beta1.AnnotationEC2NodeClassHash: "123456", + v1beta1.AnnotationEC2NodeClassHashVersion: "test", + }, + }, + Spec: corev1beta1.NodeClaimSpec{ + NodeClassRef: &corev1beta1.NodeClassReference{ + Name: nodeClass.Name, + }, + }, + }) + nodeClaim.StatusConditions().MarkTrue(corev1beta1.Drifted) + ExpectApplied(ctx, env.Client, nodeClass, nodeClaim) + + ExpectReconcileSucceeded(ctx, nodeClassController, client.ObjectKeyFromObject(nodeClass)) + nodeClaim = ExpectExists(ctx, env.Client, nodeClaim) + + // Expect ec2nodeclass-hash on the NodeClaims to stay the same + Expect(nodeClaim.Annotations).To(HaveKeyWithValue(v1beta1.AnnotationEC2NodeClassHash, "123456")) + Expect(nodeClaim.Annotations).To(HaveKeyWithValue(v1beta1.AnnotationEC2NodeClassHashVersion, v1beta1.EC2NodeClassHashVersion)) + }) }) Context("NodeClass Termination", func() { var profileName string diff --git a/pkg/providers/pricing/controller.go b/pkg/controllers/pricing/controller.go similarity index 73% rename from pkg/providers/pricing/controller.go rename to pkg/controllers/pricing/controller.go index 94ed0138181b..6e86ef5437d9 100644 --- a/pkg/providers/pricing/controller.go +++ b/pkg/controllers/pricing/controller.go @@ -16,6 +16,7 @@ package pricing import ( "context" + "fmt" "time" lop "github.com/samber/lo/parallel" @@ -23,35 +24,25 @@ import ( "sigs.k8s.io/controller-runtime/pkg/manager" "sigs.k8s.io/controller-runtime/pkg/reconcile" - corecontroller "sigs.k8s.io/karpenter/pkg/operator/controller" + "sigs.k8s.io/karpenter/pkg/operator/controller" + + "github.com/aws/karpenter-provider-aws/pkg/providers/pricing" ) type Controller struct { - pricingProvider *Provider + pricingProvider *pricing.Provider } -func NewController(pricingProvider *Provider) *Controller { +func NewController(pricingProvider *pricing.Provider) *Controller { return &Controller{ pricingProvider: pricingProvider, } } func (c *Controller) Reconcile(ctx context.Context, _ reconcile.Request) (reconcile.Result, error) { - return reconcile.Result{RequeueAfter: 12 * time.Hour}, c.updatePricing(ctx) -} - -func (c *Controller) Name() string { - return "pricing" -} - -func (c *Controller) Builder(_ context.Context, m manager.Manager) corecontroller.Builder { - return corecontroller.NewSingletonManagedBy(m) -} - -func (c *Controller) updatePricing(ctx context.Context) error { work := []func(ctx context.Context) error{ - c.pricingProvider.UpdateOnDemandPricing, c.pricingProvider.UpdateSpotPricing, + c.pricingProvider.UpdateOnDemandPricing, } errs := make([]error, len(work)) lop.ForEach(work, func(f func(ctx context.Context) error, i int) { @@ -59,6 +50,16 @@ func (c *Controller) updatePricing(ctx context.Context) error { errs[i] = err } }) + if err := multierr.Combine(errs...); err != nil { + return reconcile.Result{}, fmt.Errorf("updating pricing, %w", err) + } + return reconcile.Result{RequeueAfter: 12 * time.Hour}, nil +} + +func (c *Controller) Name() string { + return "pricing" +} - return multierr.Combine(errs...) +func (c *Controller) Builder(_ context.Context, m manager.Manager) controller.Builder { + return controller.NewSingletonManagedBy(m) } diff --git a/pkg/providers/pricing/suite_test.go b/pkg/controllers/pricing/suite_test.go similarity index 76% rename from pkg/providers/pricing/suite_test.go rename to pkg/controllers/pricing/suite_test.go index 7dd24e29283d..97a11c18eb64 100644 --- a/pkg/providers/pricing/suite_test.go +++ b/pkg/controllers/pricing/suite_test.go @@ -30,6 +30,7 @@ import ( coretest "sigs.k8s.io/karpenter/pkg/test" "github.com/aws/karpenter-provider-aws/pkg/apis" + controllerspricing "github.com/aws/karpenter-provider-aws/pkg/controllers/pricing" "github.com/aws/karpenter-provider-aws/pkg/fake" "github.com/aws/karpenter-provider-aws/pkg/operator/options" "github.com/aws/karpenter-provider-aws/pkg/providers/pricing" @@ -45,7 +46,7 @@ var ctx context.Context var stop context.CancelFunc var env *coretest.Environment var awsEnv *test.Environment -var controller *pricing.Controller +var controller *controllerspricing.Controller func TestAWS(t *testing.T) { ctx = TestContextWithLogger(t) @@ -59,7 +60,7 @@ var _ = BeforeSuite(func() { ctx = options.ToContext(ctx, test.Options()) ctx, stop = context.WithCancel(ctx) awsEnv = test.NewEnvironment(ctx, env) - controller = pricing.NewController(awsEnv.PricingProvider) + controller = controllerspricing.NewController(awsEnv.PricingProvider) }) var _ = AfterSuite(func() { @@ -123,12 +124,10 @@ var _ = Describe("Pricing", func() { price, ok := awsEnv.PricingProvider.OnDemandPrice("c98.large") Expect(ok).To(BeTrue()) Expect(price).To(BeNumerically("==", 1.20)) - Expect(getPricingEstimateMetricValue("c98.large", ec2.UsageClassTypeOnDemand, "")).To(BeNumerically("==", 1.20)) price, ok = awsEnv.PricingProvider.OnDemandPrice("c99.large") Expect(ok).To(BeTrue()) Expect(price).To(BeNumerically("==", 1.23)) - Expect(getPricingEstimateMetricValue("c99.large", ec2.UsageClassTypeOnDemand, "")).To(BeNumerically("==", 1.23)) }) It("should update spot pricing with response from the pricing API", func() { now := time.Now() @@ -171,12 +170,10 @@ var _ = Describe("Pricing", func() { price, ok := awsEnv.PricingProvider.SpotPrice("c98.large", "test-zone-1b") Expect(ok).To(BeTrue()) Expect(price).To(BeNumerically("==", 1.10)) - Expect(getPricingEstimateMetricValue("c98.large", ec2.UsageClassTypeSpot, "test-zone-1b")).To(BeNumerically("==", 1.10)) price, ok = awsEnv.PricingProvider.SpotPrice("c99.large", "test-zone-1a") Expect(ok).To(BeTrue()) Expect(price).To(BeNumerically("==", 1.23)) - Expect(getPricingEstimateMetricValue("c99.large", ec2.UsageClassTypeSpot, "test-zone-1a")).To(BeNumerically("==", 1.23)) }) It("should update zonal pricing with data from the spot pricing API", func() { now := time.Now() @@ -207,7 +204,6 @@ var _ = Describe("Pricing", func() { price, ok := awsEnv.PricingProvider.SpotPrice("c98.large", "test-zone-1a") Expect(ok).To(BeTrue()) Expect(price).To(BeNumerically("==", 1.20)) - Expect(getPricingEstimateMetricValue("c98.large", ec2.UsageClassTypeSpot, "test-zone-1a")).To(BeNumerically("==", 1.20)) _, ok = awsEnv.PricingProvider.SpotPrice("c98.large", "test-zone-1b") Expect(ok).ToNot(BeTrue()) @@ -262,18 +258,74 @@ var _ = Describe("Pricing", func() { Expect(lo.Map(inp.ProductDescriptions, func(x *string, _ int) string { return *x })). To(ContainElements("Linux/UNIX", "Linux/UNIX (Amazon VPC)")) }) -}) + It("should return static on-demand data when in isolated-vpc", func() { + ctx = options.ToContext(ctx, test.Options(test.OptionsFields{ + IsolatedVPC: lo.ToPtr(true), + })) + now := time.Now() + awsEnv.EC2API.DescribeSpotPriceHistoryOutput.Set(&ec2.DescribeSpotPriceHistoryOutput{ + SpotPriceHistory: []*ec2.SpotPrice{ + { + AvailabilityZone: aws.String("test-zone-1b"), + InstanceType: aws.String("c99.large"), + SpotPrice: aws.String("1.50"), + Timestamp: &now, + }, + { + AvailabilityZone: aws.String("test-zone-1b"), + InstanceType: aws.String("c98.large"), + SpotPrice: aws.String("1.10"), + Timestamp: &now, + }, + }, + }) + + awsEnv.PricingAPI.GetProductsOutput.Set(&awspricing.GetProductsOutput{ + // these are incorrect prices which are here to ensure that + // results from only static pricing are used + PriceList: []aws.JSONValue{ + fake.NewOnDemandPrice("c3.2xlarge", 1.20), + fake.NewOnDemandPrice("c5.xlarge", 1.23), + }, + }) + ExpectReconcileSucceeded(ctx, controller, types.NamespacedName{}) + price, ok := awsEnv.PricingProvider.OnDemandPrice("c3.2xlarge") + Expect(ok).To(BeTrue()) + Expect(price).To(BeNumerically("==", 0.420000)) + + price, ok = awsEnv.PricingProvider.SpotPrice("c98.large", "test-zone-1b") + Expect(ok).To(BeTrue()) + Expect(price).To(BeNumerically("==", 1.10)) + }) + It("should update on-demand pricing with response from the pricing API when in the CN partition", func() { + tmpPricingProvider := pricing.NewProvider(ctx, awsEnv.PricingAPI, awsEnv.EC2API, "cn-anywhere-1") + tmpController := controllerspricing.NewController(tmpPricingProvider) + + now := time.Now() + awsEnv.EC2API.DescribeSpotPriceHistoryOutput.Set(&ec2.DescribeSpotPriceHistoryOutput{ + SpotPriceHistory: []*ec2.SpotPrice{ + { + AvailabilityZone: aws.String("test-zone-1a"), + InstanceType: aws.String("c99.large"), + SpotPrice: aws.String("1.23"), + Timestamp: &now, + }, + }, + }) + awsEnv.PricingAPI.GetProductsOutput.Set(&awspricing.GetProductsOutput{ + PriceList: []aws.JSONValue{ + fake.NewOnDemandPriceWithCurrency("c98.large", 1.20, "CNY"), + fake.NewOnDemandPriceWithCurrency("c99.large", 1.23, "CNY"), + }, + }) + ExpectReconcileSucceeded(ctx, tmpController, types.NamespacedName{}) -func getPricingEstimateMetricValue(instanceType string, capacityType string, zone string) float64 { - var value *float64 - metric, ok := FindMetricWithLabelValues("karpenter_cloudprovider_instance_type_price_estimate", map[string]string{ - pricing.InstanceTypeLabel: instanceType, - pricing.CapacityTypeLabel: capacityType, - pricing.RegionLabel: fake.DefaultRegion, - pricing.TopologyLabel: zone, + price, ok := tmpPricingProvider.OnDemandPrice("c98.large") + Expect(ok).To(BeTrue()) + Expect(price).To(BeNumerically("==", 1.20)) + + price, ok = tmpPricingProvider.OnDemandPrice("c99.large") + Expect(ok).To(BeTrue()) + Expect(price).To(BeNumerically("==", 1.23)) }) - Expect(ok).To(BeTrue()) - value = metric.GetGauge().Value - Expect(value).To(Not(BeNil())) - return *value -} +}) diff --git a/pkg/errors/errors.go b/pkg/errors/errors.go index 38baec47e819..5267aad4672d 100644 --- a/pkg/errors/errors.go +++ b/pkg/errors/errors.go @@ -25,14 +25,15 @@ import ( ) const ( - launchTemplateNotFoundCode = "InvalidLaunchTemplateName.NotFoundException" + launchTemplateNameNotFoundCode = "InvalidLaunchTemplateName.NotFoundException" ) var ( // This is not an exhaustive list, add to it as needed notFoundErrorCodes = sets.New[string]( "InvalidInstanceID.NotFound", - launchTemplateNotFoundCode, + launchTemplateNameNotFoundCode, + "InvalidLaunchTemplateId.NotFound", sqs.ErrCodeQueueDoesNotExist, iam.ErrCodeNoSuchEntityException, ) @@ -102,7 +103,7 @@ func IsLaunchTemplateNotFound(err error) bool { } var awsError awserr.Error if errors.As(err, &awsError) { - return awsError.Code() == launchTemplateNotFoundCode + return awsError.Code() == launchTemplateNameNotFoundCode } return false } diff --git a/pkg/fake/cloudprovider.go b/pkg/fake/cloudprovider.go index 65ccd76b6ab6..fbc9ba7ebc35 100644 --- a/pkg/fake/cloudprovider.go +++ b/pkg/fake/cloudprovider.go @@ -19,6 +19,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "sigs.k8s.io/karpenter/pkg/apis/v1beta1" corecloudprovider "sigs.k8s.io/karpenter/pkg/cloudprovider" "sigs.k8s.io/karpenter/pkg/test" @@ -77,3 +78,7 @@ func (c *CloudProvider) Delete(context.Context, *v1beta1.NodeClaim) error { func (c *CloudProvider) Name() string { return "fake" } + +func (c *CloudProvider) GetSupportedNodeClasses() []schema.GroupVersionKind { + return []schema.GroupVersionKind{} +} diff --git a/pkg/fake/eksapi.go b/pkg/fake/eksapi.go index 4c0a48e4fd06..8d39706dd9e5 100644 --- a/pkg/fake/eksapi.go +++ b/pkg/fake/eksapi.go @@ -20,6 +20,7 @@ import ( "github.com/aws/aws-sdk-go/aws/request" "github.com/aws/aws-sdk-go/service/eks" "github.com/aws/aws-sdk-go/service/eks/eksiface" + "github.com/samber/lo" ) const () @@ -35,6 +36,10 @@ type EKSAPI struct { EKSAPIBehavior } +func NewEKSAPI() *EKSAPI { + return &EKSAPI{} +} + // Reset must be called between tests otherwise tests will pollute // each other. func (s *EKSAPI) Reset() { @@ -43,6 +48,12 @@ func (s *EKSAPI) Reset() { func (s *EKSAPI) DescribeClusterWithContext(_ context.Context, input *eks.DescribeClusterInput, _ ...request.Option) (*eks.DescribeClusterOutput, error) { return s.DescribeClusterBehavior.Invoke(input, func(*eks.DescribeClusterInput) (*eks.DescribeClusterOutput, error) { - return nil, nil + return &eks.DescribeClusterOutput{ + Cluster: &eks.Cluster{ + KubernetesNetworkConfig: &eks.KubernetesNetworkConfigResponse{ + ServiceIpv4Cidr: lo.ToPtr("10.100.0.0/16"), + }, + }, + }, nil }) } diff --git a/pkg/fake/pricingapi.go b/pkg/fake/pricingapi.go index ad0dc3ceac52..5cf0dfcf5fd1 100644 --- a/pkg/fake/pricingapi.go +++ b/pkg/fake/pricingapi.go @@ -51,6 +51,10 @@ func (p *PricingAPI) GetProductsPagesWithContext(_ aws.Context, _ *pricing.GetPr } func NewOnDemandPrice(instanceType string, price float64) aws.JSONValue { + return NewOnDemandPriceWithCurrency(instanceType, price, "USD") +} + +func NewOnDemandPriceWithCurrency(instanceType string, price float64, currency string) aws.JSONValue { return aws.JSONValue{ "product": map[string]interface{}{ "attributes": map[string]interface{}{ @@ -63,7 +67,7 @@ func NewOnDemandPrice(instanceType string, price float64) aws.JSONValue { "offerTermCode": "JRTCKXETXF", "priceDimensions": map[string]interface{}{ "JRTCKXETXF.foo.bar": map[string]interface{}{ - "pricePerUnit": map[string]interface{}{"USD": fmt.Sprintf("%f", price)}, + "pricePerUnit": map[string]interface{}{currency: fmt.Sprintf("%f", price)}, }, }, }, diff --git a/pkg/fake/utils.go b/pkg/fake/utils.go index b91df64f29a3..67eec2a7f2e6 100644 --- a/pkg/fake/utils.go +++ b/pkg/fake/utils.go @@ -15,6 +15,7 @@ limitations under the License. package fake import ( + "context" "fmt" "strings" @@ -22,6 +23,10 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" "github.com/samber/lo" + "k8s.io/apimachinery/pkg/util/sets" + + "github.com/aws/karpenter-provider-aws/pkg/operator/options" + "github.com/aws/karpenter-provider-aws/pkg/providers/pricing" ) func InstanceID() string { @@ -155,3 +160,70 @@ func matchTags(tags []*ec2.Tag, filter *ec2.Filter) bool { } return false } + +func MakeInstances() []*ec2.InstanceTypeInfo { + var instanceTypes []*ec2.InstanceTypeInfo + ctx := options.ToContext(context.Background(), &options.Options{IsolatedVPC: true}) + // Use keys from the static pricing data so that we guarantee pricing for the data + // Create uniform instance data so all of them schedule for a given pod + for _, it := range pricing.NewProvider(ctx, nil, nil, "us-east-1").InstanceTypes() { + instanceTypes = append(instanceTypes, &ec2.InstanceTypeInfo{ + InstanceType: aws.String(it), + ProcessorInfo: &ec2.ProcessorInfo{ + SupportedArchitectures: aws.StringSlice([]string{"x86_64"}), + }, + VCpuInfo: &ec2.VCpuInfo{ + DefaultCores: aws.Int64(1), + DefaultVCpus: aws.Int64(2), + }, + MemoryInfo: &ec2.MemoryInfo{ + SizeInMiB: aws.Int64(8192), + }, + NetworkInfo: &ec2.NetworkInfo{ + Ipv4AddressesPerInterface: aws.Int64(10), + DefaultNetworkCardIndex: aws.Int64(0), + NetworkCards: []*ec2.NetworkCardInfo{{ + NetworkCardIndex: lo.ToPtr(int64(0)), + MaximumNetworkInterfaces: aws.Int64(3), + }}, + }, + SupportedUsageClasses: DefaultSupportedUsageClasses, + }) + } + return instanceTypes +} + +func MakeUniqueInstancesAndFamilies(instances []*ec2.InstanceTypeInfo, numInstanceFamilies int) ([]*ec2.InstanceTypeInfo, sets.Set[string]) { + var instanceTypes []*ec2.InstanceTypeInfo + instanceFamilies := sets.Set[string]{} + for _, it := range instances { + var found bool + for instFamily := range instanceFamilies { + if strings.Split(*it.InstanceType, ".")[0] == instFamily { + found = true + break + } + } + if !found { + instanceTypes = append(instanceTypes, it) + instanceFamilies.Insert(strings.Split(*it.InstanceType, ".")[0]) + if len(instanceFamilies) == numInstanceFamilies { + break + } + } + } + return instanceTypes, instanceFamilies +} + +func MakeInstanceOfferings(instanceTypes []*ec2.InstanceTypeInfo) []*ec2.InstanceTypeOffering { + var instanceTypeOfferings []*ec2.InstanceTypeOffering + + // Create uniform instance offering data so all of them schedule for a given pod + for _, instanceType := range instanceTypes { + instanceTypeOfferings = append(instanceTypeOfferings, &ec2.InstanceTypeOffering{ + InstanceType: instanceType.InstanceType, + Location: aws.String("test-zone-1a"), + }) + } + return instanceTypeOfferings +} diff --git a/pkg/fake/zz_generated.describe_instance_types.go b/pkg/fake/zz_generated.describe_instance_types.go index 0d9aa45d555b..ad2b7da73e67 100644 --- a/pkg/fake/zz_generated.describe_instance_types.go +++ b/pkg/fake/zz_generated.describe_instance_types.go @@ -35,6 +35,7 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ BareMetal: aws.Bool(false), Hypervisor: aws.String("nitro"), ProcessorInfo: &ec2.ProcessorInfo{ + Manufacturer: aws.String("AWS"), SupportedArchitectures: aws.StringSlice([]string{"arm64"}), }, VCpuInfo: &ec2.VCpuInfo{ @@ -65,6 +66,7 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ BareMetal: aws.Bool(false), Hypervisor: aws.String("nitro"), ProcessorInfo: &ec2.ProcessorInfo{ + Manufacturer: aws.String("Intel"), SupportedArchitectures: aws.StringSlice([]string{"x86_64"}), }, VCpuInfo: &ec2.VCpuInfo{ @@ -125,6 +127,7 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ BareMetal: aws.Bool(false), Hypervisor: aws.String("nitro"), ProcessorInfo: &ec2.ProcessorInfo{ + Manufacturer: aws.String("Intel"), SupportedArchitectures: aws.StringSlice([]string{"x86_64"}), }, VCpuInfo: &ec2.VCpuInfo{ @@ -173,6 +176,7 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ BareMetal: aws.Bool(false), Hypervisor: aws.String("nitro"), ProcessorInfo: &ec2.ProcessorInfo{ + Manufacturer: aws.String("Intel"), SupportedArchitectures: aws.StringSlice([]string{"x86_64"}), }, VCpuInfo: &ec2.VCpuInfo{ @@ -212,6 +216,7 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ BareMetal: aws.Bool(false), Hypervisor: aws.String("nitro"), ProcessorInfo: &ec2.ProcessorInfo{ + Manufacturer: aws.String("Intel"), SupportedArchitectures: aws.StringSlice([]string{"x86_64"}), }, VCpuInfo: &ec2.VCpuInfo{ @@ -251,6 +256,7 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ BareMetal: aws.Bool(false), Hypervisor: aws.String("nitro"), ProcessorInfo: &ec2.ProcessorInfo{ + Manufacturer: aws.String("Intel"), SupportedArchitectures: aws.StringSlice([]string{"x86_64"}), }, VCpuInfo: &ec2.VCpuInfo{ @@ -281,6 +287,7 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ BareMetal: aws.Bool(true), Hypervisor: aws.String(""), ProcessorInfo: &ec2.ProcessorInfo{ + Manufacturer: aws.String("Intel"), SupportedArchitectures: aws.StringSlice([]string{"x86_64"}), }, VCpuInfo: &ec2.VCpuInfo{ @@ -311,6 +318,7 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ BareMetal: aws.Bool(false), Hypervisor: aws.String("nitro"), ProcessorInfo: &ec2.ProcessorInfo{ + Manufacturer: aws.String("Intel"), SupportedArchitectures: aws.StringSlice([]string{"x86_64"}), }, VCpuInfo: &ec2.VCpuInfo{ @@ -341,6 +349,7 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ BareMetal: aws.Bool(false), Hypervisor: aws.String("nitro"), ProcessorInfo: &ec2.ProcessorInfo{ + Manufacturer: aws.String("Intel"), SupportedArchitectures: aws.StringSlice([]string{"x86_64"}), }, VCpuInfo: &ec2.VCpuInfo{ @@ -381,6 +390,7 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ BareMetal: aws.Bool(false), Hypervisor: aws.String("xen"), ProcessorInfo: &ec2.ProcessorInfo{ + Manufacturer: aws.String("Intel"), SupportedArchitectures: aws.StringSlice([]string{"x86_64"}), }, VCpuInfo: &ec2.VCpuInfo{ @@ -423,6 +433,7 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ BareMetal: aws.Bool(false), Hypervisor: aws.String("nitro"), ProcessorInfo: &ec2.ProcessorInfo{ + Manufacturer: aws.String("Intel"), SupportedArchitectures: aws.StringSlice([]string{"x86_64"}), }, VCpuInfo: &ec2.VCpuInfo{ @@ -453,6 +464,7 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ BareMetal: aws.Bool(false), Hypervisor: aws.String("nitro"), ProcessorInfo: &ec2.ProcessorInfo{ + Manufacturer: aws.String("AWS"), SupportedArchitectures: aws.StringSlice([]string{"arm64"}), }, VCpuInfo: &ec2.VCpuInfo{ @@ -483,6 +495,7 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ BareMetal: aws.Bool(false), Hypervisor: aws.String("nitro"), ProcessorInfo: &ec2.ProcessorInfo{ + Manufacturer: aws.String("AWS"), SupportedArchitectures: aws.StringSlice([]string{"arm64"}), }, VCpuInfo: &ec2.VCpuInfo{ @@ -513,6 +526,7 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ BareMetal: aws.Bool(false), Hypervisor: aws.String("nitro"), ProcessorInfo: &ec2.ProcessorInfo{ + Manufacturer: aws.String("AWS"), SupportedArchitectures: aws.StringSlice([]string{"arm64"}), }, VCpuInfo: &ec2.VCpuInfo{ @@ -543,6 +557,7 @@ var defaultDescribeInstanceTypesOutput = &ec2.DescribeInstanceTypesOutput{ BareMetal: aws.Bool(false), Hypervisor: aws.String("nitro"), ProcessorInfo: &ec2.ProcessorInfo{ + Manufacturer: aws.String("Intel"), SupportedArchitectures: aws.StringSlice([]string{"x86_64"}), }, VCpuInfo: &ec2.VCpuInfo{ diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index 33d6d4528c84..4b09ee7a0927 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -45,7 +45,6 @@ import ( "k8s.io/client-go/transport" "knative.dev/pkg/logging" "knative.dev/pkg/ptr" - "sigs.k8s.io/controller-runtime/pkg/client" corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" "sigs.k8s.io/karpenter/pkg/operator" @@ -149,6 +148,7 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont ctx, cache.New(awscache.DefaultTTL, awscache.DefaultCleanupInterval), ec2api, + eks.New(sess), amiResolver, securityGroupProvider, subnetProvider, @@ -176,14 +176,6 @@ func NewOperator(ctx context.Context, operator *operator.Operator) (context.Cont launchTemplateProvider, ) - lo.Must0(operator.Manager.GetFieldIndexer().IndexField(ctx, &corev1beta1.NodeClaim{}, "spec.nodeClassRef.name", func(o client.Object) []string { - nc := o.(*corev1beta1.NodeClaim) - if nc.Spec.NodeClassRef == nil { - return []string{} - } - return []string{nc.Spec.NodeClassRef.Name} - }), "failed to setup nodeclaim indexer") - return ctx, &Operator{ Operator: operator, Session: sess, diff --git a/pkg/operator/options/options.go b/pkg/operator/options/options.go index 0aad85f970d6..84d9c38c5268 100644 --- a/pkg/operator/options/options.go +++ b/pkg/operator/options/options.go @@ -50,7 +50,7 @@ func (o *Options) AddFlags(fs *coreoptions.FlagSet) { fs.StringVar(&o.ClusterCABundle, "cluster-ca-bundle", env.WithDefaultString("CLUSTER_CA_BUNDLE", ""), "Cluster CA bundle for nodes to use for TLS connections with the API server. If not set, this is taken from the controller's TLS configuration.") fs.StringVar(&o.ClusterName, "cluster-name", env.WithDefaultString("CLUSTER_NAME", ""), "[REQUIRED] The kubernetes cluster name for resource discovery.") fs.StringVar(&o.ClusterEndpoint, "cluster-endpoint", env.WithDefaultString("CLUSTER_ENDPOINT", ""), "The external kubernetes cluster endpoint for new nodes to connect with. If not specified, will discover the cluster endpoint using DescribeCluster API.") - fs.BoolVarWithEnv(&o.IsolatedVPC, "isolated-vpc", "ISOLATED_VPC", false, "If true, then assume we can't reach AWS services which don't have a VPC endpoint. This also has the effect of disabling look-ups to the AWS pricing endpoint.") + fs.BoolVarWithEnv(&o.IsolatedVPC, "isolated-vpc", "ISOLATED_VPC", false, "If true, then assume we can't reach AWS services which don't have a VPC endpoint. This also has the effect of disabling look-ups to the AWS on-demand pricing endpoint.") fs.Float64Var(&o.VMMemoryOverheadPercent, "vm-memory-overhead-percent", env.WithDefaultFloat64("VM_MEMORY_OVERHEAD_PERCENT", 0.075), "The VM memory overhead as a percent that will be subtracted from the total memory for all instance types.") fs.StringVar(&o.InterruptionQueue, "interruption-queue", env.WithDefaultString("INTERRUPTION_QUEUE", ""), "Interruption queue is disabled if not specified. Enabling interruption handling may require additional permissions on the controller service account. Additional permissions are outlined in the docs.") fs.IntVar(&o.ReservedENIs, "reserved-enis", env.WithDefaultInt("RESERVED_ENIS", 0), "Reserved ENIs are not included in the calculations for max-pods or kube-reserved. This is most often used in the VPC CNI custom networking setup https://docs.aws.amazon.com/eks/latest/userguide/cni-custom-network.html.") diff --git a/pkg/providers/amifamily/al2023.go b/pkg/providers/amifamily/al2023.go new file mode 100644 index 000000000000..d586d66e35fd --- /dev/null +++ b/pkg/providers/amifamily/al2023.go @@ -0,0 +1,79 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package amifamily + +import ( + "fmt" + + "github.com/samber/lo" + v1 "k8s.io/api/core/v1" + corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + "sigs.k8s.io/karpenter/pkg/cloudprovider" + "sigs.k8s.io/karpenter/pkg/scheduling" + + "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily/bootstrap" +) + +type AL2023 struct { + DefaultFamily + *Options +} + +func (a AL2023) DefaultAMIs(version string) []DefaultAMIOutput { + return []DefaultAMIOutput{ + { + Query: fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/x86_64/standard/recommended/image_id", version), + Requirements: scheduling.NewRequirements( + scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureAmd64), + ), + }, + { + Query: fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/arm64/standard/recommended/image_id", version), + Requirements: scheduling.NewRequirements( + scheduling.NewRequirement(v1.LabelArchStable, v1.NodeSelectorOpIn, corev1beta1.ArchitectureArm64), + ), + }, + } +} + +func (a AL2023) UserData(kubeletConfig *corev1beta1.KubeletConfiguration, taints []v1.Taint, labels map[string]string, caBundle *string, _ []*cloudprovider.InstanceType, customUserData *string, instanceStorePolicy *v1beta1.InstanceStorePolicy) bootstrap.Bootstrapper { + return bootstrap.Nodeadm{ + Options: bootstrap.Options{ + ClusterName: a.Options.ClusterName, + ClusterEndpoint: a.Options.ClusterEndpoint, + ClusterCIDR: a.Options.ClusterCIDR, + KubeletConfig: kubeletConfig, + Taints: taints, + Labels: labels, + CABundle: caBundle, + AWSENILimitedPodDensity: false, + CustomUserData: customUserData, + InstanceStorePolicy: instanceStorePolicy, + }, + } +} + +// DefaultBlockDeviceMappings returns the default block device mappings for the AMI Family +func (a AL2023) DefaultBlockDeviceMappings() []*v1beta1.BlockDeviceMapping { + return []*v1beta1.BlockDeviceMapping{{ + DeviceName: a.EphemeralBlockDevice(), + EBS: &DefaultEBS, + }} +} + +func (a AL2023) EphemeralBlockDevice() *string { + return lo.ToPtr("/dev/xvda") +} diff --git a/pkg/providers/amifamily/bootstrap/bootstrap.go b/pkg/providers/amifamily/bootstrap/bootstrap.go index 65321c40d339..ac39772d2c53 100644 --- a/pkg/providers/amifamily/bootstrap/bootstrap.go +++ b/pkg/providers/amifamily/bootstrap/bootstrap.go @@ -25,7 +25,6 @@ import ( "knative.dev/pkg/ptr" corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" - "sigs.k8s.io/karpenter/pkg/utils/resources" "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" ) @@ -34,6 +33,7 @@ import ( type Options struct { ClusterName string ClusterEndpoint string + ClusterCIDR *string KubeletConfig *corev1beta1.KubeletConfiguration Taints []core.Taint `hash:"set"` Labels map[string]string `hash:"set"` @@ -57,8 +57,8 @@ func (o Options) kubeletExtraArgs() (args []string) { args = append(args, fmt.Sprintf("--pods-per-core=%d", ptr.Int32Value(o.KubeletConfig.PodsPerCore))) } // We have to convert some of these maps so that their values return the correct string - args = append(args, joinParameterArgs("--system-reserved", resources.StringMap(o.KubeletConfig.SystemReserved), "=")) - args = append(args, joinParameterArgs("--kube-reserved", resources.StringMap(o.KubeletConfig.KubeReserved), "=")) + args = append(args, joinParameterArgs("--system-reserved", o.KubeletConfig.SystemReserved, "=")) + args = append(args, joinParameterArgs("--kube-reserved", o.KubeletConfig.KubeReserved, "=")) args = append(args, joinParameterArgs("--eviction-hard", o.KubeletConfig.EvictionHard, "<")) args = append(args, joinParameterArgs("--eviction-soft", o.KubeletConfig.EvictionSoft, "<")) args = append(args, joinParameterArgs("--eviction-soft-grace-period", lo.MapValues(o.KubeletConfig.EvictionSoftGracePeriod, func(v metav1.Duration, _ string) string { return v.Duration.String() }), "=")) diff --git a/pkg/providers/amifamily/bootstrap/bottlerocket.go b/pkg/providers/amifamily/bootstrap/bottlerocket.go index c12b39f9ce04..c1b2d8b66962 100644 --- a/pkg/providers/amifamily/bootstrap/bottlerocket.go +++ b/pkg/providers/amifamily/bootstrap/bottlerocket.go @@ -24,8 +24,6 @@ import ( "github.com/imdario/mergo" "github.com/samber/lo" - "sigs.k8s.io/karpenter/pkg/utils/resources" - "github.com/aws/aws-sdk-go/aws" ) @@ -60,10 +58,10 @@ func (b Bottlerocket) Script() (string, error) { s.Settings.Kubernetes.ClusterDNSIP = &b.KubeletConfig.ClusterDNS[0] } if b.KubeletConfig.SystemReserved != nil { - s.Settings.Kubernetes.SystemReserved = resources.StringMap(b.KubeletConfig.SystemReserved) + s.Settings.Kubernetes.SystemReserved = b.KubeletConfig.SystemReserved } if b.KubeletConfig.KubeReserved != nil { - s.Settings.Kubernetes.KubeReserved = resources.StringMap(b.KubeletConfig.KubeReserved) + s.Settings.Kubernetes.KubeReserved = b.KubeletConfig.KubeReserved } if b.KubeletConfig.EvictionHard != nil { s.Settings.Kubernetes.EvictionHard = b.KubeletConfig.EvictionHard diff --git a/pkg/providers/amifamily/bootstrap/bottlerocketsettings.go b/pkg/providers/amifamily/bootstrap/bottlerocketsettings.go index 550922eeda7b..f2f59f0de5e6 100644 --- a/pkg/providers/amifamily/bootstrap/bottlerocketsettings.go +++ b/pkg/providers/amifamily/bootstrap/bottlerocketsettings.go @@ -35,7 +35,7 @@ type BottlerocketConfig struct { Settings BottlerocketSettings `toml:"-"` } -// BottlerocketSettings is a subset of all configuration in https://github.com/bottlerocket-os/bottlerocket/blob/develop/sources/models/src/aws-k8s-1.22/mod.rs +// BottlerocketSettings is a subset of all configuration in https://github.com/bottlerocket-os/bottlerocket/blob/d427c40931cba6e6bedc5b75e9c084a6e1818db9/sources/models/src/lib.rs#L260 // These settings apply across all K8s versions that karpenter supports. type BottlerocketSettings struct { Kubernetes BottlerocketKubernetes `toml:"kubernetes"` @@ -43,36 +43,37 @@ type BottlerocketSettings struct { // BottlerocketKubernetes is k8s specific configuration for bottlerocket api type BottlerocketKubernetes struct { - APIServer *string `toml:"api-server"` - CloudProvider *string `toml:"cloud-provider"` - ClusterCertificate *string `toml:"cluster-certificate"` - ClusterName *string `toml:"cluster-name"` - ClusterDNSIP *string `toml:"cluster-dns-ip,omitempty"` - NodeLabels map[string]string `toml:"node-labels,omitempty"` - NodeTaints map[string][]string `toml:"node-taints,omitempty"` - MaxPods *int `toml:"max-pods,omitempty"` - StaticPods map[string]BottlerocketStaticPod `toml:"static-pods,omitempty"` - EvictionHard map[string]string `toml:"eviction-hard,omitempty"` - KubeReserved map[string]string `toml:"kube-reserved,omitempty"` - SystemReserved map[string]string `toml:"system-reserved,omitempty"` - AllowedUnsafeSysctls []string `toml:"allowed-unsafe-sysctls,omitempty"` - ServerTLSBootstrap *bool `toml:"server-tls-bootstrap,omitempty"` - RegistryQPS *int `toml:"registry-qps,omitempty"` - RegistryBurst *int `toml:"registry-burst,omitempty"` - EventQPS *int `toml:"event-qps,omitempty"` - EventBurst *int `toml:"event-burst,omitempty"` - KubeAPIQPS *int `toml:"kube-api-qps,omitempty"` - KubeAPIBurst *int `toml:"kube-api-burst,omitempty"` - ContainerLogMaxSize *string `toml:"container-log-max-size,omitempty"` - ContainerLogMaxFiles *int `toml:"container-log-max-files,omitempty"` - CPUManagerPolicy *string `toml:"cpu-manager-policy,omitempty"` - CPUManagerReconcilePeriod *string `toml:"cpu-manager-reconcile-period,omitempty"` - TopologyManagerScope *string `toml:"topology-manager-scope,omitempty"` - ImageGCHighThresholdPercent *string `toml:"image-gc-high-threshold-percent,omitempty"` - ImageGCLowThresholdPercent *string `toml:"image-gc-low-threshold-percent,omitempty"` - CPUCFSQuota *bool `toml:"cpu-cfs-quota-enforced,omitempty"` - ShutdownGracePeriod *string `toml:"shutdown-grace-period,omitempty"` - ShutdownGracePeriodForCriticalPods *string `toml:"shutdown-grace-period-for-critical-pods,omitempty"` + APIServer *string `toml:"api-server"` + CloudProvider *string `toml:"cloud-provider"` + ClusterCertificate *string `toml:"cluster-certificate"` + ClusterName *string `toml:"cluster-name"` + ClusterDNSIP *string `toml:"cluster-dns-ip,omitempty"` + CredentialProviders map[string]BottlerocketCredentialProvider `toml:"credential-providers,omitempty"` + NodeLabels map[string]string `toml:"node-labels,omitempty"` + NodeTaints map[string][]string `toml:"node-taints,omitempty"` + MaxPods *int `toml:"max-pods,omitempty"` + StaticPods map[string]BottlerocketStaticPod `toml:"static-pods,omitempty"` + EvictionHard map[string]string `toml:"eviction-hard,omitempty"` + KubeReserved map[string]string `toml:"kube-reserved,omitempty"` + SystemReserved map[string]string `toml:"system-reserved,omitempty"` + AllowedUnsafeSysctls []string `toml:"allowed-unsafe-sysctls,omitempty"` + ServerTLSBootstrap *bool `toml:"server-tls-bootstrap,omitempty"` + RegistryQPS *int `toml:"registry-qps,omitempty"` + RegistryBurst *int `toml:"registry-burst,omitempty"` + EventQPS *int `toml:"event-qps,omitempty"` + EventBurst *int `toml:"event-burst,omitempty"` + KubeAPIQPS *int `toml:"kube-api-qps,omitempty"` + KubeAPIBurst *int `toml:"kube-api-burst,omitempty"` + ContainerLogMaxSize *string `toml:"container-log-max-size,omitempty"` + ContainerLogMaxFiles *int `toml:"container-log-max-files,omitempty"` + CPUManagerPolicy *string `toml:"cpu-manager-policy,omitempty"` + CPUManagerReconcilePeriod *string `toml:"cpu-manager-reconcile-period,omitempty"` + TopologyManagerScope *string `toml:"topology-manager-scope,omitempty"` + ImageGCHighThresholdPercent *string `toml:"image-gc-high-threshold-percent,omitempty"` + ImageGCLowThresholdPercent *string `toml:"image-gc-low-threshold-percent,omitempty"` + CPUCFSQuota *bool `toml:"cpu-cfs-quota-enforced,omitempty"` + ShutdownGracePeriod *string `toml:"shutdown-grace-period,omitempty"` + ShutdownGracePeriodForCriticalPods *string `toml:"shutdown-grace-period-for-critical-pods,omitempty"` } type BottlerocketStaticPod struct { @@ -80,6 +81,15 @@ type BottlerocketStaticPod struct { Manifest *string `toml:"manifest,omitempty"` } +// BottlerocketCredentialProvider is k8s specific configuration for Bottlerocket Kubelet image credential provider +// See Bottlerocket struct at https://github.com/bottlerocket-os/bottlerocket/blob/d427c40931cba6e6bedc5b75e9c084a6e1818db9/sources/models/modeled-types/src/kubernetes.rs#L1307 +type BottlerocketCredentialProvider struct { + Enabled *bool `toml:"enabled"` + CacheDuration *string `toml:"cache-duration,omitempty"` + ImagePatterns []string `toml:"image-patterns"` + Environment map[string]string `toml:"environment,omitempty"` +} + func (c *BottlerocketConfig) UnmarshalTOML(data []byte) error { // unmarshal known settings s := struct { diff --git a/pkg/providers/amifamily/bootstrap/mime/mime.go b/pkg/providers/amifamily/bootstrap/mime/mime.go new file mode 100644 index 000000000000..f41db0b95248 --- /dev/null +++ b/pkg/providers/amifamily/bootstrap/mime/mime.go @@ -0,0 +1,121 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mime + +import ( + "bytes" + "encoding/base64" + "errors" + "fmt" + "io" + "mime" + "mime/multipart" + "net/mail" + "net/textproto" + "strings" + + admapi "github.com/awslabs/amazon-eks-ami/nodeadm/api" +) + +type ContentType string + +const ( + boundary = "//" + versionHeader = "MIME-Version: 1.0" + + ContentTypeShellScript ContentType = `text/x-shellscript; charset="us-ascii"` + ContentTypeNodeConfig ContentType = "application/" + admapi.GroupName + ContentTypeMultipart ContentType = `multipart/mixed; boundary="` + boundary + `"` +) + +type Entry struct { + ContentType ContentType + Content string +} + +type Archive []Entry + +func NewArchive(content string) (Archive, error) { + archive := Archive{} + if content == "" { + return archive, nil + } + reader, err := archive.getReader(content) + if err != nil { + return nil, err + } + for { + p, err := reader.NextPart() + if err != nil { + if errors.Is(err, io.EOF) { + break + } + return nil, fmt.Errorf("parsing content, %w", err) + } + slurp, err := io.ReadAll(p) + if err != nil { + return nil, fmt.Errorf("parsing content, %s, %w", string(slurp), err) + } + archive = append(archive, Entry{ + ContentType: ContentType(p.Header.Get("Content-Type")), + Content: string(slurp), + }) + } + return archive, nil +} + +// Serialize returns a base64 encoded serialized MIME multi-part archive +func (ma Archive) Serialize() (string, error) { + buffer := bytes.Buffer{} + writer := multipart.NewWriter(&buffer) + if err := writer.SetBoundary(boundary); err != nil { + return "", err + } + buffer.WriteString(versionHeader + "\n") + buffer.WriteString(fmt.Sprintf("Content-Type: %s\n\n", ContentTypeMultipart)) + for _, entry := range ma { + partWriter, err := writer.CreatePart(textproto.MIMEHeader{ + "Content-Type": []string{string(entry.ContentType)}, + }) + if err != nil { + return "", fmt.Errorf("creating multi-part section for entry, %w", err) + } + _, err = partWriter.Write([]byte(entry.Content)) + if err != nil { + return "", fmt.Errorf("writing entry, %w", err) + } + } + if err := writer.Close(); err != nil { + return "", fmt.Errorf("terminating multi-part archive, %w", err) + } + // The mime/multipart package adds carriage returns, while the rest of our logic does not. Remove all + // carriage returns for consistency. + return base64.StdEncoding.EncodeToString([]byte(strings.ReplaceAll(buffer.String(), "\r", ""))), nil +} + +func (Archive) getReader(content string) (*multipart.Reader, error) { + mailMsg, err := mail.ReadMessage(strings.NewReader(content)) + if err != nil { + return nil, err + } + mediaType, params, err := mime.ParseMediaType(mailMsg.Header.Get("Content-Type")) + if err != nil { + return nil, fmt.Errorf("archive doesn't have Content-Type header, %w", err) + } + if !strings.HasPrefix(mediaType, "multipart/") { + return nil, fmt.Errorf("archive is not in multipart format, %w", err) + } + return multipart.NewReader(mailMsg.Body, params["boundary"]), nil +} diff --git a/pkg/providers/amifamily/bootstrap/mime/suite_test.go b/pkg/providers/amifamily/bootstrap/mime/suite_test.go new file mode 100644 index 000000000000..e0cf9fc820e6 --- /dev/null +++ b/pkg/providers/amifamily/bootstrap/mime/suite_test.go @@ -0,0 +1,97 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mime_test + +import ( + "context" + "encoding/base64" + "fmt" + "os" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/samber/lo" + . "knative.dev/pkg/logging/testing" + + "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily/bootstrap/mime" +) + +var ctx context.Context + +func TestAWS(t *testing.T) { + ctx = TestContextWithLogger(t) + RegisterFailHandler(Fail) + RunSpecs(t, "MIME Parser") +} + +var _ = Describe("MIME Parser", func() { + It("should fail to parse MIME archive with a malformed header", func() { + content, err := os.ReadFile("test_data/mime_bad_header.txt") + Expect(err).To(BeNil()) + _, err = mime.NewArchive(string(content)) + Expect(err).ToNot(BeNil()) + }) + It("should successfully parse a valid MIME archive", func() { + content, err := os.ReadFile("test_data/mime_valid.txt") + Expect(err).To(BeNil()) + archive, err := mime.NewArchive(string(content)) + Expect(err).To(BeNil()) + Expect(len(archive)).To(Equal(2)) + for ct, f := range map[mime.ContentType]string{ + mime.ContentTypeNodeConfig: "test_data/nodeconfig.txt", + mime.ContentTypeShellScript: "test_data/shell.txt", + } { + entry, ok := lo.Find(archive, func(e mime.Entry) bool { + return e.ContentType == ct + }) + Expect(ok).To(BeTrue()) + expectedContent, err := os.ReadFile(f) + Expect(err).To(BeNil()) + Expect(entry.Content).To(Equal(string(expectedContent))) + } + }) + It("should successfully serialize a MIME archive", func() { + type entryTemplate struct { + contentType mime.ContentType + file string + } + archive := mime.Archive(lo.Map([]entryTemplate{ + { + contentType: mime.ContentTypeNodeConfig, + file: "test_data/nodeconfig.txt", + }, + { + contentType: mime.ContentTypeShellScript, + file: "test_data/shell.txt", + }, + }, func(e entryTemplate, _ int) mime.Entry { + content, err := os.ReadFile(e.file) + Expect(err).To(BeNil()) + return mime.Entry{ + ContentType: e.contentType, + Content: string(content), + } + })) + fmt.Printf("%v\n", archive) + expected, err := os.ReadFile("test_data/mime_valid.txt") + Expect(err).To(BeNil()) + encoded, err := archive.Serialize() + Expect(err).To(BeNil()) + serialized, err := base64.StdEncoding.DecodeString(encoded) + Expect(err).To(BeNil()) + Expect(string(serialized)).To(Equal(string(expected))) + }) +}) diff --git a/pkg/providers/amifamily/bootstrap/mime/test_data/mime_bad_header.txt b/pkg/providers/amifamily/bootstrap/mime/test_data/mime_bad_header.txt new file mode 100644 index 000000000000..ddc8c6da4566 --- /dev/null +++ b/pkg/providers/amifamily/bootstrap/mime/test_data/mime_bad_header.txt @@ -0,0 +1,29 @@ +MIME_VERSION = 1.0 +CONTENT_TYPE = MIME + +--// +Content-Type: application/node.eks.aws + +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + cluster: + name: test-cluster + apiServerEndpoint: https://example.com + certificateAuthority: ca-bundle + cidr: 10.100.0.0/16 + kubelet: + config: + maxPods: 42 + systemReserved: + cpu: 100m + memory: 100Mi + ephemeral-storage: 1Gi + flags: + - --node-labels "foo=bar" + containerd: + config: | + [plugins."io.containerd.grpc.v1.cri".containerd] + discard_unpacked_layers = false + +--//-- diff --git a/pkg/providers/amifamily/bootstrap/mime/test_data/mime_valid.txt b/pkg/providers/amifamily/bootstrap/mime/test_data/mime_valid.txt new file mode 100644 index 000000000000..0b8c182ccccb --- /dev/null +++ b/pkg/providers/amifamily/bootstrap/mime/test_data/mime_valid.txt @@ -0,0 +1,35 @@ +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="//" + +--// +Content-Type: application/node.eks.aws + +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + cluster: + name: test-cluster + apiServerEndpoint: https://example.com + certificateAuthority: ca-bundle + cidr: 10.100.0.0/16 + kubelet: + config: + maxPods: 42 + systemReserved: + cpu: 100m + memory: 100Mi + ephemeral-storage: 1Gi + flags: + - --node-labels "foo=bar" + containerd: + config: | + [plugins."io.containerd.grpc.v1.cri".containerd] + discard_unpacked_layers = false + +--// +Content-Type: text/x-shellscript; charset="us-ascii" + +#!/bin/bash +echo "I'm a shell script!" + +--//-- diff --git a/pkg/providers/amifamily/bootstrap/mime/test_data/nodeconfig.txt b/pkg/providers/amifamily/bootstrap/mime/test_data/nodeconfig.txt new file mode 100644 index 000000000000..2197cfb55ef2 --- /dev/null +++ b/pkg/providers/amifamily/bootstrap/mime/test_data/nodeconfig.txt @@ -0,0 +1,21 @@ +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + cluster: + name: test-cluster + apiServerEndpoint: https://example.com + certificateAuthority: ca-bundle + cidr: 10.100.0.0/16 + kubelet: + config: + maxPods: 42 + systemReserved: + cpu: 100m + memory: 100Mi + ephemeral-storage: 1Gi + flags: + - --node-labels "foo=bar" + containerd: + config: | + [plugins."io.containerd.grpc.v1.cri".containerd] + discard_unpacked_layers = false diff --git a/pkg/providers/amifamily/bootstrap/mime/test_data/shell.txt b/pkg/providers/amifamily/bootstrap/mime/test_data/shell.txt new file mode 100644 index 000000000000..c2dad1b1b336 --- /dev/null +++ b/pkg/providers/amifamily/bootstrap/mime/test_data/shell.txt @@ -0,0 +1,2 @@ +#!/bin/bash +echo "I'm a shell script!" diff --git a/pkg/providers/amifamily/bootstrap/nodeadm.go b/pkg/providers/amifamily/bootstrap/nodeadm.go new file mode 100644 index 000000000000..8c82268199c6 --- /dev/null +++ b/pkg/providers/amifamily/bootstrap/nodeadm.go @@ -0,0 +1,154 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package bootstrap + +import ( + "encoding/base64" + "fmt" + "strings" + + admapi "github.com/awslabs/amazon-eks-ami/nodeadm/api" + admv1alpha1 "github.com/awslabs/amazon-eks-ami/nodeadm/api/v1alpha1" + "github.com/samber/lo" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/json" + "sigs.k8s.io/karpenter/pkg/cloudprovider" + "sigs.k8s.io/yaml" + + "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily/bootstrap/mime" +) + +type Nodeadm struct { + Options +} + +func (n Nodeadm) Script() (string, error) { + nodeConfigYAML, err := n.getNodeConfigYAML() + if err != nil { + return "", fmt.Errorf("generating NodeConfig, %w", err) + } + customEntries, err := n.parseUserData() + if err != nil { + return "", fmt.Errorf("parsing custom UserData, %w", err) + } + mimeArchive := mime.Archive(append([]mime.Entry{{ + ContentType: mime.ContentTypeNodeConfig, + Content: nodeConfigYAML, + }}, customEntries...)) + userData, err := mimeArchive.Serialize() + if err != nil { + return "", err + } + return userData, nil +} + +// getNodeConfigYAML returns the Karpenter generated NodeConfig YAML object serialized as a string +func (n Nodeadm) getNodeConfigYAML() (string, error) { + config := &admv1alpha1.NodeConfig{ + TypeMeta: metav1.TypeMeta{ + Kind: admapi.KindNodeConfig, + APIVersion: admv1alpha1.GroupVersion.String(), + }, + Spec: admv1alpha1.NodeConfigSpec{ + Cluster: admv1alpha1.ClusterDetails{ + Name: n.ClusterName, + APIServerEndpoint: n.ClusterEndpoint, + CIDR: lo.FromPtr(n.ClusterCIDR), + }, + }, + } + if lo.FromPtr(n.CABundle) != "" { + ca, err := base64.StdEncoding.DecodeString(*n.CABundle) + if err != nil { + return "", fmt.Errorf("decoding CABundle, %w", err) + } + config.Spec.Cluster.CertificateAuthority = ca + } + if cidr := lo.FromPtr(n.ClusterCIDR); cidr != "" { + config.Spec.Cluster.CIDR = cidr + } else { + return "", cloudprovider.NewNodeClassNotReadyError(fmt.Errorf("resolving cluster CIDR")) + } + if lo.FromPtr(n.InstanceStorePolicy) == v1beta1.InstanceStorePolicyRAID0 { + config.Spec.Instance.LocalStorage.Strategy = admv1alpha1.LocalStorageRAID0 + } + inlineConfig, err := n.generateInlineKubeletConfiguration() + if err != nil { + return "", err + } + config.Spec.Kubelet.Config = inlineConfig + if arg := n.nodeLabelArg(); arg != "" { + config.Spec.Kubelet.Flags = []string{arg} + } + + // Convert to YAML at the end for improved legibility. + configYAML, err := yaml.Marshal(config) + if err != nil { + return "", err + } + return fmt.Sprintf("# Karpenter Generated NodeConfig\n%s", string(configYAML)), nil +} + +// generateInlineKubeletConfiguration returns a serialized form of the KubeletConfiguration specified by the Nodeadm +// options, for use with nodeadm's NodeConfig struct. +func (n Nodeadm) generateInlineKubeletConfiguration() (map[string]runtime.RawExtension, error) { + kubeConfigJSON, err := json.Marshal(n.KubeletConfig) + if err != nil { + return nil, err + } + kubeConfigMap := map[string]runtime.RawExtension{} + err = json.Unmarshal(kubeConfigJSON, &kubeConfigMap) + if err != nil { + return nil, err + } + if len(n.Taints) != 0 { + kubeConfigMap["registerWithTaints"] = runtime.RawExtension{ + Raw: lo.Must(json.Marshal(n.Taints)), + } + } + return kubeConfigMap, nil +} + +// parseUserData returns a slice of MIMEEntrys corresponding to each entry in the custom UserData. If the custom +// UserData is not a MIME multi-part archive, the content type will be detected (NodeConfig or shell) and an entry +// will be created. +func (n Nodeadm) parseUserData() ([]mime.Entry, error) { + userData := lo.FromPtr(n.CustomUserData) + if userData == "" { + return nil, nil + } + if strings.HasPrefix(strings.TrimSpace(userData), "MIME-Version:") || + strings.HasPrefix(strings.TrimSpace(userData), "Content-Type:") { + archive, err := mime.NewArchive(userData) + if err != nil { + return nil, err + } + return archive, nil + } + // Fallback to YAML or shall script if UserData is not in MIME format. Determine the content type for the + // generated MIME header depending on the type of the custom UserData. + if err := yaml.Unmarshal([]byte(*n.CustomUserData), lo.ToPtr(map[string]interface{}{})); err == nil { + return []mime.Entry{{ + ContentType: mime.ContentTypeNodeConfig, + Content: userData, + }}, nil + } + return []mime.Entry{{ + ContentType: mime.ContentTypeShellScript, + Content: userData, + }}, nil +} diff --git a/pkg/providers/amifamily/resolver.go b/pkg/providers/amifamily/resolver.go index 14467a6d852a..4582e3aa6882 100644 --- a/pkg/providers/amifamily/resolver.go +++ b/pkg/providers/amifamily/resolver.go @@ -50,6 +50,7 @@ type Resolver struct { type Options struct { ClusterName string ClusterEndpoint string + ClusterCIDR *string InstanceProfile string CABundle *string `hash:"ignore"` InstanceStorePolicy *v1beta1.InstanceStorePolicy @@ -72,6 +73,7 @@ type LaunchTemplate struct { InstanceTypes []*cloudprovider.InstanceType `hash:"ignore"` DetailedMonitoring bool EFACount int + CapacityType string } // AMIFamily can be implemented to override the default logic for generating dynamic launch template parameters @@ -118,7 +120,7 @@ func New(amiProvider *Provider) *Resolver { // Resolve generates launch templates using the static options and dynamically generates launch template parameters. // Multiple ResolvedTemplates are returned based on the instanceTypes passed in to support special AMIs for certain instance types like GPUs. -func (r Resolver) Resolve(ctx context.Context, nodeClass *v1beta1.EC2NodeClass, nodeClaim *corev1beta1.NodeClaim, instanceTypes []*cloudprovider.InstanceType, options *Options) ([]*LaunchTemplate, error) { +func (r Resolver) Resolve(ctx context.Context, nodeClass *v1beta1.EC2NodeClass, nodeClaim *corev1beta1.NodeClaim, instanceTypes []*cloudprovider.InstanceType, capacityType string, options *Options) ([]*LaunchTemplate, error) { amiFamily := GetAMIFamily(nodeClass.Spec.AMIFamily, options) amis, err := r.amiProvider.Get(ctx, nodeClass, options) if err != nil { @@ -153,7 +155,7 @@ func (r Resolver) Resolve(ctx context.Context, nodeClass *v1beta1.EC2NodeClass, } }) for params, instanceTypes := range paramsToInstanceTypes { - resolved, err := r.resolveLaunchTemplate(nodeClass, nodeClaim, instanceTypes, amiFamily, amiID, params.maxPods, params.efaCount, options) + resolved, err := r.resolveLaunchTemplate(nodeClass, nodeClaim, instanceTypes, capacityType, amiFamily, amiID, params.maxPods, params.efaCount, options) if err != nil { return nil, err } @@ -175,6 +177,8 @@ func GetAMIFamily(amiFamily *string, options *Options) AMIFamily { return &Windows{Options: options, Version: v1beta1.Windows2022, Build: v1beta1.Windows2022Build} case v1beta1.AMIFamilyCustom: return &Custom{Options: options} + case v1beta1.AMIFamilyAL2023: + return &AL2023{Options: options} default: return &AL2{Options: options} } @@ -206,7 +210,7 @@ func (r Resolver) defaultClusterDNS(opts *Options, kubeletConfig *corev1beta1.Ku return newKubeletConfig } -func (r Resolver) resolveLaunchTemplate(nodeClass *v1beta1.EC2NodeClass, nodeClaim *corev1beta1.NodeClaim, instanceTypes []*cloudprovider.InstanceType, +func (r Resolver) resolveLaunchTemplate(nodeClass *v1beta1.EC2NodeClass, nodeClaim *corev1beta1.NodeClaim, instanceTypes []*cloudprovider.InstanceType, capacityType string, amiFamily AMIFamily, amiID string, maxPods int, efaCount int, options *Options) (*LaunchTemplate, error) { kubeletConfig := &corev1beta1.KubeletConfiguration{} if nodeClaim.Spec.Kubelet != nil { @@ -234,6 +238,7 @@ func (r Resolver) resolveLaunchTemplate(nodeClass *v1beta1.EC2NodeClass, nodeCla AMIID: amiID, InstanceTypes: instanceTypes, EFACount: efaCount, + CapacityType: capacityType, } if len(resolved.BlockDeviceMappings) == 0 { resolved.BlockDeviceMappings = amiFamily.DefaultBlockDeviceMappings() diff --git a/pkg/providers/amifamily/suite_test.go b/pkg/providers/amifamily/suite_test.go index fcf370d8fd55..7dfc5ebe0d0b 100644 --- a/pkg/providers/amifamily/suite_test.go +++ b/pkg/providers/amifamily/suite_test.go @@ -140,6 +140,16 @@ var _ = Describe("AMIProvider", func() { Expect(err).ToNot(HaveOccurred()) Expect(amis).To(HaveLen(4)) }) + It("should succeed to resolve AMIs (AL2023)", func() { + nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2023 + awsEnv.SSMAPI.Parameters = map[string]string{ + fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/x86_64/standard/recommended/image_id", version): amd64AMI, + fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/arm64/standard/recommended/image_id", version): arm64AMI, + } + amis, err := awsEnv.AMIProvider.Get(ctx, nodeClass, &amifamily.Options{}) + Expect(err).ToNot(HaveOccurred()) + Expect(amis).To(HaveLen(2)) + }) It("should succeed to resolve AMIs (Bottlerocket)", func() { nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket awsEnv.SSMAPI.Parameters = map[string]string{ @@ -199,6 +209,15 @@ var _ = Describe("AMIProvider", func() { Expect(err).ToNot(HaveOccurred()) Expect(amis).To(HaveLen(2)) }) + It("should succeed to partially resolve AMIs if all SSM aliases don't exist (AL2023)", func() { + nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2023 + awsEnv.SSMAPI.Parameters = map[string]string{ + fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/x86_64/standard/recommended/image_id", version): amd64AMI, + } + amis, err := awsEnv.AMIProvider.Get(ctx, nodeClass, &amifamily.Options{}) + Expect(err).ToNot(HaveOccurred()) + Expect(amis).To(HaveLen(1)) + }) It("should succeed to partially resolve AMIs if all SSM aliases don't exist (Bottlerocket)", func() { nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket // No GPU AMI exists for AM64 here diff --git a/pkg/providers/instance/instance.go b/pkg/providers/instance/instance.go index 7d393356da9d..46b0d57a570e 100644 --- a/pkg/providers/instance/instance.go +++ b/pkg/providers/instance/instance.go @@ -50,8 +50,6 @@ import ( ) var ( - // MaxInstanceTypes defines the number of instance type options to pass to CreateFleet - MaxInstanceTypes = 60 instanceTypeFlexibilityThreshold = 5 // falling back to on-demand without flexibility risks insufficient capacity errors instanceStateFilter = &ec2.Filter{ @@ -84,10 +82,10 @@ func NewProvider(ctx context.Context, region string, ec2api ec2iface.EC2API, una } func (p *Provider) Create(ctx context.Context, nodeClass *v1beta1.EC2NodeClass, nodeClaim *corev1beta1.NodeClaim, instanceTypes []*cloudprovider.InstanceType) (*Instance, error) { - instanceTypes = p.filterInstanceTypes(nodeClaim, instanceTypes) - instanceTypes = orderInstanceTypesByPrice(instanceTypes, scheduling.NewNodeSelectorRequirements(nodeClaim.Spec.Requirements...)) - if len(instanceTypes) > MaxInstanceTypes { - instanceTypes = instanceTypes[0:MaxInstanceTypes] + schedulingRequirements := scheduling.NewNodeSelectorRequirementsWithMinValues(nodeClaim.Spec.Requirements...) + // Only filter the instances if there are no minValues in the requirement. + if !schedulingRequirements.HasMinValues() { + instanceTypes = p.filterInstanceTypes(nodeClaim, instanceTypes) } tags := getTags(ctx, nodeClass, nodeClaim) fleetInstance, err := p.launchInstance(ctx, nodeClass, nodeClaim, instanceTypes, tags) @@ -132,6 +130,10 @@ func (p *Provider) List(ctx context.Context) ([]*Instance, error) { Name: aws.String("tag-key"), Values: aws.StringSlice([]string{corev1beta1.NodePoolLabelKey}), }, + { + Name: aws.String("tag-key"), + Values: aws.StringSlice([]string{v1beta1.LabelNodeClass}), + }, { Name: aws.String("tag-key"), Values: aws.StringSlice([]string{fmt.Sprintf("kubernetes.io/cluster/%s", options.FromContext(ctx).ClusterName)}), @@ -253,7 +255,7 @@ func getTags(ctx context.Context, nodeClass *v1beta1.EC2NodeClass, nodeClaim *co func (p *Provider) checkODFallback(nodeClaim *corev1beta1.NodeClaim, instanceTypes []*cloudprovider.InstanceType, launchTemplateConfigs []*ec2.FleetLaunchTemplateConfigRequest) error { // only evaluate for on-demand fallback if the capacity type for the request is OD and both OD and spot are allowed in requirements - if p.getCapacityType(nodeClaim, instanceTypes) != corev1beta1.CapacityTypeOnDemand || !scheduling.NewNodeSelectorRequirements(nodeClaim.Spec.Requirements...).Get(corev1beta1.CapacityTypeLabelKey).Has(corev1beta1.CapacityTypeSpot) { + if p.getCapacityType(nodeClaim, instanceTypes) != corev1beta1.CapacityTypeOnDemand || !scheduling.NewNodeSelectorRequirementsWithMinValues(nodeClaim.Spec.Requirements...).Get(corev1beta1.CapacityTypeLabelKey).Has(corev1beta1.CapacityTypeSpot) { return nil } @@ -283,7 +285,7 @@ func (p *Provider) getLaunchTemplateConfigs(ctx context.Context, nodeClass *v1be } for _, launchTemplate := range launchTemplates { launchTemplateConfig := &ec2.FleetLaunchTemplateConfigRequest{ - Overrides: p.getOverrides(launchTemplate.InstanceTypes, zonalSubnets, scheduling.NewNodeSelectorRequirements(nodeClaim.Spec.Requirements...).Get(v1.LabelTopologyZone), capacityType, launchTemplate.ImageID), + Overrides: p.getOverrides(launchTemplate.InstanceTypes, zonalSubnets, scheduling.NewNodeSelectorRequirementsWithMinValues(nodeClaim.Spec.Requirements...).Get(v1.LabelTopologyZone), capacityType, launchTemplate.ImageID), LaunchTemplateSpecification: &ec2.FleetLaunchTemplateSpecificationRequest{ LaunchTemplateName: aws.String(launchTemplate.Name), Version: aws.String("$Latest"), @@ -355,7 +357,7 @@ func (p *Provider) updateUnavailableOfferingsCache(ctx context.Context, errors [ // available offering. The AWS Cloud Provider defaults to [ on-demand ], so spot // must be explicitly included in capacity type requirements. func (p *Provider) getCapacityType(nodeClaim *corev1beta1.NodeClaim, instanceTypes []*cloudprovider.InstanceType) string { - requirements := scheduling.NewNodeSelectorRequirements(nodeClaim. + requirements := scheduling.NewNodeSelectorRequirementsWithMinValues(nodeClaim. Spec.Requirements...) if requirements.Get(corev1beta1.CapacityTypeLabelKey).Has(corev1beta1.CapacityTypeSpot) { for _, instanceType := range instanceTypes { @@ -369,25 +371,6 @@ func (p *Provider) getCapacityType(nodeClaim *corev1beta1.NodeClaim, instanceTyp return corev1beta1.CapacityTypeOnDemand } -func orderInstanceTypesByPrice(instanceTypes []*cloudprovider.InstanceType, requirements scheduling.Requirements) []*cloudprovider.InstanceType { - // Order instance types so that we get the cheapest instance types of the available offerings - sort.Slice(instanceTypes, func(i, j int) bool { - iPrice := math.MaxFloat64 - jPrice := math.MaxFloat64 - if len(instanceTypes[i].Offerings.Available().Compatible(requirements)) > 0 { - iPrice = instanceTypes[i].Offerings.Available().Compatible(requirements).Cheapest().Price - } - if len(instanceTypes[j].Offerings.Available().Compatible(requirements)) > 0 { - jPrice = instanceTypes[j].Offerings.Available().Compatible(requirements).Cheapest().Price - } - if iPrice == jPrice { - return instanceTypes[i].Name < instanceTypes[j].Name - } - return iPrice < jPrice - }) - return instanceTypes -} - // filterInstanceTypes is used to provide filtering on the list of potential instance types to further limit it to those // that make the most sense given our specific AWS cloudprovider. func (p *Provider) filterInstanceTypes(nodeClaim *corev1beta1.NodeClaim, instanceTypes []*cloudprovider.InstanceType) []*cloudprovider.InstanceType { @@ -403,7 +386,7 @@ func (p *Provider) filterInstanceTypes(nodeClaim *corev1beta1.NodeClaim, instanc // isMixedCapacityLaunch returns true if nodepools and available offerings could potentially allow either a spot or // and on-demand node to launch func (p *Provider) isMixedCapacityLaunch(nodeClaim *corev1beta1.NodeClaim, instanceTypes []*cloudprovider.InstanceType) bool { - requirements := scheduling.NewNodeSelectorRequirements(nodeClaim.Spec.Requirements...) + requirements := scheduling.NewNodeSelectorRequirementsWithMinValues(nodeClaim.Spec.Requirements...) // requirements must allow both if !requirements.Get(corev1beta1.CapacityTypeLabelKey).Has(corev1beta1.CapacityTypeSpot) || !requirements.Get(corev1beta1.CapacityTypeLabelKey).Has(corev1beta1.CapacityTypeOnDemand) { diff --git a/pkg/providers/instance/suite_test.go b/pkg/providers/instance/suite_test.go index 73d9212b6cae..e1aa36692b7e 100644 --- a/pkg/providers/instance/suite_test.go +++ b/pkg/providers/instance/suite_test.go @@ -146,6 +146,10 @@ var _ = Describe("InstanceProvider", func() { Key: aws.String(corev1beta1.NodePoolLabelKey), Value: aws.String("default"), }, + { + Key: aws.String(v1beta1.LabelNodeClass), + Value: aws.String("default"), + }, { Key: aws.String(corev1beta1.ManagedByAnnotationKey), Value: aws.String(options.FromContext(ctx).ClusterName), diff --git a/pkg/providers/instancetype/instancetype.go b/pkg/providers/instancetype/instancetype.go index 6b305e6b4381..7d11ed22678f 100644 --- a/pkg/providers/instancetype/instancetype.go +++ b/pkg/providers/instancetype/instancetype.go @@ -20,7 +20,6 @@ import ( "net/http" "sync" "sync/atomic" - "time" "github.com/mitchellh/hashstructure/v2" "github.com/patrickmn/go-cache" @@ -43,12 +42,13 @@ import ( "sigs.k8s.io/karpenter/pkg/cloudprovider" "sigs.k8s.io/karpenter/pkg/utils/pretty" + + "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily" ) const ( InstanceTypesCacheKey = "types" InstanceTypeOfferingsCacheKey = "offerings" - ZonesCacheKey = "zones" ) type Provider struct { @@ -98,9 +98,6 @@ func (p *Provider) List(ctx context.Context, kc *corev1beta1.KubeletConfiguratio if err != nil { return nil, err } - // Get zones from instancetypeOfferings - zones := p.getZones(ctx, instanceTypeOfferings) - // Constrain zones from subnets subnets, err := p.subnetProvider.List(ctx, nodeClass) if err != nil { return nil, err @@ -109,25 +106,60 @@ func (p *Provider) List(ctx context.Context, kc *corev1beta1.KubeletConfiguratio return aws.StringValue(s.AvailabilityZone) })...) + if kc == nil { + kc = &corev1beta1.KubeletConfiguration{} + } + if nodeClass == nil { + nodeClass = &v1beta1.EC2NodeClass{} + } + // Compute fully initialized instance types hash key - subnetHash, _ := hashstructure.Hash(subnets, hashstructure.FormatV2, &hashstructure.HashOptions{SlicesAsSets: true}) + subnetZonesHash, _ := hashstructure.Hash(subnetZones, hashstructure.FormatV2, &hashstructure.HashOptions{SlicesAsSets: true}) kcHash, _ := hashstructure.Hash(kc, hashstructure.FormatV2, &hashstructure.HashOptions{SlicesAsSets: true}) blockDeviceMappingsHash, _ := hashstructure.Hash(nodeClass.Spec.BlockDeviceMappings, hashstructure.FormatV2, &hashstructure.HashOptions{SlicesAsSets: true}) - key := fmt.Sprintf("%d-%d-%d-%016x-%016x-%016x-%s", p.instanceTypesSeqNum, p.instanceTypeOfferingsSeqNum, p.unavailableOfferings.SeqNum, subnetHash, kcHash, blockDeviceMappingsHash, aws.StringValue(nodeClass.Spec.AMIFamily)) + key := fmt.Sprintf("%d-%d-%d-%016x-%016x-%016x-%s-%s", + p.instanceTypesSeqNum, + p.instanceTypeOfferingsSeqNum, + p.unavailableOfferings.SeqNum, + subnetZonesHash, + kcHash, + blockDeviceMappingsHash, + aws.StringValue((*string)(nodeClass.Spec.InstanceStorePolicy)), + aws.StringValue(nodeClass.Spec.AMIFamily), + ) if item, ok := p.cache.Get(key); ok { return item.([]*cloudprovider.InstanceType), nil } + + // Get all zones across all offerings + // We don't use this in the cache key since this is produced from our instanceTypeOfferings which we do cache + allZones := sets.New[string]() + for _, offeringZones := range instanceTypeOfferings { + for zone := range offeringZones { + allZones.Insert(zone) + } + } + if p.cm.HasChanged("zones", allZones) { + logging.FromContext(ctx).With("zones", allZones.UnsortedList()).Debugf("discovered zones") + } + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) result := lo.Map(instanceTypes, func(i *ec2.InstanceTypeInfo, _ int) *cloudprovider.InstanceType { - return NewInstanceType(ctx, i, kc, p.region, nodeClass, p.createOfferings(ctx, i, instanceTypeOfferings[aws.StringValue(i.InstanceType)], zones, subnetZones)) + instanceTypeVCPU.With(prometheus.Labels{ + instanceTypeLabel: *i.InstanceType, + }).Set(float64(aws.Int64Value(i.VCpuInfo.DefaultVCpus))) + instanceTypeMemory.With(prometheus.Labels{ + instanceTypeLabel: *i.InstanceType, + }).Set(float64(aws.Int64Value(i.MemoryInfo.SizeInMiB) * 1024 * 1024)) + + // !!! Important !!! + // Any changes to the values passed into the NewInstanceType method will require making updates to the cache key + // so that Karpenter is able to cache the set of InstanceTypes based on values that alter the set of instance types + // !!! Important !!! + return NewInstanceType(ctx, i, p.region, + nodeClass.Spec.BlockDeviceMappings, nodeClass.Spec.InstanceStorePolicy, + kc.MaxPods, kc.PodsPerCore, kc.KubeReserved, kc.SystemReserved, kc.EvictionHard, kc.EvictionSoft, + amiFamily, p.createOfferings(ctx, i, instanceTypeOfferings[aws.StringValue(i.InstanceType)], allZones, subnetZones)) }) - for _, instanceType := range instanceTypes { - InstanceTypeVCPU.With(prometheus.Labels{ - InstanceTypeLabel: *instanceType.InstanceType, - }).Set(float64(aws.Int64Value(instanceType.VCpuInfo.DefaultVCpus))) - InstanceTypeMemory.With(prometheus.Labels{ - InstanceTypeLabel: *instanceType.InstanceType, - }).Set(float64(aws.Int64Value(instanceType.MemoryInfo.SizeInMiB) * 1024 * 1024)) - } p.cache.SetDefault(key, result) return result, nil } @@ -167,35 +199,21 @@ func (p *Provider) createOfferings(ctx context.Context, instanceType *ec2.Instan Price: price, Available: available, }) + instanceTypeOfferingAvailable.With(prometheus.Labels{ + instanceTypeLabel: *instanceType.InstanceType, + capacityTypeLabel: capacityType, + zoneLabel: zone, + }).Set(float64(lo.Ternary(available, 1, 0))) + instanceTypeOfferingPriceEstimate.With(prometheus.Labels{ + instanceTypeLabel: *instanceType.InstanceType, + capacityTypeLabel: capacityType, + zoneLabel: zone, + }).Set(price) } } return offerings } -func (p *Provider) getZones(ctx context.Context, instanceTypeOfferings map[string]sets.Set[string]) sets.Set[string] { - // DO NOT REMOVE THIS LOCK ---------------------------------------------------------------------------- - // We lock here so that multiple callers to getAvailabilityZones do not result in cache misses and multiple - // calls to EC2 when we could have just made one call. - // TODO @joinnis: This can be made more efficient by holding a Read lock and only obtaining the Write if not in cache - p.mu.Lock() - defer p.mu.Unlock() - if cached, ok := p.cache.Get(ZonesCacheKey); ok { - return cached.(sets.Set[string]) - } - // Get zones from offerings - zones := sets.Set[string]{} - for _, offeringZones := range instanceTypeOfferings { - for zone := range offeringZones { - zones.Insert(zone) - } - } - if p.cm.HasChanged("zones", zones) { - logging.FromContext(ctx).With("zones", zones.UnsortedList()).Debugf("discovered zones") - } - p.cache.Set(ZonesCacheKey, zones, 24*time.Hour) - return zones -} - func (p *Provider) getInstanceTypeOfferings(ctx context.Context) (map[string]sets.Set[string], error) { // DO NOT REMOVE THIS LOCK ---------------------------------------------------------------------------- // We lock here so that multiple callers to getInstanceTypeOfferings do not result in cache misses and multiple diff --git a/pkg/providers/instancetype/metrics.go b/pkg/providers/instancetype/metrics.go index 5e343aa9b12b..5b505e8f813a 100644 --- a/pkg/providers/instancetype/metrics.go +++ b/pkg/providers/instancetype/metrics.go @@ -23,12 +23,13 @@ import ( const ( cloudProviderSubsystem = "cloudprovider" + instanceTypeLabel = "instance_type" + capacityTypeLabel = "capacity_type" + zoneLabel = "zone" ) var ( - InstanceTypeLabel = "instance_type" - - InstanceTypeVCPU = prometheus.NewGaugeVec( + instanceTypeVCPU = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: metrics.Namespace, Subsystem: cloudProviderSubsystem, @@ -36,10 +37,10 @@ var ( Help: "VCPUs cores for a given instance type.", }, []string{ - InstanceTypeLabel, - }) - - InstanceTypeMemory = prometheus.NewGaugeVec( + instanceTypeLabel, + }, + ) + instanceTypeMemory = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Namespace: metrics.Namespace, Subsystem: cloudProviderSubsystem, @@ -47,10 +48,36 @@ var ( Help: "Memory, in bytes, for a given instance type.", }, []string{ - InstanceTypeLabel, + instanceTypeLabel, + }, + ) + instanceTypeOfferingAvailable = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: metrics.Namespace, + Subsystem: cloudProviderSubsystem, + Name: "instance_type_offering_available", + Help: "Instance type offering availability, based on instance type, capacity type, and zone", + }, + []string{ + instanceTypeLabel, + capacityTypeLabel, + zoneLabel, + }, + ) + instanceTypeOfferingPriceEstimate = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: metrics.Namespace, + Subsystem: cloudProviderSubsystem, + Name: "instance_type_offering_price_estimate", + Help: "Instance type offering estimated hourly price used when making informed decisions on node cost calculation, based on instance type, capacity type, and zone.", + }, + []string{ + instanceTypeLabel, + capacityTypeLabel, + zoneLabel, }) ) func init() { - crmetrics.Registry.MustRegister(InstanceTypeVCPU, InstanceTypeMemory) + crmetrics.Registry.MustRegister(instanceTypeVCPU, instanceTypeMemory, instanceTypeOfferingAvailable, instanceTypeOfferingPriceEstimate) } diff --git a/pkg/providers/instancetype/suite_test.go b/pkg/providers/instancetype/suite_test.go index 6e2802e10095..16b6f7263adf 100644 --- a/pkg/providers/instancetype/suite_test.go +++ b/pkg/providers/instancetype/suite_test.go @@ -54,9 +54,8 @@ import ( "github.com/aws/karpenter-provider-aws/pkg/cloudprovider" "github.com/aws/karpenter-provider-aws/pkg/fake" "github.com/aws/karpenter-provider-aws/pkg/operator/options" - "github.com/aws/karpenter-provider-aws/pkg/providers/instance" + "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily" "github.com/aws/karpenter-provider-aws/pkg/providers/instancetype" - "github.com/aws/karpenter-provider-aws/pkg/providers/pricing" "github.com/aws/karpenter-provider-aws/pkg/test" ) @@ -83,7 +82,7 @@ var _ = BeforeSuite(func() { cloudProvider = cloudprovider.New(awsEnv.InstanceTypesProvider, awsEnv.InstanceProvider, events.NewRecorder(&record.FakeRecorder{}), env.Client, awsEnv.AMIProvider, awsEnv.SecurityGroupProvider, awsEnv.SubnetProvider) cluster = state.NewCluster(fakeClock, env.Client, cloudProvider) - prov = provisioning.NewProvisioner(env.Client, env.KubernetesInterface.CoreV1(), events.NewRecorder(&record.FakeRecorder{}), cloudProvider, cluster) + prov = provisioning.NewProvisioner(env.Client, events.NewRecorder(&record.FakeRecorder{}), cloudProvider, cluster) }) var _ = AfterSuite(func() { @@ -112,13 +111,16 @@ var _ = Describe("InstanceTypes", func() { Spec: corev1beta1.NodePoolSpec{ Template: corev1beta1.NodeClaimTemplate{ Spec: corev1beta1.NodeClaimSpec{ - Requirements: []v1.NodeSelectorRequirement{ + Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ { - Key: corev1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.CapacityTypeOnDemand}, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: corev1beta1.CapacityTypeLabelKey, + Operator: v1.NodeSelectorOpIn, + Values: []string{corev1beta1.CapacityTypeOnDemand}, + }, }, }, + Kubelet: &corev1beta1.KubeletConfiguration{}, NodeClassRef: &corev1beta1.NodeClassReference{ Name: nodeClass.Name, }, @@ -135,11 +137,13 @@ var _ = Describe("InstanceTypes", func() { Spec: corev1beta1.NodePoolSpec{ Template: corev1beta1.NodeClaimTemplate{ Spec: corev1beta1.NodeClaimSpec{ - Requirements: []v1.NodeSelectorRequirement{ + Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ { - Key: corev1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.CapacityTypeOnDemand}, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: corev1beta1.CapacityTypeLabelKey, + Operator: v1.NodeSelectorOpIn, + Values: []string{corev1beta1.CapacityTypeOnDemand}, + }, }, }, NodeClassRef: &corev1beta1.NodeClassReference{ @@ -171,6 +175,7 @@ var _ = Describe("InstanceTypes", func() { v1beta1.LabelInstanceFamily: "g4dn", v1beta1.LabelInstanceSize: "8xlarge", v1beta1.LabelInstanceCPU: "32", + v1beta1.LabelInstanceCPUManufacturer: "intel", v1beta1.LabelInstanceMemory: "131072", v1beta1.LabelInstanceNetworkBandwidth: "50000", v1beta1.LabelInstanceGPUName: "t4", @@ -223,6 +228,7 @@ var _ = Describe("InstanceTypes", func() { v1beta1.LabelInstanceFamily: "g4dn", v1beta1.LabelInstanceSize: "8xlarge", v1beta1.LabelInstanceCPU: "32", + v1beta1.LabelInstanceCPUManufacturer: "intel", v1beta1.LabelInstanceMemory: "131072", v1beta1.LabelInstanceNetworkBandwidth: "50000", v1beta1.LabelInstanceGPUName: "t4", @@ -273,6 +279,7 @@ var _ = Describe("InstanceTypes", func() { v1beta1.LabelInstanceFamily: "inf1", v1beta1.LabelInstanceSize: "2xlarge", v1beta1.LabelInstanceCPU: "8", + v1beta1.LabelInstanceCPUManufacturer: "intel", v1beta1.LabelInstanceMemory: "16384", v1beta1.LabelInstanceNetworkBandwidth: "5000", v1beta1.LabelInstanceAcceleratorName: "inferentia", @@ -317,12 +324,12 @@ var _ = Describe("InstanceTypes", func() { ExpectNotScheduled(ctx, env.Client, pod) }) It("should order the instance types by price and only consider the cheapest ones", func() { - instances := makeFakeInstances() + instances := fake.MakeInstances() awsEnv.EC2API.DescribeInstanceTypesOutput.Set(&ec2.DescribeInstanceTypesOutput{ - InstanceTypes: makeFakeInstances(), + InstanceTypes: fake.MakeInstances(), }) awsEnv.EC2API.DescribeInstanceTypeOfferingsOutput.Set(&ec2.DescribeInstanceTypeOfferingsOutput{ - InstanceTypeOfferings: makeFakeInstanceOfferings(instances), + InstanceTypeOfferings: fake.MakeInstanceOfferings(instances), }) ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod(coretest.PodOptions{ @@ -337,7 +344,7 @@ var _ = Describe("InstanceTypes", func() { Expect(err).To(BeNil()) // Order all the instances by their price // We need some way to deterministically order them if their prices match - reqs := scheduling.NewNodeSelectorRequirements(nodePool.Spec.Template.Spec.Requirements...) + reqs := scheduling.NewNodeSelectorRequirementsWithMinValues(nodePool.Spec.Template.Spec.Requirements...) sort.Slice(its, func(i, j int) bool { iPrice := its[i].Offerings.Compatible(reqs).Cheapest().Price jPrice := its[j].Offerings.Compatible(reqs).Cheapest().Price @@ -346,35 +353,37 @@ var _ = Describe("InstanceTypes", func() { } return iPrice < jPrice }) - // Expect that the launch template overrides gives the 60 cheapest instance types - expected := sets.NewString(lo.Map(its[:instance.MaxInstanceTypes], func(i *corecloudprovider.InstanceType, _ int) string { + // Expect that the launch template overrides gives the 100 cheapest instance types + expected := sets.NewString(lo.Map(its[:100], func(i *corecloudprovider.InstanceType, _ int) string { return i.Name })...) Expect(awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) call := awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Pop() Expect(call.LaunchTemplateConfigs).To(HaveLen(1)) - Expect(call.LaunchTemplateConfigs[0].Overrides).To(HaveLen(instance.MaxInstanceTypes)) + Expect(call.LaunchTemplateConfigs[0].Overrides).To(HaveLen(60)) for _, override := range call.LaunchTemplateConfigs[0].Overrides { Expect(expected.Has(aws.StringValue(override.InstanceType))).To(BeTrue(), fmt.Sprintf("expected %s to exist in set", aws.StringValue(override.InstanceType))) } }) It("should order the instance types by price and only consider the spot types that are cheaper than the cheapest on-demand", func() { - instances := makeFakeInstances() + instances := fake.MakeInstances() awsEnv.EC2API.DescribeInstanceTypesOutput.Set(&ec2.DescribeInstanceTypesOutput{ - InstanceTypes: makeFakeInstances(), + InstanceTypes: fake.MakeInstances(), }) awsEnv.EC2API.DescribeInstanceTypeOfferingsOutput.Set(&ec2.DescribeInstanceTypeOfferingsOutput{ - InstanceTypeOfferings: makeFakeInstanceOfferings(instances), + InstanceTypeOfferings: fake.MakeInstanceOfferings(instances), }) - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ + nodePool.Spec.Template.Spec.Requirements = []corev1beta1.NodeSelectorRequirementWithMinValues{ { - Key: corev1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: []string{ - corev1beta1.CapacityTypeSpot, - corev1beta1.CapacityTypeOnDemand, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: corev1beta1.CapacityTypeLabelKey, + Operator: v1.NodeSelectorOpIn, + Values: []string{ + corev1beta1.CapacityTypeSpot, + corev1beta1.CapacityTypeOnDemand, + }, }, }, } @@ -395,7 +404,7 @@ var _ = Describe("InstanceTypes", func() { Expect(err).To(BeNil()) // Order all the instances by their price // We need some way to deterministically order them if their prices match - reqs := scheduling.NewNodeSelectorRequirements(nodePool.Spec.Template.Spec.Requirements...) + reqs := scheduling.NewNodeSelectorRequirementsWithMinValues(nodePool.Spec.Template.Spec.Requirements...) sort.Slice(its, func(i, j int) bool { iPrice := its[i].Offerings.Compatible(reqs).Cheapest().Price jPrice := its[j].Offerings.Compatible(reqs).Cheapest().Price @@ -425,6 +434,43 @@ var _ = Describe("InstanceTypes", func() { Expect(spotPrice).To(BeNumerically("<", cheapestODPrice)) } }) + It("should not remove expensive metal instanceTypeOptions if any of the requirement with minValues is provided", func() { + // Construct requirements with minValues for capacityType requirement. + nodePool.Spec.Template.Spec.Requirements = []corev1beta1.NodeSelectorRequirementWithMinValues{ + { + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: corev1beta1.CapacityTypeLabelKey, + Operator: v1.NodeSelectorOpIn, + Values: []string{corev1beta1.CapacityTypeSpot}, + }, + MinValues: lo.ToPtr(1), + }, + } + + // Apply requirements and schedule pods. + ExpectApplied(ctx, env.Client, nodePool, nodeClass) + pod := coretest.UnschedulablePod(coretest.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, + Limits: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, + }, + }) + + // Check if pods are scheduled and if CreateFleet has the expensive instance-types. + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Len()).To(Equal(1)) + call := awsEnv.EC2API.CreateFleetBehavior.CalledWithInput.Pop() + var expensiveInstanceType bool + for _, ltc := range call.LaunchTemplateConfigs { + for _, ovr := range ltc.Overrides { + if strings.Contains(aws.StringValue(ovr.InstanceType), "metal") { + expensiveInstanceType = true + } + } + } + Expect(expensiveInstanceType).To(BeTrue()) + }) It("should de-prioritize metal", func() { ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod(coretest.PodOptions{ @@ -465,9 +511,11 @@ var _ = Describe("InstanceTypes", func() { }) It("should launch on metal", func() { // add a nodePool requirement for instance type exists to remove our default filter for metal sizes - nodePool.Spec.Template.Spec.Requirements = append(nodePool.Spec.Template.Spec.Requirements, v1.NodeSelectorRequirement{ - Key: v1.LabelInstanceTypeStable, - Operator: v1.NodeSelectorOpExists, + nodePool.Spec.Template.Spec.Requirements = append(nodePool.Spec.Template.Spec.Requirements, corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1.LabelInstanceTypeStable, + Operator: v1.NodeSelectorOpExists, + }, }) ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod(coretest.PodOptions{ @@ -598,11 +646,13 @@ var _ = Describe("InstanceTypes", func() { }) It("should launch trn1 instances for AWS Neuron resource requests", func() { nodeNames := sets.NewString() - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ + nodePool.Spec.Template.Spec.Requirements = []corev1beta1.NodeSelectorRequirementWithMinValues{ { - Key: v1.LabelInstanceTypeStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{"trn1.2xlarge"}, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1.LabelInstanceTypeStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{"trn1.2xlarge"}, + }, }, } ExpectApplied(ctx, env.Client, nodePool, nodeClass) @@ -623,11 +673,13 @@ var _ = Describe("InstanceTypes", func() { Expect(nodeNames.Len()).To(Equal(1)) }) It("should launch instances for vpc.amazonaws.com/efa resource requests", func() { - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ + nodePool.Spec.Template.Spec.Requirements = []corev1beta1.NodeSelectorRequirementWithMinValues{ { - Key: v1.LabelInstanceTypeStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{"dl1.24xlarge"}, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1.LabelInstanceTypeStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{"dl1.24xlarge"}, + }, }, } ExpectApplied(ctx, env.Client, nodePool, nodeClass) @@ -681,7 +733,21 @@ var _ = Describe("InstanceTypes", func() { instanceInfo, err := awsEnv.InstanceTypesProvider.GetInstanceTypes(ctx) Expect(err).To(BeNil()) for _, info := range instanceInfo { - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Capacity.Pods().Value()).ToNot(BeNumerically("==", 110)) } }) @@ -690,38 +756,89 @@ var _ = Describe("InstanceTypes", func() { Expect(err).To(BeNil()) for _, info := range instanceInfo { - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, windowsNodeClass, nil) + amiFamily := amifamily.GetAMIFamily(windowsNodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + windowsNodeClass.Spec.BlockDeviceMappings, + windowsNodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", 110)) } }) - - It("should expose vcpu metrics for instance types", func() { - instanceInfo, err := awsEnv.InstanceTypesProvider.List(ctx, nodePool.Spec.Template.Spec.Kubelet, nodeClass) - Expect(err).To(BeNil()) - Expect(len(instanceInfo)).To(BeNumerically(">", 0)) - for _, info := range instanceInfo { - metric, ok := FindMetricWithLabelValues("karpenter_cloudprovider_instance_type_cpu_cores", map[string]string{ - instancetype.InstanceTypeLabel: info.Name, - }) - Expect(ok).To(BeTrue()) - Expect(metric).To(Not(BeNil())) - value := metric.GetGauge().Value - Expect(aws.Float64Value(value)).To(BeNumerically(">", 0)) - } - }) - It("should expose memory metrics for instance types", func() { - instanceInfo, err := awsEnv.InstanceTypesProvider.List(ctx, nodePool.Spec.Template.Spec.Kubelet, nodeClass) - Expect(err).To(BeNil()) - Expect(len(instanceInfo)).To(BeNumerically(">", 0)) - for _, info := range instanceInfo { - metric, ok := FindMetricWithLabelValues("karpenter_cloudprovider_instance_type_memory_bytes", map[string]string{ - instancetype.InstanceTypeLabel: info.Name, - }) - Expect(ok).To(BeTrue()) - Expect(metric).To(Not(BeNil())) - value := metric.GetGauge().Value - Expect(aws.Float64Value(value)).To(BeNumerically(">", 0)) - } + Context("Metrics", func() { + It("should expose vcpu metrics for instance types", func() { + instanceTypes, err := awsEnv.InstanceTypesProvider.List(ctx, nodePool.Spec.Template.Spec.Kubelet, nodeClass) + Expect(err).To(BeNil()) + Expect(len(instanceTypes)).To(BeNumerically(">", 0)) + for _, it := range instanceTypes { + metric, ok := FindMetricWithLabelValues("karpenter_cloudprovider_instance_type_cpu_cores", map[string]string{ + "instance_type": it.Name, + }) + Expect(ok).To(BeTrue()) + Expect(metric).To(Not(BeNil())) + value := metric.GetGauge().Value + Expect(aws.Float64Value(value)).To(BeNumerically(">", 0)) + } + }) + It("should expose memory metrics for instance types", func() { + instanceTypes, err := awsEnv.InstanceTypesProvider.List(ctx, nodePool.Spec.Template.Spec.Kubelet, nodeClass) + Expect(err).To(BeNil()) + Expect(len(instanceTypes)).To(BeNumerically(">", 0)) + for _, it := range instanceTypes { + metric, ok := FindMetricWithLabelValues("karpenter_cloudprovider_instance_type_memory_bytes", map[string]string{ + "instance_type": it.Name, + }) + Expect(ok).To(BeTrue()) + Expect(metric).To(Not(BeNil())) + value := metric.GetGauge().Value + Expect(aws.Float64Value(value)).To(BeNumerically(">", 0)) + } + }) + It("should expose availability metrics for instance types", func() { + instanceTypes, err := awsEnv.InstanceTypesProvider.List(ctx, nodePool.Spec.Template.Spec.Kubelet, nodeClass) + Expect(err).To(BeNil()) + Expect(len(instanceTypes)).To(BeNumerically(">", 0)) + for _, it := range instanceTypes { + for _, of := range it.Offerings { + metric, ok := FindMetricWithLabelValues("karpenter_cloudprovider_instance_type_offering_available", map[string]string{ + "instance_type": it.Name, + "capacity_type": of.CapacityType, + "zone": of.Zone, + }) + Expect(ok).To(BeTrue()) + Expect(metric).To(Not(BeNil())) + value := metric.GetGauge().Value + Expect(aws.Float64Value(value)).To(BeNumerically("==", lo.Ternary(of.Available, 1, 0))) + } + } + }) + It("should expose pricing metrics for instance types", func() { + instanceTypes, err := awsEnv.InstanceTypesProvider.List(ctx, nodePool.Spec.Template.Spec.Kubelet, nodeClass) + Expect(err).To(BeNil()) + Expect(len(instanceTypes)).To(BeNumerically(">", 0)) + for _, it := range instanceTypes { + for _, of := range it.Offerings { + metric, ok := FindMetricWithLabelValues("karpenter_cloudprovider_instance_type_offering_price_estimate", map[string]string{ + "instance_type": it.Name, + "capacity_type": of.CapacityType, + "zone": of.Zone, + }) + Expect(ok).To(BeTrue()) + Expect(metric).To(Not(BeNil())) + value := metric.GetGauge().Value + Expect(aws.Float64Value(value)).To(BeNumerically("==", of.Price)) + } + } + }) }) It("should launch instances in local zones", func() { ExpectApplied(ctx, env.Client, nodePool, nodeClass) @@ -754,20 +871,48 @@ var _ = Describe("InstanceTypes", func() { }) Context("System Reserved Resources", func() { It("should use defaults when no kubelet is specified", func() { - it := instancetype.NewInstanceType(ctx, info, &corev1beta1.KubeletConfiguration{}, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Overhead.SystemReserved.Cpu().String()).To(Equal("0")) Expect(it.Overhead.SystemReserved.Memory().String()).To(Equal("0")) Expect(it.Overhead.SystemReserved.StorageEphemeral().String()).To(Equal("0")) }) It("should override system reserved cpus when specified", func() { nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ - SystemReserved: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("2"), - v1.ResourceMemory: resource.MustParse("20Gi"), - v1.ResourceEphemeralStorage: resource.MustParse("10Gi"), + SystemReserved: map[string]string{ + string(v1.ResourceCPU): "2", + string(v1.ResourceMemory): "20Gi", + string(v1.ResourceEphemeralStorage): "10Gi", }, } - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Overhead.SystemReserved.Cpu().String()).To(Equal("2")) Expect(it.Overhead.SystemReserved.Memory().String()).To(Equal("20Gi")) Expect(it.Overhead.SystemReserved.StorageEphemeral().String()).To(Equal("10Gi")) @@ -775,24 +920,54 @@ var _ = Describe("InstanceTypes", func() { }) Context("Kube Reserved Resources", func() { It("should use defaults when no kubelet is specified", func() { - it := instancetype.NewInstanceType(ctx, info, &corev1beta1.KubeletConfiguration{}, fake.DefaultRegion, nodeClass, nil) + nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{} + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Overhead.KubeReserved.Cpu().String()).To(Equal("80m")) Expect(it.Overhead.KubeReserved.Memory().String()).To(Equal("893Mi")) Expect(it.Overhead.KubeReserved.StorageEphemeral().String()).To(Equal("1Gi")) }) It("should override kube reserved when specified", func() { - it := instancetype.NewInstanceType(ctx, info, &corev1beta1.KubeletConfiguration{ - SystemReserved: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("1"), - v1.ResourceMemory: resource.MustParse("20Gi"), - v1.ResourceEphemeralStorage: resource.MustParse("1Gi"), + nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ + SystemReserved: map[string]string{ + string(v1.ResourceCPU): "1", + string(v1.ResourceMemory): "20Gi", + string(v1.ResourceEphemeralStorage): "1Gi", }, - KubeReserved: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("2"), - v1.ResourceMemory: resource.MustParse("10Gi"), - v1.ResourceEphemeralStorage: resource.MustParse("2Gi"), + KubeReserved: map[string]string{ + string(v1.ResourceCPU): "2", + string(v1.ResourceMemory): "10Gi", + string(v1.ResourceEphemeralStorage): "2Gi", }, - }, fake.DefaultRegion, nodeClass, nil) + } + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Overhead.KubeReserved.Cpu().String()).To(Equal("2")) Expect(it.Overhead.KubeReserved.Memory().String()).To(Equal("10Gi")) Expect(it.Overhead.KubeReserved.StorageEphemeral().String()).To(Equal("2Gi")) @@ -807,88 +982,158 @@ var _ = Describe("InstanceTypes", func() { Context("Eviction Hard", func() { It("should override eviction threshold when specified as a quantity", func() { nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ - SystemReserved: v1.ResourceList{ - v1.ResourceMemory: resource.MustParse("20Gi"), + SystemReserved: map[string]string{ + string(v1.ResourceMemory): "20Gi", }, - KubeReserved: v1.ResourceList{ - v1.ResourceMemory: resource.MustParse("10Gi"), + KubeReserved: map[string]string{ + string(v1.ResourceMemory): "10Gi", }, EvictionHard: map[string]string{ instancetype.MemoryAvailable: "500Mi", }, } - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Overhead.EvictionThreshold.Memory().String()).To(Equal("500Mi")) }) It("should override eviction threshold when specified as a percentage value", func() { nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ - SystemReserved: v1.ResourceList{ - v1.ResourceMemory: resource.MustParse("20Gi"), + SystemReserved: map[string]string{ + string(v1.ResourceMemory): "20Gi", }, - KubeReserved: v1.ResourceList{ - v1.ResourceMemory: resource.MustParse("10Gi"), + KubeReserved: map[string]string{ + string(v1.ResourceMemory): "10Gi", }, EvictionHard: map[string]string{ instancetype.MemoryAvailable: "10%", }, } - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Overhead.EvictionThreshold.Memory().Value()).To(BeNumerically("~", float64(it.Capacity.Memory().Value())*0.1, 10)) }) It("should consider the eviction threshold disabled when specified as 100%", func() { nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ - SystemReserved: v1.ResourceList{ - v1.ResourceMemory: resource.MustParse("20Gi"), + SystemReserved: map[string]string{ + string(v1.ResourceMemory): "20Gi", }, - KubeReserved: v1.ResourceList{ - v1.ResourceMemory: resource.MustParse("10Gi"), + KubeReserved: map[string]string{ + string(v1.ResourceMemory): "10Gi", }, EvictionHard: map[string]string{ instancetype.MemoryAvailable: "100%", }, } - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Overhead.EvictionThreshold.Memory().String()).To(Equal("0")) }) It("should used default eviction threshold for memory when evictionHard not specified", func() { nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ - SystemReserved: v1.ResourceList{ - v1.ResourceMemory: resource.MustParse("20Gi"), + SystemReserved: map[string]string{ + string(v1.ResourceMemory): "20Gi", }, - KubeReserved: v1.ResourceList{ - v1.ResourceMemory: resource.MustParse("10Gi"), + KubeReserved: map[string]string{ + string(v1.ResourceMemory): "10Gi", }, EvictionSoft: map[string]string{ instancetype.MemoryAvailable: "50Mi", }, } - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Overhead.EvictionThreshold.Memory().String()).To(Equal("50Mi")) }) }) Context("Eviction Soft", func() { It("should override eviction threshold when specified as a quantity", func() { nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ - SystemReserved: v1.ResourceList{ - v1.ResourceMemory: resource.MustParse("20Gi"), + SystemReserved: map[string]string{ + string(v1.ResourceMemory): "20Gi", }, - KubeReserved: v1.ResourceList{ - v1.ResourceMemory: resource.MustParse("10Gi"), + KubeReserved: map[string]string{ + string(v1.ResourceMemory): "10Gi", }, EvictionSoft: map[string]string{ instancetype.MemoryAvailable: "500Mi", }, } - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Overhead.EvictionThreshold.Memory().String()).To(Equal("500Mi")) }) It("should override eviction threshold when specified as a percentage value", func() { nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ - SystemReserved: v1.ResourceList{ - v1.ResourceMemory: resource.MustParse("20Gi"), + SystemReserved: map[string]string{ + string(v1.ResourceMemory): "20Gi", }, - KubeReserved: v1.ResourceList{ - v1.ResourceMemory: resource.MustParse("10Gi"), + KubeReserved: map[string]string{ + string(v1.ResourceMemory): "10Gi", }, EvictionHard: map[string]string{ instancetype.MemoryAvailable: "5%", @@ -897,32 +1142,60 @@ var _ = Describe("InstanceTypes", func() { instancetype.MemoryAvailable: "10%", }, } - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Overhead.EvictionThreshold.Memory().Value()).To(BeNumerically("~", float64(it.Capacity.Memory().Value())*0.1, 10)) }) It("should consider the eviction threshold disabled when specified as 100%", func() { nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ - SystemReserved: v1.ResourceList{ - v1.ResourceMemory: resource.MustParse("20Gi"), + SystemReserved: map[string]string{ + string(v1.ResourceMemory): "20Gi", }, - KubeReserved: v1.ResourceList{ - v1.ResourceMemory: resource.MustParse("10Gi"), + KubeReserved: map[string]string{ + string(v1.ResourceMemory): "10Gi", }, EvictionSoft: map[string]string{ instancetype.MemoryAvailable: "100%", }, } - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Overhead.EvictionThreshold.Memory().String()).To(Equal("0")) }) It("should ignore eviction threshold when using Bottlerocket AMI", func() { nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ - SystemReserved: v1.ResourceList{ - v1.ResourceMemory: resource.MustParse("20Gi"), + SystemReserved: map[string]string{ + string(v1.ResourceMemory): "20Gi", }, - KubeReserved: v1.ResourceList{ - v1.ResourceMemory: resource.MustParse("10Gi"), + KubeReserved: map[string]string{ + string(v1.ResourceMemory): "10Gi", }, EvictionHard: map[string]string{ instancetype.MemoryAvailable: "1Gi", @@ -931,23 +1204,52 @@ var _ = Describe("InstanceTypes", func() { instancetype.MemoryAvailable: "10Gi", }, } - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Overhead.EvictionThreshold.Memory().String()).To(Equal("1Gi")) }) }) It("should take the default eviction threshold when none is specified", func() { - it := instancetype.NewInstanceType(ctx, info, &corev1beta1.KubeletConfiguration{}, fake.DefaultRegion, nodeClass, nil) + nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{} + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Overhead.EvictionThreshold.Cpu().String()).To(Equal("0")) Expect(it.Overhead.EvictionThreshold.Memory().String()).To(Equal("100Mi")) Expect(it.Overhead.EvictionThreshold.StorageEphemeral().AsApproximateFloat64()).To(BeNumerically("~", resources.Quantity("2Gi").AsApproximateFloat64())) }) It("should take the greater of evictionHard and evictionSoft for overhead as a value", func() { nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ - SystemReserved: v1.ResourceList{ - v1.ResourceMemory: resource.MustParse("20Gi"), + SystemReserved: map[string]string{ + string(v1.ResourceMemory): "20Gi", }, - KubeReserved: v1.ResourceList{ - v1.ResourceMemory: resource.MustParse("10Gi"), + KubeReserved: map[string]string{ + string(v1.ResourceMemory): "10Gi", }, EvictionSoft: map[string]string{ instancetype.MemoryAvailable: "3Gi", @@ -956,16 +1258,30 @@ var _ = Describe("InstanceTypes", func() { instancetype.MemoryAvailable: "1Gi", }, } - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Overhead.EvictionThreshold.Memory().String()).To(Equal("3Gi")) }) It("should take the greater of evictionHard and evictionSoft for overhead as a value", func() { nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ - SystemReserved: v1.ResourceList{ - v1.ResourceMemory: resource.MustParse("20Gi"), + SystemReserved: map[string]string{ + string(v1.ResourceMemory): "20Gi", }, - KubeReserved: v1.ResourceList{ - v1.ResourceMemory: resource.MustParse("10Gi"), + KubeReserved: map[string]string{ + string(v1.ResourceMemory): "10Gi", }, EvictionSoft: map[string]string{ instancetype.MemoryAvailable: "2%", @@ -974,16 +1290,30 @@ var _ = Describe("InstanceTypes", func() { instancetype.MemoryAvailable: "5%", }, } - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Overhead.EvictionThreshold.Memory().Value()).To(BeNumerically("~", float64(it.Capacity.Memory().Value())*0.05, 10)) }) It("should take the greater of evictionHard and evictionSoft for overhead with mixed percentage/value", func() { nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ - SystemReserved: v1.ResourceList{ - v1.ResourceMemory: resource.MustParse("20Gi"), + SystemReserved: map[string]string{ + string(v1.ResourceMemory): "20Gi", }, - KubeReserved: v1.ResourceList{ - v1.ResourceMemory: resource.MustParse("10Gi"), + KubeReserved: map[string]string{ + string(v1.ResourceMemory): "10Gi", }, EvictionSoft: map[string]string{ instancetype.MemoryAvailable: "10%", @@ -992,7 +1322,21 @@ var _ = Describe("InstanceTypes", func() { instancetype.MemoryAvailable: "1Gi", }, } - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Overhead.EvictionThreshold.Memory().Value()).To(BeNumerically("~", float64(it.Capacity.Memory().Value())*0.1, 10)) }) }) @@ -1001,11 +1345,39 @@ var _ = Describe("InstanceTypes", func() { Expect(err).To(BeNil()) for _, info := range instanceInfo { if *info.InstanceType == "t3.large" { - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", 35)) } if *info.InstanceType == "m6idn.32xlarge" { - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", 345)) } } @@ -1017,7 +1389,21 @@ var _ = Describe("InstanceTypes", func() { MaxPods: ptr.Int32(10), } for _, info := range instanceInfo { - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", 10)) } }) @@ -1028,7 +1414,21 @@ var _ = Describe("InstanceTypes", func() { MaxPods: ptr.Int32(10), } for _, info := range instanceInfo { - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", 10)) } }) @@ -1043,7 +1443,21 @@ var _ = Describe("InstanceTypes", func() { return *info.InstanceType == "t3.large" }) Expect(ok).To(Equal(true)) - it := instancetype.NewInstanceType(ctx, t3Large, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + t3Large, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) // t3.large // maxInterfaces = 3 // maxIPv4PerInterface = 12 @@ -1063,7 +1477,21 @@ var _ = Describe("InstanceTypes", func() { return *info.InstanceType == "t3.large" }) Expect(ok).To(Equal(true)) - it := instancetype.NewInstanceType(ctx, t3Large, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + t3Large, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) // t3.large // maxInterfaces = 3 // maxIPv4PerInterface = 12 @@ -1080,7 +1508,21 @@ var _ = Describe("InstanceTypes", func() { PodsPerCore: ptr.Int32(1), } for _, info := range instanceInfo { - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", ptr.Int64Value(info.VCpuInfo.DefaultVCpus))) } }) @@ -1092,7 +1534,21 @@ var _ = Describe("InstanceTypes", func() { MaxPods: ptr.Int32(20), } for _, info := range instanceInfo { - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", lo.Min([]int64{20, ptr.Int64Value(info.VCpuInfo.DefaultVCpus) * 4}))) } }) @@ -1104,7 +1560,21 @@ var _ = Describe("InstanceTypes", func() { PodsPerCore: ptr.Int32(1), } for _, info := range instanceInfo { - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) limitedPods := instancetype.ENILimitedPods(ctx, info) Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", limitedPods.Value())) } @@ -1117,11 +1587,39 @@ var _ = Describe("InstanceTypes", func() { } for _, info := range instanceInfo { if *info.InstanceType == "t3.large" { - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", 35)) } if *info.InstanceType == "m6idn.32xlarge" { - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, fake.DefaultRegion, nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + fake.DefaultRegion, + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) Expect(it.Capacity.Pods().Value()).To(BeNumerically("==", 345)) } } @@ -1279,10 +1777,12 @@ var _ = Describe("InstanceTypes", func() { awsEnv.EC2API.InsufficientCapacityPools.Set([]fake.CapacityPool{ {CapacityType: corev1beta1.CapacityTypeOnDemand, InstanceType: "m5.xlarge", Zone: "test-zone-1a"}, }) - nodePool.Spec.Template.Spec.Requirements = append(nodePool.Spec.Template.Spec.Requirements, v1.NodeSelectorRequirement{ - Key: v1.LabelInstanceType, - Operator: v1.NodeSelectorOpIn, - Values: []string{"m5.large", "m5.xlarge"}, + nodePool.Spec.Template.Spec.Requirements = append(nodePool.Spec.Template.Spec.Requirements, corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1.LabelInstanceType, + Operator: v1.NodeSelectorOpIn, + Values: []string{"m5.large", "m5.xlarge"}, + }, }) pods := []*v1.Pod{} for i := 0; i < 2; i++ { @@ -1360,9 +1860,9 @@ var _ = Describe("InstanceTypes", func() { } return true })).To(Succeed()) - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: corev1beta1.CapacityTypeLabelKey, Operator: v1.NodeSelectorOpIn, Values: []string{corev1beta1.CapacityTypeSpot, corev1beta1.CapacityTypeOnDemand}}, - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1a"}}, + nodePool.Spec.Template.Spec.Requirements = []corev1beta1.NodeSelectorRequirementWithMinValues{ + {NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: corev1beta1.CapacityTypeLabelKey, Operator: v1.NodeSelectorOpIn, Values: []string{corev1beta1.CapacityTypeSpot, corev1beta1.CapacityTypeOnDemand}}}, + {NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1a"}}}, } // Spot Unavailable ExpectApplied(ctx, env.Client, nodePool, nodeClass) @@ -1383,15 +1883,20 @@ var _ = Describe("InstanceTypes", func() { {CapacityType: corev1beta1.CapacityTypeSpot, InstanceType: "m5.xlarge", Zone: "test-zone-1b"}, }) nodePool.Spec.Template.Spec.Requirements = nil - nodePool.Spec.Template.Spec.Requirements = append(nodePool.Spec.Template.Spec.Requirements, v1.NodeSelectorRequirement{ - Key: v1.LabelInstanceType, - Operator: v1.NodeSelectorOpIn, - Values: []string{"m5.xlarge"}, - }) - nodePool.Spec.Template.Spec.Requirements = append(nodePool.Spec.Template.Spec.Requirements, v1.NodeSelectorRequirement{ - Key: corev1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: []string{"spot", "on-demand"}, + nodePool.Spec.Template.Spec.Requirements = append(nodePool.Spec.Template.Spec.Requirements, corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1.LabelInstanceType, + Operator: v1.NodeSelectorOpIn, + Values: []string{"m5.xlarge"}, + }, + }, + ) + nodePool.Spec.Template.Spec.Requirements = append(nodePool.Spec.Template.Spec.Requirements, corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: corev1beta1.CapacityTypeLabelKey, + Operator: v1.NodeSelectorOpIn, + Values: []string{"spot", "on-demand"}, + }, }) ExpectApplied(ctx, env.Client, nodePool, nodeClass) @@ -1433,8 +1938,8 @@ var _ = Describe("InstanceTypes", func() { Expect(node.Labels).To(HaveKeyWithValue(corev1beta1.CapacityTypeLabelKey, corev1beta1.CapacityTypeOnDemand)) }) It("should launch spot capacity if flexible to both spot and on demand", func() { - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: corev1beta1.CapacityTypeLabelKey, Operator: v1.NodeSelectorOpIn, Values: []string{corev1beta1.CapacityTypeSpot, corev1beta1.CapacityTypeOnDemand}}} + nodePool.Spec.Template.Spec.Requirements = []corev1beta1.NodeSelectorRequirementWithMinValues{ + {NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: corev1beta1.CapacityTypeLabelKey, Operator: v1.NodeSelectorOpIn, Values: []string{corev1beta1.CapacityTypeSpot, corev1beta1.CapacityTypeOnDemand}}}} ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) @@ -1455,10 +1960,10 @@ var _ = Describe("InstanceTypes", func() { }) Expect(awsEnv.PricingProvider.UpdateSpotPricing(ctx)).To(Succeed()) - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: corev1beta1.CapacityTypeLabelKey, Operator: v1.NodeSelectorOpIn, Values: []string{corev1beta1.CapacityTypeSpot}}, - {Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{"m5.large"}}, - {Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1b"}}, + nodePool.Spec.Template.Spec.Requirements = []corev1beta1.NodeSelectorRequirementWithMinValues{ + {NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: corev1beta1.CapacityTypeLabelKey, Operator: v1.NodeSelectorOpIn, Values: []string{corev1beta1.CapacityTypeSpot}}}, + {NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{"m5.large"}}}, + {NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: v1.LabelTopologyZone, Operator: v1.NodeSelectorOpIn, Values: []string{"test-zone-1b"}}}, } // Instance type with no zonal availability for spot shouldn't be scheduled @@ -1482,9 +1987,9 @@ var _ = Describe("InstanceTypes", func() { Expect(awsEnv.PricingProvider.UpdateSpotPricing(ctx)).To(Succeed()) // not restricting to the zone so we can get any zone - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ - {Key: corev1beta1.CapacityTypeLabelKey, Operator: v1.NodeSelectorOpIn, Values: []string{corev1beta1.CapacityTypeSpot}}, - {Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{"m5.large"}}, + nodePool.Spec.Template.Spec.Requirements = []corev1beta1.NodeSelectorRequirementWithMinValues{ + {NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: corev1beta1.CapacityTypeLabelKey, Operator: v1.NodeSelectorOpIn, Values: []string{corev1beta1.CapacityTypeSpot}}}, + {NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, Values: []string{"m5.large"}}}, } ExpectApplied(ctx, env.Client, nodePool, nodeClass) @@ -1540,6 +2045,22 @@ var _ = Describe("InstanceTypes", func() { Expect(*ltInput.LaunchTemplateData.BlockDeviceMappings[0].Ebs.SnapshotId).To(Equal("snap-xxxxxxxx")) }) }) + It("should default to EBS defaults when volumeSize is not defined in blockDeviceMappings for AL2023 Root volume", func() { + nodeClass.Spec.AMIFamily = aws.String(v1beta1.AMIFamilyAL2023) + awsEnv.LaunchTemplateProvider.CABundle = lo.ToPtr("Y2EtYnVuZGxlCg==") + awsEnv.LaunchTemplateProvider.ClusterCIDR.Store(lo.ToPtr("10.100.0.0/16")) + ExpectApplied(ctx, env.Client, nodePool, nodeClass) + pod := coretest.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + node := ExpectScheduled(ctx, env.Client, pod) + Expect(*node.Status.Capacity.StorageEphemeral()).To(Equal(resource.MustParse("20Gi"))) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(BeNumerically(">=", 1)) + awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.ForEach(func(ltInput *ec2.CreateLaunchTemplateInput) { + Expect(ltInput.LaunchTemplateData.BlockDeviceMappings).To(HaveLen(1)) + Expect(*ltInput.LaunchTemplateData.BlockDeviceMappings[0].DeviceName).To(Equal("/dev/xvda")) + Expect(*ltInput.LaunchTemplateData.BlockDeviceMappings[0].Ebs.SnapshotId).To(Equal("snap-xxxxxxxx")) + }) + }) It("should default to EBS defaults when volumeSize is not defined in blockDeviceMappings for Bottlerocket Root volume", func() { nodeClass.Spec.AMIFamily = aws.String(v1beta1.AMIFamilyBottlerocket) nodeClass.Spec.BlockDeviceMappings[0].DeviceName = aws.String("/dev/xvdb") @@ -1641,48 +2162,3 @@ func generateSpotPricing(cp *cloudprovider.CloudProvider, nodePool *corev1beta1. } return rsp } - -func makeFakeInstances() []*ec2.InstanceTypeInfo { - var instanceTypes []*ec2.InstanceTypeInfo - ctx := options.ToContext(context.Background(), &options.Options{IsolatedVPC: true}) - // Use keys from the static pricing data so that we guarantee pricing for the data - // Create uniform instance data so all of them schedule for a given pod - for _, it := range pricing.NewProvider(ctx, nil, nil, "us-east-1").InstanceTypes() { - instanceTypes = append(instanceTypes, &ec2.InstanceTypeInfo{ - InstanceType: aws.String(it), - ProcessorInfo: &ec2.ProcessorInfo{ - SupportedArchitectures: aws.StringSlice([]string{"x86_64"}), - }, - VCpuInfo: &ec2.VCpuInfo{ - DefaultCores: aws.Int64(1), - DefaultVCpus: aws.Int64(2), - }, - MemoryInfo: &ec2.MemoryInfo{ - SizeInMiB: aws.Int64(8192), - }, - NetworkInfo: &ec2.NetworkInfo{ - Ipv4AddressesPerInterface: aws.Int64(10), - DefaultNetworkCardIndex: aws.Int64(0), - NetworkCards: []*ec2.NetworkCardInfo{{ - NetworkCardIndex: lo.ToPtr(int64(0)), - MaximumNetworkInterfaces: aws.Int64(3), - }}, - }, - SupportedUsageClasses: fake.DefaultSupportedUsageClasses, - }) - } - return instanceTypes -} - -func makeFakeInstanceOfferings(instanceTypes []*ec2.InstanceTypeInfo) []*ec2.InstanceTypeOffering { - var instanceTypeOfferings []*ec2.InstanceTypeOffering - - // Create uniform instance offering data so all of them schedule for a given pod - for _, instanceType := range instanceTypes { - instanceTypeOfferings = append(instanceTypeOfferings, &ec2.InstanceTypeOffering{ - InstanceType: instanceType.InstanceType, - Location: aws.String("test-zone-1a"), - }) - } - return instanceTypeOfferings -} diff --git a/pkg/providers/instancetype/types.go b/pkg/providers/instancetype/types.go index 566f0554fa3f..6cebec2f523b 100644 --- a/pkg/providers/instancetype/types.go +++ b/pkg/providers/instancetype/types.go @@ -49,19 +49,20 @@ var ( instanceTypeScheme = regexp.MustCompile(`(^[a-z]+)(\-[0-9]+tb)?([0-9]+).*\.`) ) -func NewInstanceType(ctx context.Context, info *ec2.InstanceTypeInfo, kc *corev1beta1.KubeletConfiguration, - region string, nodeClass *v1beta1.EC2NodeClass, offerings cloudprovider.Offerings) *cloudprovider.InstanceType { +func NewInstanceType(ctx context.Context, info *ec2.InstanceTypeInfo, region string, + blockDeviceMappings []*v1beta1.BlockDeviceMapping, instanceStorePolicy *v1beta1.InstanceStorePolicy, maxPods *int32, podsPerCore *int32, + kubeReserved map[string]string, systemReserved map[string]string, evictionHard map[string]string, evictionSoft map[string]string, + amiFamily amifamily.AMIFamily, offerings cloudprovider.Offerings) *cloudprovider.InstanceType { - amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) it := &cloudprovider.InstanceType{ Name: aws.StringValue(info.InstanceType), Requirements: computeRequirements(info, offerings, region, amiFamily), Offerings: offerings, - Capacity: computeCapacity(ctx, info, amiFamily, nodeClass, kc), + Capacity: computeCapacity(ctx, info, amiFamily, blockDeviceMappings, instanceStorePolicy, maxPods, podsPerCore), Overhead: &cloudprovider.InstanceTypeOverhead{ - KubeReserved: kubeReservedResources(cpu(info), pods(ctx, info, amiFamily, kc), ENILimitedPods(ctx, info), amiFamily, kc), - SystemReserved: systemReservedResources(kc), - EvictionThreshold: evictionThreshold(memory(ctx, info), ephemeralStorage(info, amiFamily, nodeClass), amiFamily, kc), + KubeReserved: kubeReservedResources(cpu(info), pods(ctx, info, amiFamily, maxPods, podsPerCore), ENILimitedPods(ctx, info), amiFamily, kubeReserved), + SystemReserved: systemReservedResources(systemReserved), + EvictionThreshold: evictionThreshold(memory(ctx, info), ephemeralStorage(info, amiFamily, blockDeviceMappings, instanceStorePolicy), amiFamily, evictionHard, evictionSoft), }, } if it.Requirements.Compatible(scheduling.NewRequirements(scheduling.NewRequirement(v1.LabelOSStable, v1.NodeSelectorOpIn, string(v1.Windows)))) == nil { @@ -84,6 +85,7 @@ func computeRequirements(info *ec2.InstanceTypeInfo, offerings cloudprovider.Off scheduling.NewRequirement(corev1beta1.CapacityTypeLabelKey, v1.NodeSelectorOpIn, lo.Map(offerings.Available(), func(o cloudprovider.Offering, _ int) string { return o.CapacityType })...), // Well Known to AWS scheduling.NewRequirement(v1beta1.LabelInstanceCPU, v1.NodeSelectorOpIn, fmt.Sprint(aws.Int64Value(info.VCpuInfo.DefaultVCpus))), + scheduling.NewRequirement(v1beta1.LabelInstanceCPUManufacturer, v1.NodeSelectorOpDoesNotExist), scheduling.NewRequirement(v1beta1.LabelInstanceMemory, v1.NodeSelectorOpIn, fmt.Sprint(aws.Int64Value(info.MemoryInfo.SizeInMiB))), scheduling.NewRequirement(v1beta1.LabelInstanceNetworkBandwidth, v1.NodeSelectorOpDoesNotExist), scheduling.NewRequirement(v1beta1.LabelInstanceCategory, v1.NodeSelectorOpDoesNotExist), @@ -146,6 +148,10 @@ func computeRequirements(info *ec2.InstanceTypeInfo, offerings cloudprovider.Off requirements.Get(v1beta1.LabelInstanceAcceleratorManufacturer).Insert(lowerKabobCase("AWS")) requirements.Get(v1beta1.LabelInstanceAcceleratorCount).Insert(fmt.Sprint(awsNeurons(info))) } + // CPU Manufacturer, valid options: aws, intel, amd + if info.ProcessorInfo != nil { + requirements.Get(v1beta1.LabelInstanceCPUManufacturer).Insert(lowerKabobCase(aws.StringValue(info.ProcessorInfo.Manufacturer))) + } return requirements } @@ -169,13 +175,14 @@ func getArchitecture(info *ec2.InstanceTypeInfo) string { } func computeCapacity(ctx context.Context, info *ec2.InstanceTypeInfo, amiFamily amifamily.AMIFamily, - nodeClass *v1beta1.EC2NodeClass, kc *corev1beta1.KubeletConfiguration) v1.ResourceList { + blockDeviceMapping []*v1beta1.BlockDeviceMapping, instanceStorePolicy *v1beta1.InstanceStorePolicy, + maxPods *int32, podsPerCore *int32) v1.ResourceList { resourceList := v1.ResourceList{ v1.ResourceCPU: *cpu(info), v1.ResourceMemory: *memory(ctx, info), - v1.ResourceEphemeralStorage: *ephemeralStorage(info, amiFamily, nodeClass), - v1.ResourcePods: *pods(ctx, info, amiFamily, kc), + v1.ResourceEphemeralStorage: *ephemeralStorage(info, amiFamily, blockDeviceMapping, instanceStorePolicy), + v1.ResourcePods: *pods(ctx, info, amiFamily, maxPods, podsPerCore), v1beta1.ResourceAWSPodENI: *awsPodENI(aws.StringValue(info.InstanceType)), v1beta1.ResourceNVIDIAGPU: *nvidiaGPUs(info), v1beta1.ResourceAMDGPU: *amdGPUs(info), @@ -203,16 +210,16 @@ func memory(ctx context.Context, info *ec2.InstanceTypeInfo) *resource.Quantity } // Setting ephemeral-storage to be either the default value, what is defined in blockDeviceMappings, or the combined size of local store volumes. -func ephemeralStorage(info *ec2.InstanceTypeInfo, amiFamily amifamily.AMIFamily, nodeClass *v1beta1.EC2NodeClass) *resource.Quantity { +func ephemeralStorage(info *ec2.InstanceTypeInfo, amiFamily amifamily.AMIFamily, blockDeviceMappings []*v1beta1.BlockDeviceMapping, instanceStorePolicy *v1beta1.InstanceStorePolicy) *resource.Quantity { // If local store disks have been configured for node ephemeral-storage, use the total size of the disks. - if lo.FromPtr(nodeClass.Spec.InstanceStorePolicy) == v1beta1.InstanceStorePolicyRAID0 { + if lo.FromPtr(instanceStorePolicy) == v1beta1.InstanceStorePolicyRAID0 { if info.InstanceStorageInfo != nil && info.InstanceStorageInfo.TotalSizeInGB != nil { return resources.Quantity(fmt.Sprintf("%dG", *info.InstanceStorageInfo.TotalSizeInGB)) } } - if len(nodeClass.Spec.BlockDeviceMappings) != 0 { + if len(blockDeviceMappings) != 0 { // First check if there's a root volume configured in blockDeviceMappings. - if blockDeviceMapping, ok := lo.Find(nodeClass.Spec.BlockDeviceMappings, func(bdm *v1beta1.BlockDeviceMapping) bool { + if blockDeviceMapping, ok := lo.Find(blockDeviceMappings, func(bdm *v1beta1.BlockDeviceMapping) bool { return bdm.RootVolume }); ok && blockDeviceMapping.EBS.VolumeSize != nil { return blockDeviceMapping.EBS.VolumeSize @@ -220,11 +227,11 @@ func ephemeralStorage(info *ec2.InstanceTypeInfo, amiFamily amifamily.AMIFamily, switch amiFamily.(type) { case *amifamily.Custom: // We can't know if a custom AMI is going to have a volume size. - volumeSize := nodeClass.Spec.BlockDeviceMappings[len(nodeClass.Spec.BlockDeviceMappings)-1].EBS.VolumeSize + volumeSize := blockDeviceMappings[len(blockDeviceMappings)-1].EBS.VolumeSize return lo.Ternary(volumeSize != nil, volumeSize, amifamily.DefaultEBS.VolumeSize) default: // If a block device mapping exists in the provider for the root volume, use the volume size specified in the provider. If not, use the default - if blockDeviceMapping, ok := lo.Find(nodeClass.Spec.BlockDeviceMappings, func(bdm *v1beta1.BlockDeviceMapping) bool { + if blockDeviceMapping, ok := lo.Find(blockDeviceMappings, func(bdm *v1beta1.BlockDeviceMapping) bool { return *bdm.DeviceName == *amiFamily.EphemeralBlockDevice() }); ok && blockDeviceMapping.EBS.VolumeSize != nil { return blockDeviceMapping.EBS.VolumeSize @@ -333,14 +340,13 @@ func privateIPv4Address(info *ec2.InstanceTypeInfo) *resource.Quantity { return resources.Quantity(fmt.Sprint(capacity)) } -func systemReservedResources(kc *corev1beta1.KubeletConfiguration) v1.ResourceList { - if kc != nil && kc.SystemReserved != nil { - return kc.SystemReserved - } - return v1.ResourceList{} +func systemReservedResources(systemReserved map[string]string) v1.ResourceList { + return lo.MapEntries(systemReserved, func(k string, v string) (v1.ResourceName, resource.Quantity) { + return v1.ResourceName(k), resource.MustParse(v) + }) } -func kubeReservedResources(cpus, pods, eniLimitedPods *resource.Quantity, amiFamily amifamily.AMIFamily, kc *corev1beta1.KubeletConfiguration) v1.ResourceList { +func kubeReservedResources(cpus, pods, eniLimitedPods *resource.Quantity, amiFamily amifamily.AMIFamily, kubeReserved map[string]string) v1.ResourceList { if amiFamily.FeatureFlags().UsesENILimitedMemoryOverhead { pods = eniLimitedPods } @@ -370,28 +376,24 @@ func kubeReservedResources(cpus, pods, eniLimitedPods *resource.Quantity, amiFam resources[v1.ResourceCPU] = *cpuOverhead } } - if kc != nil && kc.KubeReserved != nil { - return lo.Assign(resources, kc.KubeReserved) - } - return resources + return lo.Assign(resources, lo.MapEntries(kubeReserved, func(k string, v string) (v1.ResourceName, resource.Quantity) { + return v1.ResourceName(k), resource.MustParse(v) + })) } -func evictionThreshold(memory *resource.Quantity, storage *resource.Quantity, amiFamily amifamily.AMIFamily, kc *corev1beta1.KubeletConfiguration) v1.ResourceList { +func evictionThreshold(memory *resource.Quantity, storage *resource.Quantity, amiFamily amifamily.AMIFamily, evictionHard map[string]string, evictionSoft map[string]string) v1.ResourceList { overhead := v1.ResourceList{ v1.ResourceMemory: resource.MustParse("100Mi"), v1.ResourceEphemeralStorage: resource.MustParse(fmt.Sprint(math.Ceil(float64(storage.Value()) / 100 * 10))), } - if kc == nil { - return overhead - } override := v1.ResourceList{} var evictionSignals []map[string]string - if kc.EvictionHard != nil { - evictionSignals = append(evictionSignals, kc.EvictionHard) + if evictionHard != nil { + evictionSignals = append(evictionSignals, evictionHard) } - if kc.EvictionSoft != nil && amiFamily.FeatureFlags().EvictionSoftEnabled { - evictionSignals = append(evictionSignals, kc.EvictionSoft) + if evictionSoft != nil && amiFamily.FeatureFlags().EvictionSoftEnabled { + evictionSignals = append(evictionSignals, evictionSoft) } for _, m := range evictionSignals { @@ -408,19 +410,19 @@ func evictionThreshold(memory *resource.Quantity, storage *resource.Quantity, am return lo.Assign(overhead, override) } -func pods(ctx context.Context, info *ec2.InstanceTypeInfo, amiFamily amifamily.AMIFamily, kc *corev1beta1.KubeletConfiguration) *resource.Quantity { +func pods(ctx context.Context, info *ec2.InstanceTypeInfo, amiFamily amifamily.AMIFamily, maxPods *int32, podsPerCore *int32) *resource.Quantity { var count int64 switch { - case kc != nil && kc.MaxPods != nil: - count = int64(ptr.Int32Value(kc.MaxPods)) + case maxPods != nil: + count = int64(ptr.Int32Value(maxPods)) case amiFamily.FeatureFlags().SupportsENILimitedPodDensity: count = ENILimitedPods(ctx, info).Value() default: count = 110 } - if kc != nil && ptr.Int32Value(kc.PodsPerCore) > 0 && amiFamily.FeatureFlags().PodsPerCoreEnabled { - count = lo.Min([]int64{int64(ptr.Int32Value(kc.PodsPerCore)) * ptr.Int64Value(info.VCpuInfo.DefaultVCpus), count}) + if ptr.Int32Value(podsPerCore) > 0 && amiFamily.FeatureFlags().PodsPerCoreEnabled { + count = lo.Min([]int64{int64(ptr.Int32Value(podsPerCore)) * ptr.Int64Value(info.VCpuInfo.DefaultVCpus), count}) } return resources.Quantity(fmt.Sprint(count)) } diff --git a/pkg/providers/instancetype/zz_generated.bandwidth.go b/pkg/providers/instancetype/zz_generated.bandwidth.go index bdc836c5cbbd..f8f92d72731f 100644 --- a/pkg/providers/instancetype/zz_generated.bandwidth.go +++ b/pkg/providers/instancetype/zz_generated.bandwidth.go @@ -571,12 +571,15 @@ var ( "c7g.16xlarge": 30000, "c7g.metal": 30000, "c7gd.16xlarge": 30000, + "c7gd.metal": 30000, "m7g.16xlarge": 30000, "m7g.metal": 30000, "m7gd.16xlarge": 30000, + "m7gd.metal": 30000, "r7g.16xlarge": 30000, "r7g.metal": 30000, "r7gd.16xlarge": 30000, + "r7gd.metal": 30000, "c6a.24xlarge": 37500, "c6i.24xlarge": 37500, "c6id.24xlarge": 37500, diff --git a/pkg/providers/instancetype/zz_generated.vpclimits.go b/pkg/providers/instancetype/zz_generated.vpclimits.go index 33a4d4e6cb3d..da5f712b9b5e 100644 --- a/pkg/providers/instancetype/zz_generated.vpclimits.go +++ b/pkg/providers/instancetype/zz_generated.vpclimits.go @@ -17,7 +17,7 @@ // so we can get this information at runtime. // Code generated by go generate; DO NOT EDIT. -// This file was generated at 2023-12-01T17:49:07Z +// This file was generated at 2024-01-29T18:28:02Z // WARNING: please add @ellistarn, @bwagner5, or @jonathan-innis from aws/karpenter to reviewers // if you are updating this file since Karpenter is depending on this file to calculate max pods. @@ -3416,14 +3416,14 @@ var Limits = map[string]*VPCLimits{ IsBareMetal: false, }, "h1.16xlarge": { - Interface: 15, + Interface: 8, IPv4PerInterface: 50, IsTrunkingCompatible: false, BranchInterface: 0, DefaultNetworkCardIndex: 0, NetworkCards: []NetworkCard{ { - MaximumNetworkInterfaces: 15, + MaximumNetworkInterfaces: 8, NetworkCardIndex: 0, }, }, diff --git a/pkg/providers/launchtemplate/launchtemplate.go b/pkg/providers/launchtemplate/launchtemplate.go index 7c0706b726b7..92abc3b08728 100644 --- a/pkg/providers/launchtemplate/launchtemplate.go +++ b/pkg/providers/launchtemplate/launchtemplate.go @@ -22,6 +22,7 @@ import ( "net" "strings" "sync" + "sync/atomic" "time" "go.uber.org/multierr" @@ -29,6 +30,8 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/ec2/ec2iface" + "github.com/aws/aws-sdk-go/service/eks" + "github.com/aws/aws-sdk-go/service/eks/eksiface" "github.com/mitchellh/hashstructure/v2" "github.com/patrickmn/go-cache" "github.com/samber/lo" @@ -64,28 +67,31 @@ type LaunchTemplate struct { type Provider struct { sync.Mutex ec2api ec2iface.EC2API + eksapi eksiface.EKSAPI amiFamily *amifamily.Resolver securityGroupProvider *securitygroup.Provider subnetProvider *subnet.Provider instanceProfileProvider *instanceprofile.Provider cache *cache.Cache - caBundle *string cm *pretty.ChangeMonitor KubeDNSIP net.IP + CABundle *string ClusterEndpoint string + ClusterCIDR atomic.Pointer[string] } -func NewProvider(ctx context.Context, cache *cache.Cache, ec2api ec2iface.EC2API, amiFamily *amifamily.Resolver, +func NewProvider(ctx context.Context, cache *cache.Cache, ec2api ec2iface.EC2API, eksapi eksiface.EKSAPI, amiFamily *amifamily.Resolver, securityGroupProvider *securitygroup.Provider, subnetProvider *subnet.Provider, instanceProfileProvider *instanceprofile.Provider, caBundle *string, startAsync <-chan struct{}, kubeDNSIP net.IP, clusterEndpoint string) *Provider { l := &Provider{ ec2api: ec2api, + eksapi: eksapi, amiFamily: amiFamily, securityGroupProvider: securityGroupProvider, subnetProvider: subnetProvider, instanceProfileProvider: instanceProfileProvider, cache: cache, - caBundle: caBundle, + CABundle: caBundle, cm: pretty.NewChangeMonitor(), KubeDNSIP: kubeDNSIP, ClusterEndpoint: clusterEndpoint, @@ -113,14 +119,14 @@ func (p *Provider) EnsureAll(ctx context.Context, nodeClass *v1beta1.EC2NodeClas if err != nil { return nil, err } - resolvedLaunchTemplates, err := p.amiFamily.Resolve(ctx, nodeClass, nodeClaim, instanceTypes, options) + resolvedLaunchTemplates, err := p.amiFamily.Resolve(ctx, nodeClass, nodeClaim, instanceTypes, capacityType, options) if err != nil { return nil, err } var launchTemplates []*LaunchTemplate for _, resolvedLaunchTemplate := range resolvedLaunchTemplates { // Ensure the launch template exists, or create it - ec2LaunchTemplate, err := p.ensureLaunchTemplate(ctx, capacityType, resolvedLaunchTemplate) + ec2LaunchTemplate, err := p.ensureLaunchTemplate(ctx, resolvedLaunchTemplate) if err != nil { return nil, err } @@ -172,6 +178,7 @@ func (p *Provider) createAMIOptions(ctx context.Context, nodeClass *v1beta1.EC2N options := &amifamily.Options{ ClusterName: options.FromContext(ctx).ClusterName, ClusterEndpoint: p.ClusterEndpoint, + ClusterCIDR: p.ClusterCIDR.Load(), InstanceProfile: instanceProfile, InstanceStorePolicy: nodeClass.Spec.InstanceStorePolicy, SecurityGroups: lo.Map(securityGroups, func(s *ec2.SecurityGroup, _ int) v1beta1.SecurityGroup { @@ -179,15 +186,18 @@ func (p *Provider) createAMIOptions(ctx context.Context, nodeClass *v1beta1.EC2N }), Tags: tags, Labels: labels, - CABundle: p.caBundle, + CABundle: p.CABundle, KubeDNSIP: p.KubeDNSIP, NodeClassName: nodeClass.Name, } - if ok, err := p.subnetProvider.CheckAnyPublicIPAssociations(ctx, nodeClass); err != nil { + if nodeClass.Spec.AssociatePublicIPAddress != nil { + options.AssociatePublicIPAddress = nodeClass.Spec.AssociatePublicIPAddress + } else if ok, err := p.subnetProvider.CheckAnyPublicIPAssociations(ctx, nodeClass); err != nil { return nil, err } else if !ok { + // when `AssociatePublicIPAddress` is not specified in the `EC2NodeClass` spec, // If all referenced subnets do not assign public IPv4 addresses to EC2 instances therein, we explicitly set - // AssociatePublicIpAddress to 'false' in the Launch Template, generated based on this configuration struct. + // AssociatePublicIPAddress to 'false' in the Launch Template, generated based on this configuration struct. // This is done to help comply with AWS account policies that require explicitly setting of that field to 'false'. // https://github.com/aws/karpenter-provider-aws/issues/3815 options.AssociatePublicIPAddress = aws.Bool(false) @@ -195,7 +205,7 @@ func (p *Provider) createAMIOptions(ctx context.Context, nodeClass *v1beta1.EC2N return options, nil } -func (p *Provider) ensureLaunchTemplate(ctx context.Context, capacityType string, options *amifamily.LaunchTemplate) (*ec2.LaunchTemplate, error) { +func (p *Provider) ensureLaunchTemplate(ctx context.Context, options *amifamily.LaunchTemplate) (*ec2.LaunchTemplate, error) { var launchTemplate *ec2.LaunchTemplate name := launchTemplateName(options) ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("launch-template-name", name)) @@ -210,7 +220,7 @@ func (p *Provider) ensureLaunchTemplate(ctx context.Context, capacityType string }) // Create LT if one doesn't exist if awserrors.IsNotFound(err) { - launchTemplate, err = p.createLaunchTemplate(ctx, capacityType, options) + launchTemplate, err = p.createLaunchTemplate(ctx, options) if err != nil { return nil, fmt.Errorf("creating launch template, %w", err) } @@ -228,7 +238,7 @@ func (p *Provider) ensureLaunchTemplate(ctx context.Context, capacityType string return launchTemplate, nil } -func (p *Provider) createLaunchTemplate(ctx context.Context, capacityType string, options *amifamily.LaunchTemplate) (*ec2.LaunchTemplate, error) { +func (p *Provider) createLaunchTemplate(ctx context.Context, options *amifamily.LaunchTemplate) (*ec2.LaunchTemplate, error) { userData, err := options.UserData.Script() if err != nil { return nil, err @@ -237,7 +247,7 @@ func (p *Provider) createLaunchTemplate(ctx context.Context, capacityType string {ResourceType: aws.String(ec2.ResourceTypeNetworkInterface), Tags: utils.MergeTags(options.Tags)}, } // Add the spot-instances-request tag if trying to launch spot capacity - if capacityType == corev1beta1.CapacityTypeSpot { + if options.CapacityType == corev1beta1.CapacityTypeSpot { launchTemplateDataTags = append(launchTemplateDataTags, &ec2.LaunchTemplateTagSpecificationRequest{ResourceType: aws.String(ec2.ResourceTypeSpotInstancesRequest), Tags: utils.MergeTags(options.Tags)}) } networkInterfaces := p.generateNetworkInterfaces(options) @@ -288,15 +298,13 @@ func (p *Provider) generateNetworkInterfaces(options *amifamily.LaunchTemplate) DeviceIndex: lo.ToPtr(lo.Ternary[int64](i == 0, 0, 1)), InterfaceType: lo.ToPtr(ec2.NetworkInterfaceTypeEfa), Groups: lo.Map(options.SecurityGroups, func(s v1beta1.SecurityGroup, _ int) *string { return aws.String(s.ID) }), + // Instances launched with multiple pre-configured network interfaces cannot set AssociatePublicIPAddress to true. This is an EC2 limitation. However, this does not apply for instances + // with a single EFA network interface, and we should support those use cases. Launch failures with multiple enis should be considered user misconfiguration. + AssociatePublicIpAddress: options.AssociatePublicIPAddress, } }) } - // If all referenced subnets do not assign public IPv4 addresses to EC2 instances therein, we explicitly set - // AssociatePublicIpAddress to 'false' in the Launch Template, generated based on this configuration struct. - // This is done to help comply with AWS account policies that require explicitly setting that field to 'false'. - // This is ignored for EFA instances since it can't be specified if you launch with multiple network interfaces. - // https://github.com/aws/karpenter-provider-aws/issues/3815 if options.AssociatePublicIPAddress != nil { return []*ec2.LaunchTemplateInstanceNetworkInterfaceSpecificationRequest{ { @@ -369,7 +377,7 @@ func (p *Provider) cachedEvictedFunc(ctx context.Context) func(string, interface return } launchTemplate := lt.(*ec2.LaunchTemplate) - if _, err := p.ec2api.DeleteLaunchTemplate(&ec2.DeleteLaunchTemplateInput{LaunchTemplateId: launchTemplate.LaunchTemplateId}); err != nil { + if _, err := p.ec2api.DeleteLaunchTemplate(&ec2.DeleteLaunchTemplateInput{LaunchTemplateId: launchTemplate.LaunchTemplateId}); awserrors.IgnoreNotFound(err) != nil { logging.FromContext(ctx).With("launch-template", launchTemplate.LaunchTemplateName).Errorf("failed to delete launch template, %v", err) return } @@ -423,3 +431,26 @@ func (p *Provider) DeleteLaunchTemplates(ctx context.Context, nodeClass *v1beta1 } return nil } + +func (p *Provider) ResolveClusterCIDR(ctx context.Context) error { + if p.ClusterCIDR.Load() != nil { + return nil + } + out, err := p.eksapi.DescribeClusterWithContext(ctx, &eks.DescribeClusterInput{ + Name: aws.String(options.FromContext(ctx).ClusterName), + }) + if err != nil { + return err + } + if ipv4CIDR := out.Cluster.KubernetesNetworkConfig.ServiceIpv4Cidr; ipv4CIDR != nil { + p.ClusterCIDR.Store(ipv4CIDR) + logging.FromContext(ctx).With("cluster-cidr", *ipv4CIDR).Debugf("discovered cluster CIDR") + return nil + } + if ipv6CIDR := out.Cluster.KubernetesNetworkConfig.ServiceIpv6Cidr; ipv6CIDR != nil { + p.ClusterCIDR.Store(ipv6CIDR) + logging.FromContext(ctx).With("cluster-cidr", *ipv6CIDR).Debugf("discovered cluster CIDR") + return nil + } + return fmt.Errorf("no CIDR found in DescribeCluster response") +} diff --git a/pkg/providers/launchtemplate/suite_test.go b/pkg/providers/launchtemplate/suite_test.go index d7189c157fc5..cbe23d5bd586 100644 --- a/pkg/providers/launchtemplate/suite_test.go +++ b/pkg/providers/launchtemplate/suite_test.go @@ -17,6 +17,7 @@ package launchtemplate_test import ( "context" "encoding/base64" + "encoding/json" "fmt" "net" "os" @@ -28,17 +29,20 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/service/ec2" + admv1alpha1 "github.com/awslabs/amazon-eks-ami/nodeadm/api/v1alpha1" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/samber/lo" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/tools/record" clock "k8s.io/utils/clock/testing" . "knative.dev/pkg/logging/testing" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/yaml" corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" "sigs.k8s.io/karpenter/pkg/controllers/provisioning" @@ -54,7 +58,9 @@ import ( "github.com/aws/karpenter-provider-aws/pkg/cloudprovider" "github.com/aws/karpenter-provider-aws/pkg/fake" "github.com/aws/karpenter-provider-aws/pkg/operator/options" + "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily" "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily/bootstrap" + "github.com/aws/karpenter-provider-aws/pkg/providers/amifamily/bootstrap/mime" "github.com/aws/karpenter-provider-aws/pkg/providers/instancetype" "github.com/aws/karpenter-provider-aws/pkg/test" ) @@ -85,7 +91,7 @@ var _ = BeforeSuite(func() { cloudProvider = cloudprovider.New(awsEnv.InstanceTypesProvider, awsEnv.InstanceProvider, events.NewRecorder(&record.FakeRecorder{}), env.Client, awsEnv.AMIProvider, awsEnv.SecurityGroupProvider, awsEnv.SubnetProvider) cluster = state.NewCluster(fakeClock, env.Client, cloudProvider) - prov = provisioning.NewProvisioner(env.Client, env.KubernetesInterface.CoreV1(), events.NewRecorder(&record.FakeRecorder{}), cloudProvider, cluster) + prov = provisioning.NewProvisioner(env.Client, events.NewRecorder(&record.FakeRecorder{}), cloudProvider, cluster) }) var _ = AfterSuite(func() { @@ -101,6 +107,7 @@ var _ = BeforeEach(func() { awsEnv.LaunchTemplateProvider.KubeDNSIP = net.ParseIP("10.0.100.10") awsEnv.LaunchTemplateProvider.ClusterEndpoint = "https://test-cluster" + awsEnv.LaunchTemplateProvider.CABundle = lo.ToPtr("ca-bundle") }) var _ = AfterEach(func() { @@ -120,13 +127,16 @@ var _ = Describe("LaunchTemplates", func() { Labels: map[string]string{coretest.DiscoveryLabel: "unspecified"}, }, Spec: corev1beta1.NodeClaimSpec{ - Requirements: []v1.NodeSelectorRequirement{ + Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ { - Key: corev1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.CapacityTypeOnDemand}, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: corev1beta1.CapacityTypeLabelKey, + Operator: v1.NodeSelectorOpIn, + Values: []string{corev1beta1.CapacityTypeOnDemand}, + }, }, }, + Kubelet: &corev1beta1.KubeletConfiguration{}, NodeClassRef: &corev1beta1.NodeClassReference{ Name: nodeClass.Name, }, @@ -141,11 +151,13 @@ var _ = Describe("LaunchTemplates", func() { Spec: corev1beta1.NodePoolSpec{ Template: corev1beta1.NodeClaimTemplate{ Spec: corev1beta1.NodeClaimSpec{ - Requirements: []v1.NodeSelectorRequirement{ + Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ { - Key: corev1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.CapacityTypeSpot}, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: corev1beta1.CapacityTypeLabelKey, + Operator: v1.NodeSelectorOpIn, + Values: []string{corev1beta1.CapacityTypeSpot}, + }, }, }, NodeClassRef: &corev1beta1.NodeClassReference{ @@ -347,11 +359,13 @@ var _ = Describe("LaunchTemplates", func() { "tag1": "tag1value", "tag2": "tag2value", } - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ + nodePool.Spec.Template.Spec.Requirements = []corev1beta1.NodeSelectorRequirementWithMinValues{ { - Key: corev1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.CapacityTypeSpot}, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: corev1beta1.CapacityTypeLabelKey, + Operator: v1.NodeSelectorOpIn, + Values: []string{corev1beta1.CapacityTypeSpot}, + }, }, } ExpectApplied(ctx, env.Client, nodePool, nodeClass) @@ -408,6 +422,22 @@ var _ = Describe("LaunchTemplates", func() { Expect(ltInput.LaunchTemplateData.BlockDeviceMappings[0].Ebs.Iops).To(BeNil()) }) }) + It("should default AL2023 block device mappings", func() { + nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2023 + awsEnv.LaunchTemplateProvider.CABundle = lo.ToPtr("Y2EtYnVuZGxlCg==") + awsEnv.LaunchTemplateProvider.ClusterCIDR.Store(lo.ToPtr("10.100.0.0/16")) + ExpectApplied(ctx, env.Client, nodePool, nodeClass) + pod := coretest.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(BeNumerically(">=", 1)) + awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.ForEach(func(ltInput *ec2.CreateLaunchTemplateInput) { + Expect(len(ltInput.LaunchTemplateData.BlockDeviceMappings)).To(Equal(1)) + Expect(*ltInput.LaunchTemplateData.BlockDeviceMappings[0].Ebs.VolumeSize).To(Equal(int64(20))) + Expect(*ltInput.LaunchTemplateData.BlockDeviceMappings[0].Ebs.VolumeType).To(Equal("gp3")) + Expect(ltInput.LaunchTemplateData.BlockDeviceMappings[0].Ebs.Iops).To(BeNil()) + }) + }) It("should use custom block device mapping", func() { nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 nodeClass.Spec.BlockDeviceMappings = []*v1beta1.BlockDeviceMapping{ @@ -794,7 +824,22 @@ var _ = Describe("LaunchTemplates", func() { })) nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, "", nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + "", + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) + overhead := it.Overhead.Total() Expect(overhead.Memory().String()).To(Equal("993Mi")) }) @@ -832,7 +877,22 @@ var _ = Describe("LaunchTemplates", func() { })) nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, "", nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + "", + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) + overhead := it.Overhead.Total() Expect(overhead.Memory().String()).To(Equal("993Mi")) }) @@ -843,7 +903,21 @@ var _ = Describe("LaunchTemplates", func() { nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyBottlerocket nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{MaxPods: lo.ToPtr[int32](110)} - it := instancetype.NewInstanceType(ctx, info, nodePool.Spec.Template.Spec.Kubelet, "", nodeClass, nil) + amiFamily := amifamily.GetAMIFamily(nodeClass.Spec.AMIFamily, &amifamily.Options{}) + it := instancetype.NewInstanceType(ctx, + info, + "", + nodeClass.Spec.BlockDeviceMappings, + nodeClass.Spec.InstanceStorePolicy, + nodePool.Spec.Template.Spec.Kubelet.MaxPods, + nodePool.Spec.Template.Spec.Kubelet.PodsPerCore, + nodePool.Spec.Template.Spec.Kubelet.KubeReserved, + nodePool.Spec.Template.Spec.Kubelet.SystemReserved, + nodePool.Spec.Template.Spec.Kubelet.EvictionHard, + nodePool.Spec.Template.Spec.Kubelet.EvictionSoft, + amiFamily, + nil, + ) overhead := it.Overhead.Total() Expect(overhead.Memory().String()).To(Equal("1565Mi")) }) @@ -866,10 +940,10 @@ var _ = Describe("LaunchTemplates", func() { }) It("should specify --system-reserved when overriding system reserved values", func() { nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ - SystemReserved: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("500m"), - v1.ResourceMemory: resource.MustParse("1Gi"), - v1.ResourceEphemeralStorage: resource.MustParse("2Gi"), + SystemReserved: map[string]string{ + string(v1.ResourceCPU): "500m", + string(v1.ResourceMemory): "1Gi", + string(v1.ResourceEphemeralStorage): "2Gi", }, } ExpectApplied(ctx, env.Client, nodePool, nodeClass) @@ -887,16 +961,16 @@ var _ = Describe("LaunchTemplates", func() { rem := string(userData)[(i + len(arg)):] i = strings.Index(rem, "'") for k, v := range nodePool.Spec.Template.Spec.Kubelet.SystemReserved { - Expect(rem[:i]).To(ContainSubstring(fmt.Sprintf("%v=%v", k.String(), v.String()))) + Expect(rem[:i]).To(ContainSubstring(fmt.Sprintf("%v=%v", k, v))) } }) }) It("should specify --kube-reserved when overriding system reserved values", func() { nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ - KubeReserved: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("500m"), - v1.ResourceMemory: resource.MustParse("1Gi"), - v1.ResourceEphemeralStorage: resource.MustParse("2Gi"), + KubeReserved: map[string]string{ + string(v1.ResourceCPU): "500m", + string(v1.ResourceMemory): "1Gi", + string(v1.ResourceEphemeralStorage): "2Gi", }, } ExpectApplied(ctx, env.Client, nodePool, nodeClass) @@ -914,7 +988,7 @@ var _ = Describe("LaunchTemplates", func() { rem := string(userData)[(i + len(arg)):] i = strings.Index(rem, "'") for k, v := range nodePool.Spec.Template.Spec.Kubelet.KubeReserved { - Expect(rem[:i]).To(ContainSubstring(fmt.Sprintf("%v=%v", k.String(), v.String()))) + Expect(rem[:i]).To(ContainSubstring(fmt.Sprintf("%v=%v", k, v))) } }) }) @@ -1161,10 +1235,10 @@ var _ = Describe("LaunchTemplates", func() { It("should override system reserved values in user data", func() { ExpectApplied(ctx, env.Client, nodeClass) nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ - SystemReserved: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("2"), - v1.ResourceMemory: resource.MustParse("3Gi"), - v1.ResourceEphemeralStorage: resource.MustParse("10Gi"), + SystemReserved: map[string]string{ + string(v1.ResourceCPU): "2", + string(v1.ResourceMemory): "3Gi", + string(v1.ResourceEphemeralStorage): "10Gi", }, } ExpectApplied(ctx, env.Client, nodePool) @@ -1186,10 +1260,10 @@ var _ = Describe("LaunchTemplates", func() { It("should override kube reserved values in user data", func() { ExpectApplied(ctx, env.Client, nodeClass) nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ - KubeReserved: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("2"), - v1.ResourceMemory: resource.MustParse("3Gi"), - v1.ResourceEphemeralStorage: resource.MustParse("10Gi"), + KubeReserved: map[string]string{ + string(v1.ResourceCPU): "2", + string(v1.ResourceMemory): "3Gi", + string(v1.ResourceEphemeralStorage): "10Gi", }, } ExpectApplied(ctx, env.Client, nodePool) @@ -1380,6 +1454,207 @@ var _ = Describe("LaunchTemplates", func() { ExpectLaunchTemplatesCreatedWithUserData(expectedUserData) }) }) + Context("AL2023", func() { + BeforeEach(func() { + nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2023 + + // base64 encoded version of "ca-bundle" to ensure the nodeadm bootstrap provider can decode successfully + awsEnv.LaunchTemplateProvider.CABundle = lo.ToPtr("Y2EtYnVuZGxlCg==") + awsEnv.LaunchTemplateProvider.ClusterCIDR.Store(lo.ToPtr("10.100.0.0/16")) + }) + Context("Kubelet", func() { + It("should specify taints in the KubeletConfiguration when specified in NodePool", func() { + desiredTaints := []v1.Taint{ + { + Key: "test-taint-1", + Effect: v1.TaintEffectNoSchedule, + }, + { + Key: "test-taint-2", + Effect: v1.TaintEffectNoExecute, + }, + } + nodePool.Spec.Template.Spec.Taints = desiredTaints + ExpectApplied(ctx, env.Client, nodePool, nodeClass) + pod := coretest.UnschedulablePod(coretest.UnscheduleablePodOptions(coretest.PodOptions{ + Tolerations: []v1.Toleration{{ + Operator: v1.TolerationOpExists, + }}, + })) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + for _, userData := range ExpectUserDataExistsFromCreatedLaunchTemplates() { + configs := ExpectUserDataCreatedWithNodeConfigs(userData) + Expect(len(configs)).To(Equal(1)) + taintsRaw, ok := configs[0].Spec.Kubelet.Config["registerWithTaints"] + Expect(ok).To(BeTrue()) + taints := []v1.Taint{} + Expect(yaml.Unmarshal(taintsRaw.Raw, &taints)).To(Succeed()) + Expect(len(taints)).To(Equal(2)) + Expect(taints).To(ContainElements(lo.Map(desiredTaints, func(t v1.Taint, _ int) interface{} { + return interface{}(t) + }))) + } + }) + It("should specify labels in the Kubelet flags when specified in NodePool", func() { + desiredLabels := map[string]string{ + "test-label-1": "value-1", + "test-label-2": "value-2", + } + nodePool.Spec.Template.Labels = desiredLabels + + ExpectApplied(ctx, env.Client, nodePool, nodeClass) + pod := coretest.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + for _, userData := range ExpectUserDataExistsFromCreatedLaunchTemplates() { + configs := ExpectUserDataCreatedWithNodeConfigs(userData) + Expect(len(configs)).To(Equal(1)) + labelFlag, ok := lo.Find(configs[0].Spec.Kubelet.Flags, func(flag string) bool { + return strings.HasPrefix(flag, "--node-labels") + }) + Expect(ok).To(BeTrue()) + for label, value := range desiredLabels { + Expect(labelFlag).To(ContainSubstring(fmt.Sprintf("%s=%s", label, value))) + } + } + }) + DescribeTable( + "should specify KubletConfiguration field when specified in NodePool", + func(field string, kc corev1beta1.KubeletConfiguration) { + nodePool.Spec.Template.Spec.Kubelet = &kc + ExpectApplied(ctx, env.Client, nodePool, nodeClass) + pod := coretest.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + + // Convert provided KubeletConfiguration to an InlineConfig for comparison with NodeConfig + inlineConfig := func() map[string]runtime.RawExtension { + configYAML, err := yaml.Marshal(kc) + Expect(err).To(BeNil()) + configMap := map[string]interface{}{} + Expect(yaml.Unmarshal(configYAML, &configMap)).To(Succeed()) + return lo.MapValues(configMap, func(v interface{}, _ string) runtime.RawExtension { + val, err := json.Marshal(v) + Expect(err).To(BeNil()) + return runtime.RawExtension{Raw: val} + }) + }() + for _, userData := range ExpectUserDataExistsFromCreatedLaunchTemplates() { + configs := ExpectUserDataCreatedWithNodeConfigs(userData) + Expect(len(configs)).To(Equal(1)) + Expect(configs[0].Spec.Kubelet.Config[field]).To(Equal(inlineConfig[field])) + } + }, + Entry("systemReserved", "systemReserved", corev1beta1.KubeletConfiguration{ + SystemReserved: map[string]string{ + string(v1.ResourceCPU): "500m", + string(v1.ResourceMemory): "1Gi", + string(v1.ResourceEphemeralStorage): "2Gi", + }, + }), + Entry("kubeReserved", "kubeReserved", corev1beta1.KubeletConfiguration{ + KubeReserved: map[string]string{ + string(v1.ResourceCPU): "500m", + string(v1.ResourceMemory): "1Gi", + string(v1.ResourceEphemeralStorage): "2Gi", + }, + }), + Entry("evictionHard", "evictionHard", corev1beta1.KubeletConfiguration{ + EvictionHard: map[string]string{ + "memory.available": "10%", + "nodefs.available": "15%", + "nodefs.inodesFree": "5%", + }, + }), + Entry("evictionSoft", "evictionSoft", corev1beta1.KubeletConfiguration{ + EvictionSoft: map[string]string{ + "memory.available": "10%", + "nodefs.available": "15%", + "nodefs.inodesFree": "5%", + }, + EvictionSoftGracePeriod: map[string]metav1.Duration{ + "memory.available": {Duration: time.Minute}, + "nodefs.available": {Duration: time.Second * 180}, + "nodefs.inodesFree": {Duration: time.Minute * 5}, + }, + }), + Entry("evictionSoftGracePeriod", "evictionSoftGracePeriod", corev1beta1.KubeletConfiguration{ + EvictionSoft: map[string]string{ + "memory.available": "10%", + "nodefs.available": "15%", + "nodefs.inodesFree": "5%", + }, + EvictionSoftGracePeriod: map[string]metav1.Duration{ + "memory.available": {Duration: time.Minute}, + "nodefs.available": {Duration: time.Second * 180}, + "nodefs.inodesFree": {Duration: time.Minute * 5}, + }, + }), + Entry("evictionMaxPodGracePeriod", "evictionMaxPodGracePeriod", corev1beta1.KubeletConfiguration{ + EvictionMaxPodGracePeriod: lo.ToPtr[int32](300), + }), + Entry("podsPerCore", "podsPerCore", corev1beta1.KubeletConfiguration{ + PodsPerCore: lo.ToPtr[int32](2), + }), + Entry("clusterDNS", "clusterDNS", corev1beta1.KubeletConfiguration{ + ClusterDNS: []string{"10.0.100.0"}, + }), + Entry("imageGCHighThresholdPercent", "imageGCHighThresholdPercent", corev1beta1.KubeletConfiguration{ + ImageGCHighThresholdPercent: lo.ToPtr[int32](50), + }), + Entry("imageGCLowThresholdPercent", "imageGCLowThresholdPercent", corev1beta1.KubeletConfiguration{ + ImageGCLowThresholdPercent: lo.ToPtr[int32](50), + }), + Entry("cpuCFSQuota", "cpuCFSQuota", corev1beta1.KubeletConfiguration{ + CPUCFSQuota: lo.ToPtr(false), + }), + ) + }) + It("should set LocalDiskStrategy to Raid0 when specified by the InstanceStorePolicy", func() { + nodeClass.Spec.InstanceStorePolicy = lo.ToPtr(v1beta1.InstanceStorePolicyRAID0) + ExpectApplied(ctx, env.Client, nodeClass, nodePool) + pod := coretest.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + for _, userData := range ExpectUserDataExistsFromCreatedLaunchTemplates() { + configs := ExpectUserDataCreatedWithNodeConfigs(userData) + Expect(len(configs)).To(Equal(1)) + Expect(configs[0].Spec.Instance.LocalStorage.Strategy).To(Equal(admv1alpha1.LocalStorageRAID0)) + } + }) + DescribeTable( + "should merge custom user data", + func(inputFile *string, mergedFile string) { + if inputFile != nil { + content, err := os.ReadFile("testdata/" + *inputFile) + Expect(err).To(BeNil()) + nodeClass.Spec.UserData = lo.ToPtr(string(content)) + } + nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{MaxPods: lo.ToPtr[int32](110)} + ExpectApplied(ctx, env.Client, nodeClass, nodePool) + pod := coretest.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + content, err := os.ReadFile("testdata/" + mergedFile) + Expect(err).To(BeNil()) + expectedUserData := fmt.Sprintf(string(content), corev1beta1.NodePoolLabelKey, nodePool.Name) + ExpectLaunchTemplatesCreatedWithUserData(expectedUserData) + }, + Entry("MIME", lo.ToPtr("al2023_mime_userdata_input.golden"), "al2023_mime_userdata_merged.golden"), + Entry("YAML", lo.ToPtr("al2023_yaml_userdata_input.golden"), "al2023_yaml_userdata_merged.golden"), + Entry("shell", lo.ToPtr("al2023_shell_userdata_input.golden"), "al2023_shell_userdata_merged.golden"), + Entry("empty", nil, "al2023_userdata_unmerged.golden"), + ) + It("should fail to create launch templates if cluster CIDR is unresolved", func() { + awsEnv.LaunchTemplateProvider.ClusterCIDR.Store(nil) + ExpectApplied(ctx, env.Client, nodeClass, nodePool) + pod := coretest.UnschedulablePod() + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectNotScheduled(ctx, env.Client, pod) + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(Equal(0)) + }) + }) Context("Custom AMI Selector", func() { It("should use ami selector specified in EC2NodeClass", func() { nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{Tags: map[string]string{"*": "*"}}} @@ -1502,11 +1777,13 @@ var _ = Describe("LaunchTemplates", func() { }}) nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{Tags: map[string]string{"*": "*"}}} ExpectApplied(ctx, env.Client, nodeClass) - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ + nodePool.Spec.Template.Spec.Requirements = []corev1beta1.NodeSelectorRequirementWithMinValues{ { - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.ArchitectureAmd64}, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{corev1beta1.ArchitectureAmd64}, + }, }, } ExpectApplied(ctx, env.Client, nodePool) @@ -1560,8 +1837,8 @@ var _ = Describe("LaunchTemplates", func() { Expect(*input.LaunchTemplateData.ImageId).To(ContainSubstring("test-ami")) }) }) - Context("Subnet-based Launch Template Configration", func() { - It("should explicitly set 'AssignPublicIPv4' to false in the Launch Template", func() { + Context("Public IP Association", func() { + It("should explicitly set 'AssociatePublicIPAddress' to false in the Launch Template", func() { nodeClass.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ {Tags: map[string]string{"Name": "test-subnet-1"}}, {Tags: map[string]string{"Name": "test-subnet-3"}}, @@ -1573,9 +1850,10 @@ var _ = Describe("LaunchTemplates", func() { input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() Expect(*input.LaunchTemplateData.NetworkInterfaces[0].AssociatePublicIpAddress).To(BeFalse()) }) - - It("should not explicitly set 'AssignPublicIPv4' when the subnets are configured to assign public IPv4 addresses", func() { - nodeClass.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{{Tags: map[string]string{"Name": "test-subnet-2"}}} + It("should not explicitly set 'AssociatePublicIPAddress' when the subnets are configured to assign public IPv4 addresses", func() { + nodeClass.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{ + {Tags: map[string]string{"Name": "test-subnet-2"}}, + } ExpectApplied(ctx, env.Client, nodePool, nodeClass) pod := coretest.UnschedulablePod() ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) @@ -1583,6 +1861,27 @@ var _ = Describe("LaunchTemplates", func() { input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() Expect(len(input.LaunchTemplateData.NetworkInterfaces)).To(BeNumerically("==", 0)) }) + DescribeTable( + "should set 'AssociatePublicIPAddress' based on EC2NodeClass", + func(setValue, expectedValue, isEFA bool) { + nodeClass.Spec.AssociatePublicIPAddress = lo.ToPtr(setValue) + ExpectApplied(ctx, env.Client, nodePool, nodeClass) + pod := coretest.UnschedulablePod(lo.Ternary(isEFA, coretest.PodOptions{ + ResourceRequirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{v1beta1.ResourceEFA: resource.MustParse("2")}, + Limits: v1.ResourceList{v1beta1.ResourceEFA: resource.MustParse("2")}, + }, + }, coretest.PodOptions{})) + ExpectProvisioned(ctx, env.Client, cluster, cloudProvider, prov, pod) + ExpectScheduled(ctx, env.Client, pod) + input := awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Pop() + Expect(*input.LaunchTemplateData.NetworkInterfaces[0].AssociatePublicIpAddress).To(Equal(expectedValue)) + }, + Entry("AssociatePublicIPAddress is true", true, true, false), + Entry("AssociatePublicIPAddress is false", false, false, false), + Entry("AssociatePublicIPAddress is true (EFA)", true, true, true), + Entry("AssociatePublicIPAddress is false (EFA)", false, false, true), + ) }) Context("Kubelet Args", func() { It("should specify the --dns-cluster-ip flag when clusterDNSIP is set", func() { @@ -1596,7 +1895,7 @@ var _ = Describe("LaunchTemplates", func() { }) Context("Windows Custom UserData", func() { BeforeEach(func() { - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{{Key: v1.LabelOSStable, Operator: v1.NodeSelectorOpIn, Values: []string{string(v1.Windows)}}} + nodePool.Spec.Template.Spec.Requirements = []corev1beta1.NodeSelectorRequirementWithMinValues{{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: v1.LabelOSStable, Operator: v1.NodeSelectorOpIn, Values: []string{string(v1.Windows)}}}} nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyWindows2022 nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{MaxPods: lo.ToPtr[int32](110)} }) @@ -1718,3 +2017,32 @@ func ExpectLaunchTemplatesCreatedWithUserData(expected string) { ExpectWithOffset(2, actualUserData).To(Equal(expectedUserData)) }) } + +func ExpectUserDataExistsFromCreatedLaunchTemplates() []string { + GinkgoHelper() + Expect(awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.Len()).To(BeNumerically(">=", 1)) + userDatas := []string{} + awsEnv.EC2API.CalledWithCreateLaunchTemplateInput.ForEach(func(input *ec2.CreateLaunchTemplateInput) { + userData, err := base64.StdEncoding.DecodeString(*input.LaunchTemplateData.UserData) + ExpectWithOffset(2, err).To(BeNil()) + userDatas = append(userDatas, string(userData)) + }) + return userDatas +} + +func ExpectUserDataCreatedWithNodeConfigs(userData string) []admv1alpha1.NodeConfig { + GinkgoHelper() + archive, err := mime.NewArchive(userData) + Expect(err).To(BeNil()) + nodeConfigs := lo.FilterMap([]mime.Entry(archive), func(entry mime.Entry, _ int) (admv1alpha1.NodeConfig, bool) { + config := admv1alpha1.NodeConfig{} + if entry.ContentType != mime.ContentTypeNodeConfig { + return config, false + } + err := yaml.Unmarshal([]byte(entry.Content), &config) + Expect(err).To(BeNil()) + return config, true + }) + Expect(len(nodeConfigs)).To(BeNumerically(">=", 1)) + return nodeConfigs +} diff --git a/pkg/providers/launchtemplate/testdata/al2023_mime_userdata_input.golden b/pkg/providers/launchtemplate/testdata/al2023_mime_userdata_input.golden new file mode 100644 index 000000000000..926a1286c0b1 --- /dev/null +++ b/pkg/providers/launchtemplate/testdata/al2023_mime_userdata_input.golden @@ -0,0 +1,23 @@ +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="//" + +--// +Content-Type: application/node.eks.aws + +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + cluster: + name: test-cluster + clusterEndpoint: https://test-cluster + certificateAuthority: cluster-ca + cidr: 10.100.0.0/16 + kubelet: + config: + maxPods: 42 +--// +Content-Type: text/x-shellscript; charset="us-ascii" + +#!/bin/bash +echo "Hello, AL2023!" +--// diff --git a/pkg/providers/launchtemplate/testdata/al2023_mime_userdata_merged.golden b/pkg/providers/launchtemplate/testdata/al2023_mime_userdata_merged.golden new file mode 100644 index 000000000000..c65e73381035 --- /dev/null +++ b/pkg/providers/launchtemplate/testdata/al2023_mime_userdata_merged.golden @@ -0,0 +1,48 @@ +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="//" + +--// +Content-Type: application/node.eks.aws + +# Karpenter Generated NodeConfig +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +metadata: + creationTimestamp: null +spec: + cluster: + apiServerEndpoint: https://test-cluster + certificateAuthority: Y2EtYnVuZGxlCg== + cidr: 10.100.0.0/16 + name: test-cluster + containerd: {} + instance: + localStorage: {} + kubelet: + config: + clusterDNS: + - 10.0.100.10 + maxPods: 110 + flags: + - --node-labels="karpenter.sh/capacity-type=on-demand,%s=%s,testing/cluster=unspecified" + +--// +Content-Type: application/node.eks.aws + +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + cluster: + name: test-cluster + clusterEndpoint: https://test-cluster + certificateAuthority: cluster-ca + cidr: 10.100.0.0/16 + kubelet: + config: + maxPods: 42 +--// +Content-Type: text/x-shellscript; charset="us-ascii" + +#!/bin/bash +echo "Hello, AL2023!" +--//-- diff --git a/pkg/providers/launchtemplate/testdata/al2023_shell_userdata_input.golden b/pkg/providers/launchtemplate/testdata/al2023_shell_userdata_input.golden new file mode 100644 index 000000000000..73cc0afff04d --- /dev/null +++ b/pkg/providers/launchtemplate/testdata/al2023_shell_userdata_input.golden @@ -0,0 +1,2 @@ +#!/bin/bash +echo "Hello, AL2023!" diff --git a/pkg/providers/launchtemplate/testdata/al2023_shell_userdata_merged.golden b/pkg/providers/launchtemplate/testdata/al2023_shell_userdata_merged.golden new file mode 100644 index 000000000000..6dd0bb7ac6dd --- /dev/null +++ b/pkg/providers/launchtemplate/testdata/al2023_shell_userdata_merged.golden @@ -0,0 +1,34 @@ +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="//" + +--// +Content-Type: application/node.eks.aws + +# Karpenter Generated NodeConfig +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +metadata: + creationTimestamp: null +spec: + cluster: + apiServerEndpoint: https://test-cluster + certificateAuthority: Y2EtYnVuZGxlCg== + cidr: 10.100.0.0/16 + name: test-cluster + containerd: {} + instance: + localStorage: {} + kubelet: + config: + clusterDNS: + - 10.0.100.10 + maxPods: 110 + flags: + - --node-labels="karpenter.sh/capacity-type=on-demand,%s=%s,testing/cluster=unspecified" + +--// +Content-Type: text/x-shellscript; charset="us-ascii" + +#!/bin/bash +echo "Hello, AL2023!" +--//-- diff --git a/pkg/providers/launchtemplate/testdata/al2023_userdata_unmerged.golden b/pkg/providers/launchtemplate/testdata/al2023_userdata_unmerged.golden new file mode 100644 index 000000000000..191107c780fb --- /dev/null +++ b/pkg/providers/launchtemplate/testdata/al2023_userdata_unmerged.golden @@ -0,0 +1,29 @@ +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="//" + +--// +Content-Type: application/node.eks.aws + +# Karpenter Generated NodeConfig +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +metadata: + creationTimestamp: null +spec: + cluster: + apiServerEndpoint: https://test-cluster + certificateAuthority: Y2EtYnVuZGxlCg== + cidr: 10.100.0.0/16 + name: test-cluster + containerd: {} + instance: + localStorage: {} + kubelet: + config: + clusterDNS: + - 10.0.100.10 + maxPods: 110 + flags: + - --node-labels="karpenter.sh/capacity-type=on-demand,%s=%s,testing/cluster=unspecified" + +--//-- diff --git a/pkg/providers/launchtemplate/testdata/al2023_yaml_userdata_input.golden b/pkg/providers/launchtemplate/testdata/al2023_yaml_userdata_input.golden new file mode 100644 index 000000000000..37a2e1684269 --- /dev/null +++ b/pkg/providers/launchtemplate/testdata/al2023_yaml_userdata_input.golden @@ -0,0 +1,11 @@ +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + cluster: + name: test-cluster + clusterEndpoint: https://test-cluster + certificateAuthority: cluster-ca + cidr: 10.100.0.0/16 + kubelet: + config: + maxPods: 42 diff --git a/pkg/providers/launchtemplate/testdata/al2023_yaml_userdata_merged.golden b/pkg/providers/launchtemplate/testdata/al2023_yaml_userdata_merged.golden new file mode 100644 index 000000000000..724f80b5b79d --- /dev/null +++ b/pkg/providers/launchtemplate/testdata/al2023_yaml_userdata_merged.golden @@ -0,0 +1,43 @@ +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="//" + +--// +Content-Type: application/node.eks.aws + +# Karpenter Generated NodeConfig +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +metadata: + creationTimestamp: null +spec: + cluster: + apiServerEndpoint: https://test-cluster + certificateAuthority: Y2EtYnVuZGxlCg== + cidr: 10.100.0.0/16 + name: test-cluster + containerd: {} + instance: + localStorage: {} + kubelet: + config: + clusterDNS: + - 10.0.100.10 + maxPods: 110 + flags: + - --node-labels="karpenter.sh/capacity-type=on-demand,%s=%s,testing/cluster=unspecified" + +--// +Content-Type: application/node.eks.aws + +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + cluster: + name: test-cluster + clusterEndpoint: https://test-cluster + certificateAuthority: cluster-ca + cidr: 10.100.0.0/16 + kubelet: + config: + maxPods: 42 +--//-- diff --git a/pkg/providers/pricing/metrics.go b/pkg/providers/pricing/metrics.go deleted file mode 100644 index 60c053580439..000000000000 --- a/pkg/providers/pricing/metrics.go +++ /dev/null @@ -1,50 +0,0 @@ -/* -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package pricing - -import ( - "github.com/prometheus/client_golang/prometheus" - crmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" - - "sigs.k8s.io/karpenter/pkg/metrics" -) - -const ( - cloudProviderSubsystem = "cloudprovider" -) - -var ( - InstanceTypeLabel = "instance_type" - CapacityTypeLabel = "capacity_type" - RegionLabel = "region" - TopologyLabel = "zone" - InstancePriceEstimate = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: metrics.Namespace, - Subsystem: cloudProviderSubsystem, - Name: "instance_type_price_estimate", - Help: "Estimated hourly price used when making informed decisions on node cost calculation. This is updated once on startup and then every 12 hours.", - }, - []string{ - InstanceTypeLabel, - CapacityTypeLabel, - RegionLabel, - TopologyLabel, - }) -) - -func init() { - crmetrics.Registry.MustRegister(InstancePriceEstimate) -} diff --git a/pkg/providers/pricing/pricing.go b/pkg/providers/pricing/pricing.go index e46cff66f9bd..3db8b0625aaf 100644 --- a/pkg/providers/pricing/pricing.go +++ b/pkg/providers/pricing/pricing.go @@ -25,13 +25,14 @@ import ( "sync" "time" + "github.com/aws/karpenter-provider-aws/pkg/operator/options" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/ec2/ec2iface" "github.com/aws/aws-sdk-go/service/pricing" "github.com/aws/aws-sdk-go/service/pricing/pricingiface" - "github.com/prometheus/client_golang/prometheus" "github.com/samber/lo" "go.uber.org/multierr" "knative.dev/pkg/logging" @@ -53,8 +54,10 @@ type Provider struct { region string cm *pretty.ChangeMonitor - mu sync.RWMutex - onDemandPrices map[string]float64 + muOnDemand sync.RWMutex + onDemandPrices map[string]float64 + + muSpot sync.RWMutex spotPrices map[string]zonal spotPricingUpdated bool } @@ -108,16 +111,18 @@ func NewProvider(_ context.Context, pricing pricingiface.PricingAPI, ec2Api ec2i // InstanceTypes returns the list of all instance types for which either a spot or on-demand price is known. func (p *Provider) InstanceTypes() []string { - p.mu.RLock() - defer p.mu.RUnlock() + p.muOnDemand.RLock() + p.muSpot.RLock() + defer p.muOnDemand.RUnlock() + defer p.muSpot.RUnlock() return lo.Union(lo.Keys(p.onDemandPrices), lo.Keys(p.spotPrices)) } // OnDemandPrice returns the last known on-demand price for a given instance type, returning an error if there is no // known on-demand pricing for the instance type. func (p *Provider) OnDemandPrice(instanceType string) (float64, bool) { - p.mu.RLock() - defer p.mu.RUnlock() + p.muOnDemand.RLock() + defer p.muOnDemand.RUnlock() price, ok := p.onDemandPrices[instanceType] if !ok { return 0.0, false @@ -128,8 +133,8 @@ func (p *Provider) OnDemandPrice(instanceType string) (float64, bool) { // SpotPrice returns the last known spot price for a given instance type and zone, returning an error // if there is no known spot pricing for that instance type or zone func (p *Provider) SpotPrice(instanceType string, zone string) (float64, bool) { - p.mu.RLock() - defer p.mu.RUnlock() + p.muSpot.RLock() + defer p.muSpot.RUnlock() if val, ok := p.spotPrices[instanceType]; ok { if !p.spotPricingUpdated { return val.defaultPrice, true @@ -148,6 +153,18 @@ func (p *Provider) UpdateOnDemandPricing(ctx context.Context) error { var onDemandPrices, onDemandMetalPrices map[string]float64 var onDemandErr, onDemandMetalErr error + // if we are in isolated vpc, skip updating on demand pricing + // as pricing api may not be available + if options.FromContext(ctx).IsolatedVPC { + if p.cm.HasChanged("on-demand-prices", nil) { + logging.FromContext(ctx).Debug("running in an isolated VPC, on-demand pricing information will not be updated") + } + return nil + } + + p.muOnDemand.Lock() + defer p.muOnDemand.Unlock() + wg.Add(1) go func() { defer wg.Done() @@ -183,8 +200,6 @@ func (p *Provider) UpdateOnDemandPricing(ctx context.Context) error { wg.Wait() - p.mu.Lock() - defer p.mu.Unlock() err := multierr.Append(onDemandErr, onDemandMetalErr) if err != nil { return fmt.Errorf("retreiving on-demand pricing data, %w", err) @@ -195,14 +210,6 @@ func (p *Provider) UpdateOnDemandPricing(ctx context.Context) error { } p.onDemandPrices = lo.Assign(onDemandPrices, onDemandMetalPrices) - for instanceType, price := range p.onDemandPrices { - InstancePriceEstimate.With(prometheus.Labels{ - InstanceTypeLabel: instanceType, - CapacityTypeLabel: ec2.UsageClassTypeOnDemand, - RegionLabel: p.region, - TopologyLabel: "", - }).Set(price) - } if p.cm.HasChanged("on-demand-prices", p.onDemandPrices) { logging.FromContext(ctx).With("instance-type-count", len(p.onDemandPrices)).Debugf("updated on-demand pricing") } @@ -243,18 +250,51 @@ func (p *Provider) fetchOnDemandPricing(ctx context.Context, additionalFilters . Value: aws.String("OnDemand"), }}, additionalFilters...) - if err := p.pricing.GetProductsPagesWithContext(ctx, &pricing.GetProductsInput{ - Filters: filters, - ServiceCode: aws.String("AmazonEC2")}, p.onDemandPage(prices)); err != nil { + + err := p.pricing.GetProductsPagesWithContext( + ctx, + &pricing.GetProductsInput{ + Filters: filters, + ServiceCode: aws.String("AmazonEC2"), + }, + p.onDemandPage(ctx, prices), + ) + if err != nil { return nil, err } + return prices, nil } +func (p *Provider) spotPage(ctx context.Context, prices map[string]map[string]float64) func(output *ec2.DescribeSpotPriceHistoryOutput, b bool) bool { + return func(output *ec2.DescribeSpotPriceHistoryOutput, b bool) bool { + for _, sph := range output.SpotPriceHistory { + spotPriceStr := aws.StringValue(sph.SpotPrice) + spotPrice, err := strconv.ParseFloat(spotPriceStr, 64) + // these errors shouldn't occur, but if pricing API does have an error, we ignore the record + if err != nil { + logging.FromContext(ctx).Debugf("unable to parse price record %#v", sph) + continue + } + if sph.Timestamp == nil { + continue + } + instanceType := aws.StringValue(sph.InstanceType) + az := aws.StringValue(sph.AvailabilityZone) + _, ok := prices[instanceType] + if !ok { + prices[instanceType] = map[string]float64{} + } + prices[instanceType][az] = spotPrice + } + return true + } +} + // turning off cyclo here, it measures as a 12 due to all of the type checks of the pricing data which returns a deeply // nested map[string]interface{} // nolint: gocyclo -func (p *Provider) onDemandPage(prices map[string]float64) func(output *pricing.GetProductsOutput, b bool) bool { +func (p *Provider) onDemandPage(ctx context.Context, prices map[string]float64) func(output *pricing.GetProductsOutput, b bool) bool { // this isn't the full pricing struct, just the portions we care about type priceItem struct { Product struct { @@ -273,19 +313,19 @@ func (p *Provider) onDemandPage(prices map[string]float64) func(output *pricing. return func(output *pricing.GetProductsOutput, b bool) bool { currency := "USD" - if p.region == "cn-north-1" { + if strings.HasPrefix(p.region, "cn-") { currency = "CNY" } for _, outer := range output.PriceList { var buf bytes.Buffer enc := json.NewEncoder(&buf) if err := enc.Encode(outer); err != nil { - logging.FromContext(context.Background()).Errorf("encoding %s", err) + logging.FromContext(ctx).Errorf("encoding %s", err) } dec := json.NewDecoder(&buf) var pItem priceItem if err := dec.Decode(&pItem); err != nil { - logging.FromContext(context.Background()).Errorf("decoding %s", err) + logging.FromContext(ctx).Errorf("decoding %s", err) } if pItem.Product.Attributes.InstanceType == "" { continue @@ -306,43 +346,22 @@ func (p *Provider) onDemandPage(prices map[string]float64) func(output *pricing. // nolint: gocyclo func (p *Provider) UpdateSpotPricing(ctx context.Context) error { - totalOfferings := 0 - prices := map[string]map[string]float64{} - err := p.ec2.DescribeSpotPriceHistoryPagesWithContext(ctx, &ec2.DescribeSpotPriceHistoryInput{ - ProductDescriptions: []*string{aws.String("Linux/UNIX"), aws.String("Linux/UNIX (Amazon VPC)")}, - // get the latest spot price for each instance type - StartTime: aws.Time(time.Now()), - }, func(output *ec2.DescribeSpotPriceHistoryOutput, b bool) bool { - for _, sph := range output.SpotPriceHistory { - spotPriceStr := aws.StringValue(sph.SpotPrice) - spotPrice, err := strconv.ParseFloat(spotPriceStr, 64) - // these errors shouldn't occur, but if pricing API does have an error, we ignore the record - if err != nil { - logging.FromContext(ctx).Debugf("unable to parse price record %#v", sph) - continue - } - if sph.Timestamp == nil { - continue - } - instanceType := aws.StringValue(sph.InstanceType) - az := aws.StringValue(sph.AvailabilityZone) - _, ok := prices[instanceType] - if !ok { - prices[instanceType] = map[string]float64{} - } - prices[instanceType][az] = spotPrice - InstancePriceEstimate.With(prometheus.Labels{ - InstanceTypeLabel: instanceType, - CapacityTypeLabel: ec2.UsageClassTypeSpot, - RegionLabel: p.region, - TopologyLabel: az, - }).Set(spotPrice) - } - return true - }) - p.mu.Lock() - defer p.mu.Unlock() + + p.muSpot.Lock() + defer p.muSpot.Unlock() + err := p.ec2.DescribeSpotPriceHistoryPagesWithContext( + ctx, + &ec2.DescribeSpotPriceHistoryInput{ + ProductDescriptions: []*string{ + aws.String("Linux/UNIX"), + aws.String("Linux/UNIX (Amazon VPC)"), + }, + // get the latest spot price for each instance type + StartTime: aws.Time(time.Now()), + }, + p.spotPage(ctx, prices), + ) if err != nil { return fmt.Errorf("retrieving spot pricing data, %w", err) @@ -350,6 +369,8 @@ func (p *Provider) UpdateSpotPricing(ctx context.Context) error { if len(prices) == 0 { return fmt.Errorf("no spot pricing found") } + + totalOfferings := 0 for it, zoneData := range prices { if _, ok := p.spotPrices[it]; !ok { p.spotPrices[it] = newZonalPricing(0) @@ -371,9 +392,11 @@ func (p *Provider) UpdateSpotPricing(ctx context.Context) error { func (p *Provider) LivenessProbe(_ *http.Request) error { // ensure we don't deadlock and nolint for the empty critical section - p.mu.Lock() + p.muOnDemand.Lock() + p.muSpot.Lock() //nolint: staticcheck - p.mu.Unlock() + p.muOnDemand.Unlock() + p.muSpot.Unlock() return nil } diff --git a/pkg/providers/pricing/zz_generated.pricing_aws.go b/pkg/providers/pricing/zz_generated.pricing_aws.go index 18849e4d1cdd..91bce11f094d 100644 --- a/pkg/providers/pricing/zz_generated.pricing_aws.go +++ b/pkg/providers/pricing/zz_generated.pricing_aws.go @@ -16,7 +16,7 @@ limitations under the License. package pricing -// generated at 2023-12-11T13:07:33Z for us-east-1 +// generated at 2024-03-11T13:05:10Z for us-east-1 var InitialOnDemandPricesAWS = map[string]map[string]float64{ // us-east-1 @@ -86,7 +86,8 @@ var InitialOnDemandPricesAWS = map[string]map[string]float64{ "c7g.xlarge": 0.145000, // c7gd family "c7gd.12xlarge": 2.177300, "c7gd.16xlarge": 2.903000, "c7gd.2xlarge": 0.362900, "c7gd.4xlarge": 0.725800, - "c7gd.8xlarge": 1.451500, "c7gd.large": 0.090700, "c7gd.medium": 0.045400, "c7gd.xlarge": 0.181400, + "c7gd.8xlarge": 1.451500, "c7gd.large": 0.090700, "c7gd.medium": 0.045400, "c7gd.metal": 2.903000, + "c7gd.xlarge": 0.181400, // c7gn family "c7gn.12xlarge": 2.995200, "c7gn.16xlarge": 3.993600, "c7gn.2xlarge": 0.499200, "c7gn.4xlarge": 0.998400, "c7gn.8xlarge": 1.996800, "c7gn.large": 0.124800, "c7gn.medium": 0.062400, "c7gn.xlarge": 0.249600, @@ -226,7 +227,8 @@ var InitialOnDemandPricesAWS = map[string]map[string]float64{ "m7g.xlarge": 0.163200, // m7gd family "m7gd.12xlarge": 2.562800, "m7gd.16xlarge": 3.417100, "m7gd.2xlarge": 0.427100, "m7gd.4xlarge": 0.854300, - "m7gd.8xlarge": 1.708600, "m7gd.large": 0.106800, "m7gd.medium": 0.053400, "m7gd.xlarge": 0.213600, + "m7gd.8xlarge": 1.708600, "m7gd.large": 0.106800, "m7gd.medium": 0.053400, "m7gd.metal": 3.417100, + "m7gd.xlarge": 0.213600, // m7i-flex family "m7i-flex.2xlarge": 0.383040, "m7i-flex.4xlarge": 0.766080, "m7i-flex.8xlarge": 1.532160, "m7i-flex.large": 0.095760, "m7i-flex.xlarge": 0.191520, @@ -316,7 +318,8 @@ var InitialOnDemandPricesAWS = map[string]map[string]float64{ "r7g.xlarge": 0.214200, // r7gd family "r7gd.12xlarge": 3.265900, "r7gd.16xlarge": 4.354600, "r7gd.2xlarge": 0.544300, "r7gd.4xlarge": 1.088600, - "r7gd.8xlarge": 2.177300, "r7gd.large": 0.136100, "r7gd.medium": 0.068000, "r7gd.xlarge": 0.272200, + "r7gd.8xlarge": 2.177300, "r7gd.large": 0.136100, "r7gd.medium": 0.068000, "r7gd.metal": 4.354600, + "r7gd.xlarge": 0.272200, // r7i family "r7i.12xlarge": 3.175200, "r7i.16xlarge": 4.233600, "r7i.24xlarge": 6.350400, "r7i.2xlarge": 0.529200, "r7i.48xlarge": 12.700800, "r7i.4xlarge": 1.058400, "r7i.8xlarge": 2.116800, "r7i.large": 0.132300, diff --git a/pkg/providers/pricing/zz_generated.pricing_aws_us_gov.go b/pkg/providers/pricing/zz_generated.pricing_aws_us_gov.go index f45503fb332c..099208bd90d9 100644 --- a/pkg/providers/pricing/zz_generated.pricing_aws_us_gov.go +++ b/pkg/providers/pricing/zz_generated.pricing_aws_us_gov.go @@ -16,7 +16,7 @@ limitations under the License. package pricing -// generated at 2023-12-11T13:07:38Z for us-east-1 +// generated at 2024-03-18T13:06:23Z for us-east-1 var InitialOnDemandPricesUSGov = map[string]map[string]float64{ // us-gov-east-1 @@ -74,8 +74,7 @@ var InitialOnDemandPricesUSGov = map[string]map[string]float64{ "inf1.24xlarge": 5.953000, "inf1.2xlarge": 0.456000, "inf1.6xlarge": 1.488000, "inf1.xlarge": 0.288000, // m5 family "m5.12xlarge": 2.904000, "m5.16xlarge": 3.872000, "m5.24xlarge": 5.808000, "m5.2xlarge": 0.484000, - "m5.4xlarge": 0.968000, "m5.8xlarge": 1.936000, "m5.large": 0.121000, "m5.metal": 5.808000, - "m5.xlarge": 0.242000, + "m5.4xlarge": 0.968000, "m5.8xlarge": 1.936000, "m5.large": 0.121000, "m5.xlarge": 0.242000, // m5a family "m5a.12xlarge": 2.616000, "m5a.16xlarge": 3.488000, "m5a.24xlarge": 5.232000, "m5a.2xlarge": 0.436000, "m5a.4xlarge": 0.872000, "m5a.8xlarge": 1.744000, "m5a.large": 0.109000, "m5a.xlarge": 0.218000, @@ -95,6 +94,10 @@ var InitialOnDemandPricesUSGov = map[string]map[string]float64{ "m6g.12xlarge": 2.323200, "m6g.16xlarge": 3.097600, "m6g.2xlarge": 0.387200, "m6g.4xlarge": 0.774400, "m6g.8xlarge": 1.548800, "m6g.large": 0.096800, "m6g.medium": 0.048400, "m6g.metal": 3.283500, "m6g.xlarge": 0.193600, + // m6gd family + "m6gd.12xlarge": 2.745600, "m6gd.16xlarge": 3.660800, "m6gd.2xlarge": 0.457600, "m6gd.4xlarge": 0.915200, + "m6gd.8xlarge": 1.830400, "m6gd.large": 0.114400, "m6gd.medium": 0.057200, "m6gd.metal": 3.880400, + "m6gd.xlarge": 0.228800, // m6i family "m6i.12xlarge": 2.904000, "m6i.16xlarge": 3.872000, "m6i.24xlarge": 5.808000, "m6i.2xlarge": 0.484000, "m6i.32xlarge": 7.744000, "m6i.4xlarge": 0.968000, "m6i.8xlarge": 1.936000, "m6i.large": 0.121000, @@ -287,6 +290,10 @@ var InitialOnDemandPricesUSGov = map[string]map[string]float64{ "m6g.12xlarge": 2.323200, "m6g.16xlarge": 3.097600, "m6g.2xlarge": 0.387200, "m6g.4xlarge": 0.774400, "m6g.8xlarge": 1.548800, "m6g.large": 0.096800, "m6g.medium": 0.048400, "m6g.metal": 3.283500, "m6g.xlarge": 0.193600, + // m6gd family + "m6gd.12xlarge": 2.745600, "m6gd.16xlarge": 3.660800, "m6gd.2xlarge": 0.457600, "m6gd.4xlarge": 0.915200, + "m6gd.8xlarge": 1.830400, "m6gd.large": 0.114400, "m6gd.medium": 0.057200, "m6gd.metal": 3.880400, + "m6gd.xlarge": 0.228800, // m6i family "m6i.12xlarge": 2.904000, "m6i.16xlarge": 3.872000, "m6i.24xlarge": 5.808000, "m6i.2xlarge": 0.484000, "m6i.32xlarge": 7.744000, "m6i.4xlarge": 0.968000, "m6i.8xlarge": 1.936000, "m6i.large": 0.121000, diff --git a/pkg/providers/version/version.go b/pkg/providers/version/version.go index 814c6eb3af5d..bde13ae72765 100644 --- a/pkg/providers/version/version.go +++ b/pkg/providers/version/version.go @@ -33,7 +33,7 @@ const ( // If a user runs a karpenter image on a k8s version outside the min and max, // One error message will be fired to notify MinK8sVersion = "1.23" - MaxK8sVersion = "1.28" + MaxK8sVersion = "1.29" ) // Provider get the APIServer version. This will be initialized at start up and allows karpenter to have an understanding of the cluster version diff --git a/pkg/test/environment.go b/pkg/test/environment.go index 5e8a7de6c85a..21cc4dcfa50c 100644 --- a/pkg/test/environment.go +++ b/pkg/test/environment.go @@ -52,6 +52,7 @@ func init() { type Environment struct { // API EC2API *fake.EC2API + EKSAPI *fake.EKSAPI SSMAPI *fake.SSMAPI IAMAPI *fake.IAMAPI PricingAPI *fake.PricingAPI @@ -82,6 +83,7 @@ type Environment struct { func NewEnvironment(ctx context.Context, env *coretest.Environment) *Environment { // API ec2api := fake.NewEC2API() + eksapi := fake.NewEKSAPI() ssmapi := fake.NewSSMAPI() iamapi := fake.NewIAMAPI() @@ -110,6 +112,7 @@ func NewEnvironment(ctx context.Context, env *coretest.Environment) *Environment ctx, launchTemplateCache, ec2api, + eksapi, amiResolver, securityGroupProvider, subnetProvider, @@ -131,6 +134,7 @@ func NewEnvironment(ctx context.Context, env *coretest.Environment) *Environment return &Environment{ EC2API: ec2api, + EKSAPI: eksapi, SSMAPI: ssmapi, IAMAPI: iamapi, PricingAPI: fakePricingAPI, @@ -159,6 +163,7 @@ func NewEnvironment(ctx context.Context, env *coretest.Environment) *Environment func (env *Environment) Reset() { env.EC2API.Reset() + env.EKSAPI.Reset() env.SSMAPI.Reset() env.IAMAPI.Reset() env.PricingAPI.Reset() diff --git a/test/cloudformation/grafana_cloudformation.yaml b/test/cloudformation/grafana_cloudformation.yaml index 781c2257d6a3..57179eb1ab10 100644 --- a/test/cloudformation/grafana_cloudformation.yaml +++ b/test/cloudformation/grafana_cloudformation.yaml @@ -19,23 +19,21 @@ Resources: IdpMetadata: Xml: >- - - - - - - MIIF8TCCBNmgAwIBAgIQA6qh6xZ6zQnKXfkqO7LtCDANBgkqhkiG9w0BAQsFADA8MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRwwGgYDVQQDExNBbWF6b24gUlNBIDIwNDggTTAxMB4XDTIzMDUwMTAwMDAwMFoXDTI0MDMxMzIzNTk1OVowLTErMCkGA1UEAxMic2FtbC5pZHAtaW50ZWcuZmVkZXJhdGUuYW1hem9uLmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAIiQuQyI1SSwfeJyXGvA9pYrfORAg3uL38TtjJstX/8Mcqj3yi8OHM9iFq7B+FWOkQ/rvehSUsbPNTVCcS2rnnOCN4MoBX4g3W4ajqKUhmuAnmwqDSZrjXfoVan6khufPqw94rI2n65Au2ryIPfit5Z6umbQOROQFI//mCSO6CDtkmV5Clv5gXoP357AodpYnnKk91e0/m7+/o8MwNjkEerFzsZSroT6QpWRQrkOt+QlBIanCSaJXmrFmEsKQEIyYg7VWUPuCDKgFTbdsqFrYVUE/DhiIszn/3legIGMeAP1lWuH4W0ctsrZwq2zB16i8+fcx0nGMeyHhG0Zyg9DgwMCAwEAAaOCAvwwggL4MB8GA1UdIwQYMBaAFIG4DmOKiRIY5fo7O1CVn+blkBOFMB0GA1UdDgQWBBQA8q7Wb5Jzqe/D7/jr8q7SwH5wnDAtBgNVHREEJjAkgiJzYW1sLmlkcC1pbnRlZy5mZWRlcmF0ZS5hbWF6b24uY29tMA4GA1UdDwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwOwYDVR0fBDQwMjAwoC6gLIYqaHR0cDovL2NybC5yMm0wMS5hbWF6b250cnVzdC5jb20vcjJtMDEuY3JsMBMGA1UdIAQMMAowCAYGZ4EMAQIBMHUGCCsGAQUFBwEBBGkwZzAtBggrBgEFBQcwAYYhaHR0cDovL29jc3AucjJtMDEuYW1hem9udHJ1c3QuY29tMDYGCCsGAQUFBzAChipodHRwOi8vY3J0LnIybTAxLmFtYXpvbnRydXN0LmNvbS9yMm0wMS5jZXIwDAYDVR0TAQH/BAIwADCCAX8GCisGAQQB1nkCBAIEggFvBIIBawFpAHYAdv+IPwq2+5VRwmHM9Ye6NLSkzbsp3GhCCp/mZ0xaOnQAAAGH1vlxigAABAMARzBFAiB8y7GKkz+F1qPeAN6LG08XVVosH8J55Y7h3+bIY8GtlQIhANCHaorJyXR+jzHN5A9b24pJPP4Q3o6yIJr0SyR8rnE5AHcAc9meiRtMlnigIH1HneayxhzQUV5xGSqMa4AQesF3crUAAAGH1vlxhwAABAMASDBGAiEA2X5gFq/rptL7toKXCbI80w9A80qx6ip6fyfyss8zPdQCIQC273DZFdVNOeejp9w9MQj/dJTkTn7N/nto0xY0YOwG7gB2AEiw42vapkc0D+VqAvqdMOscUgHLVt0sgdm7v6s52IRzAAABh9b5cWMAAAQDAEcwRQIhAJrDp/DtSB9oOR+cVxxacoHM+rFbNov0SkWV9PCckBwBAiBd7FKRY/zwErUpBdadDw7fyjxbAjwKgT7nIzDf6ZImfzANBgkqhkiG9w0BAQsFAAOCAQEA0Oo6RNNWN+8vCIqTP5qh+CbbhXgdNcHeW3o/Rqhdje8+D2teeoSCVm17/D7Gdz7rVJiElDCk5D533Zi9ahvKyblVfJlzvSisSVSXBBR/Lhz2+wEsRmTFXdEI+AZtOVpUfbCFqPzVkeoPcVvLs71OWEx9JdlzZL4gX7D7dhQEVt7I7cy1GqaGEE4lakXPecDeFFJtCq1RYVmk0r9o9x4Uq2UnSG8WwUU6Da4S3zcKJsfw9j0c449VY7OvopUiy8lYEMQoBxXhaBVDokGYX2SYiKVHobWTIJs9dOloLs5P+K83vRWbBjhYzKK5pdfzxeyyrdG+fvwOyYtv08z9tijBmg== - - - - - - urn:oasis:names:tc:SAML:2.0:nameid-format:unspecified - - - + + + + + + MIIF7zCCBNegAwIBAgIQArB9/E3FC/Q11QHMC3RpxTANBgkqhkiG9w0BAQsFADA8MQswCQYDVQQGEwJVUzEPMA0GA1UEChMGQW1hem9uMRwwGgYDVQQDExNBbWF6b24gUlNBIDIwNDggTTAxMB4XDTI0MDIwMTAwMDAwMFoXDTI1MDExMTIzNTk1OVowLTErMCkGA1UEAxMic2FtbC5pZHAtaW50ZWcuZmVkZXJhdGUuYW1hem9uLmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALT2EKHnas0RGAZVYt99Bx+vixgPRdo2ObxzWZBl1pR4AwHwirvi0CKbLDk8Kb+WrMfbdLw1eSve7DgEutbjfqByaGJdWwvIVysdKqGABeDTuZMiImxzusJYP/id2/CChLDpGZb8h4rlYg/NFu7uIIOmdb33OtJEkPtv3j79VNXi0CStng7OU0VJU/pQQZOM52DV1Ru1QmZ7ySyUZxYfXI6vZK8dOgOB88XTCtDCJRPVB17FnjNVxnPrad34cozVwIgYRVYYjp19OuGi62XK8qAtH1zWrsoFqUC5B40TUgIBpSvrzNDoSFJxIGVS+VYtI0hqrA5ZtEBwupVX/8qdhnkCAwEAAaOCAvowggL2MB8GA1UdIwQYMBaAFIG4DmOKiRIY5fo7O1CVn+blkBOFMB0GA1UdDgQWBBR9mYxuFkmM61T2uARRR3W9O1dQVTAtBgNVHREEJjAkgiJzYW1sLmlkcC1pbnRlZy5mZWRlcmF0ZS5hbWF6b24uY29tMBMGA1UdIAQMMAowCAYGZ4EMAQIBMA4GA1UdDwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwOwYDVR0fBDQwMjAwoC6gLIYqaHR0cDovL2NybC5yMm0wMS5hbWF6b250cnVzdC5jb20vcjJtMDEuY3JsMHUGCCsGAQUFBwEBBGkwZzAtBggrBgEFBQcwAYYhaHR0cDovL29jc3AucjJtMDEuYW1hem9udHJ1c3QuY29tMDYGCCsGAQUFBzAChipodHRwOi8vY3J0LnIybTAxLmFtYXpvbnRydXN0LmNvbS9yMm0wMS5jZXIwDAYDVR0TAQH/BAIwADCCAX0GCisGAQQB1nkCBAIEggFtBIIBaQFnAHYATnWjJ1yaEMM4W2zU3z9S6x3w4I4bjWnAsfpksWKaOd8AAAGNZFWvUQAABAMARzBFAiEA4TjLW3+0X54p91BrxQTEJHObRliJkn84aFy9at8j3ioCIATXmS+7Q8dPlYBTCJYn8yax2wGBwloaQ1K//OVjKPRDAHYAPxdLT9ciR1iUHWUchL4NEu2QN38fhWrrwb8ohez4ZG4AAAGNZFWvkgAABAMARzBFAiBWUA0dEBEPn726fJz03HVHQ2vTzSUdscLs5hjBnDvVvgIhAN/vHS+4oqR5zOQId3lLKf4WOykV8lr4nYDhstgU9Gr4AHUAfVkeEuF4KnscYWd8Xv340IdcFKBOlZ65Ay/ZDowuebgAAAGNZFWvhAAABAMARjBEAiAwGll9pIluhmNvE7XWVll04ELP+psr79pdj2R45FSBhgIgYpoh0B+GTJfdGakPU39bl2Yk73pyN0vn43weN0JuTtIwDQYJKoZIhvcNAQELBQADggEBAC1gAmyJ57M/8FYYU49OCQv9Lv9GqLN0i2KuSlWSCZ7wqplm0wsvyRs+6hKJB9qM3D2QWShdmq+cignDVDndqhr30VhhYqQLxXJS1FfluhB92SuqPUJm72LY7pgd36ZvaGcMbWyKFHp0PxiiKPQeqZJzfTIQiZBbv5Usa/0zRguLn8LymlUE5VeBW3K1fihYUC5Z0x3Dv0+ZQouQfnkTcnetJisD3zPQvuJ8h62R0kLoz2GrxzW88NmchngRfh2aAUKQHYHKWJQmD83GTh7r9lFeeWYrGj3gDZ2YBMmrRhF4cj+HcskLqRDUvHR+uBj35b0IqAMKVDrJyKFh47+WkQ8= + + + + + + urn:oasis:names:tc:SAML:2.0:nameid-format:unspecified + + + AssertionAttributes: Name: displayName diff --git a/test/cloudformation/iam_cloudformation.yaml b/test/cloudformation/iam_cloudformation.yaml index e73a54933aeb..757354f702e6 100644 --- a/test/cloudformation/iam_cloudformation.yaml +++ b/test/cloudformation/iam_cloudformation.yaml @@ -49,7 +49,7 @@ Resources: - !Sub "arn:${AWS::Partition}:cloudformation:*:${AWS::AccountId}:stack/eksctl-*" Condition: StringEquals: - aws:RequestedRegion: + aws:RequestedRegion: Ref: Regions - Effect: Allow Action: @@ -64,28 +64,28 @@ Resources: Resource: !Sub "arn:${AWS::Partition}:fis:*:${AWS::AccountId}:action/*" Condition: StringEquals: - aws:RequestedRegion: + aws:RequestedRegion: Ref: Regions - Effect: Allow Action: - fis:CreateExperimentTemplate - fis:DeleteExperimentTemplate - fis:StartExperiment - Resource: + Resource: - !Sub "arn:${AWS::Partition}:fis:*:${AWS::AccountId}:experiment-template/*" Condition: StringEquals: - aws:RequestedRegion: + aws:RequestedRegion: Ref: Regions - Effect: Allow Action: - fis:GetExperiment - fis:StartExperiment - Resource: + Resource: - !Sub "arn:${AWS::Partition}:fis:*:${AWS::AccountId}:experiment/*" Condition: StringEquals: - aws:RequestedRegion: + aws:RequestedRegion: Ref: Regions - Effect: Allow Action: @@ -105,11 +105,11 @@ Resources: - sqs:RemovePermission - sqs:DeleteQueue - sqs:UntagQueue - Resource: + Resource: - !Sub "arn:${AWS::Partition}:sqs:*:${AWS::AccountId}:*" Condition: StringEquals: - aws:RequestedRegion: + aws:RequestedRegion: Ref: Regions - Effect: Allow Action: @@ -122,7 +122,7 @@ Resources: Resource: !Sub "arn:${AWS::Partition}:events:*:${AWS::AccountId}:rule/*" Condition: StringEquals: - aws:RequestedRegion: + aws:RequestedRegion: Ref: Regions - Effect: Allow Action: timestream:WriteRecords @@ -183,7 +183,7 @@ Resources: - iam:ListInstanceProfiles - iam:ListInstanceProfileTags Resource: - - !Sub "arn:${AWS::Partition}:iam::${AWS::AccountId}:instance-profile/*" + - !Sub "arn:${AWS::Partition}:iam::${AWS::AccountId}:instance-profile/*" - Effect: Allow Action: - iam:AddRoleToInstanceProfile @@ -236,7 +236,7 @@ Resources: Resource: !Sub "arn:${AWS::Partition}:eks:*:${AWS::AccountId}:cluster/*" Condition: StringEquals: - aws:RequestedRegion: + aws:RequestedRegion: Ref: Regions - Effect: Allow Action: @@ -246,7 +246,7 @@ Resources: - !Sub "arn:${AWS::Partition}:eks:*:${AWS::AccountId}:addon/*" Condition: StringEquals: - aws:RequestedRegion: + aws:RequestedRegion: Ref: Regions - Effect: Allow Action: @@ -256,7 +256,15 @@ Resources: Resource: !Sub "arn:${AWS::Partition}:eks:*:${AWS::AccountId}:nodegroup/*" Condition: StringEquals: - aws:RequestedRegion: + aws:RequestedRegion: + Ref: Regions + - Effect: Allow + Action: + - eks:TagResource + Resource: !Sub "arn:${AWS::Partition}:eks:*:${AWS::AccountId}:podidentityassociation/*" + Condition: + StringEquals: + aws:RequestedRegion: Ref: Regions # GithubActionsPermissionsBoundary includes all permissions needed for all designated roles provisioned by the GithubActions # CI task. This includes the cluster ServiceRoles that are generated by EKSCTL and all roles generated with IRSA to interface from the @@ -421,9 +429,9 @@ Resources: Statement: - Effect: Allow Action: ec2:SendSpotInstanceInterruptions - Resource: + Resource: - !Sub "arn:${AWS::Partition}:ec2:*:${AWS::AccountId}:instance/*" Condition: StringEquals: - aws:RequestedRegion: - Ref: Regions \ No newline at end of file + aws:RequestedRegion: + Ref: Regions diff --git a/test/hack/e2e_scripts/clean_private_cluster.sh b/test/hack/e2e_scripts/clean_private_cluster.sh new file mode 100755 index 000000000000..d749b66dad67 --- /dev/null +++ b/test/hack/e2e_scripts/clean_private_cluster.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +# Delete instance profile +aws iam remove-role-from-instance-profile --instance-profile-name "KarpenterNodeInstanceProfile-${CLUSTER_NAME}" --role-name "KarpenterNodeRole-${CLUSTER_NAME}" +aws iam delete-instance-profile --instance-profile-name "KarpenterNodeInstanceProfile-${CLUSTER_NAME}" + +# Delete private registry policy for pull through cache +aws iam delete-role-policy --role-name "${NODE_ROLE}" --policy-name "PullThroughCachePolicy" + +# Delete cluster +eksctl delete cluster --name "${CLUSTER_NAME}" --force + +#Delete manually created VPC endpoints +endpoints=$(aws ec2 describe-vpc-endpoints --filters Name=vpc-id,Values="${CLUSTER_VPC_ID}" Name=tag:testing/cluster,Values="${CLUSTER_NAME}" --query "VpcEndpoints") +echo "$endpoints" | jq '.[].VpcEndpointId' -r | +while read -r endpointID; +do + aws ec2 delete-vpc-endpoints --vpc-endpoint-ids "$endpointID" + sleep 1 +done + +#Remove codebuild security group ingress from cluster security group +aws ec2 revoke-security-group-ingress --group-id "${EKS_CLUSTER_SG}" --protocol all --source-group "${SG_CB}" + +# Delete route table entry for cluster +subnet_config=$(aws ec2 describe-subnets --filters Name=vpc-id,Values="${VPC_CB}" Name=tag:aws-cdk:subnet-type,Values=Private --query "Subnets") +echo "$subnet_config" | jq '.[].SubnetId' -r | +while read -r subnet; +do + ROUTE_TABLE_ID=$((aws ec2 describe-route-tables --filters Name=vpc-id,Values="${VPC_CB}" Name=association.subnet-id,Values="$subnet" --query "RouteTables[0].RouteTableId") | jq -r) + aws ec2 delete-route --route-table-id "$ROUTE_TABLE_ID" --destination-cidr-block 192.168.0.0/16 +done + +# Delete VPC peering connection +aws ec2 delete-vpc-peering-connection --vpc-peering-connection-id "${VPC_PEERING_CONNECTION_ID}" \ No newline at end of file diff --git a/test/hack/e2e_scripts/configure_private_cluster.sh b/test/hack/e2e_scripts/configure_private_cluster.sh new file mode 100755 index 000000000000..a6babed1d227 --- /dev/null +++ b/test/hack/e2e_scripts/configure_private_cluster.sh @@ -0,0 +1,83 @@ +# Add the SQS and SSM VPC endpoints if we are creating a private cluster +# We need to grab all of the VPC details for the cluster in order to add the endpoint +# Add inbound rules for codeBuild security group, create temporary access entry +VPC_CONFIG=$(aws eks describe-cluster --name "$CLUSTER_NAME" --query "cluster.resourcesVpcConfig") +VPC_ID=$(echo "$VPC_CONFIG" | jq .vpcId -r) +echo CLUSTER_VPC_ID="$VPC_ID" >> "$GITHUB_ENV" +SUBNET_IDS=($(echo "$VPC_CONFIG" | jq '.subnetIds | join(" ")' -r)) +SHARED_NODE_SG=$((aws ec2 describe-security-groups --filters Name=tag:aws:cloudformation:stack-name,Values=eksctl-"$CLUSTER_NAME"-cluster Name=tag:aws:cloudformation:logical-id,Values=ClusterSharedNodeSecurityGroup --query "SecurityGroups[0]") | jq .GroupId -r) +eks_cluster_sg=$((aws ec2 describe-security-groups --filters Name=tag:aws:eks:cluster-name,Values="$CLUSTER_NAME" --query "SecurityGroups[0]") | jq .GroupId -r) +echo EKS_CLUSTER_SG="$eks_cluster_sg" >> "$GITHUB_ENV" + +for SERVICE in "com.amazonaws.$REGION.ssm" "com.amazonaws.$REGION.eks" "com.amazonaws.$REGION.sqs"; do + aws ec2 create-vpc-endpoint \ + --vpc-id "${VPC_ID}" \ + --vpc-endpoint-type Interface \ + --service-name "${SERVICE}" \ + --subnet-ids "${SUBNET_IDS[@]}" \ + --security-group-ids "${eks_cluster_sg}" \ + --tag-specifications "ResourceType=vpc-endpoint,Tags=[{Key=testing/type,Value=e2e},{Key=testing/cluster,Value=$CLUSTER_NAME},{Key=github.com/run-url,Value=https://github.com/$REPOSITORY/actions/runs/$RUN_ID},{Key=karpenter.sh/discovery,Value=$CLUSTER_NAME}]" +done + +# VPC peering request from codebuild +aws ec2 create-vpc-peering-connection --vpc-id "${CODEBUILD_VPC}" --peer-vpc-id "${VPC_ID}" --tag-specifications "ResourceType=vpc-peering-connection,Tags=[{Key=testing/type,Value=e2e},{Key=testing/cluster,Value=$CLUSTER_NAME},{Key=github.com/run-url,Value=https://github.com/$REPOSITORY/actions/runs/$RUN_ID},{Key=karpenter.sh/discovery,Value=$CLUSTER_NAME}]" +vpc_peering_connection_id=$((aws ec2 describe-vpc-peering-connections --filters Name=accepter-vpc-info.vpc-id,Values="${VPC_ID}" --query "VpcPeeringConnections[0]") | jq .VpcPeeringConnectionId -r) +aws ec2 accept-vpc-peering-connection --vpc-peering-connection-id "${vpc_peering_connection_id}" +echo VPC_PEERING_CONNECTION_ID="$vpc_peering_connection_id" >> "$GITHUB_ENV" + +# Modify route table for codebuild vpc +subnet_config=$(aws ec2 describe-subnets --filters Name=vpc-id,Values="${CODEBUILD_VPC}" Name=tag:aws-cdk:subnet-type,Values=Private --query "Subnets") +echo "$subnet_config" | jq '.[].SubnetId' -r | +while read -r subnet; +do + ROUTE_TABLE_ID=$((aws ec2 describe-route-tables --filters Name=vpc-id,Values="${CODEBUILD_VPC}" Name=association.subnet-id,Values="$subnet" --query "RouteTables[0].RouteTableId") | jq -r) + aws ec2 create-route --route-table-id "$ROUTE_TABLE_ID" --destination-cidr-block 192.168.0.0/16 --vpc-peering-connection-id "$vpc_peering_connection_id" +done + + +# Modify route table for cluster vpc +CLUSTER_ROUTE_TABLE=$(aws ec2 describe-route-tables --filters Name=vpc-id,Values="${VPC_ID}" Name=association.main,Values=false --query "RouteTables") +echo "$CLUSTER_ROUTE_TABLE" | jq '.[].RouteTableId' -r | +while read -r routeTableId; +do + aws ec2 create-route --route-table-id $routeTableId --destination-cidr-block 10.0.0.0/16 --vpc-peering-connection-id "$vpc_peering_connection_id" +done + +aws ec2 authorize-security-group-ingress --group-id "${SHARED_NODE_SG}" --protocol all --source-group "${CODEBUILD_SG}" +aws ec2 authorize-security-group-ingress --group-id "${eks_cluster_sg}" --protocol all --source-group "${CODEBUILD_SG}" + +# There is currently no VPC private endpoint for the IAM API. Therefore, we need to +# provision and manage an instance profile manually. +aws iam create-instance-profile --instance-profile-name "KarpenterNodeInstanceProfile-${CLUSTER_NAME}" --tags Key=testing/cluster,Value="$CLUSTER_NAME" +aws iam add-role-to-instance-profile --instance-profile-name "KarpenterNodeInstanceProfile-${CLUSTER_NAME}" --role-name "KarpenterNodeRole-${CLUSTER_NAME}" + +#Create private registry policy for pull through cache +MANAGED_NG=$(aws eks list-nodegroups --cluster-name "${CLUSTER_NAME}" --query nodegroups --output text) +node_role=$(aws eks describe-nodegroup --cluster-name "${CLUSTER_NAME}" --nodegroup-name "${MANAGED_NG}" --query nodegroup.nodeRole --output text | cut -d '/' -f 2) +echo NODE_ROLE="$node_role" >> "$GITHUB_ENV" +cat <> policy.json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "PullThroughCache", + "Effect": "Allow", + "Action": [ + "ecr:BatchImportUpstreamImage", + "ecr:CreateRepository" + ], + "Resource": [ + "arn:aws:ecr:$REGION:$ACCOUNT_ID:repository/ecr-public/*", + "arn:aws:ecr:$REGION:$ACCOUNT_ID:repository/k8s/*", + "arn:aws:ecr:$REGION:$ACCOUNT_ID:repository/quay/*" + ] + } + ] +} +EOF +aws iam put-role-policy --role-name "${node_role}" --policy-name "PullThroughCachePolicy" --policy-document file://policy.json + +# Use pull through cache to pull images that are needed for the tests to run as it requires a route to the internet for the first time +docker pull "$ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/k8s/pause:3.6" +docker pull "$ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/ecr-public/eks-distro/kubernetes/pause:3.2" +docker pull "$ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/ecr-public/docker/library/alpine:latest" \ No newline at end of file diff --git a/test/hack/e2e_scripts/diff_karpenter.sh b/test/hack/e2e_scripts/diff_karpenter.sh new file mode 100755 index 000000000000..42d6e5274a0f --- /dev/null +++ b/test/hack/e2e_scripts/diff_karpenter.sh @@ -0,0 +1,9 @@ +CHART="oci://$ECR_ACCOUNT_ID.dkr.ecr.$ECR_REGION.amazonaws.com/karpenter/snapshot/karpenter" +if (( "$PRIVATE_CLUSTER" == 'true' )); then + CHART="oci://$ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/karpenter/snapshot/karpenter" +fi + +helm diff upgrade --namespace kube-system \ +karpenter "${CHART}" \ +--version 0-$(git rev-parse HEAD) \ +--reuse-values --three-way-merge --detailed-exitcode diff --git a/test/hack/e2e_scripts/install_helm.sh b/test/hack/e2e_scripts/install_helm.sh new file mode 100755 index 000000000000..afa71b811479 --- /dev/null +++ b/test/hack/e2e_scripts/install_helm.sh @@ -0,0 +1,4 @@ +TEMPDIR=$(mktemp -d) +curl -fsSL -o "${TEMPDIR}/get_helm.sh" https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 +chmod 700 "${TEMPDIR}/get_helm.sh" +"${TEMPDIR}/get_helm.sh" --version "$HELM_VERSION" diff --git a/test/hack/e2e_scripts/install_karpenter.sh b/test/hack/e2e_scripts/install_karpenter.sh new file mode 100755 index 000000000000..dbe1aba20d71 --- /dev/null +++ b/test/hack/e2e_scripts/install_karpenter.sh @@ -0,0 +1,43 @@ +aws eks update-kubeconfig --name "$CLUSTER_NAME" + +# Parse minor version to determine whether to enable the webhooks +K8S_VERSION_MINOR="${K8S_VERSION#*.}" +WEBHOOK_ENABLED=false +if (( K8S_VERSION_MINOR < 25 )); then + WEBHOOK_ENABLED=true +fi + +CHART="oci://$ECR_ACCOUNT_ID.dkr.ecr.$ECR_REGION.amazonaws.com/karpenter/snapshot/karpenter" +ADDITIONAL_FLAGS="" +if (( "$PRIVATE_CLUSTER" == 'true' )); then + CHART="oci://$ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/karpenter/snapshot/karpenter" + ADDITIONAL_FLAGS="--set .Values.controller.image.repository=$ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/karpenter/snapshot/controller --set .Values.controller.image.digest=\"\"" +fi + +# Remove service account annotation when dropping support for 1.23 +helm upgrade --install karpenter "${CHART}" \ + -n kube-system \ + --version "0-$(git rev-parse HEAD)" \ + --set logLevel=debug \ + --set webhook.enabled=${WEBHOOK_ENABLED} \ + --set settings.isolatedVPC=${PRIVATE_CLUSTER} \ + --set serviceAccount.annotations."eks\.amazonaws\.com/role-arn"="arn:aws:iam::$ACCOUNT_ID:role/karpenter-irsa-$CLUSTER_NAME" \ + $ADDITIONAL_FLAGS \ + --set settings.clusterName="$CLUSTER_NAME" \ + --set settings.interruptionQueue="$CLUSTER_NAME" \ + --set settings.featureGates.spotToSpotConsolidation=true \ + --set controller.resources.requests.cpu=3 \ + --set controller.resources.requests.memory=3Gi \ + --set controller.resources.limits.cpu=3 \ + --set controller.resources.limits.memory=3Gi \ + --set serviceMonitor.enabled=true \ + --set serviceMonitor.additionalLabels.scrape=enabled \ + --set "serviceMonitor.endpointConfig.relabelings[0].targetLabel=clusterName" \ + --set "serviceMonitor.endpointConfig.relabelings[0].replacement=$CLUSTER_NAME" \ + --set "serviceMonitor.endpointConfig.relabelings[1].targetLabel=gitRef" \ + --set "serviceMonitor.endpointConfig.relabelings[1].replacement=$(git rev-parse HEAD)" \ + --set "serviceMonitor.endpointConfig.relabelings[2].targetLabel=mostRecentTag" \ + --set "serviceMonitor.endpointConfig.relabelings[2].replacement=$(git describe --abbrev=0 --tags)" \ + --set "serviceMonitor.endpointConfig.relabelings[3].targetLabel=commitsAfterTag" \ + --set "serviceMonitor.endpointConfig.relabelings[3].replacement=\"$(git describe --tags | cut -d '-' -f 2)\"" \ + --wait diff --git a/test/hack/e2e_scripts/install_prometheus.sh b/test/hack/e2e_scripts/install_prometheus.sh new file mode 100755 index 000000000000..d05e463dd223 --- /dev/null +++ b/test/hack/e2e_scripts/install_prometheus.sh @@ -0,0 +1,31 @@ +# Remove service account annotation when dropping support for 1.23 + +CHART=prometheus-community/kube-prometheus-stack +VALUES=./.github/actions/e2e/install-prometheus/values.yaml +ENABLED=true +ADDITIONAL_FLAGS="" +if (( "$PRIVATE_CLUSTER" == 'true' )); then + CHART=$(find . -name kube-prometheus-stack*) + ENABLED=false + ADDITIONAL_FLAGS="--set prometheusOperator.admissionWebhooks.patch.image.registry=$ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/k8s --set prometheusOperator.prometheusConfigReloader.image.registry=$ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/quay --set prometheusOperator.image.registry=$ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/quay --set kube-state-metrics.image.registry=$ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/k8s --set alertmanager.alertmanagerSpec.image.registry=$ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/quay --set prometheus.prometheusSpec.image.registry=$ACCOUNT_ID.dkr.ecr.$REGION.amazonaws.com/quay --set grafana.enabled=$ENABLED" +fi + +helm upgrade --install prometheus "${CHART}" \ +-n prometheus \ +-f ${VALUES} \ +--set prometheus.prometheusSpec.remoteWrite[0].url=https://aps-workspaces.$PROMETHEUS_REGION.amazonaws.com/workspaces/$WORKSPACE_ID/api/v1/remote_write \ +--set prometheus.prometheusSpec.remoteWrite[0].sigv4.region=$PROMETHEUS_REGION \ +--set prometheus.serviceAccount.annotations."eks\.amazonaws\.com/role-arn"="arn:aws:iam::$ACCOUNT_ID:role/prometheus-irsa-$CLUSTER_NAME" \ +$ADDITIONAL_FLAGS \ +--set "kubelet.serviceMonitor.cAdvisorRelabelings[0].targetLabel=metrics_path" \ +--set "kubelet.serviceMonitor.cAdvisorRelabelings[0].action=replace" \ +--set "kubelet.serviceMonitor.cAdvisorRelabelings[0].sourceLabels[0]=__metrics_path__" \ +--set "kubelet.serviceMonitor.cAdvisorRelabelings[1].targetLabel=clusterName" \ +--set "kubelet.serviceMonitor.cAdvisorRelabelings[1].replacement=$CLUSTER_NAME" \ +--set "kubelet.serviceMonitor.cAdvisorRelabelings[2].targetLabel=gitRef" \ +--set "kubelet.serviceMonitor.cAdvisorRelabelings[2].replacement=$(git rev-parse HEAD)" \ +--set "kubelet.serviceMonitor.cAdvisorRelabelings[3].targetLabel=mostRecentTag" \ +--set "kubelet.serviceMonitor.cAdvisorRelabelings[3].replacement=$(git describe --abbrev=0 --tags)" \ +--set "kubelet.serviceMonitor.cAdvisorRelabelings[4].targetLabel=commitsAfterTag" \ +--set "kubelet.serviceMonitor.cAdvisorRelabelings[4].replacement=\"$(git describe --tags | cut -d '-' -f 2)\"" \ +--wait diff --git a/test/hack/e2e_scripts/noderole_bootstrap_permission.sh b/test/hack/e2e_scripts/noderole_bootstrap_permission.sh new file mode 100755 index 000000000000..1fa0d9de5be9 --- /dev/null +++ b/test/hack/e2e_scripts/noderole_bootstrap_permission.sh @@ -0,0 +1,7 @@ +eksctl create iamidentitymapping \ +--username system:node:{{EC2PrivateDNSName}} \ +--cluster "$CLUSTER_NAME" \ +--arn "arn:aws:iam::$ACCOUNT_ID:role/KarpenterNodeRole-$CLUSTER_NAME" \ +--group system:bootstrappers \ +--group system:nodes \ +--group eks:kube-proxy-windows diff --git a/test/hack/resource/clean/main.go b/test/hack/resource/clean/main.go index be115a6b855b..fc44495a1984 100644 --- a/test/hack/resource/clean/main.go +++ b/test/hack/resource/clean/main.go @@ -26,6 +26,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/iam" "github.com/samber/lo" "go.uber.org/zap" + "golang.org/x/exp/slices" "github.com/aws/karpenter-provider-aws/test/hack/resource/pkg/metrics" "github.com/aws/karpenter-provider-aws/test/hack/resource/pkg/resourcetypes" @@ -33,6 +34,10 @@ import ( const sweeperCleanedResourcesTableName = "sweeperCleanedResources" +var excludedClusters = []string{ + "soak-periodic-2213793", +} + func main() { expiration := flag.String("expiration", "12h", "define the expirationTTL of the resources") clusterName := flag.String("cluster-name", "", "define cluster name to cleanup") @@ -70,6 +75,7 @@ func main() { resourcetypes.NewOIDC(iamClient), resourcetypes.NewInstanceProfile(iamClient), resourcetypes.NewStack(cloudFormationClient), + resourcetypes.NewVPCPeeringConnection(ec2Client), } for i := range resourceTypes { @@ -78,8 +84,8 @@ func main() { var err error // If there's no cluster defined, clean up all expired resources. otherwise, only cleanup the resources associated with the cluster if lo.FromPtr(clusterName) == "" { - ids, err = resourceTypes[i].GetExpired(ctx, expirationTime) - } else { + ids, err = resourceTypes[i].GetExpired(ctx, expirationTime, excludedClusters) + } else if !slices.Contains(excludedClusters, lo.FromPtr(clusterName)) { ids, err = resourceTypes[i].Get(ctx, lo.FromPtr(clusterName)) } if err != nil { diff --git a/test/hack/resource/count/main.go b/test/hack/resource/count/main.go index 2696fcc5e76b..e80c271759f2 100644 --- a/test/hack/resource/count/main.go +++ b/test/hack/resource/count/main.go @@ -50,6 +50,7 @@ func main() { resourcetypes.NewOIDC(iamClient), resourcetypes.NewInstanceProfile(iamClient), resourcetypes.NewStack(cloudFormationClient), + resourcetypes.NewVPCPeeringConnection(ec2Client), } for i := range resourceTypes { diff --git a/test/hack/resource/go.mod b/test/hack/resource/go.mod index 30966932745d..af0f422f889a 100644 --- a/test/hack/resource/go.mod +++ b/test/hack/resource/go.mod @@ -1,6 +1,6 @@ module github.com/aws/karpenter-provider-aws/test/hack/resource -go 1.21 +go 1.22 require ( github.com/aws/aws-sdk-go-v2 v1.22.1 @@ -12,6 +12,8 @@ require ( github.com/samber/lo v1.38.1 go.uber.org/multierr v1.11.0 go.uber.org/zap v1.24.0 + golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 + k8s.io/api v0.29.2 ) require ( @@ -26,10 +28,21 @@ require ( github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.19.2 // indirect github.com/aws/smithy-go v1.16.0 // indirect + github.com/go-logr/logr v1.3.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/google/gofuzz v1.2.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect - github.com/pkg/errors v0.9.1 // indirect - github.com/stretchr/testify v1.8.1 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect go.uber.org/atomic v1.7.0 // indirect - golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 // indirect + golang.org/x/net v0.19.0 // indirect + golang.org/x/text v0.14.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect + k8s.io/apimachinery v0.29.2 // indirect + k8s.io/klog/v2 v2.110.1 // indirect + k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect ) diff --git a/test/hack/resource/go.sum b/test/hack/resource/go.sum index e03db99812ba..3bdbdcc58303 100644 --- a/test/hack/resource/go.sum +++ b/test/hack/resource/go.sum @@ -45,26 +45,50 @@ github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZx github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/samber/lo v1.38.1 h1:j2XEAqXKb09Am4ebOg31SpvzUTTs6EN3VfgeLUhPdXM= github.com/samber/lo v1.38.1/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= @@ -73,12 +97,58 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 h1:3MTrJm4PyNL9NBqvYDSj3DHl46qQakyfqfWo4jgfaEM= golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/api v0.29.2 h1:hBC7B9+MU+ptchxEqTNW2DkUosJpp1P+Wn6YncZ474A= +k8s.io/api v0.29.2/go.mod h1:sdIaaKuU7P44aoyyLlikSLayT6Vb7bvJNCX105xZXY0= +k8s.io/apimachinery v0.29.2 h1:EWGpfJ856oj11C52NRCHuU7rFDwxev48z+6DSlGNsV8= +k8s.io/apimachinery v0.29.2/go.mod h1:6HVkd1FwxIagpYrHSwJlQqZI3G9LfYWRPAkUvLnXTKU= +k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= +k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= +sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/test/hack/resource/pkg/resourcetypes/eni.go b/test/hack/resource/pkg/resourcetypes/eni.go index 52201a213768..ac759d2d8db7 100644 --- a/test/hack/resource/pkg/resourcetypes/eni.go +++ b/test/hack/resource/pkg/resourcetypes/eni.go @@ -23,6 +23,7 @@ import ( ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/samber/lo" "go.uber.org/multierr" + "golang.org/x/exp/slices" ) type ENI struct { @@ -41,7 +42,7 @@ func (e *ENI) Global() bool { return false } -func (e *ENI) GetExpired(ctx context.Context, expirationTime time.Time) (ids []string, err error) { +func (e *ENI) GetExpired(ctx context.Context, expirationTime time.Time, excludedClusters []string) (ids []string, err error) { var nextToken *string for { out, err := e.ec2Client.DescribeNetworkInterfaces(ctx, &ec2.DescribeNetworkInterfacesInput{ @@ -58,6 +59,12 @@ func (e *ENI) GetExpired(ctx context.Context, expirationTime time.Time) (ids []s } for _, ni := range out.NetworkInterfaces { + clusterName, found := lo.Find(ni.TagSet, func(tag ec2types.Tag) bool { + return *tag.Key == k8sClusterTag + }) + if found && slices.Contains(excludedClusters, lo.FromPtr(clusterName.Value)) { + continue + } creationDate, found := lo.Find(ni.TagSet, func(tag ec2types.Tag) bool { return *tag.Key == "node.k8s.amazonaws.com/createdAt" }) diff --git a/test/hack/resource/pkg/resourcetypes/instance.go b/test/hack/resource/pkg/resourcetypes/instance.go index 2e8c557014fe..089bda1b8d59 100644 --- a/test/hack/resource/pkg/resourcetypes/instance.go +++ b/test/hack/resource/pkg/resourcetypes/instance.go @@ -21,6 +21,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/ec2" ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/samber/lo" + "golang.org/x/exp/slices" ) type Instance struct { @@ -39,7 +40,7 @@ func (i *Instance) Global() bool { return false } -func (i *Instance) GetExpired(ctx context.Context, expirationTime time.Time) (ids []string, err error) { +func (i *Instance) GetExpired(ctx context.Context, expirationTime time.Time, excludedClusters []string) (ids []string, err error) { var nextToken *string for { out, err := i.ec2Client.DescribeInstances(ctx, &ec2.DescribeInstancesInput{ @@ -61,6 +62,12 @@ func (i *Instance) GetExpired(ctx context.Context, expirationTime time.Time) (id for _, res := range out.Reservations { for _, instance := range res.Instances { + clusterName, found := lo.Find(instance.Tags, func(tag ec2types.Tag) bool { + return *tag.Key == karpenterTestingTag + }) + if found && slices.Contains(excludedClusters, lo.FromPtr(clusterName.Value)) { + continue + } if lo.FromPtr(instance.LaunchTime).Before(expirationTime) { ids = append(ids, lo.FromPtr(instance.InstanceId)) } diff --git a/test/hack/resource/pkg/resourcetypes/instanceprofile.go b/test/hack/resource/pkg/resourcetypes/instanceprofile.go index 29c645876fd7..fa205f7ccc32 100644 --- a/test/hack/resource/pkg/resourcetypes/instanceprofile.go +++ b/test/hack/resource/pkg/resourcetypes/instanceprofile.go @@ -16,13 +16,15 @@ package resourcetypes import ( "context" - "strings" "time" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/iam" + iamtypes "github.com/aws/aws-sdk-go-v2/service/iam/types" "github.com/samber/lo" "go.uber.org/multierr" + "golang.org/x/exp/slices" + v1 "k8s.io/api/core/v1" ) type InstanceProfile struct { @@ -41,7 +43,7 @@ func (ip *InstanceProfile) Global() bool { return true } -func (ip *InstanceProfile) GetExpired(ctx context.Context, expirationTime time.Time) (names []string, err error) { +func (ip *InstanceProfile) GetExpired(ctx context.Context, expirationTime time.Time, excludedClusters []string) (names []string, err error) { out, err := ip.iamClient.ListInstanceProfiles(ctx, &iam.ListInstanceProfilesInput{}) if err != nil { return names, err @@ -49,10 +51,6 @@ func (ip *InstanceProfile) GetExpired(ctx context.Context, expirationTime time.T errs := make([]error, len(out.InstanceProfiles)) for i := range out.InstanceProfiles { - // Checking to make sure we are only list resources in the given region - if !strings.Contains(lo.FromPtr(out.InstanceProfiles[i].Arn), lo.Must(config.LoadDefaultConfig(ctx)).Region) { - continue - } profiles, err := ip.iamClient.ListInstanceProfileTags(ctx, &iam.ListInstanceProfileTagsInput{ InstanceProfileName: out.InstanceProfiles[i].InstanceProfileName, }) @@ -61,6 +59,18 @@ func (ip *InstanceProfile) GetExpired(ctx context.Context, expirationTime time.T continue } + clusterName, _ := lo.Find(profiles.Tags, func(tag iamtypes.Tag) bool { + return lo.FromPtr(tag.Key) == karpenterTestingTag + }) + // Checking to make sure we are only list resources in the given region + region, _ := lo.Find(profiles.Tags, func(tag iamtypes.Tag) bool { + return lo.FromPtr(tag.Key) == v1.LabelTopologyZone + }) + + if slices.Contains(excludedClusters, lo.FromPtr(clusterName.Value)) || lo.FromPtr(region.Value) != lo.Must(config.LoadDefaultConfig(ctx)).Region { + continue + } + for _, t := range profiles.Tags { // Since we can only get the date of the instance profile (not the exact time the instance profile was created) // we add a day to the time that it was created to account for the worst-case of the instance profile being created @@ -128,6 +138,7 @@ func (ip *InstanceProfile) Cleanup(ctx context.Context, names []string) ([]strin if err != nil { errs = multierr.Append(errs, err) } + deleted = append(deleted, names[i]) } return deleted, errs } diff --git a/test/hack/resource/pkg/resourcetypes/launchtemplate.go b/test/hack/resource/pkg/resourcetypes/launchtemplate.go index d4b6b964f12b..18cfaccbf46f 100644 --- a/test/hack/resource/pkg/resourcetypes/launchtemplate.go +++ b/test/hack/resource/pkg/resourcetypes/launchtemplate.go @@ -22,6 +22,7 @@ import ( ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/samber/lo" "go.uber.org/multierr" + "golang.org/x/exp/slices" ) type LaunchTemplate struct { @@ -40,7 +41,7 @@ func (lt *LaunchTemplate) Global() bool { return false } -func (lt *LaunchTemplate) GetExpired(ctx context.Context, expirationTime time.Time) (names []string, err error) { +func (lt *LaunchTemplate) GetExpired(ctx context.Context, expirationTime time.Time, excludedClusters []string) (names []string, err error) { var nextToken *string for { out, err := lt.ec2Client.DescribeLaunchTemplates(ctx, &ec2.DescribeLaunchTemplatesInput{ @@ -57,6 +58,12 @@ func (lt *LaunchTemplate) GetExpired(ctx context.Context, expirationTime time.Ti } for _, launchTemplate := range out.LaunchTemplates { + clusterName, found := lo.Find(launchTemplate.Tags, func(tag ec2types.Tag) bool { + return *tag.Key == k8sClusterTag + }) + if found && slices.Contains(excludedClusters, lo.FromPtr(clusterName.Value)) { + continue + } if lo.FromPtr(launchTemplate.CreateTime).Before(expirationTime) { names = append(names, lo.FromPtr(launchTemplate.LaunchTemplateName)) } diff --git a/test/hack/resource/pkg/resourcetypes/oidc.go b/test/hack/resource/pkg/resourcetypes/oidc.go index 92a8b2843b38..51fc42b16a8f 100644 --- a/test/hack/resource/pkg/resourcetypes/oidc.go +++ b/test/hack/resource/pkg/resourcetypes/oidc.go @@ -21,8 +21,10 @@ import ( "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/iam" + iamtypes "github.com/aws/aws-sdk-go-v2/service/iam/types" "github.com/samber/lo" "go.uber.org/multierr" + "golang.org/x/exp/slices" ) type OIDC struct { @@ -41,7 +43,7 @@ func (o *OIDC) Global() bool { return true } -func (o *OIDC) GetExpired(ctx context.Context, expirationTime time.Time) (names []string, err error) { +func (o *OIDC) GetExpired(ctx context.Context, expirationTime time.Time, excludedClusters []string) (names []string, err error) { out, err := o.iamClient.ListOpenIDConnectProviders(ctx, &iam.ListOpenIDConnectProvidersInput{}) if err != nil { return names, err @@ -61,6 +63,13 @@ func (o *OIDC) GetExpired(ctx context.Context, expirationTime time.Time) (names continue } + clusterName, found := lo.Find(oicd.Tags, func(tag iamtypes.Tag) bool { + return *tag.Key == k8sClusterTag + }) + if found && slices.Contains(excludedClusters, lo.FromPtr(clusterName.Value)) { + continue + } + for _, t := range oicd.Tags { if lo.FromPtr(t.Key) == githubRunURLTag && oicd.CreateDate.Before(expirationTime) { names = append(names, lo.FromPtr(out.OpenIDConnectProviderList[i].Arn)) diff --git a/test/hack/resource/pkg/resourcetypes/resourcetypes.go b/test/hack/resource/pkg/resourcetypes/resourcetypes.go index bf4f84bc94b1..ce8a7bfd9441 100644 --- a/test/hack/resource/pkg/resourcetypes/resourcetypes.go +++ b/test/hack/resource/pkg/resourcetypes/resourcetypes.go @@ -39,7 +39,7 @@ type Type interface { // Get returns all resources of the type associated with the clusterName Get(ctx context.Context, clusterName string) (ids []string, err error) // GetExpired returns all resources of the type that were provisioned before the expirationTime - GetExpired(ctx context.Context, expirationTime time.Time) (ids []string, err error) + GetExpired(ctx context.Context, expirationTime time.Time, excludedClusters []string) (ids []string, err error) // Cleanup deletes all resources of the type by id and returns the resource ids it succeeded to delete // In general, if all resources can't be deleted by id with a single API call (like with DeleteInstances) // you should call the requests synchronously to avoid rate limiting against the number of requests made diff --git a/test/hack/resource/pkg/resourcetypes/securitygroup.go b/test/hack/resource/pkg/resourcetypes/securitygroup.go index e9b57096f26d..e7224ce694dd 100644 --- a/test/hack/resource/pkg/resourcetypes/securitygroup.go +++ b/test/hack/resource/pkg/resourcetypes/securitygroup.go @@ -23,6 +23,7 @@ import ( ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/samber/lo" "go.uber.org/multierr" + "golang.org/x/exp/slices" ) type SecurityGroup struct { @@ -41,7 +42,7 @@ func (sg *SecurityGroup) Global() bool { return false } -func (sg *SecurityGroup) GetExpired(ctx context.Context, expirationTime time.Time) (ids []string, err error) { +func (sg *SecurityGroup) GetExpired(ctx context.Context, expirationTime time.Time, excludedClusters []string) (ids []string, err error) { var nextToken *string for { out, err := sg.ec2Client.DescribeSecurityGroups(ctx, &ec2.DescribeSecurityGroupsInput{ @@ -58,6 +59,12 @@ func (sg *SecurityGroup) GetExpired(ctx context.Context, expirationTime time.Tim } for _, sgroup := range out.SecurityGroups { + clusterName, found := lo.Find(sgroup.Tags, func(tag ec2types.Tag) bool { + return *tag.Key == karpenterTestingTag + }) + if found && slices.Contains(excludedClusters, lo.FromPtr(clusterName.Value)) { + continue + } creationDate, found := lo.Find(sgroup.Tags, func(tag ec2types.Tag) bool { return *tag.Key == "creation-date" }) diff --git a/test/hack/resource/pkg/resourcetypes/stack.go b/test/hack/resource/pkg/resourcetypes/stack.go index 7d42ec1225c1..fed6ecece85f 100644 --- a/test/hack/resource/pkg/resourcetypes/stack.go +++ b/test/hack/resource/pkg/resourcetypes/stack.go @@ -22,6 +22,7 @@ import ( cloudformationtypes "github.com/aws/aws-sdk-go-v2/service/cloudformation/types" "github.com/samber/lo" "go.uber.org/multierr" + "golang.org/x/exp/slices" ) type Stack struct { @@ -40,7 +41,7 @@ func (s *Stack) Global() bool { return false } -func (s *Stack) GetExpired(ctx context.Context, expirationTime time.Time) (names []string, err error) { +func (s *Stack) GetExpired(ctx context.Context, expirationTime time.Time, excludedClusters []string) (names []string, err error) { var nextToken *string for { out, err := s.cloudFormationClient.DescribeStacks(ctx, &cloudformation.DescribeStacksInput{ @@ -55,6 +56,12 @@ func (s *Stack) GetExpired(ctx context.Context, expirationTime time.Time) (names s.StackStatus == cloudformationtypes.StackStatusDeleteInProgress }) for _, stack := range stacks { + clusterName, found := lo.Find(stack.Tags, func(tag cloudformationtypes.Tag) bool { + return *tag.Key == karpenterTestingTag + }) + if found && slices.Contains(excludedClusters, lo.FromPtr(clusterName.Value)) { + continue + } if _, found := lo.Find(stack.Tags, func(t cloudformationtypes.Tag) bool { return lo.FromPtr(t.Key) == karpenterTestingTag || lo.FromPtr(t.Key) == githubRunURLTag }); found && lo.FromPtr(stack.CreationTime).Before(expirationTime) { diff --git a/test/hack/resource/pkg/resourcetypes/vpc_endpoint.go b/test/hack/resource/pkg/resourcetypes/vpc_endpoint.go index 16729d27b35d..a87d8a3b599c 100644 --- a/test/hack/resource/pkg/resourcetypes/vpc_endpoint.go +++ b/test/hack/resource/pkg/resourcetypes/vpc_endpoint.go @@ -21,6 +21,7 @@ import ( "github.com/aws/aws-sdk-go-v2/service/ec2" ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" "github.com/samber/lo" + "golang.org/x/exp/slices" ) type VPCEndpoint struct { @@ -85,7 +86,7 @@ func (v *VPCEndpoint) CountAll(ctx context.Context) (count int, err error) { return count, err } -func (v *VPCEndpoint) GetExpired(ctx context.Context, expirationTime time.Time) (ids []string, err error) { +func (v *VPCEndpoint) GetExpired(ctx context.Context, expirationTime time.Time, excludedClusters []string) (ids []string, err error) { var nextToken *string for { out, err := v.ec2Client.DescribeVpcEndpoints(ctx, &ec2.DescribeVpcEndpointsInput{ @@ -101,6 +102,12 @@ func (v *VPCEndpoint) GetExpired(ctx context.Context, expirationTime time.Time) return ids, err } for _, endpoint := range out.VpcEndpoints { + clusterName, found := lo.Find(endpoint.Tags, func(tag ec2types.Tag) bool { + return *tag.Key == k8sClusterTag + }) + if found && slices.Contains(excludedClusters, lo.FromPtr(clusterName.Value)) { + continue + } if endpoint.CreationTimestamp.Before(expirationTime) { ids = append(ids, lo.FromPtr(endpoint.VpcEndpointId)) } diff --git a/test/hack/resource/pkg/resourcetypes/vpc_peering_connection.go b/test/hack/resource/pkg/resourcetypes/vpc_peering_connection.go new file mode 100644 index 000000000000..b7c9648c609c --- /dev/null +++ b/test/hack/resource/pkg/resourcetypes/vpc_peering_connection.go @@ -0,0 +1,133 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resourcetypes + +import ( + "context" + "time" + + "github.com/aws/aws-sdk-go-v2/service/ec2" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/samber/lo" + "golang.org/x/exp/slices" +) + +type VPCPeeringConnection struct { + ec2Client *ec2.Client +} + +func NewVPCPeeringConnection(ec2Client *ec2.Client) *VPCPeeringConnection { + return &VPCPeeringConnection{ec2Client: ec2Client} +} + +func (v *VPCPeeringConnection) String() string { + return "VPCPeeringConnection" +} + +func (v *VPCPeeringConnection) Global() bool { + return false +} + +func (v *VPCPeeringConnection) Get(ctx context.Context, clusterName string) (ids []string, err error) { + var nextToken *string + for { + out, err := v.ec2Client.DescribeVpcPeeringConnections(ctx, &ec2.DescribeVpcPeeringConnectionsInput{ + Filters: []ec2types.Filter{ + { + Name: lo.ToPtr("tag:" + karpenterTestingTag), + Values: []string{clusterName}, + }, + }, + NextToken: nextToken, + }) + if err != nil { + return ids, err + } + for _, connection := range out.VpcPeeringConnections { + ids = append(ids, lo.FromPtr(connection.VpcPeeringConnectionId)) + } + nextToken = out.NextToken + if nextToken == nil { + break + } + } + return ids, err +} + +func (v *VPCPeeringConnection) CountAll(ctx context.Context) (count int, err error) { + var nextToken *string + for { + out, err := v.ec2Client.DescribeVpcPeeringConnections(ctx, &ec2.DescribeVpcPeeringConnectionsInput{ + NextToken: nextToken, + }) + if err != nil { + return count, err + } + + count += len(out.VpcPeeringConnections) + + nextToken = out.NextToken + if nextToken == nil { + break + } + } + return count, err +} + +func (v *VPCPeeringConnection) GetExpired(ctx context.Context, expirationTime time.Time, excludedClusters []string) (ids []string, err error) { + var nextToken *string + for { + out, err := v.ec2Client.DescribeVpcPeeringConnections(ctx, &ec2.DescribeVpcPeeringConnectionsInput{ + Filters: []ec2types.Filter{ + { + Name: lo.ToPtr("tag-key"), + Values: []string{karpenterTestingTag}, + }, + }, + NextToken: nextToken, + }) + if err != nil { + return ids, err + } + for _, connection := range out.VpcPeeringConnections { + clusterName, found := lo.Find(connection.Tags, func(tag ec2types.Tag) bool { + return *tag.Key == k8sClusterTag + }) + if found && slices.Contains(excludedClusters, lo.FromPtr(clusterName.Value)) { + continue + } + if connection.ExpirationTime.Before(expirationTime) { + ids = append(ids, lo.FromPtr(connection.VpcPeeringConnectionId)) + } + } + nextToken = out.NextToken + if nextToken == nil { + break + } + } + return ids, err +} + +// Cleanup any old VPC peering connections that were provisioned as part of testing +func (v *VPCPeeringConnection) Cleanup(ctx context.Context, ids []string) ([]string, error) { + for _, id := range ids { + if _, err := v.ec2Client.DeleteVpcPeeringConnection(ctx, &ec2.DeleteVpcPeeringConnectionInput{ + VpcPeeringConnectionId: lo.ToPtr(id), + }); err != nil { + return nil, err + } + } + return ids, nil +} diff --git a/test/hack/soak/get_clusters.go b/test/hack/soak/get_clusters.go index abed1c068f59..1278f933753b 100644 --- a/test/hack/soak/get_clusters.go +++ b/test/hack/soak/get_clusters.go @@ -18,6 +18,7 @@ import ( "context" "encoding/json" "fmt" + "slices" "strings" "time" @@ -35,6 +36,8 @@ type cluster struct { const expirationTTL = time.Hour * 168 // 7 days +var excludedClustersCleanup = []string{} + func main() { ctx := context.Background() cfg := lo.Must(config.LoadDefaultConfig(ctx)) @@ -54,9 +57,14 @@ func main() { if strings.HasPrefix(c, "soak-periodic-") { outputList = append(outputList, &cluster{ - Name: c, - GitRef: clusterDetails.Cluster.Tags["test/git_ref"], - Cleanup: clusterDetails.Cluster.CreatedAt.Before(expirationTime)}) + Name: c, + GitRef: clusterDetails.Cluster.Tags["test/git_ref"], + Cleanup: lo.Ternary( + slices.Contains(excludedClustersCleanup, c), + false, + clusterDetails.Cluster.CreatedAt.Before(expirationTime), + ), + }) } } diff --git a/test/hack/soak/go.mod b/test/hack/soak/go.mod index 2a9582b9fc72..cfe28ff8f3d1 100644 --- a/test/hack/soak/go.mod +++ b/test/hack/soak/go.mod @@ -1,6 +1,6 @@ module github.com/aws/karpenter/test/hack/soak -go 1.21 +go 1.22 require ( github.com/aws/aws-sdk-go v1.47.9 diff --git a/test/pkg/debug/node.go b/test/pkg/debug/node.go index 8d27be82e280..5fd1d0e3f4ff 100644 --- a/test/pkg/debug/node.go +++ b/test/pkg/debug/node.go @@ -62,7 +62,7 @@ func (c *NodeController) Reconcile(ctx context.Context, req reconcile.Request) ( } func (c *NodeController) GetInfo(ctx context.Context, n *v1.Node) string { - pods, _ := nodeutils.GetNodePods(ctx, c.kubeClient, n) + pods, _ := nodeutils.GetPods(ctx, c.kubeClient, n) return fmt.Sprintf("ready=%s schedulable=%t initialized=%s pods=%d taints=%v", nodeutils.GetCondition(n, v1.NodeReady).Status, !n.Spec.Unschedulable, n.Labels[v1beta1.NodeInitializedLabelKey], len(pods), n.Spec.Taints) } diff --git a/test/pkg/environment/aws/environment.go b/test/pkg/environment/aws/environment.go index c243c7c07205..1b1dc4476ab5 100644 --- a/test/pkg/environment/aws/environment.go +++ b/test/pkg/environment/aws/environment.go @@ -19,6 +19,8 @@ import ( "os" "testing" + coretest "sigs.k8s.io/karpenter/pkg/test" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/client" "github.com/aws/aws-sdk-go/aws/endpoints" @@ -53,11 +55,14 @@ func init() { corev1beta1.NormalizedLabels = lo.Assign(corev1beta1.NormalizedLabels, map[string]string{"topology.ebs.csi.aws.com/zone": corev1.LabelTopologyZone}) } -const WindowsDefaultImage = "mcr.microsoft.com/oss/kubernetes/pause:3.9" +var WindowsDefaultImage = "mcr.microsoft.com/oss/kubernetes/pause:3.9" + +var EphemeralInitContainerImage = "alpine" // ExcludedInstanceFamilies denotes instance families that have issues during resource registration due to compatibility -// issues with versions of the VPR Resource Controller -var ExcludedInstanceFamilies = []string{"m7a", "r7a", "c7a", "r7i"} +// issues with versions of the VPR Resource Controller. +// TODO: jmdeal@ remove a1 from exclusion list once Karpenter implicitly filters a1 instances for AL2023 AMI family (incompatible) +var ExcludedInstanceFamilies = []string{"m7a", "r7a", "c7a", "r7i", "a1"} type Environment struct { *common.Environment @@ -76,6 +81,7 @@ type Environment struct { ClusterName string ClusterEndpoint string InterruptionQueue string + PrivateCluster bool } func NewEnvironment(t *testing.T) *Environment { @@ -105,6 +111,12 @@ func NewEnvironment(t *testing.T) *Environment { ClusterName: lo.Must(os.LookupEnv("CLUSTER_NAME")), ClusterEndpoint: lo.Must(os.LookupEnv("CLUSTER_ENDPOINT")), } + + if _, awsEnv.PrivateCluster = os.LookupEnv("PRIVATE_CLUSTER"); awsEnv.PrivateCluster { + WindowsDefaultImage = fmt.Sprintf("857221689048.dkr.ecr.%s.amazonaws.com/k8s/pause:3.6", awsEnv.Region) + EphemeralInitContainerImage = fmt.Sprintf("857221689048.dkr.ecr.%s.amazonaws.com/ecr-public/docker/library/alpine:latest", awsEnv.Region) + coretest.DefaultImage = fmt.Sprintf("857221689048.dkr.ecr.%s.amazonaws.com/ecr-public/eks-distro/kubernetes/pause:3.2", awsEnv.Region) + } // Initialize the provider only if the INTERRUPTION_QUEUE environment variable is defined if v, ok := os.LookupEnv("INTERRUPTION_QUEUE"); ok { awsEnv.SQSProvider = lo.Must(sqs.NewProvider(env.Context, servicesqs.New(session), v)) @@ -122,7 +134,7 @@ func GetTimeStreamAPI(session *session.Session) timestreamwriteiface.TimestreamW func (env *Environment) DefaultEC2NodeClass() *v1beta1.EC2NodeClass { nodeClass := test.EC2NodeClass() - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 + nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2023 nodeClass.Spec.Tags = map[string]string{ "testing/cluster": env.ClusterName, } @@ -136,6 +148,11 @@ func (env *Environment) DefaultEC2NodeClass() *v1beta1.EC2NodeClass { Tags: map[string]string{"karpenter.sh/discovery": env.ClusterName}, }, } + if env.PrivateCluster { + nodeClass.Spec.Role = "" + nodeClass.Spec.InstanceProfile = lo.ToPtr(fmt.Sprintf("KarpenterNodeInstanceProfile-%s", env.ClusterName)) + return nodeClass + } nodeClass.Spec.Role = fmt.Sprintf("KarpenterNodeRole-%s", env.ClusterName) return nodeClass } diff --git a/test/pkg/environment/aws/expectations.go b/test/pkg/environment/aws/expectations.go index 6703a651c17d..2b69ccd47d64 100644 --- a/test/pkg/environment/aws/expectations.go +++ b/test/pkg/environment/aws/expectations.go @@ -338,6 +338,14 @@ func (env *Environment) GetK8sVersion(offset int) string { return fmt.Sprintf("%s.%d", serverVersion.Major, minorVersion-offset) } +func (env *Environment) GetK8sMinorVersion(offset int) (int, error) { + version, err := strconv.Atoi(strings.Split(env.GetK8sVersion(offset), ".")[1]) + if err != nil { + return 0, err + } + return version, nil +} + func (env *Environment) GetCustomAMI(amiPath string, versionOffset int) string { version := env.GetK8sVersion(versionOffset) parameter, err := env.SSMAPI.GetParameter(&ssm.GetParameterInput{ diff --git a/test/pkg/environment/common/environment.go b/test/pkg/environment/common/environment.go index d7473c894dd5..908a734904be 100644 --- a/test/pkg/environment/common/environment.go +++ b/test/pkg/environment/common/environment.go @@ -142,26 +142,42 @@ func (env *Environment) DefaultNodePool(nodeClass *v1beta1.EC2NodeClass) *corev1 nodePool.Spec.Template.Spec.NodeClassRef = &corev1beta1.NodeClassReference{ Name: nodeClass.Name, } - nodePool.Spec.Template.Spec.Requirements = []v1.NodeSelectorRequirement{ + nodePool.Spec.Template.Spec.Requirements = []corev1beta1.NodeSelectorRequirementWithMinValues{ { - Key: v1.LabelOSStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{string(v1.Linux)}, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1.LabelOSStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{string(v1.Linux)}, + }, }, { - Key: corev1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.CapacityTypeOnDemand}, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: corev1beta1.CapacityTypeLabelKey, + Operator: v1.NodeSelectorOpIn, + Values: []string{corev1beta1.CapacityTypeOnDemand}, + }, }, { - Key: v1beta1.LabelInstanceCategory, - Operator: v1.NodeSelectorOpIn, - Values: []string{"c", "m", "r"}, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceCategory, + Operator: v1.NodeSelectorOpIn, + Values: []string{"c", "m", "r"}, + }, }, { - Key: v1beta1.LabelInstanceGeneration, - Operator: v1.NodeSelectorOpGt, - Values: []string{"2"}, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceGeneration, + Operator: v1.NodeSelectorOpGt, + Values: []string{"2"}, + }, + }, + // Filter out a1 instance types, which are incompatible with AL2023 AMIs + { + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceFamily, + Operator: v1.NodeSelectorOpNotIn, + Values: []string{"a1"}, + }, }, } nodePool.Spec.Disruption.ConsolidateAfter = &corev1beta1.NillableDuration{} diff --git a/test/pkg/environment/common/expectations.go b/test/pkg/environment/common/expectations.go index c4ceddd16d74..adbc5cdda0ee 100644 --- a/test/pkg/environment/common/expectations.go +++ b/test/pkg/environment/common/expectations.go @@ -19,6 +19,7 @@ import ( "encoding/base64" "fmt" "io" + "math" "strings" "time" @@ -134,7 +135,7 @@ func (env *Environment) ExpectSettingsReplaced(vars ...v1.EnvVar) { if !equality.Semantic.DeepEqual(d, stored) { By("replacing environment variables for karpenter deployment") - Expect(env.Client.Patch(env.Context, d, client.MergeFrom(stored))).To(Succeed()) + Expect(env.Client.Patch(env.Context, d, client.StrategicMergeFrom(stored))).To(Succeed()) env.EventuallyExpectKarpenterRestarted() } } @@ -158,7 +159,7 @@ func (env *Environment) ExpectSettingsOverridden(vars ...v1.EnvVar) { } if !equality.Semantic.DeepEqual(d, stored) { By("overriding environment variables for karpenter deployment") - Expect(env.Client.Patch(env.Context, d, client.MergeFrom(stored))).To(Succeed()) + Expect(env.Client.Patch(env.Context, d, client.StrategicMergeFrom(stored))).To(Succeed()) env.EventuallyExpectKarpenterRestarted() } } @@ -178,7 +179,7 @@ func (env *Environment) ExpectSettingsRemoved(vars ...v1.EnvVar) { }) if !equality.Semantic.DeepEqual(d, stored) { By("removing environment variables for karpenter deployment") - Expect(env.Client.Patch(env.Context, d, client.MergeFrom(stored))).To(Succeed()) + Expect(env.Client.Patch(env.Context, d, client.StrategicMergeFrom(stored))).To(Succeed()) env.EventuallyExpectKarpenterRestarted() } } @@ -260,11 +261,12 @@ func (env *Environment) ExpectPrefixDelegationDisabled() { "ENABLE_PREFIX_DELEGATION", "false", "aws-node") } -func (env *Environment) ExpectExists(obj client.Object) { +func (env *Environment) ExpectExists(obj client.Object) client.Object { GinkgoHelper() Eventually(func(g Gomega) { g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(obj), obj)).To(Succeed()) }).WithTimeout(time.Second * 5).Should(Succeed()) + return obj } func (env *Environment) EventuallyExpectBound(pods ...*v1.Pod) { @@ -327,7 +329,7 @@ func (env *Environment) EventuallyExpectRollout(name, namespace string) { "kubectl.kubernetes.io/restartedAt": time.Now().Format(time.RFC3339), } deploy.Spec.Template.Annotations = lo.Assign(deploy.Spec.Template.Annotations, restartedAtAnnotation) - Expect(env.Client.Patch(env.Context, deploy, client.MergeFrom(stored))).To(Succeed()) + Expect(env.Client.Patch(env.Context, deploy, client.StrategicMergeFrom(stored))).To(Succeed()) By("waiting for the newly generated deployment to rollout") Eventually(func(g Gomega) { @@ -468,13 +470,35 @@ func (env *Environment) ExpectCreatedNodeCount(comparator string, count int) []* return createdNodes } +func (env *Environment) ExpectNodeCount(comparator string, count int) { + GinkgoHelper() + + nodeList := &v1.NodeList{} + Expect(env.Client.List(env, nodeList, client.HasLabels{test.DiscoveryLabel})).To(Succeed()) + Expect(len(nodeList.Items)).To(BeNumerically(comparator, count)) +} + +func (env *Environment) ExpectNodeClaimCount(comparator string, count int) { + GinkgoHelper() + + nodeClaimList := &corev1beta1.NodeClaimList{} + Expect(env.Client.List(env, nodeClaimList, client.HasLabels{test.DiscoveryLabel})).To(Succeed()) + Expect(len(nodeClaimList.Items)).To(BeNumerically(comparator, count)) +} + +func NodeClaimNames(nodeClaims []*corev1beta1.NodeClaim) []string { + return lo.Map(nodeClaims, func(n *corev1beta1.NodeClaim, index int) string { + return n.Name + }) +} + func NodeNames(nodes []*v1.Node) []string { return lo.Map(nodes, func(n *v1.Node, index int) string { return n.Name }) } -func (env *Environment) ConsistentlyExpectNodeCount(comparator string, count int, duration string) []*v1.Node { +func (env *Environment) ConsistentlyExpectNodeCount(comparator string, count int, duration time.Duration) []*v1.Node { GinkgoHelper() By(fmt.Sprintf("expecting nodes to be %s to %d for %s", comparator, count, duration)) nodeList := &v1.NodeList{} @@ -482,29 +506,38 @@ func (env *Environment) ConsistentlyExpectNodeCount(comparator string, count int g.Expect(env.Client.List(env, nodeList, client.HasLabels{test.DiscoveryLabel})).To(Succeed()) g.Expect(len(nodeList.Items)).To(BeNumerically(comparator, count), fmt.Sprintf("expected %d nodes, had %d (%v) for %s", count, len(nodeList.Items), NodeNames(lo.ToSlicePtr(nodeList.Items)), duration)) - }, duration).Should(Succeed()) + }, duration.String()).Should(Succeed()) return lo.ToSlicePtr(nodeList.Items) } -func (env *Environment) ConsistentlyExpectNoDisruptions(nodeCount int, duration string) { +func (env *Environment) ConsistentlyExpectNoDisruptions(nodeCount int, duration time.Duration) (taintedNodes []*v1.Node) { + GinkgoHelper() + return env.ConsistentlyExpectDisruptionsWithNodeCount(0, nodeCount, duration) +} + +// ConsistentlyExpectDisruptionsWithNodeCount will continually ensure that there are exactly disruptingNodes with totalNodes (including replacements and existing nodes) +func (env *Environment) ConsistentlyExpectDisruptionsWithNodeCount(disruptingNodes, totalNodes int, duration time.Duration) (taintedNodes []*v1.Node) { GinkgoHelper() + nodes := []v1.Node{} Consistently(func(g Gomega) { // Ensure we don't change our NodeClaims nodeClaimList := &corev1beta1.NodeClaimList{} g.Expect(env.Client.List(env, nodeClaimList, client.HasLabels{test.DiscoveryLabel})).To(Succeed()) - g.Expect(nodeClaimList.Items).To(HaveLen(nodeCount)) + g.Expect(nodeClaimList.Items).To(HaveLen(totalNodes)) nodeList := &v1.NodeList{} g.Expect(env.Client.List(env, nodeList, client.HasLabels{test.DiscoveryLabel})).To(Succeed()) - g.Expect(nodeList.Items).To(HaveLen(nodeCount)) + g.Expect(nodeList.Items).To(HaveLen(totalNodes)) - for _, node := range nodeList.Items { - _, ok := lo.Find(node.Spec.Taints, func(t v1.Taint) bool { + nodes = lo.Filter(nodeList.Items, func(n v1.Node, _ int) bool { + _, ok := lo.Find(n.Spec.Taints, func(t v1.Taint) bool { return corev1beta1.IsDisruptingTaint(t) }) - g.Expect(ok).To(BeFalse()) - } + return ok + }) + g.Expect(nodes).To(HaveLen(disruptingNodes)) }, duration).Should(Succeed()) + return lo.ToSlicePtr(nodes) } func (env *Environment) EventuallyExpectTaintedNodeCount(comparator string, count int) []*v1.Node { @@ -530,6 +563,18 @@ func (env *Environment) EventuallyExpectNodesUntaintedWithTimeout(timeout time.D }).WithTimeout(timeout).Should(Succeed()) } +func (env *Environment) EventuallyExpectNodeClaimCount(comparator string, count int) []*corev1beta1.NodeClaim { + GinkgoHelper() + By(fmt.Sprintf("waiting for nodes to be %s to %d", comparator, count)) + nodeClaimList := &corev1beta1.NodeClaimList{} + Eventually(func(g Gomega) { + g.Expect(env.Client.List(env, nodeClaimList, client.HasLabels{test.DiscoveryLabel})).To(Succeed()) + g.Expect(len(nodeClaimList.Items)).To(BeNumerically(comparator, count), + fmt.Sprintf("expected %d nodeclaims, had %d (%v)", count, len(nodeClaimList.Items), NodeClaimNames(lo.ToSlicePtr(nodeClaimList.Items)))) + }).Should(Succeed()) + return lo.ToSlicePtr(nodeClaimList.Items) +} + func (env *Environment) EventuallyExpectNodeCount(comparator string, count int) []*v1.Node { GinkgoHelper() By(fmt.Sprintf("waiting for nodes to be %s to %d", comparator, count)) @@ -651,6 +696,18 @@ func (env *Environment) EventuallyExpectDrifted(nodeClaims ...*corev1beta1.NodeC }).Should(Succeed()) } +func (env *Environment) ConsistentlyExpectNodeClaimsNotDrifted(duration time.Duration, nodeClaims ...*corev1beta1.NodeClaim) { + GinkgoHelper() + nodeClaimNames := lo.Map(nodeClaims, func(nc *corev1beta1.NodeClaim, _ int) string { return nc.Name }) + By(fmt.Sprintf("consistently expect nodeclaims %s not to be drifted for %s", nodeClaimNames, duration)) + Consistently(func(g Gomega) { + for _, nc := range nodeClaims { + g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(nc), nc)).To(Succeed()) + g.Expect(nc.StatusConditions().GetCondition(corev1beta1.Drifted)).To(BeNil()) + } + }, duration).Should(Succeed()) +} + func (env *Environment) EventuallyExpectEmpty(nodeClaims ...*corev1beta1.NodeClaim) { GinkgoHelper() Eventually(func(g Gomega) { @@ -694,7 +751,7 @@ func (env *Environment) printControllerLogs(options *v1.PodLogOptions) { temp := options.DeepCopy() // local version of the log options fmt.Printf("------- pod/%s -------\n", pod.Name) - if pod.Status.ContainerStatuses[0].RestartCount > 0 { + if len(pod.Status.ContainerStatuses) > 0 && pod.Status.ContainerStatuses[0].RestartCount > 0 { fmt.Printf("[PREVIOUS CONTAINER LOGS]\n") temp.Previous = true } @@ -732,7 +789,7 @@ func (env *Environment) ExpectDaemonSetEnvironmentVariableUpdated(obj client.Obj Expect(len(ds.Spec.Template.Spec.Containers)).To(BeNumerically("==", 1)) containers = append(containers, ds.Spec.Template.Spec.Containers[0].Name) } - patch := client.MergeFrom(ds.DeepCopy()) + patch := client.StrategicMergeFrom(ds.DeepCopy()) containerNames := sets.New(containers...) for ci := range ds.Spec.Template.Spec.Containers { c := &ds.Spec.Template.Spec.Containers[ci] @@ -751,17 +808,63 @@ func (env *Environment) ExpectDaemonSetEnvironmentVariableUpdated(obj client.Obj Expect(env.Client.Patch(env.Context, ds, patch)).To(Succeed()) } -func (env *Environment) ExpectHealthyPodsForNode(nodeName string) []*v1.Pod { +// ForcePodsToSpread ensures that currently scheduled pods get spread evenly across all passed nodes by deleting pods off of existing +// nodes and waiting them to reschedule. This is useful for scenarios where you want to force the nodes be underutilized +// but you want to keep a consistent count of nodes rather than leaving around empty ones. +func (env *Environment) ForcePodsToSpread(nodes ...*v1.Node) { + GinkgoHelper() + + // Get the total count of pods across + podCount := 0 + for _, n := range nodes { + podCount += len(env.ExpectActivePodsForNode(n.Name)) + } + maxPodsPerNode := int(math.Ceil(float64(podCount) / float64(len(nodes)))) + + By(fmt.Sprintf("forcing %d pods to spread across %d nodes", podCount, len(nodes))) + start := time.Now() + for { + var nodePods []*v1.Pod + node, found := lo.Find(nodes, func(n *v1.Node) bool { + nodePods = env.ExpectActivePodsForNode(n.Name) + return len(nodePods) > maxPodsPerNode + }) + if !found { + break + } + // Set the nodes to unschedulable so that the pods won't reschedule. + Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(node), node)).To(Succeed()) + stored := node.DeepCopy() + node.Spec.Unschedulable = true + Expect(env.Client.Patch(env.Context, node, client.StrategicMergeFrom(stored))).To(Succeed()) + for _, pod := range nodePods[maxPodsPerNode:] { + env.ExpectDeleted(pod) + } + Eventually(func(g Gomega) { + g.Expect(len(env.ExpectActivePodsForNode(node.Name))).To(Or(Equal(maxPodsPerNode), Equal(maxPodsPerNode-1))) + }).WithTimeout(5 * time.Second).Should(Succeed()) + + // TODO: Consider moving this time check to an Eventually poll. This gets a little tricker with helper functions + // since you need to make sure that your Expectation helper functions are scoped to to your "g Gomega" scope + // so that you don't fail the first time you get a failure on your expectation + if time.Since(start) > time.Minute*15 { + Fail("forcing pods to spread failed due to a timeout") + } + } + for _, n := range nodes { + stored := n.DeepCopy() + n.Spec.Unschedulable = false + Expect(env.Client.Patch(env.Context, n, client.StrategicMergeFrom(stored))).To(Succeed()) + } +} + +func (env *Environment) ExpectActivePodsForNode(nodeName string) []*v1.Pod { GinkgoHelper() podList := &v1.PodList{} Expect(env.Client.List(env, podList, client.MatchingFields{"spec.nodeName": nodeName}, client.HasLabels{test.DiscoveryLabel})).To(Succeed()) - // Return the healthy pods return lo.Filter(lo.ToSlicePtr(podList.Items), func(p *v1.Pod, _ int) bool { - _, found := lo.Find(p.Status.Conditions, func(cond v1.PodCondition) bool { - return cond.Type == v1.PodReady && cond.Status == v1.ConditionTrue - }) - return found + return p.DeletionTimestamp.IsZero() }) } diff --git a/test/pkg/environment/common/setup.go b/test/pkg/environment/common/setup.go index dda838310832..cc233cc561ce 100644 --- a/test/pkg/environment/common/setup.go +++ b/test/pkg/environment/common/setup.go @@ -126,7 +126,9 @@ func (env *Environment) CleanupObjects(cleanableObjects ...client.Object) { workqueue.ParallelizeUntil(env, 50, len(metaList.Items), func(i int) { defer GinkgoRecover() g.Expect(env.ExpectTestingFinalizerRemoved(&metaList.Items[i])).To(Succeed()) - g.Expect(client.IgnoreNotFound(env.Client.Delete(env, &metaList.Items[i], client.PropagationPolicy(metav1.DeletePropagationForeground)))).To(Succeed()) + g.Expect(client.IgnoreNotFound(env.Client.Delete(env, &metaList.Items[i], + client.PropagationPolicy(metav1.DeletePropagationForeground), + &client.DeleteOptions{GracePeriodSeconds: lo.ToPtr(int64(0))}))).To(Succeed()) }) // If the deletes eventually succeed, we should have no elements here at the end of the test g.Expect(env.Client.List(env, metaList, client.HasLabels([]string{test.DiscoveryLabel}), client.Limit(1))).To(Succeed()) @@ -143,13 +145,18 @@ func (env *Environment) ExpectTestingFinalizerRemoved(obj client.Object) error { if err := env.Client.Get(env, client.ObjectKeyFromObject(obj), metaObj); err != nil { return client.IgnoreNotFound(err) } - deepCopy := metaObj.DeepCopy() metaObj.Finalizers = lo.Reject(metaObj.Finalizers, func(finalizer string, _ int) bool { return finalizer == TestingFinalizer }) if !equality.Semantic.DeepEqual(metaObj, deepCopy) { + // If the Group is the "core" APIs, then we can strategic merge patch + // CRDs do not currently have support for strategic merge patching, so we can't blindly do it + // https://kubernetes.io/docs/concepts/extend-kubernetes/api-extension/custom-resources/#advanced-features-and-flexibility:~:text=Yes-,strategic%2Dmerge%2Dpatch,-The%20new%20endpoints + if metaObj.GroupVersionKind().Group == "" { + return client.IgnoreNotFound(env.Client.Patch(env, metaObj, client.StrategicMergeFrom(deepCopy))) + } return client.IgnoreNotFound(env.Client.Patch(env, metaObj, client.MergeFrom(deepCopy))) } return nil diff --git a/test/suites/chaos/suite_test.go b/test/suites/chaos/suite_test.go index 7cdf6cbbc74d..9011876d9a36 100644 --- a/test/suites/chaos/suite_test.go +++ b/test/suites/chaos/suite_test.go @@ -74,10 +74,12 @@ var _ = Describe("Chaos", func() { ctx, cancel := context.WithCancel(env.Context) defer cancel() - nodePool = coretest.ReplaceRequirements(nodePool, v1.NodeSelectorRequirement{ - Key: corev1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.CapacityTypeSpot}, + nodePool = coretest.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: corev1beta1.CapacityTypeLabelKey, + Operator: v1.NodeSelectorOpIn, + Values: []string{corev1beta1.CapacityTypeSpot}, + }, }) nodePool.Spec.Disruption.ConsolidationPolicy = corev1beta1.ConsolidationPolicyWhenUnderutilized nodePool.Spec.Disruption.ConsolidateAfter = nil @@ -148,7 +150,7 @@ func (t *taintAdder) Reconcile(ctx context.Context, req reconcile.Request) (reco if err := t.kubeClient.Get(ctx, req.NamespacedName, node); err != nil { return reconcile.Result{}, client.IgnoreNotFound(err) } - mergeFrom := client.MergeFrom(node.DeepCopy()) + mergeFrom := client.StrategicMergeFrom(node.DeepCopy()) taint := v1.Taint{ Key: "test", Value: "true", diff --git a/test/suites/consolidation/suite_test.go b/test/suites/consolidation/suite_test.go index 8c3257e2e597..3ed5408c5aa0 100644 --- a/test/suites/consolidation/suite_test.go +++ b/test/suites/consolidation/suite_test.go @@ -22,10 +22,12 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/samber/lo" + appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "sigs.k8s.io/controller-runtime/pkg/client" corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" "sigs.k8s.io/karpenter/pkg/test" @@ -63,6 +65,337 @@ var _ = AfterEach(func() { env.Cleanup() }) var _ = AfterEach(func() { env.AfterEach() }) var _ = Describe("Consolidation", func() { + Context("Budgets", func() { + var nodePool *corev1beta1.NodePool + var dep *appsv1.Deployment + var selector labels.Selector + var numPods int32 + BeforeEach(func() { + nodePool = env.DefaultNodePool(nodeClass) + nodePool.Spec.Disruption.ConsolidateAfter = nil + + numPods = 5 + dep = test.Deployment(test.DeploymentOptions{ + Replicas: numPods, + PodOptions: test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": "regular-app"}, + }, + ResourceRequirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, + }, + }, + }) + selector = labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) + }) + It("should respect budgets for empty delete consolidation", func() { + nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{ + { + Nodes: "40%", + }, + } + + // Hostname anti-affinity to require one pod on each node + dep.Spec.Template.Spec.Affinity = &v1.Affinity{ + PodAntiAffinity: &v1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + { + LabelSelector: dep.Spec.Selector, + TopologyKey: v1.LabelHostname, + }, + }, + }, + } + env.ExpectCreated(nodeClass, nodePool, dep) + + env.EventuallyExpectCreatedNodeClaimCount("==", 5) + nodes := env.EventuallyExpectCreatedNodeCount("==", 5) + env.EventuallyExpectHealthyPodCount(selector, int(numPods)) + + By("adding finalizers to the nodes to prevent termination") + for _, node := range nodes { + Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(node), node)).To(Succeed()) + node.Finalizers = append(node.Finalizers, common.TestingFinalizer) + env.ExpectUpdated(node) + } + + dep.Spec.Replicas = lo.ToPtr[int32](1) + By("making the nodes empty") + // Update the deployment to only contain 1 replica. + env.ExpectUpdated(dep) + + // Ensure that we get two nodes tainted, and they have overlap during the drift + env.EventuallyExpectTaintedNodeCount("==", 2) + nodes = env.ConsistentlyExpectDisruptionsWithNodeCount(2, 5, 5*time.Second) + + // Remove the finalizer from each node so that we can terminate + for _, node := range nodes { + Expect(env.ExpectTestingFinalizerRemoved(node)).To(Succeed()) + } + + // After the deletion timestamp is set and all pods are drained + // the node should be gone + env.EventuallyExpectNotFound(nodes[0], nodes[1]) + + // This check ensures that we are consolidating nodes at the same time + env.EventuallyExpectTaintedNodeCount("==", 2) + nodes = env.ConsistentlyExpectDisruptionsWithNodeCount(2, 3, 5*time.Second) + + for _, node := range nodes { + Expect(env.ExpectTestingFinalizerRemoved(node)).To(Succeed()) + } + env.EventuallyExpectNotFound(nodes[0], nodes[1]) + + // Expect there to only be one node remaining for the last replica + env.ExpectNodeCount("==", 1) + }) + It("should respect budgets for non-empty delete consolidation", func() { + // This test will hold consolidation until we are ready to execute it + nodePool.Spec.Disruption.ConsolidateAfter = &corev1beta1.NillableDuration{} + + nodePool = test.ReplaceRequirements(nodePool, + corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: v1beta1.LabelInstanceSize, + Operator: v1.NodeSelectorOpIn, + Values: []string{"2xlarge"}, + }, + }, + ) + // We're expecting to create 3 nodes, so we'll expect to see at most 2 nodes deleting at one time. + nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{ + Nodes: "50%", + }} + numPods = 9 + dep = test.Deployment(test.DeploymentOptions{ + Replicas: numPods, + PodOptions: test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{"app": "large-app"}, + }, + // Each 2xlarge has 8 cpu, so each node should fit no more than 3 pods. + ResourceRequirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("2100m"), + }, + }, + }, + }) + selector = labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) + env.ExpectCreated(nodeClass, nodePool, dep) + + env.EventuallyExpectCreatedNodeClaimCount("==", 3) + nodes := env.EventuallyExpectCreatedNodeCount("==", 3) + env.EventuallyExpectHealthyPodCount(selector, int(numPods)) + + By("scaling down the deployment") + // Update the deployment to a third of the replicas. + dep.Spec.Replicas = lo.ToPtr[int32](3) + env.ExpectUpdated(dep) + + env.ForcePodsToSpread(nodes...) + env.EventuallyExpectHealthyPodCount(selector, 3) + + By("cordoning and adding finalizer to the nodes") + // Add a finalizer to each node so that we can stop termination disruptions + for _, node := range nodes { + Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(node), node)).To(Succeed()) + node.Finalizers = append(node.Finalizers, common.TestingFinalizer) + env.ExpectUpdated(node) + } + + By("enabling consolidation") + nodePool.Spec.Disruption.ConsolidateAfter = nil + env.ExpectUpdated(nodePool) + + // Ensure that we get two nodes tainted, and they have overlap during consolidation + env.EventuallyExpectTaintedNodeCount("==", 2) + nodes = env.ConsistentlyExpectDisruptionsWithNodeCount(2, 3, 5*time.Second) + + for _, node := range nodes { + Expect(env.ExpectTestingFinalizerRemoved(node)).To(Succeed()) + } + env.EventuallyExpectNotFound(nodes[0], nodes[1]) + env.ExpectNodeCount("==", 1) + }) + It("should respect budgets for non-empty replace consolidation", func() { + appLabels := map[string]string{"app": "large-app"} + // This test will hold consolidation until we are ready to execute it + nodePool.Spec.Disruption.ConsolidateAfter = &corev1beta1.NillableDuration{} + + nodePool = test.ReplaceRequirements(nodePool, + corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceSize, + Operator: v1.NodeSelectorOpIn, + Values: []string{"xlarge", "2xlarge"}, + }, + }, + // Add an Exists operator so that we can select on a fake partition later + corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: "test-partition", + Operator: v1.NodeSelectorOpExists, + }, + }, + ) + nodePool.Labels = appLabels + // We're expecting to create 5 nodes, so we'll expect to see at most 3 nodes deleting at one time. + nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{ + Nodes: "3", + }} + + ds := test.DaemonSet(test.DaemonSetOptions{ + Selector: appLabels, + PodOptions: test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Labels: appLabels, + }, + // Each 2xlarge has 8 cpu, so each node should fit no more than 1 pod since each node will have. + // an equivalently sized daemonset + ResourceRequirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("3"), + }, + }, + }, + }) + + env.ExpectCreated(ds) + + // Make 5 pods all with different deployments and different test partitions, so that each pod can be put + // on a separate node. + selector = labels.SelectorFromSet(appLabels) + numPods = 5 + deployments := make([]*appsv1.Deployment, numPods) + for i := range lo.Range(int(numPods)) { + deployments[i] = test.Deployment(test.DeploymentOptions{ + Replicas: 1, + PodOptions: test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Labels: appLabels, + }, + NodeSelector: map[string]string{"test-partition": fmt.Sprintf("%d", i)}, + // Each 2xlarge has 8 cpu, so each node should fit no more than 1 pod since each node will have. + // an equivalently sized daemonset + ResourceRequirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("3"), + }, + }, + }, + }) + } + + env.ExpectCreated(nodeClass, nodePool, deployments[0], deployments[1], deployments[2], deployments[3], deployments[4]) + + originalNodeClaims := env.EventuallyExpectCreatedNodeClaimCount("==", 5) + originalNodes := env.EventuallyExpectCreatedNodeCount("==", 5) + + // Check that all daemonsets and deployment pods are online + env.EventuallyExpectHealthyPodCount(selector, int(numPods)*2) + + By("cordoning and adding finalizer to the nodes") + // Add a finalizer to each node so that we can stop termination disruptions + for _, node := range originalNodes { + Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(node), node)).To(Succeed()) + node.Finalizers = append(node.Finalizers, common.TestingFinalizer) + env.ExpectUpdated(node) + } + + // Delete the daemonset so that the nodes can be consolidated to smaller size + env.ExpectDeleted(ds) + // Check that all daemonsets and deployment pods are online + env.EventuallyExpectHealthyPodCount(selector, int(numPods)) + + By("enabling consolidation") + nodePool.Spec.Disruption.ConsolidateAfter = nil + env.ExpectUpdated(nodePool) + + // Ensure that we get three nodes tainted, and they have overlap during the consolidation + env.EventuallyExpectTaintedNodeCount("==", 3) + env.EventuallyExpectNodeClaimCount("==", 8) + env.EventuallyExpectNodeCount("==", 8) + env.ConsistentlyExpectDisruptionsWithNodeCount(3, 8, 5*time.Second) + + for _, node := range originalNodes { + Expect(env.ExpectTestingFinalizerRemoved(node)).To(Succeed()) + } + + // Eventually expect all the nodes to be rolled and completely removed + // Since this completes the disruption operation, this also ensures that we aren't leaking nodes into subsequent + // tests since nodeclaims that are actively replacing but haven't brought-up nodes yet can register nodes later + env.EventuallyExpectNotFound(lo.Map(originalNodes, func(n *v1.Node, _ int) client.Object { return n })...) + env.EventuallyExpectNotFound(lo.Map(originalNodeClaims, func(n *corev1beta1.NodeClaim, _ int) client.Object { return n })...) + env.ExpectNodeClaimCount("==", 5) + env.ExpectNodeCount("==", 5) + }) + It("should not allow consolidation if the budget is fully blocking", func() { + // We're going to define a budget that doesn't allow any consolidation to happen + nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{ + Nodes: "0", + }} + + // Hostname anti-affinity to require one pod on each node + dep.Spec.Template.Spec.Affinity = &v1.Affinity{ + PodAntiAffinity: &v1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + { + LabelSelector: dep.Spec.Selector, + TopologyKey: v1.LabelHostname, + }, + }, + }, + } + env.ExpectCreated(nodeClass, nodePool, dep) + + env.EventuallyExpectCreatedNodeClaimCount("==", 5) + env.EventuallyExpectCreatedNodeCount("==", 5) + env.EventuallyExpectHealthyPodCount(selector, int(numPods)) + + dep.Spec.Replicas = lo.ToPtr[int32](1) + By("making the nodes empty") + // Update the deployment to only contain 1 replica. + env.ExpectUpdated(dep) + + env.ConsistentlyExpectNoDisruptions(5, time.Minute) + }) + It("should not allow consolidation if the budget is fully blocking during a scheduled time", func() { + // We're going to define a budget that doesn't allow any drift to happen + // This is going to be on a schedule that only lasts 30 minutes, whose window starts 15 minutes before + // the current time and extends 15 minutes past the current time + // Times need to be in UTC since the karpenter containers were built in UTC time + windowStart := time.Now().Add(-time.Minute * 15).UTC() + nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{ + Nodes: "0", + Schedule: lo.ToPtr(fmt.Sprintf("%d %d * * *", windowStart.Minute(), windowStart.Hour())), + Duration: &metav1.Duration{Duration: time.Minute * 30}, + }} + + // Hostname anti-affinity to require one pod on each node + dep.Spec.Template.Spec.Affinity = &v1.Affinity{ + PodAntiAffinity: &v1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + { + LabelSelector: dep.Spec.Selector, + TopologyKey: v1.LabelHostname, + }, + }, + }, + } + env.ExpectCreated(nodeClass, nodePool, dep) + + env.EventuallyExpectCreatedNodeClaimCount("==", 5) + env.EventuallyExpectCreatedNodeCount("==", 5) + env.EventuallyExpectHealthyPodCount(selector, int(numPods)) + + dep.Spec.Replicas = lo.ToPtr[int32](1) + By("making the nodes empty") + // Update the deployment to only contain 1 replica. + env.ExpectUpdated(dep) + + env.ConsistentlyExpectNoDisruptions(5, time.Minute) + }) + }) DescribeTable("should consolidate nodes (delete)", Label(debug.NoWatch), Label(debug.NoEvents), func(spotToSpot bool) { nodePool := test.NodePool(corev1beta1.NodePool{ @@ -74,23 +407,30 @@ var _ = Describe("Consolidation", func() { }, Template: corev1beta1.NodeClaimTemplate{ Spec: corev1beta1.NodeClaimSpec{ - Requirements: []v1.NodeSelectorRequirement{ + Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ { - Key: corev1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: lo.Ternary(spotToSpot, []string{corev1beta1.CapacityTypeSpot}, []string{corev1beta1.CapacityTypeOnDemand}), + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: corev1beta1.CapacityTypeLabelKey, + Operator: v1.NodeSelectorOpIn, + Values: lo.Ternary(spotToSpot, []string{corev1beta1.CapacityTypeSpot}, []string{corev1beta1.CapacityTypeOnDemand}), + }, }, { - Key: v1beta1.LabelInstanceSize, - Operator: v1.NodeSelectorOpIn, - Values: []string{"medium", "large", "xlarge"}, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceSize, + Operator: v1.NodeSelectorOpIn, + Values: []string{"medium", "large", "xlarge"}, + }, }, { - Key: v1beta1.LabelInstanceFamily, - Operator: v1.NodeSelectorOpNotIn, - // remove some cheap burstable and the odd c1 instance types so we have - // more control over what gets provisioned - Values: []string{"t2", "t3", "c1", "t3a", "t4g"}, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceFamily, + Operator: v1.NodeSelectorOpNotIn, + // remove some cheap burstable and the odd c1 instance types so we have + // more control over what gets provisioned + // TODO: jmdeal@ remove a1 from exclusion list once Karpenter implicitly filters a1 instances for AL2023 AMI family (incompatible) + Values: []string{"t2", "t3", "c1", "t3a", "t4g", "a1"}, + }, }, }, NodeClassRef: &corev1beta1.NodeClassReference{Name: nodeClass.Name}, @@ -146,16 +486,29 @@ var _ = Describe("Consolidation", func() { }, Template: corev1beta1.NodeClaimTemplate{ Spec: corev1beta1.NodeClaimSpec{ - Requirements: []v1.NodeSelectorRequirement{ + Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ { - Key: corev1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: lo.Ternary(spotToSpot, []string{corev1beta1.CapacityTypeSpot}, []string{corev1beta1.CapacityTypeOnDemand}), + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: corev1beta1.CapacityTypeLabelKey, + Operator: v1.NodeSelectorOpIn, + Values: lo.Ternary(spotToSpot, []string{corev1beta1.CapacityTypeSpot}, []string{corev1beta1.CapacityTypeOnDemand}), + }, }, { - Key: v1beta1.LabelInstanceSize, - Operator: v1.NodeSelectorOpIn, - Values: []string{"large", "2xlarge"}, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceSize, + Operator: v1.NodeSelectorOpIn, + Values: []string{"large", "2xlarge"}, + }, + }, + { + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceFamily, + Operator: v1.NodeSelectorOpNotIn, + // remove some cheap burstable and the odd c1 / a1 instance types so we have + // more control over what gets provisioned + Values: []string{"t2", "t3", "c1", "t3a", "t4g", "a1"}, + }, }, }, NodeClassRef: &corev1beta1.NodeClassReference{Name: nodeClass.Name}, @@ -269,16 +622,29 @@ var _ = Describe("Consolidation", func() { }, Template: corev1beta1.NodeClaimTemplate{ Spec: corev1beta1.NodeClaimSpec{ - Requirements: []v1.NodeSelectorRequirement{ + Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ + { + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: corev1beta1.CapacityTypeLabelKey, + Operator: v1.NodeSelectorOpIn, + Values: []string{corev1beta1.CapacityTypeOnDemand}, + }, + }, { - Key: corev1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.CapacityTypeOnDemand}, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceSize, + Operator: v1.NodeSelectorOpIn, + Values: []string{"large"}, + }, }, { - Key: v1beta1.LabelInstanceSize, - Operator: v1.NodeSelectorOpIn, - Values: []string{"large"}, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceFamily, + Operator: v1.NodeSelectorOpNotIn, + // remove some cheap burstable and the odd c1 / a1 instance types so we have + // more control over what gets provisioned + Values: []string{"t2", "t3", "c1", "t3a", "t4g", "a1"}, + }, }, }, NodeClassRef: &corev1beta1.NodeClassReference{Name: nodeClass.Name}, @@ -324,14 +690,18 @@ var _ = Describe("Consolidation", func() { // instance than on-demand nodePool.Spec.Disruption.ConsolidateAfter = nil test.ReplaceRequirements(nodePool, - v1.NodeSelectorRequirement{ - Key: corev1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpExists, + corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: corev1beta1.CapacityTypeLabelKey, + Operator: v1.NodeSelectorOpExists, + }, }, - v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceSize, - Operator: v1.NodeSelectorOpIn, - Values: []string{"large"}, + corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceSize, + Operator: v1.NodeSelectorOpIn, + Values: []string{"large"}, + }, }, ) env.ExpectUpdated(nodePool) diff --git a/test/suites/drift/suite_test.go b/test/suites/drift/suite_test.go index 6a3d65037fc8..49ceb84a55d0 100644 --- a/test/suites/drift/suite_test.go +++ b/test/suites/drift/suite_test.go @@ -17,6 +17,8 @@ package drift_test import ( "fmt" "sort" + "strconv" + "strings" "testing" "time" @@ -30,6 +32,7 @@ import ( "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/sets" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/karpenter/pkg/utils/resources" awssdk "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" @@ -77,7 +80,7 @@ var _ = Describe("Drift", func() { var selector labels.Selector var numPods int BeforeEach(func() { - amdAMI = env.GetCustomAMI("/aws/service/eks/optimized-ami/%s/amazon-linux-2/recommended/image_id", 1) + amdAMI = env.GetCustomAMI("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/x86_64/standard/recommended/image_id", 1) numPods = 1 // Add pods with a do-not-disrupt annotation so that we can check node metadata before we disrupt dep = coretest.Deployment(coretest.DeploymentOptions{ @@ -95,16 +98,16 @@ var _ = Describe("Drift", func() { }, }) selector = labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) - - env.ExpectSettingsOverridden(v1.EnvVar{Name: "FEATURE_GATES", Value: "Drift=true"}) }) Context("Budgets", func() { It("should respect budgets for empty drift", func() { nodePool = coretest.ReplaceRequirements(nodePool, - v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceSize, - Operator: v1.NodeSelectorOpIn, - Values: []string{"2xlarge"}, + corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceSize, + Operator: v1.NodeSelectorOpIn, + Values: []string{"2xlarge"}, + }, }, ) // We're expecting to create 3 nodes, so we'll expect to see 2 nodes deleting at one time. @@ -135,7 +138,6 @@ var _ = Describe("Drift", func() { nodeClaims := env.EventuallyExpectCreatedNodeClaimCount("==", 3) nodes := env.EventuallyExpectCreatedNodeCount("==", 3) env.EventuallyExpectHealthyPodCount(selector, int(numPods)) - env.Monitor.Reset() // Reset the monitor so that we can expect a single node to be spun up after expiration // List nodes so that we get any updated information on the nodes. If we don't // we have the potential to over-write any changes Karpenter makes to the nodes. @@ -158,7 +160,9 @@ var _ = Describe("Drift", func() { env.EventuallyExpectDrifted(nodeClaims...) - nodes = env.EventuallyExpectTaintedNodeCount("==", 2) + // Ensure that we get two nodes tainted, and they have overlap during the drift + env.EventuallyExpectTaintedNodeCount("==", 2) + nodes = env.ConsistentlyExpectDisruptionsWithNodeCount(2, 3, 5*time.Second) // Remove the finalizer from each node so that we can terminate for _, node := range nodes { @@ -175,10 +179,12 @@ var _ = Describe("Drift", func() { }) It("should respect budgets for non-empty delete drift", func() { nodePool = coretest.ReplaceRequirements(nodePool, - v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceSize, - Operator: v1.NodeSelectorOpIn, - Values: []string{"2xlarge"}, + corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceSize, + Operator: v1.NodeSelectorOpIn, + Values: []string{"2xlarge"}, + }, }, ) // We're expecting to create 3 nodes, so we'll expect to see at most 2 nodes deleting at one time. @@ -209,36 +215,15 @@ var _ = Describe("Drift", func() { nodeClaims := env.EventuallyExpectCreatedNodeClaimCount("==", 3) nodes := env.EventuallyExpectCreatedNodeCount("==", 3) env.EventuallyExpectHealthyPodCount(selector, int(numPods)) - env.Monitor.Reset() // Reset the monitor so that we can expect a single node to be spun up after drift By("scaling down the deployment") // Update the deployment to a third of the replicas. dep.Spec.Replicas = lo.ToPtr[int32](3) env.ExpectUpdated(dep) - By("spreading the pods to each of the nodes") + // First expect there to be 3 pods, then try to spread the pods. env.EventuallyExpectHealthyPodCount(selector, 3) - // Delete pods from the deployment until each node has one pod. - var nodePods []*v1.Pod - for { - node, found := lo.Find(nodes, func(n *v1.Node) bool { - nodePods = env.ExpectHealthyPodsForNode(n.Name) - return len(nodePods) > 1 - }) - if !found { - break - } - // Set the nodes to unschedulable so that the pods won't reschedule. - Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(node), node)).To(Succeed()) - node.Spec.Unschedulable = true - env.ExpectUpdated(node) - for _, pod := range nodePods[1:] { - env.ExpectDeleted(pod) - } - Eventually(func(g Gomega) { - g.Expect(len(env.ExpectHealthyPodsForNode(node.Name))).To(Equal(1)) - }).WithTimeout(5 * time.Second).Should(Succeed()) - } + env.ForcePodsToSpread(nodes...) env.EventuallyExpectHealthyPodCount(selector, 3) By("cordoning and adding finalizer to the nodes") @@ -246,8 +231,6 @@ var _ = Describe("Drift", func() { for _, node := range nodes { Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(node), node)).To(Succeed()) node.Finalizers = append(node.Finalizers, common.TestingFinalizer) - // Set nodes as unschedulable so that pod nomination doesn't delay disruption for the second disruption action - node.Spec.Unschedulable = true env.ExpectUpdated(node) } @@ -266,11 +249,9 @@ var _ = Describe("Drift", func() { env.ExpectUpdated(pod) } - // Mark one node as schedulable so the other two nodes can schedule to this node and delete. - Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(nodes[0]), nodes[0])).To(Succeed()) - nodes[0].Spec.Unschedulable = false - env.ExpectUpdated(nodes[0]) - nodes = env.EventuallyExpectTaintedNodeCount("==", 2) + // Ensure that we get two nodes tainted, and they have overlap during the drift + env.EventuallyExpectTaintedNodeCount("==", 2) + nodes = env.ConsistentlyExpectDisruptionsWithNodeCount(2, 3, 30*time.Second) By("removing the finalizer from the nodes") Expect(env.ExpectTestingFinalizerRemoved(nodes[0])).To(Succeed()) @@ -281,85 +262,90 @@ var _ = Describe("Drift", func() { env.EventuallyExpectNotFound(nodes[0], nodes[1]) }) It("should respect budgets for non-empty replace drift", func() { + appLabels := map[string]string{"app": "large-app"} + nodePool = coretest.ReplaceRequirements(nodePool, - v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceSize, - Operator: v1.NodeSelectorOpIn, - Values: []string{"2xlarge"}, + corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceSize, + Operator: v1.NodeSelectorOpIn, + Values: []string{"xlarge"}, + }, + }, + // Add an Exists operator so that we can select on a fake partition later + corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: "test-partition", + Operator: v1.NodeSelectorOpExists, + }, }, ) - // We're expecting to create 3 nodes, so we'll expect to see at most 2 nodes deleting at one time. + nodePool.Labels = appLabels + // We're expecting to create 5 nodes, so we'll expect to see at most 3 nodes deleting at one time. nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{ - Nodes: "50%", + Nodes: "3", }} - var numPods int32 = 3 - dep = coretest.Deployment(coretest.DeploymentOptions{ - Replicas: numPods, - PodOptions: coretest.PodOptions{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - corev1beta1.DoNotDisruptAnnotationKey: "true", + + // Make 5 pods all with different deployments and different test partitions, so that each pod can be put + // on a separate node. + selector = labels.SelectorFromSet(appLabels) + numPods = 5 + deployments := make([]*appsv1.Deployment, numPods) + for i := range lo.Range(numPods) { + deployments[i] = coretest.Deployment(coretest.DeploymentOptions{ + Replicas: 1, + PodOptions: coretest.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Labels: appLabels, }, - Labels: map[string]string{"app": "large-app"}, - }, - // Each 2xlarge has 8 cpu, so each node should fit no more than 3 pods. - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("5"), + NodeSelector: map[string]string{"test-partition": fmt.Sprintf("%d", i)}, + // Each xlarge has 4 cpu, so each node should fit no more than 1 pod. + ResourceRequirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("3"), + }, }, }, - }, - }) - selector = labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) - env.ExpectCreated(nodeClass, nodePool, dep) + }) + } - nodeClaims := env.EventuallyExpectCreatedNodeClaimCount("==", 3) - nodes := env.EventuallyExpectCreatedNodeCount("==", 3) - env.EventuallyExpectHealthyPodCount(selector, int(numPods)) - env.Monitor.Reset() // Reset the monitor so that we can expect a single node to be spun up after drift + env.ExpectCreated(nodeClass, nodePool, deployments[0], deployments[1], deployments[2], deployments[3], deployments[4]) + + originalNodeClaims := env.EventuallyExpectCreatedNodeClaimCount("==", 5) + originalNodes := env.EventuallyExpectCreatedNodeCount("==", 5) + + // Check that all deployment pods are online + env.EventuallyExpectHealthyPodCount(selector, numPods) By("cordoning and adding finalizer to the nodes") // Add a finalizer to each node so that we can stop termination disruptions - for _, node := range nodes { + for _, node := range originalNodes { Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(node), node)).To(Succeed()) node.Finalizers = append(node.Finalizers, common.TestingFinalizer) - // Set nodes as unschedulable so that pod nomination doesn't delay disruption for the second disruption action env.ExpectUpdated(node) } - By("drifting the nodes") - // Drift the nodeclaims - nodePool.Spec.Template.Annotations = map[string]string{"test": "annotation"} + By("drifting the nodepool") + nodePool.Spec.Template.Annotations = lo.Assign(nodePool.Spec.Template.Annotations, map[string]string{"test-annotation": "drift"}) env.ExpectUpdated(nodePool) - env.EventuallyExpectDrifted(nodeClaims...) + // Ensure that we get three nodes tainted, and they have overlap during the drift + env.EventuallyExpectTaintedNodeCount("==", 3) + env.EventuallyExpectNodeClaimCount("==", 8) + env.EventuallyExpectNodeCount("==", 8) + env.ConsistentlyExpectDisruptionsWithNodeCount(3, 8, 5*time.Second) - By("enabling disruption by removing the do not disrupt annotation") - pods := env.EventuallyExpectHealthyPodCount(selector, 3) - // Remove the do-not-disrupt annotation so that the nodes are now disruptable - for _, pod := range pods { - delete(pod.Annotations, corev1beta1.DoNotDisruptAnnotationKey) - env.ExpectUpdated(pod) + for _, node := range originalNodes { + Expect(env.ExpectTestingFinalizerRemoved(node)).To(Succeed()) } - // Expect two nodes tainted, and 2 nodes created - tainted := env.EventuallyExpectTaintedNodeCount("==", 2) - env.EventuallyExpectCreatedNodeCount("==", 2) - - Expect(env.ExpectTestingFinalizerRemoved(tainted[0])).To(Succeed()) - Expect(env.ExpectTestingFinalizerRemoved(tainted[1])).To(Succeed()) - - env.EventuallyExpectNotFound(tainted[0], tainted[1]) - - // Expect one node tainted and a one more new node created. - tainted = env.EventuallyExpectTaintedNodeCount("==", 1) - env.EventuallyExpectCreatedNodeCount("==", 3) - - Expect(env.ExpectTestingFinalizerRemoved(tainted[0])).To(Succeed()) - - // After the deletion timestamp is set and all pods are drained - // the node should be gone - env.EventuallyExpectNotFound(nodes[0], nodes[1], nodes[2]) + // Eventually expect all the nodes to be rolled and completely removed + // Since this completes the disruption operation, this also ensures that we aren't leaking nodes into subsequent + // tests since nodeclaims that are actively replacing but haven't brought-up nodes yet can register nodes later + env.EventuallyExpectNotFound(lo.Map(originalNodes, func(n *v1.Node, _ int) client.Object { return n })...) + env.EventuallyExpectNotFound(lo.Map(originalNodeClaims, func(n *corev1beta1.NodeClaim, _ int) client.Object { return n })...) + env.ExpectNodeClaimCount("==", 5) + env.ExpectNodeCount("==", 5) }) It("should not allow drift if the budget is fully blocking", func() { // We're going to define a budget that doesn't allow any drift to happen @@ -380,7 +366,7 @@ var _ = Describe("Drift", func() { env.ExpectUpdated(nodePool) env.EventuallyExpectDrifted(nodeClaim) - env.ConsistentlyExpectNoDisruptions(1, "1m") + env.ConsistentlyExpectNoDisruptions(1, time.Minute) }) It("should not allow drift if the budget is fully blocking during a scheduled time", func() { // We're going to define a budget that doesn't allow any drift to happen @@ -407,19 +393,20 @@ var _ = Describe("Drift", func() { env.ExpectUpdated(nodePool) env.EventuallyExpectDrifted(nodeClaim) - env.ConsistentlyExpectNoDisruptions(1, "1m") + env.ConsistentlyExpectNoDisruptions(1, time.Minute) }) }) It("should disrupt nodes that have drifted due to AMIs", func() { - // choose an old static image - parameter, err := env.SSMAPI.GetParameter(&ssm.GetParameterInput{ - Name: awssdk.String("/aws/service/eks/optimized-ami/1.23/amazon-linux-2/amazon-eks-node-1.23-v20230322/image_id"), - }) + // Choose and old, static image. The 1.23 image is incompatible with EKS 1.29 so fallback to a newer image. + parameterName := lo.Ternary(lo.Must(strconv.Atoi(strings.Split(env.GetK8sVersion(0), ".")[1])) >= 29, + "/aws/service/eks/optimized-ami/1.27/amazon-linux-2023/x86_64/standard/amazon-eks-node-al2023-x86_64-standard-1.27-v20240307/image_id", + "/aws/service/eks/optimized-ami/1.23/amazon-linux-2023/arm64/standard/amazon-eks-node-al2023-arm64-standard-1.23-v20240307/image_id", + ) + parameter, err := env.SSMAPI.GetParameter(&ssm.GetParameterInput{Name: awssdk.String(parameterName)}) Expect(err).To(BeNil()) oldCustomAMI := *parameter.Parameter.Value - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyCustom + nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2023 nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ID: oldCustomAMI}} - nodeClass.Spec.UserData = awssdk.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", env.ClusterName)) env.ExpectCreated(dep, nodeClass, nodePool) pod := env.EventuallyExpectHealthyPodCount(selector, numPods)[0] @@ -440,11 +427,11 @@ var _ = Describe("Drift", func() { It("should return drifted if the AMI no longer matches the existing NodeClaims instance type", func() { version := env.GetK8sVersion(1) armParameter, err := env.SSMAPI.GetParameter(&ssm.GetParameterInput{ - Name: awssdk.String(fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2-arm64/recommended/image_id", version)), + Name: awssdk.String(fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/arm64/standard/recommended/image_id", version)), }) Expect(err).To(BeNil()) armAMI := *armParameter.Parameter.Value - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 + nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2023 nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ID: armAMI}} env.ExpectCreated(dep, nodeClass, nodePool) @@ -466,15 +453,18 @@ var _ = Describe("Drift", func() { It("should not disrupt nodes that have drifted without the featureGate enabled", func() { version := env.GetK8sVersion(1) env.ExpectSettingsOverridden(v1.EnvVar{Name: "FEATURE_GATES", Value: "Drift=false"}) - // choose an old static image + // Choose an old static image (AL2023 AMIs don't exist for 1.22) + parameterName := lo.Ternary(lo.Must(strconv.Atoi(strings.Split(env.GetK8sVersion(0), ".")[1])) == 23, + "/aws/service/eks/optimized-ami/1.23/amazon-linux-2023/arm64/standard/amazon-eks-node-al2023-arm64-standard-1.23-v20240307/image_id", + fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/arm64/standard/recommended/image_id", version), + ) parameter, err := env.SSMAPI.GetParameter(&ssm.GetParameterInput{ - Name: awssdk.String(fmt.Sprintf("/aws/service/eks/optimized-ami/%s/amazon-linux-2-arm64/recommended/image_id", version)), + Name: awssdk.String(parameterName), }) Expect(err).To(BeNil()) oldCustomAMI := *parameter.Parameter.Value - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyCustom + nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2023 nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{{ID: oldCustomAMI}} - nodeClass.Spec.UserData = awssdk.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", env.ClusterName)) env.ExpectCreated(dep, nodeClass, nodePool) env.EventuallyExpectHealthyPodCount(selector, numPods) @@ -628,7 +618,7 @@ var _ = Describe("Drift", func() { g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(nodeTwo), nodeTwo)).To(Succeed()) stored := nodeTwo.DeepCopy() nodeTwo.Spec.Taints = lo.Reject(nodeTwo.Spec.Taints, func(t v1.Taint, _ int) bool { return t.Key == "example.com/another-taint-2" }) - g.Expect(env.Client.Patch(env.Context, nodeTwo, client.MergeFrom(stored))).To(Succeed()) + g.Expect(env.Client.Patch(env.Context, nodeTwo, client.StrategicMergeFrom(stored))).To(Succeed()) }).Should(Succeed()) } env.EventuallyExpectNotFound(pod, node) @@ -664,7 +654,7 @@ var _ = Describe("Drift", func() { }), Entry("NodeRequirements", corev1beta1.NodeClaimTemplate{ Spec: corev1beta1.NodeClaimSpec{ - Requirements: []v1.NodeSelectorRequirement{{Key: corev1beta1.CapacityTypeLabelKey, Operator: v1.NodeSelectorOpIn, Values: []string{corev1beta1.CapacityTypeSpot}}}, + Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: corev1beta1.CapacityTypeLabelKey, Operator: v1.NodeSelectorOpIn, Values: []string{corev1beta1.CapacityTypeSpot}}}}, }, }), ) @@ -686,14 +676,14 @@ var _ = Describe("Drift", func() { env.EventuallyExpectNotFound(pod, node) env.EventuallyExpectHealthyPodCount(selector, numPods) }, - Entry("UserData", v1beta1.EC2NodeClassSpec{UserData: awssdk.String("#!/bin/bash\n/etc/eks/bootstrap.sh")}), + Entry("UserData", v1beta1.EC2NodeClassSpec{UserData: awssdk.String("#!/bin/bash\necho \"Hello, AL2023\"")}), Entry("Tags", v1beta1.EC2NodeClassSpec{Tags: map[string]string{"keyTag-test-3": "valueTag-test-3"}}), Entry("MetadataOptions", v1beta1.EC2NodeClassSpec{MetadataOptions: &v1beta1.MetadataOptions{HTTPTokens: awssdk.String("required"), HTTPPutResponseHopLimit: awssdk.Int64(10)}}), Entry("BlockDeviceMappings", v1beta1.EC2NodeClassSpec{BlockDeviceMappings: []*v1beta1.BlockDeviceMapping{ { DeviceName: awssdk.String("/dev/xvda"), EBS: &v1beta1.BlockDevice{ - VolumeSize: resource.NewScaledQuantity(20, resource.Giga), + VolumeSize: resources.Quantity("20Gi"), VolumeType: awssdk.String("gp3"), Encrypted: awssdk.Bool(true), }, @@ -732,6 +722,134 @@ var _ = Describe("Drift", func() { env.EventuallyExpectNotFound(pod, node) env.EventuallyExpectHealthyPodCount(selector, numPods) }) + It("should drift the EC2NodeClass on BlockDeviceMappings volume size update", func() { + nodeClass.Spec.BlockDeviceMappings = []*v1beta1.BlockDeviceMapping{ + { + DeviceName: awssdk.String("/dev/xvda"), + EBS: &v1beta1.BlockDevice{ + VolumeSize: resources.Quantity("20Gi"), + VolumeType: awssdk.String("gp3"), + Encrypted: awssdk.Bool(true), + }, + }, + } + env.ExpectCreated(dep, nodeClass, nodePool) + pod := env.EventuallyExpectHealthyPodCount(selector, numPods)[0] + nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0] + node := env.ExpectCreatedNodeCount("==", 1)[0] + + nodeClass.Spec.BlockDeviceMappings[0].EBS.VolumeSize = resources.Quantity("100Gi") + env.ExpectCreatedOrUpdated(nodeClass) + + By("validating the drifted status condition has propagated") + Eventually(func(g Gomega) { + g.Expect(env.Client.Get(env, client.ObjectKeyFromObject(nodeClaim), nodeClaim)).To(Succeed()) + g.Expect(nodeClaim.StatusConditions().GetCondition(corev1beta1.Drifted)).ToNot(BeNil()) + g.Expect(nodeClaim.StatusConditions().GetCondition(corev1beta1.Drifted).IsTrue()).To(BeTrue()) + }).Should(Succeed()) + + delete(pod.Annotations, corev1beta1.DoNotDisruptAnnotationKey) + env.ExpectUpdated(pod) + env.EventuallyExpectNotFound(pod, node) + env.EventuallyExpectHealthyPodCount(selector, numPods) + }) + It("should update the nodepool-hash annotation on the nodepool and nodeclaim when the nodepool's nodepool-hash-version annotation does not match the controller hash version", func() { + env.ExpectCreated(dep, nodeClass, nodePool) + env.EventuallyExpectHealthyPodCount(selector, numPods) + nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0] + nodePool = env.ExpectExists(nodePool).(*corev1beta1.NodePool) + expectedHash := nodePool.Hash() + + By(fmt.Sprintf("expect nodepool %s and nodeclaim %s to contain %s and %s annotations", nodePool.Name, nodeClaim.Name, corev1beta1.NodePoolHashAnnotationKey, corev1beta1.NodePoolHashVersionAnnotationKey)) + Eventually(func(g Gomega) { + g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(nodePool), nodePool)).To(Succeed()) + g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(nodeClaim), nodeClaim)).To(Succeed()) + + g.Expect(nodePool.Annotations).To(HaveKeyWithValue(corev1beta1.NodePoolHashAnnotationKey, expectedHash)) + g.Expect(nodePool.Annotations).To(HaveKeyWithValue(corev1beta1.NodePoolHashVersionAnnotationKey, corev1beta1.NodePoolHashVersion)) + g.Expect(nodeClaim.Annotations).To(HaveKeyWithValue(corev1beta1.NodePoolHashAnnotationKey, expectedHash)) + g.Expect(nodeClaim.Annotations).To(HaveKeyWithValue(corev1beta1.NodePoolHashVersionAnnotationKey, corev1beta1.NodePoolHashVersion)) + }).WithTimeout(30 * time.Second).Should(Succeed()) + + nodePool.Annotations = lo.Assign(nodePool.Annotations, map[string]string{ + corev1beta1.NodePoolHashAnnotationKey: "test-hash-1", + corev1beta1.NodePoolHashVersionAnnotationKey: "test-hash-version-1", + }) + // Updating `nodePool.Spec.Template.Annotations` would normally trigger drift on all nodeclaims owned by the + // nodepool. However, the nodepool-hash-version does not match the controller hash version, so we will see that + // none of the nodeclaims will be drifted and all nodeclaims will have an updated `nodepool-hash` and `nodepool-hash-version` annotation + nodePool.Spec.Template.Annotations = lo.Assign(nodePool.Spec.Template.Annotations, map[string]string{ + "test-key": "test-value", + }) + nodeClaim.Annotations = lo.Assign(nodePool.Annotations, map[string]string{ + corev1beta1.NodePoolHashAnnotationKey: "test-hash-2", + corev1beta1.NodePoolHashVersionAnnotationKey: "test-hash-version-2", + }) + + // The nodeclaim will need to be updated first, as the hash controller will only be triggered on changes to the nodepool + env.ExpectUpdated(nodeClaim, nodePool) + expectedHash = nodePool.Hash() + + // Expect all nodeclaims not to be drifted and contain an updated `nodepool-hash` and `nodepool-hash-version` annotation + Eventually(func(g Gomega) { + g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(nodePool), nodePool)).To(Succeed()) + g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(nodeClaim), nodeClaim)).To(Succeed()) + + g.Expect(nodePool.Annotations).To(HaveKeyWithValue(corev1beta1.NodePoolHashAnnotationKey, expectedHash)) + g.Expect(nodePool.Annotations).To(HaveKeyWithValue(corev1beta1.NodePoolHashVersionAnnotationKey, corev1beta1.NodePoolHashVersion)) + g.Expect(nodeClaim.Annotations).To(HaveKeyWithValue(corev1beta1.NodePoolHashAnnotationKey, expectedHash)) + g.Expect(nodeClaim.Annotations).To(HaveKeyWithValue(corev1beta1.NodePoolHashVersionAnnotationKey, corev1beta1.NodePoolHashVersion)) + }) + }) + It("should update the ec2nodeclass-hash annotation on the ec2nodeclass and nodeclaim when the ec2nodeclass's ec2nodeclass-hash-version annotation does not match the controller hash version", func() { + env.ExpectCreated(dep, nodeClass, nodePool) + env.EventuallyExpectHealthyPodCount(selector, numPods) + nodeClaim := env.EventuallyExpectCreatedNodeClaimCount("==", 1)[0] + nodeClass = env.ExpectExists(nodeClass).(*v1beta1.EC2NodeClass) + expectedHash := nodeClass.Hash() + + By(fmt.Sprintf("expect nodeclass %s and nodeclaim %s to contain %s and %s annotations", nodeClass.Name, nodeClaim.Name, v1beta1.AnnotationEC2NodeClassHash, v1beta1.AnnotationEC2NodeClassHashVersion)) + Eventually(func(g Gomega) { + g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(nodeClass), nodeClass)).To(Succeed()) + g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(nodeClaim), nodeClaim)).To(Succeed()) + + g.Expect(nodeClass.Annotations).To(HaveKeyWithValue(v1beta1.AnnotationEC2NodeClassHash, expectedHash)) + g.Expect(nodeClass.Annotations).To(HaveKeyWithValue(v1beta1.AnnotationEC2NodeClassHashVersion, v1beta1.EC2NodeClassHashVersion)) + g.Expect(nodeClaim.Annotations).To(HaveKeyWithValue(v1beta1.AnnotationEC2NodeClassHash, expectedHash)) + g.Expect(nodeClaim.Annotations).To(HaveKeyWithValue(v1beta1.AnnotationEC2NodeClassHashVersion, v1beta1.EC2NodeClassHashVersion)) + }).WithTimeout(30 * time.Second).Should(Succeed()) + + nodeClass.Annotations = lo.Assign(nodeClass.Annotations, map[string]string{ + v1beta1.AnnotationEC2NodeClassHash: "test-hash-1", + v1beta1.AnnotationEC2NodeClassHashVersion: "test-hash-version-1", + }) + // Updating `nodeClass.Spec.Tags` would normally trigger drift on all nodeclaims using the + // nodeclass. However, the ec2nodeclass-hash-version does not match the controller hash version, so we will see that + // none of the nodeclaims will be drifted and all nodeclaims will have an updated `ec2nodeclass-hash` and `ec2nodeclass-hash-version` annotation + nodeClass.Spec.Tags = lo.Assign(nodeClass.Spec.Tags, map[string]string{ + "test-key": "test-value", + }) + nodeClaim.Annotations = lo.Assign(nodePool.Annotations, map[string]string{ + v1beta1.AnnotationEC2NodeClassHash: "test-hash-2", + v1beta1.AnnotationEC2NodeClassHashVersion: "test-hash-version-2", + }) + + // The nodeclaim will need to be updated first, as the hash controller will only be triggered on changes to the nodeclass + env.ExpectUpdated(nodeClaim, nodeClass) + expectedHash = nodeClass.Hash() + + // Expect all nodeclaims not to be drifted and contain an updated `nodepool-hash` and `nodepool-hash-version` annotation + Eventually(func(g Gomega) { + g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(nodeClass), nodeClass)).To(Succeed()) + g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(nodeClaim), nodeClaim)).To(Succeed()) + + g.Expect(nodeClass.Annotations).To(HaveKeyWithValue(v1beta1.AnnotationEC2NodeClassHash, expectedHash)) + g.Expect(nodeClass.Annotations).To(HaveKeyWithValue(v1beta1.AnnotationEC2NodeClassHashVersion, v1beta1.EC2NodeClassHashVersion)) + g.Expect(nodeClaim.Annotations).To(HaveKeyWithValue(v1beta1.AnnotationEC2NodeClassHash, expectedHash)) + g.Expect(nodeClaim.Annotations).To(HaveKeyWithValue(v1beta1.AnnotationEC2NodeClassHashVersion, v1beta1.EC2NodeClassHashVersion)) + }).WithTimeout(30 * time.Second).Should(Succeed()) + env.ConsistentlyExpectNodeClaimsNotDrifted(time.Minute, nodeClaim) + }) Context("Failure", func() { It("should not continue to drift if a node never registers", func() { // launch a new nodeClaim @@ -853,7 +971,6 @@ var _ = Describe("Drift", func() { MatchLabels: map[string]string{"app": "inflate"}, }}, }, - ReadinessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{ HTTPGet: &v1.HTTPGetAction{ @@ -882,7 +999,7 @@ var _ = Describe("Drift", func() { env.ExpectUpdated(nodePool) env.EventuallyExpectDrifted(nodeClaims...) - env.ConsistentlyExpectNoDisruptions(int(numPods), "1m") + env.ConsistentlyExpectNoDisruptions(int(numPods), time.Minute) }) }) }) diff --git a/test/suites/expiration/suite_test.go b/test/suites/expiration/suite_test.go index 6ceb1c365f8d..168381716de6 100644 --- a/test/suites/expiration/suite_test.go +++ b/test/suites/expiration/suite_test.go @@ -93,12 +93,99 @@ var _ = Describe("Expiration", func() { selector = labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) }) Context("Budgets", func() { + // Two nodes, both expired or both drifted, the more drifted one with a pre-stop pod that sleeps for 300 seconds, + // and we consistently ensure that the second node is not tainted == disrupted. + It("should not continue to disrupt nodes that have been the target of pod nomination", func() { + coretest.ReplaceRequirements(nodePool, + corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceSize, + Operator: v1.NodeSelectorOpIn, + Values: []string{"2xlarge"}, + }, + }, + ) + nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{ + Nodes: "100%", + }} + nodePool.Spec.Disruption.ExpireAfter = corev1beta1.NillableDuration{} + + // Create a deployment with one pod to create one node. + dep = coretest.Deployment(coretest.DeploymentOptions{ + Replicas: 1, + PodOptions: coretest.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + corev1beta1.DoNotDisruptAnnotationKey: "true", + }, + Labels: map[string]string{"app": "large-app"}, + }, + // Each 2xlarge has 8 cpu, so each node should fit 2 pods. + ResourceRequirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("3"), + }, + }, + Command: []string{"sh", "-c", "sleep 3600"}, + Image: "alpine:latest", + PreStopSleep: lo.ToPtr(int64(300)), + TerminationGracePeriodSeconds: lo.ToPtr(int64(500)), + }, + }) + selector = labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) + env.ExpectCreated(nodeClass, nodePool, dep) + + env.EventuallyExpectCreatedNodeClaimCount("==", 1) + env.EventuallyExpectCreatedNodeCount("==", 1) + env.EventuallyExpectHealthyPodCount(selector, 1) + + // Set the node to unschedulable so that we can create another node with one pod. + node := env.EventuallyExpectNodeCount("==", 1)[0] + node.Spec.Unschedulable = true + env.ExpectUpdated(node) + + dep.Spec.Replicas = lo.ToPtr(int32(2)) + env.ExpectUpdated(dep) + + ncs := env.EventuallyExpectCreatedNodeClaimCount("==", 2) + env.EventuallyExpectCreatedNodeCount("==", 2) + pods := env.EventuallyExpectHealthyPodCount(selector, 2) + env.Monitor.Reset() // Reset the monitor so that we can expect a single node to be spun up after expiration + + node = env.ExpectExists(node).(*v1.Node) + node.Spec.Unschedulable = false + env.ExpectUpdated(node) + + By("enabling expiration") + nodePool.Spec.Disruption.ExpireAfter = corev1beta1.NillableDuration{Duration: lo.ToPtr(time.Second * 30)} + env.ExpectUpdated(nodePool) + + // Expect that both of the nodes are expired, but not being disrupted + env.EventuallyExpectExpired(ncs...) + env.ConsistentlyExpectNoDisruptions(2, 30*time.Second) + + By("removing the do not disrupt annotations") + // Remove the do not disrupt annotation from the two pods + for _, p := range pods { + p := env.ExpectExists(p).(*v1.Pod) + delete(p.Annotations, corev1beta1.DoNotDisruptAnnotationKey) + env.ExpectUpdated(p) + } + env.EventuallyExpectTaintedNodeCount("==", 1) + + By("expecting only one disruption for 60s") + // Expect only one node being disrupted as the other node should continue to be nominated. + // As the pod has a 300s pre-stop sleep. + env.ConsistentlyExpectDisruptionsWithNodeCount(1, 2, time.Minute) + }) It("should respect budgets for empty expiration", func() { coretest.ReplaceRequirements(nodePool, - v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceSize, - Operator: v1.NodeSelectorOpIn, - Values: []string{"2xlarge"}, + corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceSize, + Operator: v1.NodeSelectorOpIn, + Values: []string{"2xlarge"}, + }, }, ) nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{ @@ -151,7 +238,8 @@ var _ = Describe("Expiration", func() { env.EventuallyExpectExpired(nodeClaims...) // Expect that two nodes are tainted. - nodes = env.EventuallyExpectTaintedNodeCount("==", 2) + env.EventuallyExpectTaintedNodeCount("==", 2) + nodes = env.ConsistentlyExpectDisruptionsWithNodeCount(2, 3, 5*time.Second) // Remove finalizers for _, node := range nodes { @@ -171,10 +259,12 @@ var _ = Describe("Expiration", func() { }) It("should respect budgets for non-empty delete expiration", func() { nodePool = coretest.ReplaceRequirements(nodePool, - v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceSize, - Operator: v1.NodeSelectorOpIn, - Values: []string{"2xlarge"}, + corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceSize, + Operator: v1.NodeSelectorOpIn, + Values: []string{"2xlarge"}, + }, }, ) // We're expecting to create 3 nodes, so we'll expect to see at most 2 nodes deleting at one time. @@ -207,36 +297,15 @@ var _ = Describe("Expiration", func() { nodeClaims := env.EventuallyExpectCreatedNodeClaimCount("==", 3) nodes := env.EventuallyExpectCreatedNodeCount("==", 3) env.EventuallyExpectHealthyPodCount(selector, numPods) - env.Monitor.Reset() // Reset the monitor so that we can expect a single node to be spun up after expiration By("scaling down the deployment") // Update the deployment to a third of the replicas. dep.Spec.Replicas = lo.ToPtr[int32](3) env.ExpectUpdated(dep) - By("spreading the pods to each of the nodes") + // First expect there to be 3 pods, then try to spread the pods. env.EventuallyExpectHealthyPodCount(selector, 3) - // Delete pods from the deployment until each node has one pod. - var nodePods []*v1.Pod - for { - node, found := lo.Find(nodes, func(n *v1.Node) bool { - nodePods = env.ExpectHealthyPodsForNode(n.Name) - return len(nodePods) > 1 - }) - if !found { - break - } - // Set the nodes to unschedulable so that the pods won't reschedule. - Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(node), node)).To(Succeed()) - node.Spec.Unschedulable = true - env.ExpectUpdated(node) - for _, pod := range nodePods[1:] { - env.ExpectDeleted(pod) - } - Eventually(func(g Gomega) { - g.Expect(len(env.ExpectHealthyPodsForNode(node.Name))).To(Equal(1)) - }).WithTimeout(5 * time.Second).Should(Succeed()) - } + env.ForcePodsToSpread(nodes...) env.EventuallyExpectHealthyPodCount(selector, 3) By("cordoning and adding finalizer to the nodes") @@ -244,8 +313,6 @@ var _ = Describe("Expiration", func() { for _, node := range nodes { Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(node), node)).To(Succeed()) node.Finalizers = append(node.Finalizers, common.TestingFinalizer) - // Set nodes as unschedulable so that pod nomination doesn't delay disruption for the second disruption action - node.Spec.Unschedulable = true env.ExpectUpdated(node) } @@ -264,11 +331,9 @@ var _ = Describe("Expiration", func() { env.ExpectUpdated(pod) } - // Mark one node as schedulable so the other two nodes can schedule to this node and delete. - Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(nodes[0]), nodes[0])).To(Succeed()) - nodes[0].Spec.Unschedulable = false - env.ExpectUpdated(nodes[0]) - nodes = env.EventuallyExpectTaintedNodeCount("==", 2) + // Ensure that we get two nodes tainted, and they have overlap during the expiration + env.EventuallyExpectTaintedNodeCount("==", 2) + nodes = env.ConsistentlyExpectDisruptionsWithNodeCount(2, 3, 5*time.Second) By("removing the finalizer from the nodes") Expect(env.ExpectTestingFinalizerRemoved(nodes[0])).To(Succeed()) @@ -279,79 +344,78 @@ var _ = Describe("Expiration", func() { env.EventuallyExpectNotFound(nodes[0], nodes[1]) }) It("should respect budgets for non-empty replace expiration", func() { + appLabels := map[string]string{"app": "large-app"} + nodePool = coretest.ReplaceRequirements(nodePool, - v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceSize, - Operator: v1.NodeSelectorOpIn, - Values: []string{"2xlarge"}, + corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceSize, + Operator: v1.NodeSelectorOpIn, + Values: []string{"xlarge"}, + }, + }, + // Add an Exists operator so that we can select on a fake partition later + corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: "test-partition", + Operator: v1.NodeSelectorOpExists, + }, }, ) - // We're expecting to create 3 nodes, so we'll expect to see at most 2 nodes deleting at one time. + nodePool.Labels = appLabels + // We're expecting to create 5 nodes, so we'll expect to see at most 3 nodes deleting at one time. nodePool.Spec.Disruption.Budgets = []corev1beta1.Budget{{ - Nodes: "50%", + Nodes: "3", }} - nodePool.Spec.Disruption.ExpireAfter = corev1beta1.NillableDuration{} - numPods = 3 - dep = coretest.Deployment(coretest.DeploymentOptions{ - Replicas: int32(numPods), - PodOptions: coretest.PodOptions{ - ObjectMeta: metav1.ObjectMeta{ - Annotations: map[string]string{ - corev1beta1.DoNotDisruptAnnotationKey: "true", + + // Make 5 pods all with different deployments and different test partitions, so that each pod can be put + // on a separate node. + selector = labels.SelectorFromSet(appLabels) + numPods = 5 + deployments := make([]*appsv1.Deployment, numPods) + for i := range lo.Range(numPods) { + deployments[i] = coretest.Deployment(coretest.DeploymentOptions{ + Replicas: 1, + PodOptions: coretest.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Labels: appLabels, }, - Labels: map[string]string{"app": "large-app"}, - }, - // Each 2xlarge has 8 cpu, so each node should fit no more than 3 pods. - ResourceRequirements: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("5"), + NodeSelector: map[string]string{"test-partition": fmt.Sprintf("%d", i)}, + // Each xlarge has 4 cpu, so each node should fit no more than 1 pod. + ResourceRequirements: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("3"), + }, }, }, - }, - }) - selector = labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) - env.ExpectCreated(nodeClass, nodePool, dep) + }) + } - nodeClaims := env.EventuallyExpectCreatedNodeClaimCount("==", 3) - nodes := env.EventuallyExpectCreatedNodeCount("==", 3) + env.ExpectCreated(nodeClass, nodePool, deployments[0], deployments[1], deployments[2], deployments[3], deployments[4]) + + env.EventuallyExpectCreatedNodeClaimCount("==", 5) + nodes := env.EventuallyExpectCreatedNodeCount("==", 5) + + // Check that all daemonsets and deployment pods are online env.EventuallyExpectHealthyPodCount(selector, numPods) - env.Monitor.Reset() // Reset the monitor so that we can expect a single node to be spun up after drift By("cordoning and adding finalizer to the nodes") // Add a finalizer to each node so that we can stop termination disruptions for _, node := range nodes { Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(node), node)).To(Succeed()) node.Finalizers = append(node.Finalizers, common.TestingFinalizer) - // Set nodes as unschedulable so that pod nomination doesn't delay disruption for the second disruption action env.ExpectUpdated(node) } - By("expiring the nodes") - // Expire the nodeclaims - nodePool.Spec.Disruption.ExpireAfter = corev1beta1.NillableDuration{Duration: lo.ToPtr(time.Second * 90)} + By("enabling expiration") + nodePool.Spec.Disruption.ExpireAfter = corev1beta1.NillableDuration{Duration: lo.ToPtr(30 * time.Second)} env.ExpectUpdated(nodePool) - env.EventuallyExpectExpired(nodeClaims...) - - By("enabling disruption by removing the do not disrupt annotation") - pods := env.EventuallyExpectHealthyPodCount(selector, 3) - // Remove the do-not-disrupt annotation so that the nodes are now disruptable - for _, pod := range pods { - delete(pod.Annotations, corev1beta1.DoNotDisruptAnnotationKey) - env.ExpectUpdated(pod) - } - - // Expect two nodes tainted and two nodes created - tainted := env.EventuallyExpectTaintedNodeCount("==", 2) - env.EventuallyExpectCreatedNodeCount("==", 2) - - Expect(env.ExpectTestingFinalizerRemoved(tainted[0])).To(Succeed()) - Expect(env.ExpectTestingFinalizerRemoved(tainted[1])).To(Succeed()) - - env.EventuallyExpectNotFound(tainted[0], tainted[1]) - - tainted = env.EventuallyExpectTaintedNodeCount("==", 1) - env.EventuallyExpectCreatedNodeCount("==", 3) + // Ensure that we get two nodes tainted, and they have overlap during the expiration + env.EventuallyExpectTaintedNodeCount("==", 3) + env.EventuallyExpectNodeClaimCount("==", 8) + env.EventuallyExpectNodeCount("==", 8) + nodes = env.ConsistentlyExpectDisruptionsWithNodeCount(3, 8, 5*time.Second) // Set the expireAfter to "Never" to make sure new node isn't deleted // This is CRITICAL since it prevents nodes that are immediately spun up from immediately being expired and @@ -359,11 +423,12 @@ var _ = Describe("Expiration", func() { nodePool.Spec.Disruption.ExpireAfter.Duration = nil env.ExpectUpdated(nodePool) - Expect(env.ExpectTestingFinalizerRemoved(tainted[0])).To(Succeed()) + for _, node := range nodes { + Expect(env.ExpectTestingFinalizerRemoved(node)).To(Succeed()) + } - // After the deletion timestamp is set and all pods are drained - // the node should be gone env.EventuallyExpectNotFound(nodes[0], nodes[1], nodes[2]) + env.ExpectNodeCount("==", 5) }) It("should not allow expiration if the budget is fully blocking", func() { // We're going to define a budget that doesn't allow any expirations to happen @@ -379,7 +444,7 @@ var _ = Describe("Expiration", func() { env.EventuallyExpectHealthyPodCount(selector, numPods) env.EventuallyExpectExpired(nodeClaim) - env.ConsistentlyExpectNoDisruptions(1, "1m") + env.ConsistentlyExpectNoDisruptions(1, time.Minute) }) It("should not allow expiration if the budget is fully blocking during a scheduled time", func() { // We're going to define a budget that doesn't allow any expirations to happen @@ -401,7 +466,7 @@ var _ = Describe("Expiration", func() { env.EventuallyExpectHealthyPodCount(selector, numPods) env.EventuallyExpectExpired(nodeClaim) - env.ConsistentlyExpectNoDisruptions(1, "1m") + env.ConsistentlyExpectNoDisruptions(1, time.Minute) }) }) It("should expire the node after the expiration is reached", func() { @@ -420,7 +485,7 @@ var _ = Describe("Expiration", func() { env.ExpectUpdated(pod) } - // Eventually the node will be set as unschedulable, which means its actively being deprovisioned + // Eventually the node will be tainted, which means its actively being disrupted Eventually(func(g Gomega) { g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(node), node)).Should(Succeed()) _, ok := lo.Find(node.Spec.Taints, func(t v1.Taint) bool { @@ -484,7 +549,7 @@ var _ = Describe("Expiration", func() { env.ExpectUpdated(pod) } - // Eventually the node will be set as unschedulable, which means its actively being deprovisioned + // Eventually the node will be tainted, which means its actively being disrupted Eventually(func(g Gomega) { g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(node), node)).Should(Succeed()) _, ok := lo.Find(node.Spec.Taints, func(t v1.Taint) bool { @@ -595,7 +660,7 @@ var _ = Describe("Expiration", func() { g.Expect(env.Client.Get(env.Context, client.ObjectKeyFromObject(node), node)).To(Succeed()) stored := node.DeepCopy() node.Spec.Taints = lo.Reject(node.Spec.Taints, func(t v1.Taint, _ int) bool { return t.Key == "example.com/taint" }) - g.Expect(env.Client.Patch(env.Context, node, client.MergeFrom(stored))).To(Succeed()) + g.Expect(env.Client.Patch(env.Context, node, client.StrategicMergeFrom(stored))).To(Succeed()) } }).Should(Succeed()) @@ -641,7 +706,6 @@ var _ = Describe("Expiration", func() { MatchLabels: map[string]string{"app": "inflate"}, }}, }, - ReadinessProbe: &v1.Probe{ ProbeHandler: v1.ProbeHandler{ HTTPGet: &v1.HTTPGetAction{ @@ -666,7 +730,7 @@ var _ = Describe("Expiration", func() { env.EventuallyExpectBoundPodCount(selector, int(numPods)) env.EventuallyExpectExpired(nodeClaims...) - env.ConsistentlyExpectNoDisruptions(int(numPods), "1m") + env.ConsistentlyExpectNoDisruptions(int(numPods), time.Minute) }) }) }) diff --git a/test/suites/integration/ami_test.go b/test/suites/integration/ami_test.go index 53eb65e0d6b7..96c3bdc57a1c 100644 --- a/test/suites/integration/ami_test.go +++ b/test/suites/integration/ami_test.go @@ -24,12 +24,15 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/ssm" + "github.com/samber/lo" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" coretest "sigs.k8s.io/karpenter/pkg/test" + corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" awsenv "github.com/aws/karpenter-provider-aws/test/pkg/environment/aws" @@ -40,7 +43,7 @@ import ( var _ = Describe("AMI", func() { var customAMI string BeforeEach(func() { - customAMI = env.GetCustomAMI("/aws/service/eks/optimized-ami/%s/amazon-linux-2/recommended/image_id", 1) + customAMI = env.GetCustomAMI("/aws/service/eks/optimized-ami/%s/amazon-linux-2023/x86_64/standard/recommended/image_id", 1) }) It("should use the AMI defined by the AMI Selector Terms", func() { @@ -58,12 +61,7 @@ var _ = Describe("AMI", func() { }) It("should use the most recent AMI when discovering multiple", func() { // choose an old static image - parameter, err := env.SSMAPI.GetParameter(&ssm.GetParameterInput{ - Name: aws.String("/aws/service/eks/optimized-ami/1.23/amazon-linux-2/amazon-eks-node-1.23-v20230322/image_id"), - }) - Expect(err).To(BeNil()) - oldCustomAMI := *parameter.Parameter.Value - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyCustom + oldCustomAMI := env.GetCustomAMI("/aws/service/eks/optimized-ami/%[1]s/amazon-linux-2023/x86_64/standard/amazon-eks-node-al2023-x86_64-standard-%[1]s-v20240307/image_id", 1) nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ { ID: customAMI, @@ -72,7 +70,6 @@ var _ = Describe("AMI", func() { ID: oldCustomAMI, }, } - nodeClass.Spec.UserData = aws.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", env.ClusterName)) pod := coretest.Pod() env.ExpectCreated(pod, nodeClass, nodePool) @@ -87,14 +84,12 @@ var _ = Describe("AMI", func() { }) Expect(err).To(BeNil()) Expect(output.Images).To(HaveLen(1)) - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyCustom nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ { Name: *output.Images[0].Name, Owner: "fakeOwnerValue", }, } - nodeClass.Spec.UserData = aws.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", env.ClusterName)) pod := coretest.Pod() env.ExpectCreated(pod, nodeClass, nodePool) @@ -108,13 +103,11 @@ var _ = Describe("AMI", func() { Expect(err).To(BeNil()) Expect(output.Images).To(HaveLen(1)) - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyCustom nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ { Name: *output.Images[0].Name, }, } - nodeClass.Spec.UserData = aws.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", env.ClusterName)) pod := coretest.Pod() env.ExpectCreated(pod, nodeClass, nodePool) @@ -124,13 +117,11 @@ var _ = Describe("AMI", func() { env.ExpectInstance(pod.Spec.NodeName).To(HaveField("ImageId", HaveValue(Equal(customAMI)))) }) It("should support ami selector ids", func() { - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyCustom nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ { ID: customAMI, }, } - nodeClass.Spec.UserData = aws.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", env.ClusterName)) pod := coretest.Pod() env.ExpectCreated(pod, nodeClass, nodePool) @@ -142,6 +133,14 @@ var _ = Describe("AMI", func() { Context("AMIFamily", func() { It("should provision a node using the AL2 family", func() { + pod := coretest.Pod() + nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 + env.ExpectCreated(nodeClass, nodePool, pod) + env.EventuallyExpectHealthy(pod) + env.ExpectCreatedNodeCount("==", 1) + }) + It("should provision a node using the AL2023 family", func() { + nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2023 pod := coretest.Pod() env.ExpectCreated(nodeClass, nodePool, pod) env.EventuallyExpectHealthy(pod) @@ -156,13 +155,26 @@ var _ = Describe("AMI", func() { }) It("should provision a node using the Ubuntu family", func() { nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyUbuntu + // TODO (jmdeal@): remove once 22.04 AMIs are supported + if env.GetK8sVersion(0) == "1.29" { + nodeClass.Spec.AMISelectorTerms = lo.Map([]string{ + "/aws/service/canonical/ubuntu/eks/20.04/1.28/stable/current/amd64/hvm/ebs-gp2/ami-id", + "/aws/service/canonical/ubuntu/eks/20.04/1.28/stable/current/arm64/hvm/ebs-gp2/ami-id", + }, func(arg string, _ int) v1beta1.AMISelectorTerm { + parameter, err := env.SSMAPI.GetParameter(&ssm.GetParameterInput{Name: lo.ToPtr(arg)}) + Expect(err).To(BeNil()) + return v1beta1.AMISelectorTerm{ID: *parameter.Parameter.Value} + }) + } // TODO: remove requirements after Ubuntu fixes bootstrap script issue w/ // new instance types not included in the max-pods.txt file. (https://github.com/aws/karpenter-provider-aws/issues/4472) nodePool = coretest.ReplaceRequirements(nodePool, - v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceFamily, - Operator: v1.NodeSelectorOpNotIn, - Values: awsenv.ExcludedInstanceFamilies, + corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceFamily, + Operator: v1.NodeSelectorOpNotIn, + Values: awsenv.ExcludedInstanceFamilies, + }, }, ) pod := coretest.Pod() @@ -172,9 +184,10 @@ var _ = Describe("AMI", func() { }) It("should support Custom AMIFamily with AMI Selectors", func() { nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyCustom + al2AMI := env.GetCustomAMI("/aws/service/eks/optimized-ami/%s/amazon-linux-2/recommended/image_id", 1) nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ { - ID: customAMI, + ID: al2AMI, }, } nodeClass.Spec.UserData = aws.String(fmt.Sprintf("#!/bin/bash\n/etc/eks/bootstrap.sh '%s'", env.ClusterName)) @@ -184,7 +197,7 @@ var _ = Describe("AMI", func() { env.EventuallyExpectHealthy(pod) env.ExpectCreatedNodeCount("==", 1) - env.ExpectInstance(pod.Spec.NodeName).To(HaveField("ImageId", HaveValue(Equal(customAMI)))) + env.ExpectInstance(pod.Spec.NodeName).To(HaveField("ImageId", HaveValue(Equal(al2AMI)))) }) It("should have the EC2NodeClass status for AMIs using wildcard", func() { nodeClass.Spec.AMISelectorTerms = []v1beta1.AMISelectorTerm{ @@ -213,6 +226,7 @@ var _ = Describe("AMI", func() { It("should merge UserData contents for AL2 AMIFamily", func() { content, err := os.ReadFile("testdata/al2_userdata_input.sh") Expect(err).ToNot(HaveOccurred()) + nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 nodeClass.Spec.UserData = aws.String(string(content)) nodePool.Spec.Template.Spec.Taints = []v1.Taint{{Key: "example.com", Value: "value", Effect: "NoExecute"}} nodePool.Spec.Template.Spec.StartupTaints = []v1.Taint{{Key: "example.com", Value: "value", Effect: "NoSchedule"}} @@ -233,6 +247,7 @@ var _ = Describe("AMI", func() { It("should merge non-MIME UserData contents for AL2 AMIFamily", func() { content, err := os.ReadFile("testdata/al2_no_mime_userdata_input.sh") Expect(err).ToNot(HaveOccurred()) + nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 nodeClass.Spec.UserData = aws.String(string(content)) nodePool.Spec.Template.Spec.Taints = []v1.Taint{{Key: "example.com", Value: "value", Effect: "NoExecute"}} nodePool.Spec.Template.Spec.StartupTaints = []v1.Taint{{Key: "example.com", Value: "value", Effect: "NoSchedule"}} @@ -284,15 +299,19 @@ var _ = Describe("AMI", func() { // TODO: remove this requirement once VPC RC rolls out m7a.*, r7a.* ENI data (https://github.com/aws/karpenter-provider-aws/issues/4472) nodePool = coretest.ReplaceRequirements(nodePool, - v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceFamily, - Operator: v1.NodeSelectorOpNotIn, - Values: awsenv.ExcludedInstanceFamilies, + corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceFamily, + Operator: v1.NodeSelectorOpNotIn, + Values: awsenv.ExcludedInstanceFamilies, + }, }, - v1.NodeSelectorRequirement{ - Key: v1.LabelOSStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{string(v1.Windows)}, + corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1.LabelOSStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{string(v1.Windows)}, + }, }, ) pod := coretest.Pod(coretest.PodOptions{ diff --git a/test/suites/integration/block_device_mappings_test.go b/test/suites/integration/block_device_mappings_test.go index f5336a0a090b..56e0643eeaa0 100644 --- a/test/suites/integration/block_device_mappings_test.go +++ b/test/suites/integration/block_device_mappings_test.go @@ -31,7 +31,7 @@ var _ = Describe("BlockDeviceMappings", func() { { DeviceName: aws.String("/dev/xvda"), EBS: &v1beta1.BlockDevice{ - VolumeSize: resources.Quantity("10G"), + VolumeSize: resources.Quantity("20Gi"), VolumeType: aws.String("io2"), IOPS: aws.Int64(1000), Encrypted: aws.Bool(true), @@ -51,7 +51,7 @@ var _ = Describe("BlockDeviceMappings", func() { Expect(instance.BlockDeviceMappings[0].Ebs).To(HaveField("DeleteOnTermination", HaveValue(BeTrue()))) volume := env.GetVolume(instance.BlockDeviceMappings[0].Ebs.VolumeId) Expect(volume).To(HaveField("Encrypted", HaveValue(BeTrue()))) - Expect(volume).To(HaveField("Size", HaveValue(Equal(int64(10))))) // Convert G -> Gib (rounded up) + Expect(volume).To(HaveField("Size", HaveValue(Equal(int64(20))))) Expect(volume).To(HaveField("Iops", HaveValue(Equal(int64(1000))))) Expect(volume).To(HaveField("VolumeType", HaveValue(Equal("io2")))) }) diff --git a/test/suites/integration/emptiness_test.go b/test/suites/integration/emptiness_test.go index d583e4a07b11..a4aca9792b0d 100644 --- a/test/suites/integration/emptiness_test.go +++ b/test/suites/integration/emptiness_test.go @@ -69,7 +69,7 @@ var _ = Describe("Emptiness", func() { env.ExpectDeleted(dep) env.EventuallyExpectEmpty(nodeClaim) - env.ConsistentlyExpectNoDisruptions(1, "1m") + env.ConsistentlyExpectNoDisruptions(1, time.Minute) }) It("should not allow emptiness if the budget is fully blocking during a scheduled time", func() { // We're going to define a budget that doesn't allow any emptiness disruption to happen @@ -93,7 +93,7 @@ var _ = Describe("Emptiness", func() { env.ExpectDeleted(dep) env.EventuallyExpectEmpty(nodeClaim) - env.ConsistentlyExpectNoDisruptions(1, "1m") + env.ConsistentlyExpectNoDisruptions(1, time.Minute) }) }) It("should terminate an empty node", func() { @@ -111,7 +111,7 @@ var _ = Describe("Emptiness", func() { By("making the nodeclaim empty") persisted := deployment.DeepCopy() deployment.Spec.Replicas = ptr.Int32(0) - Expect(env.Client.Patch(env, deployment, client.MergeFrom(persisted))).To(Succeed()) + Expect(env.Client.Patch(env, deployment, client.StrategicMergeFrom(persisted))).To(Succeed()) env.EventuallyExpectEmpty(nodeClaim) diff --git a/test/suites/integration/extended_resources_test.go b/test/suites/integration/extended_resources_test.go index 28884c03b13d..49c2e8d3c7ae 100644 --- a/test/suites/integration/extended_resources_test.go +++ b/test/suites/integration/extended_resources_test.go @@ -28,6 +28,8 @@ import ( "sigs.k8s.io/karpenter/pkg/test" + corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" awsenv "github.com/aws/karpenter-provider-aws/test/pkg/environment/aws" @@ -36,9 +38,15 @@ import ( ) var _ = Describe("Extended Resources", func() { + BeforeEach(func() { + if env.PrivateCluster { + Skip("skipping Extended Resources test for private cluster") + } + }) It("should provision nodes for a deployment that requests nvidia.com/gpu", func() { ExpectNvidiaDevicePluginCreated() - + // TODO: jmdeal@ remove AL2 pin once AL2023 accelerated AMIs are available + nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 numPods := 1 dep := test.Deployment(test.DeploymentOptions{ Replicas: int32(numPods), @@ -57,9 +65,11 @@ var _ = Describe("Extended Resources", func() { }, }) selector := labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) - test.ReplaceRequirements(nodePool, v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceCategory, - Operator: v1.NodeSelectorOpExists, + test.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceCategory, + Operator: v1.NodeSelectorOpExists, + }, }) env.ExpectCreated(nodeClass, nodePool, dep) env.EventuallyExpectHealthyPodCount(selector, numPods) @@ -87,10 +97,11 @@ var _ = Describe("Extended Resources", func() { }, }) selector := labels.SelectorFromSet(dep.Spec.Selector.MatchLabels) - test.ReplaceRequirements(nodePool, v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceCategory, - Operator: v1.NodeSelectorOpExists, - }) + test.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceCategory, + Operator: v1.NodeSelectorOpExists, + }}) env.ExpectCreated(nodeClass, nodePool, dep) env.EventuallyExpectHealthyPodCount(selector, numPods) env.ExpectCreatedNodeCount("==", 1) @@ -103,10 +114,12 @@ var _ = Describe("Extended Resources", func() { }) // TODO: remove this requirement once VPC RC rolls out m7a.*, r7a.* ENI data (https://github.com/aws/karpenter-provider-aws/issues/4472) test.ReplaceRequirements(nodePool, - v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceFamily, - Operator: v1.NodeSelectorOpNotIn, - Values: awsenv.ExcludedInstanceFamilies, + corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceFamily, + Operator: v1.NodeSelectorOpNotIn, + Values: awsenv.ExcludedInstanceFamilies, + }, }, ) numPods := 1 @@ -225,6 +238,8 @@ var _ = Describe("Extended Resources", func() { } // Only select private subnets since instances with multiple network instances at launch won't get a public IP. nodeClass.Spec.SubnetSelectorTerms[0].Tags["Name"] = "*Private*" + // TODO: jmdeal@ remove AL2 pin once AL2023 accelerated AMIs are available + nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyAL2 numPods := 1 dep := test.Deployment(test.DeploymentOptions{ diff --git a/test/suites/integration/instance_profile_test.go b/test/suites/integration/instance_profile_test.go index 1dd44f424080..4c2e7b860a89 100644 --- a/test/suites/integration/instance_profile_test.go +++ b/test/suites/integration/instance_profile_test.go @@ -30,6 +30,11 @@ import ( ) var _ = Describe("InstanceProfile Generation", func() { + BeforeEach(func() { + if env.PrivateCluster { + Skip("skipping InstanceProfile Generation test for private cluster") + } + }) It("should generate the InstanceProfile when setting the role", func() { pod := coretest.Pod() env.ExpectCreated(nodePool, nodeClass, pod) diff --git a/test/suites/integration/kubelet_config_test.go b/test/suites/integration/kubelet_config_test.go index f6775611c90e..a79804cdab0d 100644 --- a/test/suites/integration/kubelet_config_test.go +++ b/test/suites/integration/kubelet_config_test.go @@ -26,6 +26,9 @@ import ( corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + "github.com/aws/aws-sdk-go/service/ssm" + "github.com/samber/lo" + "github.com/aws/karpenter-provider-aws/test/pkg/environment/aws" "sigs.k8s.io/karpenter/pkg/test" @@ -33,6 +36,7 @@ import ( "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" ) var _ = Describe("KubeletConfiguration Overrides", func() { @@ -42,15 +46,15 @@ var _ = Describe("KubeletConfiguration Overrides", func() { nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ MaxPods: ptr.Int32(110), PodsPerCore: ptr.Int32(10), - SystemReserved: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("200m"), - v1.ResourceMemory: resource.MustParse("200Mi"), - v1.ResourceEphemeralStorage: resource.MustParse("1Gi"), + SystemReserved: map[string]string{ + string(v1.ResourceCPU): "200m", + string(v1.ResourceMemory): "200Mi", + string(v1.ResourceEphemeralStorage): "1Gi", }, - KubeReserved: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("200m"), - v1.ResourceMemory: resource.MustParse("200Mi"), - v1.ResourceEphemeralStorage: resource.MustParse("1Gi"), + KubeReserved: map[string]string{ + string(v1.ResourceCPU): "200m", + string(v1.ResourceMemory): "200Mi", + string(v1.ResourceEphemeralStorage): "1Gi", }, EvictionHard: map[string]string{ "memory.available": "5%", @@ -85,6 +89,17 @@ var _ = Describe("KubeletConfiguration Overrides", func() { DescribeTable("Linux AMIFamilies", func(amiFamily *string) { nodeClass.Spec.AMIFamily = amiFamily + // TODO (jmdeal@): remove once 22.04 AMIs are supported + if *amiFamily == v1beta1.AMIFamilyUbuntu && env.GetK8sVersion(0) == "1.29" { + nodeClass.Spec.AMISelectorTerms = lo.Map([]string{ + "/aws/service/canonical/ubuntu/eks/20.04/1.28/stable/current/amd64/hvm/ebs-gp2/ami-id", + "/aws/service/canonical/ubuntu/eks/20.04/1.28/stable/current/arm64/hvm/ebs-gp2/ami-id", + }, func(arg string, _ int) v1beta1.AMISelectorTerm { + parameter, err := env.SSMAPI.GetParameter(&ssm.GetParameterInput{Name: lo.ToPtr(arg)}) + Expect(err).To(BeNil()) + return v1beta1.AMISelectorTerm{ID: *parameter.Parameter.Value} + }) + } pod := test.Pod(test.PodOptions{ NodeSelector: map[string]string{ v1.LabelOSStable: string(v1.Linux), @@ -96,6 +111,7 @@ var _ = Describe("KubeletConfiguration Overrides", func() { env.ExpectCreatedNodeCount("==", 1) }, Entry("when the AMIFamily is AL2", &v1beta1.AMIFamilyAL2), + Entry("when the AMIFamily is AL2023", &v1beta1.AMIFamilyAL2023), Entry("when the AMIFamily is Ubuntu", &v1beta1.AMIFamilyUbuntu), Entry("when the AMIFamily is Bottlerocket", &v1beta1.AMIFamilyBottlerocket), ) @@ -112,15 +128,19 @@ var _ = Describe("KubeletConfiguration Overrides", func() { // properly scoped // TODO: remove this requirement once VPC RC rolls out m7a.*, r7a.*, c7a.* ENI data (https://github.com/aws/karpenter-provider-aws/issues/4472) test.ReplaceRequirements(nodePool, - v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceFamily, - Operator: v1.NodeSelectorOpNotIn, - Values: aws.ExcludedInstanceFamilies, + corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceFamily, + Operator: v1.NodeSelectorOpNotIn, + Values: aws.ExcludedInstanceFamilies, + }, }, - v1.NodeSelectorRequirement{ - Key: v1.LabelOSStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{string(v1.Windows)}, + corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1.LabelOSStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{string(v1.Windows)}, + }, }, ) pod := test.Pod(test.PodOptions{ @@ -168,10 +188,12 @@ var _ = Describe("KubeletConfiguration Overrides", func() { // PodsPerCore needs to account for the daemonsets that will run on the nodes // This will have 4 pods available on each node (2 taken by daemonset pods) test.ReplaceRequirements(nodePool, - v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceCPU, - Operator: v1.NodeSelectorOpIn, - Values: []string{"2"}, + corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceCPU, + Operator: v1.NodeSelectorOpIn, + Values: []string{"2"}, + }, }, ) numPods := 4 @@ -211,10 +233,12 @@ var _ = Describe("KubeletConfiguration Overrides", func() { // All pods should schedule to a single node since we are ignoring podsPerCore value // This would normally schedule to 3 nodes if not using Bottlerocket test.ReplaceRequirements(nodePool, - v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceCPU, - Operator: v1.NodeSelectorOpIn, - Values: []string{"2"}, + corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceCPU, + Operator: v1.NodeSelectorOpIn, + Values: []string{"2"}, + }, }, ) diff --git a/test/suites/integration/network_interface_test.go b/test/suites/integration/network_interface_test.go new file mode 100644 index 000000000000..8a5976fed031 --- /dev/null +++ b/test/suites/integration/network_interface_test.go @@ -0,0 +1,54 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/samber/lo" + "sigs.k8s.io/karpenter/pkg/test" + + "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" +) + +var _ = Describe("NetworkInterfaces", func() { + DescribeTable( + "should correctly configure public IP assignment on instances", + func(associatePublicIPAddress *bool) { + nodeClass.Spec.SubnetSelectorTerms = []v1beta1.SubnetSelectorTerm{{ + Tags: map[string]string{ + "Name": "*Private*", + "karpenter.sh/discovery": env.ClusterName, + }, + }} + nodeClass.Spec.AssociatePublicIPAddress = associatePublicIPAddress + + pod := test.Pod() + env.ExpectCreated(pod, nodeClass, nodePool) + env.EventuallyExpectHealthy(pod) + env.ExpectCreatedNodeCount("==", 1) + instance := env.GetInstance(pod.Spec.NodeName) + + if lo.FromPtr(associatePublicIPAddress) { + Expect(instance.PublicIpAddress).ToNot(BeNil()) + } else { + Expect(instance.PublicIpAddress).To(BeNil()) + } + }, + // Only tests private subnets since nodes w/o a public IP address in a public subnet will be unable to join the cluster + Entry("AssociatePublicIPAddress not specified by the user while using a private subnet", nil), + Entry("AssociatePublicIPAddress set true by user while using a private subnet", lo.ToPtr(true)), + ) +}) diff --git a/test/suites/integration/scheduling_test.go b/test/suites/integration/scheduling_test.go index ff6b16dbca88..9c9dba0f6a26 100644 --- a/test/suites/integration/scheduling_test.go +++ b/test/suites/integration/scheduling_test.go @@ -20,6 +20,7 @@ import ( "github.com/samber/lo" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/sets" @@ -42,13 +43,17 @@ var _ = Describe("Scheduling", Ordered, ContinueOnFailure, func() { BeforeEach(func() { // Make the NodePool requirements fully flexible, so we can match well-known label keys nodePool = test.ReplaceRequirements(nodePool, - v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceCategory, - Operator: v1.NodeSelectorOpExists, + corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceCategory, + Operator: v1.NodeSelectorOpExists, + }, }, - v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceGeneration, - Operator: v1.NodeSelectorOpExists, + corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceGeneration, + Operator: v1.NodeSelectorOpExists, + }, }, ) }) @@ -59,6 +64,7 @@ var _ = Describe("Scheduling", Ordered, ContinueOnFailure, func() { // Ensure that we're exercising all well known labels Expect(lo.Keys(selectors)).To(ContainElements(append(corev1beta1.WellKnownLabels.UnsortedList(), lo.Keys(corev1beta1.NormalizedLabels)...))) }) + It("should apply annotations to the node", func() { nodePool.Spec.Template.Annotations = map[string]string{ "foo": "bar", @@ -70,353 +76,483 @@ var _ = Describe("Scheduling", Ordered, ContinueOnFailure, func() { env.ExpectCreatedNodeCount("==", 1) Expect(env.GetNode(pod.Spec.NodeName).Annotations).To(And(HaveKeyWithValue("foo", "bar"), HaveKeyWithValue(corev1beta1.DoNotDisruptAnnotationKey, "true"))) }) - It("should support well-known labels for instance type selection", func() { - nodeSelector := map[string]string{ - // Well Known - corev1beta1.NodePoolLabelKey: nodePool.Name, - v1.LabelInstanceTypeStable: "c5.large", - // Well Known to AWS - v1beta1.LabelInstanceHypervisor: "nitro", - v1beta1.LabelInstanceCategory: "c", - v1beta1.LabelInstanceGeneration: "5", - v1beta1.LabelInstanceFamily: "c5", - v1beta1.LabelInstanceSize: "large", - v1beta1.LabelInstanceCPU: "2", - v1beta1.LabelInstanceMemory: "4096", - v1beta1.LabelInstanceNetworkBandwidth: "750", - } - selectors.Insert(lo.Keys(nodeSelector)...) // Add node selector keys to selectors used in testing to ensure we test all labels - requirements := lo.MapToSlice(nodeSelector, func(key string, value string) v1.NodeSelectorRequirement { - return v1.NodeSelectorRequirement{Key: key, Operator: v1.NodeSelectorOpIn, Values: []string{value}} + + Context("Labels", func() { + It("should support well-known labels for instance type selection", func() { + nodeSelector := map[string]string{ + // Well Known + corev1beta1.NodePoolLabelKey: nodePool.Name, + v1.LabelInstanceTypeStable: "c5.large", + // Well Known to AWS + v1beta1.LabelInstanceHypervisor: "nitro", + v1beta1.LabelInstanceCategory: "c", + v1beta1.LabelInstanceGeneration: "5", + v1beta1.LabelInstanceFamily: "c5", + v1beta1.LabelInstanceSize: "large", + v1beta1.LabelInstanceCPU: "2", + v1beta1.LabelInstanceCPUManufacturer: "intel", + v1beta1.LabelInstanceMemory: "4096", + v1beta1.LabelInstanceNetworkBandwidth: "750", + } + selectors.Insert(lo.Keys(nodeSelector)...) // Add node selector keys to selectors used in testing to ensure we test all labels + requirements := lo.MapToSlice(nodeSelector, func(key string, value string) v1.NodeSelectorRequirement { + return v1.NodeSelectorRequirement{Key: key, Operator: v1.NodeSelectorOpIn, Values: []string{value}} + }) + deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ + NodeSelector: nodeSelector, + NodePreferences: requirements, + NodeRequirements: requirements, + }}) + env.ExpectCreated(nodeClass, nodePool, deployment) + env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) + env.ExpectCreatedNodeCount("==", 1) }) - deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ - NodeSelector: nodeSelector, - NodePreferences: requirements, - NodeRequirements: requirements, - }}) - env.ExpectCreated(nodeClass, nodePool, deployment) - env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) - env.ExpectCreatedNodeCount("==", 1) - }) - It("should support well-known labels for local NVME storage", func() { - selectors.Insert(v1beta1.LabelInstanceLocalNVME) // Add node selector keys to selectors used in testing to ensure we test all labels - deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ - NodePreferences: []v1.NodeSelectorRequirement{ - { - Key: v1beta1.LabelInstanceLocalNVME, - Operator: v1.NodeSelectorOpGt, - Values: []string{"0"}, + It("should support well-known labels for local NVME storage", func() { + selectors.Insert(v1beta1.LabelInstanceLocalNVME) // Add node selector keys to selectors used in testing to ensure we test all labels + deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ + NodePreferences: []v1.NodeSelectorRequirement{ + { + Key: v1beta1.LabelInstanceLocalNVME, + Operator: v1.NodeSelectorOpGt, + Values: []string{"0"}, + }, }, - }, - NodeRequirements: []v1.NodeSelectorRequirement{ - { - Key: v1beta1.LabelInstanceLocalNVME, - Operator: v1.NodeSelectorOpGt, - Values: []string{"0"}, + NodeRequirements: []v1.NodeSelectorRequirement{ + { + Key: v1beta1.LabelInstanceLocalNVME, + Operator: v1.NodeSelectorOpGt, + Values: []string{"0"}, + }, }, - }, - }}) - env.ExpectCreated(nodeClass, nodePool, deployment) - env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) - env.ExpectCreatedNodeCount("==", 1) - }) - It("should support well-known labels for encryption in transit", func() { - selectors.Insert(v1beta1.LabelInstanceEncryptionInTransitSupported) // Add node selector keys to selectors used in testing to ensure we test all labels - deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ - NodePreferences: []v1.NodeSelectorRequirement{ - { - Key: v1beta1.LabelInstanceEncryptionInTransitSupported, - Operator: v1.NodeSelectorOpIn, - Values: []string{"true"}, + }}) + env.ExpectCreated(nodeClass, nodePool, deployment) + env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) + env.ExpectCreatedNodeCount("==", 1) + }) + It("should support well-known labels for encryption in transit", func() { + selectors.Insert(v1beta1.LabelInstanceEncryptionInTransitSupported) // Add node selector keys to selectors used in testing to ensure we test all labels + deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ + NodePreferences: []v1.NodeSelectorRequirement{ + { + Key: v1beta1.LabelInstanceEncryptionInTransitSupported, + Operator: v1.NodeSelectorOpIn, + Values: []string{"true"}, + }, }, - }, - NodeRequirements: []v1.NodeSelectorRequirement{ - { - Key: v1beta1.LabelInstanceEncryptionInTransitSupported, - Operator: v1.NodeSelectorOpIn, - Values: []string{"true"}, + NodeRequirements: []v1.NodeSelectorRequirement{ + { + Key: v1beta1.LabelInstanceEncryptionInTransitSupported, + Operator: v1.NodeSelectorOpIn, + Values: []string{"true"}, + }, }, - }, - }}) - env.ExpectCreated(nodeClass, nodePool, deployment) - env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) - env.ExpectCreatedNodeCount("==", 1) - }) - It("should support well-known deprecated labels", func() { - nodeSelector := map[string]string{ - // Deprecated Labels - v1.LabelFailureDomainBetaRegion: env.Region, - v1.LabelFailureDomainBetaZone: fmt.Sprintf("%sa", env.Region), - "topology.ebs.csi.aws.com/zone": fmt.Sprintf("%sa", env.Region), - - "beta.kubernetes.io/arch": "amd64", - "beta.kubernetes.io/os": "linux", - v1.LabelInstanceType: "c5.large", - } - selectors.Insert(lo.Keys(nodeSelector)...) // Add node selector keys to selectors used in testing to ensure we test all labels - requirements := lo.MapToSlice(nodeSelector, func(key string, value string) v1.NodeSelectorRequirement { - return v1.NodeSelectorRequirement{Key: key, Operator: v1.NodeSelectorOpIn, Values: []string{value}} + }}) + env.ExpectCreated(nodeClass, nodePool, deployment) + env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) + env.ExpectCreatedNodeCount("==", 1) }) - deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ - NodeSelector: nodeSelector, - NodePreferences: requirements, - NodeRequirements: requirements, - }}) - env.ExpectCreated(nodeClass, nodePool, deployment) - env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) - env.ExpectCreatedNodeCount("==", 1) - }) - It("should support well-known labels for topology and architecture", func() { - nodeSelector := map[string]string{ - // Well Known - corev1beta1.NodePoolLabelKey: nodePool.Name, - v1.LabelTopologyRegion: env.Region, - v1.LabelTopologyZone: fmt.Sprintf("%sa", env.Region), - v1.LabelOSStable: "linux", - v1.LabelArchStable: "amd64", - corev1beta1.CapacityTypeLabelKey: corev1beta1.CapacityTypeOnDemand, - } - selectors.Insert(lo.Keys(nodeSelector)...) // Add node selector keys to selectors used in testing to ensure we test all labels - requirements := lo.MapToSlice(nodeSelector, func(key string, value string) v1.NodeSelectorRequirement { - return v1.NodeSelectorRequirement{Key: key, Operator: v1.NodeSelectorOpIn, Values: []string{value}} + It("should support well-known deprecated labels", func() { + nodeSelector := map[string]string{ + // Deprecated Labels + v1.LabelFailureDomainBetaRegion: env.Region, + v1.LabelFailureDomainBetaZone: fmt.Sprintf("%sa", env.Region), + "topology.ebs.csi.aws.com/zone": fmt.Sprintf("%sa", env.Region), + + "beta.kubernetes.io/arch": "amd64", + "beta.kubernetes.io/os": "linux", + v1.LabelInstanceType: "c5.large", + } + selectors.Insert(lo.Keys(nodeSelector)...) // Add node selector keys to selectors used in testing to ensure we test all labels + requirements := lo.MapToSlice(nodeSelector, func(key string, value string) v1.NodeSelectorRequirement { + return v1.NodeSelectorRequirement{Key: key, Operator: v1.NodeSelectorOpIn, Values: []string{value}} + }) + deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ + NodeSelector: nodeSelector, + NodePreferences: requirements, + NodeRequirements: requirements, + }}) + env.ExpectCreated(nodeClass, nodePool, deployment) + env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) + env.ExpectCreatedNodeCount("==", 1) }) - deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ - NodeSelector: nodeSelector, - NodePreferences: requirements, - NodeRequirements: requirements, - }}) - env.ExpectCreated(nodeClass, nodePool, deployment) - env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) - env.ExpectCreatedNodeCount("==", 1) - }) - It("should support well-known labels for a gpu (nvidia)", func() { - nodeSelector := map[string]string{ - v1beta1.LabelInstanceGPUName: "t4", - v1beta1.LabelInstanceGPUMemory: "16384", - v1beta1.LabelInstanceGPUManufacturer: "nvidia", - v1beta1.LabelInstanceGPUCount: "1", - } - selectors.Insert(lo.Keys(nodeSelector)...) // Add node selector keys to selectors used in testing to ensure we test all labels - requirements := lo.MapToSlice(nodeSelector, func(key string, value string) v1.NodeSelectorRequirement { - return v1.NodeSelectorRequirement{Key: key, Operator: v1.NodeSelectorOpIn, Values: []string{value}} + It("should support well-known labels for topology and architecture", func() { + nodeSelector := map[string]string{ + // Well Known + corev1beta1.NodePoolLabelKey: nodePool.Name, + v1.LabelTopologyRegion: env.Region, + v1.LabelTopologyZone: fmt.Sprintf("%sa", env.Region), + v1.LabelOSStable: "linux", + v1.LabelArchStable: "amd64", + corev1beta1.CapacityTypeLabelKey: corev1beta1.CapacityTypeOnDemand, + } + selectors.Insert(lo.Keys(nodeSelector)...) // Add node selector keys to selectors used in testing to ensure we test all labels + requirements := lo.MapToSlice(nodeSelector, func(key string, value string) v1.NodeSelectorRequirement { + return v1.NodeSelectorRequirement{Key: key, Operator: v1.NodeSelectorOpIn, Values: []string{value}} + }) + deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ + NodeSelector: nodeSelector, + NodePreferences: requirements, + NodeRequirements: requirements, + }}) + env.ExpectCreated(nodeClass, nodePool, deployment) + env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) + env.ExpectCreatedNodeCount("==", 1) }) - deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ - NodeSelector: nodeSelector, - NodePreferences: requirements, - NodeRequirements: requirements, - }}) - env.ExpectCreated(nodeClass, nodePool, deployment) - env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) - env.ExpectCreatedNodeCount("==", 1) - }) - It("should support well-known labels for an accelerator (inferentia)", func() { - nodeSelector := map[string]string{ - v1beta1.LabelInstanceAcceleratorName: "inferentia", - v1beta1.LabelInstanceAcceleratorManufacturer: "aws", - v1beta1.LabelInstanceAcceleratorCount: "1", - } - selectors.Insert(lo.Keys(nodeSelector)...) // Add node selector keys to selectors used in testing to ensure we test all labels - requirements := lo.MapToSlice(nodeSelector, func(key string, value string) v1.NodeSelectorRequirement { - return v1.NodeSelectorRequirement{Key: key, Operator: v1.NodeSelectorOpIn, Values: []string{value}} + It("should support well-known labels for a gpu (nvidia)", func() { + nodeSelector := map[string]string{ + v1beta1.LabelInstanceGPUName: "t4", + v1beta1.LabelInstanceGPUMemory: "16384", + v1beta1.LabelInstanceGPUManufacturer: "nvidia", + v1beta1.LabelInstanceGPUCount: "1", + } + selectors.Insert(lo.Keys(nodeSelector)...) // Add node selector keys to selectors used in testing to ensure we test all labels + requirements := lo.MapToSlice(nodeSelector, func(key string, value string) v1.NodeSelectorRequirement { + return v1.NodeSelectorRequirement{Key: key, Operator: v1.NodeSelectorOpIn, Values: []string{value}} + }) + deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ + NodeSelector: nodeSelector, + NodePreferences: requirements, + NodeRequirements: requirements, + }}) + env.ExpectCreated(nodeClass, nodePool, deployment) + env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) + env.ExpectCreatedNodeCount("==", 1) }) - deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ - NodeSelector: nodeSelector, - NodePreferences: requirements, - NodeRequirements: requirements, - }}) - env.ExpectCreated(nodeClass, nodePool, deployment) - env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) - env.ExpectCreatedNodeCount("==", 1) - }) - It("should support well-known labels for windows-build version", func() { - env.ExpectWindowsIPAMEnabled() - DeferCleanup(func() { - env.ExpectWindowsIPAMDisabled() + It("should support well-known labels for an accelerator (inferentia)", func() { + nodeSelector := map[string]string{ + v1beta1.LabelInstanceAcceleratorName: "inferentia", + v1beta1.LabelInstanceAcceleratorManufacturer: "aws", + v1beta1.LabelInstanceAcceleratorCount: "1", + } + selectors.Insert(lo.Keys(nodeSelector)...) // Add node selector keys to selectors used in testing to ensure we test all labels + requirements := lo.MapToSlice(nodeSelector, func(key string, value string) v1.NodeSelectorRequirement { + return v1.NodeSelectorRequirement{Key: key, Operator: v1.NodeSelectorOpIn, Values: []string{value}} + }) + deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ + NodeSelector: nodeSelector, + NodePreferences: requirements, + NodeRequirements: requirements, + }}) + env.ExpectCreated(nodeClass, nodePool, deployment) + env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) + env.ExpectCreatedNodeCount("==", 1) }) + It("should support well-known labels for windows-build version", func() { + env.ExpectWindowsIPAMEnabled() + DeferCleanup(func() { + env.ExpectWindowsIPAMDisabled() + }) - nodeSelector := map[string]string{ - // Well Known - v1.LabelWindowsBuild: v1beta1.Windows2022Build, - v1.LabelOSStable: string(v1.Windows), // Specify the OS to enable vpc-resource-controller to inject the PrivateIPv4Address resource - } - selectors.Insert(lo.Keys(nodeSelector)...) // Add node selector keys to selectors used in testing to ensure we test all labels - requirements := lo.MapToSlice(nodeSelector, func(key string, value string) v1.NodeSelectorRequirement { - return v1.NodeSelectorRequirement{Key: key, Operator: v1.NodeSelectorOpIn, Values: []string{value}} + nodeSelector := map[string]string{ + // Well Known + v1.LabelWindowsBuild: v1beta1.Windows2022Build, + v1.LabelOSStable: string(v1.Windows), // Specify the OS to enable vpc-resource-controller to inject the PrivateIPv4Address resource + } + selectors.Insert(lo.Keys(nodeSelector)...) // Add node selector keys to selectors used in testing to ensure we test all labels + requirements := lo.MapToSlice(nodeSelector, func(key string, value string) v1.NodeSelectorRequirement { + return v1.NodeSelectorRequirement{Key: key, Operator: v1.NodeSelectorOpIn, Values: []string{value}} + }) + deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ + NodeSelector: nodeSelector, + NodePreferences: requirements, + NodeRequirements: requirements, + Image: aws.WindowsDefaultImage, + }}) + nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyWindows2022 + // TODO: remove this requirement once VPC RC rolls out m7a.*, r7a.* ENI data (https://github.com/aws/karpenter-provider-aws/issues/4472) + test.ReplaceRequirements(nodePool, + corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceFamily, + Operator: v1.NodeSelectorOpNotIn, + Values: aws.ExcludedInstanceFamilies, + }, + }, + corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1.LabelOSStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{string(v1.Windows)}, + }, + }, + ) + env.ExpectCreated(nodeClass, nodePool, deployment) + env.EventuallyExpectHealthyPodCountWithTimeout(time.Minute*15, labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) + env.ExpectCreatedNodeCount("==", 1) }) - deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ - NodeSelector: nodeSelector, - NodePreferences: requirements, - NodeRequirements: requirements, - Image: aws.WindowsDefaultImage, - }}) - nodeClass.Spec.AMIFamily = &v1beta1.AMIFamilyWindows2022 - // TODO: remove this requirement once VPC RC rolls out m7a.*, r7a.* ENI data (https://github.com/aws/karpenter-provider-aws/issues/4472) - test.ReplaceRequirements(nodePool, - v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceFamily, - Operator: v1.NodeSelectorOpNotIn, - Values: aws.ExcludedInstanceFamilies, - }, - v1.NodeSelectorRequirement{ - Key: v1.LabelOSStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{string(v1.Windows)}, - }, + DescribeTable("should support restricted label domain exceptions", func(domain string) { + // Assign labels to the nodepool so that it has known values + test.ReplaceRequirements(nodePool, + corev1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: domain + "/team", Operator: v1.NodeSelectorOpExists}}, + corev1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: domain + "/custom-label", Operator: v1.NodeSelectorOpExists}}, + corev1beta1.NodeSelectorRequirementWithMinValues{NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: "subdomain." + domain + "/custom-label", Operator: v1.NodeSelectorOpExists}}, + ) + nodeSelector := map[string]string{ + domain + "/team": "team-1", + domain + "/custom-label": "custom-value", + "subdomain." + domain + "/custom-label": "custom-value", + } + selectors.Insert(lo.Keys(nodeSelector)...) // Add node selector keys to selectors used in testing to ensure we test all labels + requirements := lo.MapToSlice(nodeSelector, func(key string, value string) v1.NodeSelectorRequirement { + return v1.NodeSelectorRequirement{Key: key, Operator: v1.NodeSelectorOpIn, Values: []string{value}} + }) + deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ + NodeSelector: nodeSelector, + NodePreferences: requirements, + NodeRequirements: requirements, + }}) + env.ExpectCreated(nodeClass, nodePool, deployment) + env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) + node := env.ExpectCreatedNodeCount("==", 1)[0] + // Ensure that the requirements/labels specified above are propagated onto the node + for k, v := range nodeSelector { + Expect(node.Labels).To(HaveKeyWithValue(k, v)) + } + }, + Entry("node-restriction.kuberentes.io", "node-restriction.kuberentes.io"), + Entry("node.kubernetes.io", "node.kubernetes.io"), + Entry("kops.k8s.io", "kops.k8s.io"), ) - env.ExpectCreated(nodeClass, nodePool, deployment) - env.EventuallyExpectHealthyPodCountWithTimeout(time.Minute*15, labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) - env.ExpectCreatedNodeCount("==", 1) }) - DescribeTable("should support restricted label domain exceptions", func(domain string) { - // Assign labels to the nodepool so that it has known values - test.ReplaceRequirements(nodePool, - v1.NodeSelectorRequirement{Key: domain + "/team", Operator: v1.NodeSelectorOpExists}, - v1.NodeSelectorRequirement{Key: domain + "/custom-label", Operator: v1.NodeSelectorOpExists}, - v1.NodeSelectorRequirement{Key: "subdomain." + domain + "/custom-label", Operator: v1.NodeSelectorOpExists}, - ) - nodeSelector := map[string]string{ - domain + "/team": "team-1", - domain + "/custom-label": "custom-value", - "subdomain." + domain + "/custom-label": "custom-value", - } - selectors.Insert(lo.Keys(nodeSelector)...) // Add node selector keys to selectors used in testing to ensure we test all labels - requirements := lo.MapToSlice(nodeSelector, func(key string, value string) v1.NodeSelectorRequirement { - return v1.NodeSelectorRequirement{Key: key, Operator: v1.NodeSelectorOpIn, Values: []string{value}} - }) - deployment := test.Deployment(test.DeploymentOptions{Replicas: 1, PodOptions: test.PodOptions{ - NodeSelector: nodeSelector, - NodePreferences: requirements, - NodeRequirements: requirements, - }}) - env.ExpectCreated(nodeClass, nodePool, deployment) - env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) - node := env.ExpectCreatedNodeCount("==", 1)[0] - // Ensure that the requirements/labels specified above are propagated onto the node - for k, v := range nodeSelector { - Expect(node.Labels).To(HaveKeyWithValue(k, v)) - } - }, - Entry("node-restriction.kuberentes.io", "node-restriction.kuberentes.io"), - Entry("node.kubernetes.io", "node.kubernetes.io"), - Entry("kops.k8s.io", "kops.k8s.io"), - ) - It("should provision a node for naked pods", func() { - pod := test.Pod() - env.ExpectCreated(nodeClass, nodePool, pod) - env.EventuallyExpectHealthy(pod) - env.ExpectCreatedNodeCount("==", 1) - }) - It("should provision a node for a deployment", Label(debug.NoWatch), Label(debug.NoEvents), func() { - deployment := test.Deployment(test.DeploymentOptions{Replicas: 50}) - env.ExpectCreated(nodeClass, nodePool, deployment) - env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) - env.ExpectCreatedNodeCount("<=", 2) // should probably all land on a single node, but at worst two depending on batching - }) - It("should provision a node for a self-affinity deployment", func() { - // just two pods as they all need to land on the same node - podLabels := map[string]string{"test": "self-affinity"} - deployment := test.Deployment(test.DeploymentOptions{ - Replicas: 2, - PodOptions: test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{ - Labels: podLabels, - }, - PodRequirements: []v1.PodAffinityTerm{ - { - LabelSelector: &metav1.LabelSelector{MatchLabels: podLabels}, - TopologyKey: v1.LabelHostname, + Context("Provisioning", func() { + It("should provision a node for naked pods", func() { + pod := test.Pod() + + env.ExpectCreated(nodeClass, nodePool, pod) + env.EventuallyExpectHealthy(pod) + env.ExpectCreatedNodeCount("==", 1) + }) + It("should provision a node for a deployment", Label(debug.NoWatch), Label(debug.NoEvents), func() { + deployment := test.Deployment(test.DeploymentOptions{Replicas: 50}) + env.ExpectCreated(nodeClass, nodePool, deployment) + env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), int(*deployment.Spec.Replicas)) + env.ExpectCreatedNodeCount("<=", 2) // should probably all land on a single node, but at worst two depending on batching + }) + It("should provision a node for a self-affinity deployment", func() { + // just two pods as they all need to land on the same node + podLabels := map[string]string{"test": "self-affinity"} + deployment := test.Deployment(test.DeploymentOptions{ + Replicas: 2, + PodOptions: test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Labels: podLabels, + }, + PodRequirements: []v1.PodAffinityTerm{ + { + LabelSelector: &metav1.LabelSelector{MatchLabels: podLabels}, + TopologyKey: v1.LabelHostname, + }, }, }, - }, - }) + }) - env.ExpectCreated(nodeClass, nodePool, deployment) - env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), 2) - env.ExpectCreatedNodeCount("==", 1) - }) - It("should provision three nodes for a zonal topology spread", func() { - // one pod per zone - podLabels := map[string]string{"test": "zonal-spread"} - deployment := test.Deployment(test.DeploymentOptions{ - Replicas: 3, - PodOptions: test.PodOptions{ - ObjectMeta: metav1.ObjectMeta{ - Labels: podLabels, - }, - TopologySpreadConstraints: []v1.TopologySpreadConstraint{ - { - MaxSkew: 1, - TopologyKey: v1.LabelTopologyZone, - WhenUnsatisfiable: v1.DoNotSchedule, - LabelSelector: &metav1.LabelSelector{MatchLabels: podLabels}, - MinDomains: lo.ToPtr(int32(3)), + env.ExpectCreated(nodeClass, nodePool, deployment) + env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(deployment.Spec.Selector.MatchLabels), 2) + env.ExpectCreatedNodeCount("==", 1) + }) + It("should provision three nodes for a zonal topology spread", func() { + // one pod per zone + podLabels := map[string]string{"test": "zonal-spread"} + deployment := test.Deployment(test.DeploymentOptions{ + Replicas: 3, + PodOptions: test.PodOptions{ + ObjectMeta: metav1.ObjectMeta{ + Labels: podLabels, + }, + TopologySpreadConstraints: []v1.TopologySpreadConstraint{ + { + MaxSkew: 1, + TopologyKey: v1.LabelTopologyZone, + WhenUnsatisfiable: v1.DoNotSchedule, + LabelSelector: &metav1.LabelSelector{MatchLabels: podLabels}, + MinDomains: lo.ToPtr(int32(3)), + }, }, }, - }, - }) + }) - env.ExpectCreated(nodeClass, nodePool, deployment) - env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(podLabels), 3) - // Karpenter will launch three nodes, however if all three nodes don't get register with the cluster at the same time, two pods will be placed on one node. - // This can result in a case where all 3 pods are healthy, while there are only two created nodes. - // In that case, we still expect to eventually have three nodes. - env.EventuallyExpectNodeCount("==", 3) - }) - It("should provision a node using a NodePool with higher priority", func() { - nodePoolLowPri := test.NodePool(corev1beta1.NodePool{ - Spec: corev1beta1.NodePoolSpec{ - Weight: ptr.Int32(10), - Template: corev1beta1.NodeClaimTemplate{ - Spec: corev1beta1.NodeClaimSpec{ - NodeClassRef: &corev1beta1.NodeClassReference{ - Name: nodeClass.Name, - }, - Requirements: []v1.NodeSelectorRequirement{ - { - Key: v1.LabelOSStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{string(v1.Linux)}, + env.ExpectCreated(nodeClass, nodePool, deployment) + env.EventuallyExpectHealthyPodCount(labels.SelectorFromSet(podLabels), 3) + // Karpenter will launch three nodes, however if all three nodes don't get register with the cluster at the same time, two pods will be placed on one node. + // This can result in a case where all 3 pods are healthy, while there are only two created nodes. + // In that case, we still expect to eventually have three nodes. + env.EventuallyExpectNodeCount("==", 3) + }) + It("should provision a node using a NodePool with higher priority", func() { + nodePoolLowPri := test.NodePool(corev1beta1.NodePool{ + Spec: corev1beta1.NodePoolSpec{ + Weight: ptr.Int32(10), + Template: corev1beta1.NodeClaimTemplate{ + Spec: corev1beta1.NodeClaimSpec{ + NodeClassRef: &corev1beta1.NodeClassReference{ + Name: nodeClass.Name, }, - { - Key: v1.LabelInstanceTypeStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{"t3.nano"}, + Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ + { + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1.LabelOSStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{string(v1.Linux)}, + }, + }, + { + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1.LabelInstanceTypeStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{"t3.nano"}, + }, + }, }, }, }, }, - }, - }) - nodePoolHighPri := test.NodePool(corev1beta1.NodePool{ - Spec: corev1beta1.NodePoolSpec{ - Weight: ptr.Int32(100), - Template: corev1beta1.NodeClaimTemplate{ - Spec: corev1beta1.NodeClaimSpec{ - NodeClassRef: &corev1beta1.NodeClassReference{ - Name: nodeClass.Name, - }, - Requirements: []v1.NodeSelectorRequirement{ - { - Key: v1.LabelOSStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{string(v1.Linux)}, + }) + nodePoolHighPri := test.NodePool(corev1beta1.NodePool{ + Spec: corev1beta1.NodePoolSpec{ + Weight: ptr.Int32(100), + Template: corev1beta1.NodeClaimTemplate{ + Spec: corev1beta1.NodeClaimSpec{ + NodeClassRef: &corev1beta1.NodeClassReference{ + Name: nodeClass.Name, }, - { - Key: v1.LabelInstanceTypeStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{"c5.large"}, + Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ + { + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1.LabelOSStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{string(v1.Linux)}, + }, + }, + { + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1.LabelInstanceTypeStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{"c5.large"}, + }, + }, }, }, }, }, + }) + pod := test.Pod() + env.ExpectCreated(pod, nodeClass, nodePoolLowPri, nodePoolHighPri) + env.EventuallyExpectHealthy(pod) + env.ExpectCreatedNodeCount("==", 1) + Expect(ptr.StringValue(env.GetInstance(pod.Spec.NodeName).InstanceType)).To(Equal("c5.large")) + Expect(env.GetNode(pod.Spec.NodeName).Labels[corev1beta1.NodePoolLabelKey]).To(Equal(nodePoolHighPri.Name)) + }) + + DescribeTable( + "should provision a right-sized node when a pod has InitContainers (cpu)", + func(expectedNodeCPU string, containerRequirements v1.ResourceRequirements, initContainers ...v1.Container) { + if version, err := env.GetK8sMinorVersion(0); err != nil || version < 29 { + Skip("native sidecar containers are only enabled on EKS 1.29+") + } + test.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceCPU, + Operator: v1.NodeSelectorOpIn, + Values: []string{"1", "2"}, + }, + }, corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceCategory, + Operator: v1.NodeSelectorOpNotIn, + Values: []string{"t"}, + }, + }) + pod := test.Pod(test.PodOptions{ + InitContainers: initContainers, + ResourceRequirements: containerRequirements, + }) + env.ExpectCreated(nodePool, nodeClass, pod) + env.EventuallyExpectHealthy(pod) + node := env.ExpectCreatedNodeCount("==", 1)[0] + Expect(node.ObjectMeta.GetLabels()[v1beta1.LabelInstanceCPU]).To(Equal(expectedNodeCPU)) }, + Entry("sidecar requirements + later init requirements do exceed container requirements", "2", v1.ResourceRequirements{ + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("400m")}, + }, ephemeralInitContainer(v1.ResourceRequirements{ + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("300m")}, + }), v1.Container{ + RestartPolicy: lo.ToPtr(v1.ContainerRestartPolicyAlways), + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("350m")}, + }, + }, ephemeralInitContainer(v1.ResourceRequirements{ + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1")}, + })), + Entry("sidecar requirements + later init requirements do not exceed container requirements", "1", v1.ResourceRequirements{ + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("400m")}, + }, ephemeralInitContainer(v1.ResourceRequirements{ + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("300m")}, + }), v1.Container{ + RestartPolicy: lo.ToPtr(v1.ContainerRestartPolicyAlways), + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("350m")}, + }, + }, ephemeralInitContainer(v1.ResourceRequirements{ + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("300m")}, + })), + Entry("init container requirements exceed all later requests", "2", v1.ResourceRequirements{ + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("400m")}, + }, v1.Container{ + RestartPolicy: lo.ToPtr(v1.ContainerRestartPolicyAlways), + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("100m")}, + }, + }, ephemeralInitContainer(v1.ResourceRequirements{ + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("1500m")}, + }), v1.Container{ + RestartPolicy: lo.ToPtr(v1.ContainerRestartPolicyAlways), + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{v1.ResourceCPU: resource.MustParse("100m")}, + }, + }), + ) + It("should provision a right-sized node when a pod has InitContainers (mixed resources)", func() { + if version, err := env.GetK8sMinorVersion(0); err != nil || version < 29 { + Skip("native sidecar containers are only enabled on EKS 1.29+") + } + test.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceCategory, + Operator: v1.NodeSelectorOpNotIn, + Values: []string{"t"}, + }, + }) + pod := test.Pod(test.PodOptions{ + InitContainers: []v1.Container{ + { + RestartPolicy: lo.ToPtr(v1.ContainerRestartPolicyAlways), + Resources: v1.ResourceRequirements{Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse("128Mi"), + }}, + }, + ephemeralInitContainer(v1.ResourceRequirements{Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("50m"), + v1.ResourceMemory: resource.MustParse("4Gi"), + }}), + }, + ResourceRequirements: v1.ResourceRequirements{Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse("128Mi"), + }}, + }) + env.ExpectCreated(nodePool, nodeClass, pod) + env.EventuallyExpectHealthy(pod) }) - pod := test.Pod() - env.ExpectCreated(pod, nodeClass, nodePoolLowPri, nodePoolHighPri) - env.EventuallyExpectHealthy(pod) - env.ExpectCreatedNodeCount("==", 1) - Expect(ptr.StringValue(env.GetInstance(pod.Spec.NodeName).InstanceType)).To(Equal("c5.large")) - Expect(env.GetNode(pod.Spec.NodeName).Labels[corev1beta1.NodePoolLabelKey]).To(Equal(nodePoolHighPri.Name)) }) }) + +func ephemeralInitContainer(requirements v1.ResourceRequirements) v1.Container { + return v1.Container{ + Image: aws.EphemeralInitContainerImage, + Command: []string{"/bin/sh"}, + Args: []string{"-c", "sleep 5"}, + Resources: requirements, + } +} diff --git a/test/suites/integration/storage_test.go b/test/suites/integration/storage_test.go index 1fdd9ef57bd2..8c6f4b5e781f 100644 --- a/test/suites/integration/storage_test.go +++ b/test/suites/integration/storage_test.go @@ -241,7 +241,7 @@ func ExpectSetEBSDriverLimit(limit int) { containers[i].Args = append(containers[i].Args, fmt.Sprintf("--volume-attach-limit=%d", limit)) break } - Expect(env.Client.Patch(env.Context, ds, client.MergeFrom(stored))).To(Succeed()) + Expect(env.Client.Patch(env.Context, ds, client.StrategicMergeFrom(stored))).To(Succeed()) } func ExpectRemoveEBSDriverLimit() { @@ -260,5 +260,5 @@ func ExpectRemoveEBSDriverLimit() { }) break } - Expect(env.Client.Patch(env.Context, ds, client.MergeFrom(stored))).To(Succeed()) + Expect(env.Client.Patch(env.Context, ds, client.StrategicMergeFrom(stored))).To(Succeed()) } diff --git a/test/suites/integration/subnet_test.go b/test/suites/integration/subnet_test.go index 5483f8f5cd73..3312f91c33c6 100644 --- a/test/suites/integration/subnet_test.go +++ b/test/suites/integration/subnet_test.go @@ -26,6 +26,8 @@ import ( "sigs.k8s.io/karpenter/pkg/test" + corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" "github.com/aws/karpenter-provider-aws/test/pkg/environment/aws" @@ -100,11 +102,12 @@ var _ = Describe("Subnets", func() { Expect(len(subnets)).ToNot(Equal(0)) shuffledAZs := lo.Shuffle(lo.Keys(subnets)) - test.ReplaceRequirements(nodePool, v1.NodeSelectorRequirement{ - Key: v1.LabelZoneFailureDomainStable, - Operator: "In", - Values: []string{shuffledAZs[0]}, - }) + test.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1.LabelZoneFailureDomainStable, + Operator: "In", + Values: []string{shuffledAZs[0]}, + }}) pod := test.Pod() env.ExpectCreated(pod, nodeClass, nodePool) diff --git a/test/suites/integration/tags_test.go b/test/suites/integration/tags_test.go index 50bb2c9184f7..ab56b5abd312 100644 --- a/test/suites/integration/tags_test.go +++ b/test/suites/integration/tags_test.go @@ -15,6 +15,7 @@ limitations under the License. package integration_test import ( + "fmt" "time" "github.com/aws/aws-sdk-go/service/ec2" @@ -65,11 +66,12 @@ var _ = Describe("Tags", func() { } }) It("should tag spot instance requests when creating resources", func() { - coretest.ReplaceRequirements(nodePool, v1.NodeSelectorRequirement{ - Key: corev1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.CapacityTypeSpot}, - }) + coretest.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: corev1beta1.CapacityTypeLabelKey, + Operator: v1.NodeSelectorOpIn, + Values: []string{corev1beta1.CapacityTypeSpot}, + }}) nodeClass.Spec.Tags = map[string]string{"TestTag": "TestVal"} pod := coretest.Pod() @@ -107,6 +109,10 @@ var _ = Describe("Tags", func() { nodeClass = test.EC2NodeClass(*nodeClass, v1beta1.EC2NodeClass{Spec: v1beta1.EC2NodeClassSpec{ Tags: map[string]string{"Name": "custom-name", "testing/cluster": env.ClusterName}, }}) + if env.PrivateCluster { + nodeClass.Spec.Role = "" + nodeClass.Spec.InstanceProfile = lo.ToPtr(fmt.Sprintf("KarpenterNodeInstanceProfile-%s", env.ClusterName)) + } nodePool = coretest.NodePool(*nodePool, corev1beta1.NodePool{ Spec: corev1beta1.NodePoolSpec{ Template: corev1beta1.NodeClaimTemplate{ diff --git a/test/suites/integration/utilization_test.go b/test/suites/integration/utilization_test.go index 73e59386e56d..7599de17d2c8 100644 --- a/test/suites/integration/utilization_test.go +++ b/test/suites/integration/utilization_test.go @@ -23,6 +23,8 @@ import ( "sigs.k8s.io/karpenter/pkg/test" + corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" "github.com/aws/karpenter-provider-aws/test/pkg/debug" @@ -32,14 +34,18 @@ import ( var _ = Describe("Utilization", Label(debug.NoWatch), Label(debug.NoEvents), func() { It("should provision one pod per node", func() { test.ReplaceRequirements(nodePool, - v1.NodeSelectorRequirement{ - Key: v1.LabelInstanceTypeStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{"t3.small"}, + corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1.LabelInstanceTypeStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{"t3.small"}, + }, }, - v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceCategory, - Operator: v1.NodeSelectorOpExists, + corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceCategory, + Operator: v1.NodeSelectorOpExists, + }, }, ) deployment := test.Deployment(test.DeploymentOptions{ diff --git a/test/suites/integration/validation_test.go b/test/suites/integration/validation_test.go index 873e540977d4..790a880ce29b 100644 --- a/test/suites/integration/validation_test.go +++ b/test/suites/integration/validation_test.go @@ -58,43 +58,48 @@ var _ = Describe("Validation", func() { Expect(env.Client.Create(env.Context, nodePool)).To(Succeed()) }) It("should error when a requirement references a restricted label (karpenter.sh/nodepool)", func() { - nodePool = coretest.ReplaceRequirements(nodePool, v1.NodeSelectorRequirement{ - Key: corev1beta1.NodePoolLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: []string{"default"}, - }) + nodePool = coretest.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: corev1beta1.NodePoolLabelKey, + Operator: v1.NodeSelectorOpIn, + Values: []string{"default"}, + }}) Expect(env.Client.Create(env.Context, nodePool)).ToNot(Succeed()) }) It("should error when a requirement uses In but has no values", func() { - nodePool = coretest.ReplaceRequirements(nodePool, v1.NodeSelectorRequirement{ - Key: v1.LabelInstanceTypeStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{}, - }) + nodePool = coretest.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1.LabelInstanceTypeStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{}, + }}) Expect(env.Client.Create(env.Context, nodePool)).ToNot(Succeed()) }) It("should error when a requirement uses an unknown operator", func() { - nodePool = coretest.ReplaceRequirements(nodePool, v1.NodeSelectorRequirement{ - Key: corev1beta1.CapacityTypeLabelKey, - Operator: "within", - Values: []string{corev1beta1.CapacityTypeSpot}, - }) + nodePool = coretest.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: corev1beta1.CapacityTypeLabelKey, + Operator: "within", + Values: []string{corev1beta1.CapacityTypeSpot}, + }}) Expect(env.Client.Create(env.Context, nodePool)).ToNot(Succeed()) }) It("should error when Gt is used with multiple integer values", func() { - nodePool = coretest.ReplaceRequirements(nodePool, v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceMemory, - Operator: v1.NodeSelectorOpGt, - Values: []string{"1000000", "2000000"}, - }) + nodePool = coretest.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceMemory, + Operator: v1.NodeSelectorOpGt, + Values: []string{"1000000", "2000000"}, + }}) Expect(env.Client.Create(env.Context, nodePool)).ToNot(Succeed()) }) It("should error when Lt is used with multiple integer values", func() { - nodePool = coretest.ReplaceRequirements(nodePool, v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceMemory, - Operator: v1.NodeSelectorOpLt, - Values: []string{"1000000", "2000000"}, - }) + nodePool = coretest.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceMemory, + Operator: v1.NodeSelectorOpLt, + Values: []string{"1000000", "2000000"}, + }}) Expect(env.Client.Create(env.Context, nodePool)).ToNot(Succeed()) }) It("should error when ttlSecondAfterEmpty is negative", func() { @@ -124,6 +129,50 @@ var _ = Describe("Validation", func() { } Expect(env.Client.Create(env.Context, nodePool)).ToNot(Succeed()) }) + It("should error when minValues for a requirement key is negative", func() { + nodePool = coretest.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1.LabelInstanceTypeStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{"insance-type-1", "insance-type-2"}, + }, + MinValues: lo.ToPtr(-1)}, + ) + Expect(env.Client.Create(env.Context, nodePool)).ToNot(Succeed()) + }) + It("should error when minValues for a requirement key is zero", func() { + nodePool = coretest.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1.LabelInstanceTypeStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{"insance-type-1", "insance-type-2"}, + }, + MinValues: lo.ToPtr(0)}, + ) + Expect(env.Client.Create(env.Context, nodePool)).ToNot(Succeed()) + }) + It("should error when minValues for a requirement key is more than 50", func() { + nodePool = coretest.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1.LabelInstanceTypeStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{"insance-type-1", "insance-type-2"}, + }, + MinValues: lo.ToPtr(51)}, + ) + Expect(env.Client.Create(env.Context, nodePool)).ToNot(Succeed()) + }) + It("should error when minValues for a requirement key is greater than the values specified within In operator", func() { + nodePool = coretest.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1.LabelInstanceTypeStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{"insance-type-1", "insance-type-2"}, + }, + MinValues: lo.ToPtr(3)}, + ) + Expect(env.Client.Create(env.Context, nodePool)).ToNot(Succeed()) + }) }) Context("EC2NodeClass", func() { It("should error when amiSelectorTerms are not defined for amiFamily Custom", func() { @@ -151,6 +200,12 @@ var _ = Describe("Validation", func() { nodeClass.Spec.Tags = map[string]string{fmt.Sprintf("kubernetes.io/cluster/%s", env.ClusterName): "owned"} Expect(env.Client.Create(env.Context, nodeClass)).ToNot(Succeed()) + + nodeClass.Spec.Tags = map[string]string{"karpenter.sh/nodeclaim": "custom-value"} + Expect(env.Client.Create(env.Context, nodeClass)).ToNot(Succeed()) + + nodeClass.Spec.Tags = map[string]string{"karpenter.k8s.aws/ec2nodeclass": "custom-value"} + Expect(env.Client.Create(env.Context, nodeClass)).ToNot(Succeed()) }) It("should fail when securityGroupSelectorTerms has id and other filters", func() { nodeClass.Spec.SecurityGroupSelectorTerms = []v1beta1.SecurityGroupSelectorTerm{ diff --git a/test/suites/interruption/suite_test.go b/test/suites/interruption/suite_test.go index bd338cd39899..f2adf66c1437 100644 --- a/test/suites/interruption/suite_test.go +++ b/test/suites/interruption/suite_test.go @@ -70,11 +70,12 @@ var _ = AfterEach(func() { env.AfterEach() }) var _ = Describe("Interruption", func() { It("should terminate the spot instance and spin-up a new node on spot interruption warning", func() { By("Creating a single healthy node with a healthy deployment") - nodePool = coretest.ReplaceRequirements(nodePool, v1.NodeSelectorRequirement{ - Key: corev1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.CapacityTypeSpot}, - }) + nodePool = coretest.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: corev1beta1.CapacityTypeLabelKey, + Operator: v1.NodeSelectorOpIn, + Values: []string{corev1beta1.CapacityTypeSpot}, + }}) numPods := 1 dep := coretest.Deployment(coretest.DeploymentOptions{ Replicas: int32(numPods), diff --git a/test/suites/ipv6/suite_test.go b/test/suites/ipv6/suite_test.go index cb2a00e5e7d4..f1b2f33ea104 100644 --- a/test/suites/ipv6/suite_test.go +++ b/test/suites/ipv6/suite_test.go @@ -51,14 +51,18 @@ var _ = BeforeEach(func() { nodeClass = env.DefaultEC2NodeClass() nodePool = env.DefaultNodePool(nodeClass) nodePool = coretest.ReplaceRequirements(nodePool, - v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceCategory, - Operator: v1.NodeSelectorOpExists, + corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceCategory, + Operator: v1.NodeSelectorOpExists, + }, }, - v1.NodeSelectorRequirement{ - Key: v1.LabelInstanceTypeStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{"t3a.small"}, + corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1.LabelInstanceTypeStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{"t3a.small"}, + }, }, ) }) diff --git a/test/suites/localzone/suite_test.go b/test/suites/localzone/suite_test.go index 6da05a67e88f..53b0d4940773 100644 --- a/test/suites/localzone/suite_test.go +++ b/test/suites/localzone/suite_test.go @@ -66,11 +66,12 @@ var _ = BeforeEach(func() { }, }) nodePool = env.DefaultNodePool(nodeClass) - nodePool.Spec.Template.Spec.Requirements = append(nodePool.Spec.Template.Spec.Requirements, v1.NodeSelectorRequirement{ - Key: v1.LabelTopologyZone, - Operator: v1.NodeSelectorOpIn, - Values: lo.Keys(lo.PickByValues(env.GetZones(), []string{"local-zone"})), - }) + nodePool.Spec.Template.Spec.Requirements = append(nodePool.Spec.Template.Spec.Requirements, corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1.LabelTopologyZone, + Operator: v1.NodeSelectorOpIn, + Values: lo.Keys(lo.PickByValues(env.GetZones(), []string{"local-zone"})), + }}) }) var _ = AfterEach(func() { env.Cleanup() }) var _ = AfterEach(func() { env.AfterEach() }) diff --git a/test/suites/nodeclaim/garbage_collection_test.go b/test/suites/nodeclaim/garbage_collection_test.go index 7317c93c88e1..35d5dd4700bc 100644 --- a/test/suites/nodeclaim/garbage_collection_test.go +++ b/test/suites/nodeclaim/garbage_collection_test.go @@ -28,6 +28,7 @@ import ( corev1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" coretest "sigs.k8s.io/karpenter/pkg/test" + "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" awserrors "github.com/aws/karpenter-provider-aws/pkg/errors" "github.com/aws/karpenter-provider-aws/pkg/utils" environmentaws "github.com/aws/karpenter-provider-aws/test/pkg/environment/aws" @@ -84,6 +85,10 @@ var _ = Describe("GarbageCollection", func() { Key: aws.String(corev1beta1.NodePoolLabelKey), Value: aws.String(nodePool.Name), }, + { + Key: aws.String(v1beta1.LabelNodeClass), + Value: aws.String(nodeClass.Name), + }, }, }, }, diff --git a/test/suites/nodeclaim/nodeclaim_test.go b/test/suites/nodeclaim/nodeclaim_test.go index 60edb27208b6..bc36bd8fd380 100644 --- a/test/suites/nodeclaim/nodeclaim_test.go +++ b/test/suites/nodeclaim/nodeclaim_test.go @@ -40,16 +40,20 @@ var _ = Describe("StandaloneNodeClaim", func() { It("should create a standard NodeClaim within the 'c' instance family", func() { nodeClaim := test.NodeClaim(corev1beta1.NodeClaim{ Spec: corev1beta1.NodeClaimSpec{ - Requirements: []v1.NodeSelectorRequirement{ + Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ { - Key: v1beta1.LabelInstanceCategory, - Operator: v1.NodeSelectorOpIn, - Values: []string{"c"}, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceCategory, + Operator: v1.NodeSelectorOpIn, + Values: []string{"c"}, + }, }, { - Key: corev1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.CapacityTypeOnDemand}, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: corev1beta1.CapacityTypeLabelKey, + Operator: v1.NodeSelectorOpIn, + Values: []string{corev1beta1.CapacityTypeOnDemand}, + }, }, }, NodeClassRef: &corev1beta1.NodeClassReference{ @@ -115,15 +119,15 @@ var _ = Describe("StandaloneNodeClaim", func() { Kubelet: &corev1beta1.KubeletConfiguration{ MaxPods: lo.ToPtr[int32](110), PodsPerCore: lo.ToPtr[int32](10), - SystemReserved: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("200m"), - v1.ResourceMemory: resource.MustParse("200Mi"), - v1.ResourceEphemeralStorage: resource.MustParse("1Gi"), + SystemReserved: map[string]string{ + string(v1.ResourceCPU): "200m", + string(v1.ResourceMemory): "200Mi", + string(v1.ResourceEphemeralStorage): "1Gi", }, - KubeReserved: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("200m"), - v1.ResourceMemory: resource.MustParse("200Mi"), - v1.ResourceEphemeralStorage: resource.MustParse("1Gi"), + KubeReserved: map[string]string{ + string(v1.ResourceCPU): "200m", + string(v1.ResourceMemory): "200Mi", + string(v1.ResourceEphemeralStorage): "1Gi", }, EvictionHard: map[string]string{ "memory.available": "5%", @@ -189,16 +193,20 @@ var _ = Describe("StandaloneNodeClaim", func() { It("should remove the cloudProvider NodeClaim when the cluster NodeClaim is deleted", func() { nodeClaim := test.NodeClaim(corev1beta1.NodeClaim{ Spec: corev1beta1.NodeClaimSpec{ - Requirements: []v1.NodeSelectorRequirement{ + Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ { - Key: v1beta1.LabelInstanceCategory, - Operator: v1.NodeSelectorOpIn, - Values: []string{"c"}, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceCategory, + Operator: v1.NodeSelectorOpIn, + Values: []string{"c"}, + }, }, { - Key: corev1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.CapacityTypeOnDemand}, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: corev1beta1.CapacityTypeLabelKey, + Operator: v1.NodeSelectorOpIn, + Values: []string{corev1beta1.CapacityTypeOnDemand}, + }, }, }, NodeClassRef: &corev1beta1.NodeClassReference{ @@ -224,16 +232,20 @@ var _ = Describe("StandaloneNodeClaim", func() { It("should delete a NodeClaim from the node termination finalizer", func() { nodeClaim := test.NodeClaim(corev1beta1.NodeClaim{ Spec: corev1beta1.NodeClaimSpec{ - Requirements: []v1.NodeSelectorRequirement{ + Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ { - Key: v1beta1.LabelInstanceCategory, - Operator: v1.NodeSelectorOpIn, - Values: []string{"c"}, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceCategory, + Operator: v1.NodeSelectorOpIn, + Values: []string{"c"}, + }, }, { - Key: corev1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.CapacityTypeOnDemand}, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: corev1beta1.CapacityTypeLabelKey, + Operator: v1.NodeSelectorOpIn, + Values: []string{corev1beta1.CapacityTypeOnDemand}, + }, }, }, NodeClassRef: &corev1beta1.NodeClassReference{ @@ -270,21 +282,27 @@ var _ = Describe("StandaloneNodeClaim", func() { nodeClaim := test.NodeClaim(corev1beta1.NodeClaim{ Spec: corev1beta1.NodeClaimSpec{ - Requirements: []v1.NodeSelectorRequirement{ + Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ { - Key: v1beta1.LabelInstanceCategory, - Operator: v1.NodeSelectorOpIn, - Values: []string{"c"}, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceCategory, + Operator: v1.NodeSelectorOpIn, + Values: []string{"c"}, + }, }, { - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{"amd64"}, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{"amd64"}, + }, }, { - Key: corev1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.CapacityTypeOnDemand}, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: corev1beta1.CapacityTypeLabelKey, + Operator: v1.NodeSelectorOpIn, + Values: []string{corev1beta1.CapacityTypeOnDemand}, + }, }, }, NodeClassRef: &corev1beta1.NodeClassReference{ @@ -315,21 +333,27 @@ var _ = Describe("StandaloneNodeClaim", func() { nodeClaim := test.NodeClaim(corev1beta1.NodeClaim{ Spec: corev1beta1.NodeClaimSpec{ - Requirements: []v1.NodeSelectorRequirement{ + Requirements: []corev1beta1.NodeSelectorRequirementWithMinValues{ { - Key: v1beta1.LabelInstanceCategory, - Operator: v1.NodeSelectorOpIn, - Values: []string{"c"}, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceCategory, + Operator: v1.NodeSelectorOpIn, + Values: []string{"c"}, + }, }, { - Key: v1.LabelArchStable, - Operator: v1.NodeSelectorOpIn, - Values: []string{"amd64"}, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1.LabelArchStable, + Operator: v1.NodeSelectorOpIn, + Values: []string{"amd64"}, + }, }, { - Key: corev1beta1.CapacityTypeLabelKey, - Operator: v1.NodeSelectorOpIn, - Values: []string{corev1beta1.CapacityTypeOnDemand}, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: corev1beta1.CapacityTypeLabelKey, + Operator: v1.NodeSelectorOpIn, + Values: []string{corev1beta1.CapacityTypeOnDemand}, + }, }, }, NodeClassRef: &corev1beta1.NodeClassReference{ diff --git a/test/suites/scale/deprovisioning_test.go b/test/suites/scale/deprovisioning_test.go index 38fdb29a9408..cb7086bbee08 100644 --- a/test/suites/scale/deprovisioning_test.go +++ b/test/suites/scale/deprovisioning_test.go @@ -85,11 +85,11 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), Label(debug.NoEvents), nodeClass = env.DefaultEC2NodeClass() nodePool = env.DefaultNodePool(nodeClass) nodePool.Spec.Limits = nil - test.ReplaceRequirements(nodePool, v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceHypervisor, - Operator: v1.NodeSelectorOpIn, - Values: []string{"nitro"}, - }) + test.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{Key: v1beta1.LabelInstanceHypervisor, + Operator: v1.NodeSelectorOpIn, + Values: []string{"nitro"}, + }}) deploymentOptions = test.DeploymentOptions{ PodOptions: test.PodOptions{ ResourceRequirements: v1.ResourceRequirements{ @@ -438,11 +438,12 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), Label(debug.NoEvents), replicas := replicasPerNode * expectedNodeCount // Add in a instance type size requirement that's larger than the smallest that fits the pods. - test.ReplaceRequirements(nodePool, v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceSize, - Operator: v1.NodeSelectorOpIn, - Values: []string{"2xlarge"}, - }) + test.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceSize, + Operator: v1.NodeSelectorOpIn, + Values: []string{"2xlarge"}, + }}) deployment.Spec.Replicas = lo.ToPtr[int32](int32(replicas)) // Hostname anti-affinity to require one pod on each node @@ -485,7 +486,7 @@ var _ = Describe("Deprovisioning", Label(debug.NoWatch), Label(debug.NoEvents), // the requirements wide-open should cause deletes and increase our utilization on the cluster nodePool.Spec.Disruption.ConsolidationPolicy = corev1beta1.ConsolidationPolicyWhenUnderutilized nodePool.Spec.Disruption.ConsolidateAfter = nil - nodePool.Spec.Template.Spec.Requirements = lo.Reject(nodePool.Spec.Template.Spec.Requirements, func(r v1.NodeSelectorRequirement, _ int) bool { + nodePool.Spec.Template.Spec.Requirements = lo.Reject(nodePool.Spec.Template.Spec.Requirements, func(r corev1beta1.NodeSelectorRequirementWithMinValues, _ int) bool { return r.Key == v1beta1.LabelInstanceSize }) env.ExpectUpdated(nodePool) diff --git a/test/suites/scale/provisioning_test.go b/test/suites/scale/provisioning_test.go index 8fb4d68fa6a6..79c54896a9b5 100644 --- a/test/suites/scale/provisioning_test.go +++ b/test/suites/scale/provisioning_test.go @@ -48,11 +48,12 @@ var _ = Describe("Provisioning", Label(debug.NoWatch), Label(debug.NoEvents), fu nodeClass = env.DefaultEC2NodeClass() nodePool = env.DefaultNodePool(nodeClass) nodePool.Spec.Limits = nil - test.ReplaceRequirements(nodePool, v1.NodeSelectorRequirement{ - Key: v1beta1.LabelInstanceHypervisor, - Operator: v1.NodeSelectorOpIn, - Values: []string{"nitro"}, - }) + test.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceHypervisor, + Operator: v1.NodeSelectorOpIn, + Values: []string{"nitro"}, + }}) deployment = test.Deployment(test.DeploymentOptions{ PodOptions: test.PodOptions{ ResourceRequirements: v1.ResourceRequirements{ @@ -115,6 +116,62 @@ var _ = Describe("Provisioning", Label(debug.NoWatch), Label(debug.NoEvents), fu aws.PodDensityDimension: strconv.Itoa(replicasPerNode), }) }, SpecTimeout(time.Minute*30)) + It("should scale successfully on a node-dense scale-up with minValues in the NodePool requirement", Label(debug.NoEvents), func(_ context.Context) { + // Disable Prefix Delegation for the node-dense scale-up to not exhaust the IPs + // This is required because of the number of Warm ENIs that will be created and the number of IPs + // that will be allocated across this large number of nodes, despite the fact that the ENI CIDR space will + // be extremely under-utilized + env.ExpectPrefixDelegationDisabled() + DeferCleanup(func() { + env.ExpectPrefixDelegationEnabled() + }) + + replicasPerNode := 1 + expectedNodeCount := 500 + replicas := replicasPerNode * expectedNodeCount + + deployment.Spec.Replicas = lo.ToPtr[int32](int32(replicas)) + // Hostname anti-affinity to require one pod on each node + deployment.Spec.Template.Spec.Affinity = &v1.Affinity{ + PodAntiAffinity: &v1.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []v1.PodAffinityTerm{ + { + LabelSelector: deployment.Spec.Selector, + TopologyKey: v1.LabelHostname, + }, + }, + }, + } + + test.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ + // minValues is restricted to 30 to have enough instance types to be sent to launch API and not make this test flaky. + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1.LabelInstanceTypeStable, + Operator: v1.NodeSelectorOpExists, + }, + MinValues: lo.ToPtr(30), + }) + + By("waiting for the deployment to deploy all of its pods") + env.ExpectCreated(deployment) + env.EventuallyExpectPendingPodCount(selector, replicas) + + env.MeasureProvisioningDurationFor(func() { + By("kicking off provisioning by applying the nodePool and nodeClass") + env.ExpectCreated(nodePool, nodeClass) + + env.EventuallyExpectCreatedNodeClaimCount("==", expectedNodeCount) + env.EventuallyExpectCreatedNodeCount("==", expectedNodeCount) + env.EventuallyExpectInitializedNodeCount("==", expectedNodeCount) + env.EventuallyExpectHealthyPodCount(selector, replicas) + }, map[string]string{ + aws.TestCategoryDimension: testGroup, + aws.TestNameDimension: "node-dense", + aws.ProvisionedNodeCountDimension: strconv.Itoa(expectedNodeCount), + aws.DeprovisionedNodeCountDimension: strconv.Itoa(0), + aws.PodDensityDimension: strconv.Itoa(replicasPerNode), + }) + }, SpecTimeout(time.Minute*30)) It("should scale successfully on a pod-dense scale-up", func(_ context.Context) { replicasPerNode := 110 maxPodDensity := replicasPerNode + dsCount @@ -124,14 +181,64 @@ var _ = Describe("Provisioning", Label(debug.NoWatch), Label(debug.NoEvents), fu nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ MaxPods: lo.ToPtr[int32](int32(maxPodDensity)), } - test.ReplaceRequirements(nodePool, v1.NodeSelectorRequirement{ + test.ReplaceRequirements(nodePool, corev1beta1.NodeSelectorRequirementWithMinValues{ // With Prefix Delegation enabled, .large instances can have 434 pods. - Key: v1beta1.LabelInstanceSize, - Operator: v1.NodeSelectorOpIn, - Values: []string{"large"}, + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceSize, + Operator: v1.NodeSelectorOpIn, + Values: []string{"large"}, + }, }, ) + env.MeasureProvisioningDurationFor(func() { + By("waiting for the deployment to deploy all of its pods") + env.ExpectCreated(deployment) + env.EventuallyExpectPendingPodCount(selector, replicas) + + By("kicking off provisioning by applying the nodePool and nodeClass") + env.ExpectCreated(nodePool, nodeClass) + + env.EventuallyExpectCreatedNodeClaimCount("==", expectedNodeCount) + env.EventuallyExpectCreatedNodeCount("==", expectedNodeCount) + env.EventuallyExpectInitializedNodeCount("==", expectedNodeCount) + env.EventuallyExpectHealthyPodCount(selector, replicas) + }, map[string]string{ + aws.TestCategoryDimension: testGroup, + aws.TestNameDimension: "pod-dense", + aws.ProvisionedNodeCountDimension: strconv.Itoa(expectedNodeCount), + aws.DeprovisionedNodeCountDimension: strconv.Itoa(0), + aws.PodDensityDimension: strconv.Itoa(replicasPerNode), + }) + }, SpecTimeout(time.Minute*30)) + It("should scale successfully on a pod-dense scale-up with minValues in the NodePool requirement", func(_ context.Context) { + replicasPerNode := 110 + maxPodDensity := replicasPerNode + dsCount + expectedNodeCount := 60 + replicas := replicasPerNode * expectedNodeCount + deployment.Spec.Replicas = lo.ToPtr[int32](int32(replicas)) + nodePool.Spec.Template.Spec.Kubelet = &corev1beta1.KubeletConfiguration{ + MaxPods: lo.ToPtr[int32](int32(maxPodDensity)), + } + test.ReplaceRequirements(nodePool, + corev1beta1.NodeSelectorRequirementWithMinValues{ + // With Prefix Delegation enabled, .large instances can have 434 pods. + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1beta1.LabelInstanceSize, + Operator: v1.NodeSelectorOpIn, + Values: []string{"large"}, + }, + }, + corev1beta1.NodeSelectorRequirementWithMinValues{ + // minValues is restricted to 30 to have enough instance types to be sent to launch API and not make this test flaky. + NodeSelectorRequirement: v1.NodeSelectorRequirement{ + Key: v1.LabelInstanceTypeStable, + Operator: v1.NodeSelectorOpExists, + }, + MinValues: lo.ToPtr(30), + }, + ) + env.MeasureProvisioningDurationFor(func() { By("waiting for the deployment to deploy all of its pods") env.ExpectCreated(deployment) diff --git a/tools/allocatable-diff/go.mod b/tools/allocatable-diff/go.mod index 8112260be942..8bcdeb37a91f 100644 --- a/tools/allocatable-diff/go.mod +++ b/tools/allocatable-diff/go.mod @@ -1,6 +1,6 @@ module github.com/aws/karpenter-provider-aws/tools/allocatable-diff -go 1.21 +go 1.22 require ( github.com/aws/karpenter-provider-aws v0.33.1-0.20231206223517-f73ccfa65419 @@ -81,7 +81,7 @@ require ( google.golang.org/genproto/googleapis/api v0.0.0-20231009173412-8bfb1ae86b6c // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20231009173412-8bfb1ae86b6c // indirect google.golang.org/grpc v1.58.3 // indirect - google.golang.org/protobuf v1.31.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/tools/allocatable-diff/go.sum b/tools/allocatable-diff/go.sum index 3250e38d9da4..5a614287ebfd 100644 --- a/tools/allocatable-diff/go.sum +++ b/tools/allocatable-diff/go.sum @@ -670,8 +670,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/tools/kompat/go.mod b/tools/kompat/go.mod index 1296ab885f6c..1887902dc01a 100644 --- a/tools/kompat/go.mod +++ b/tools/kompat/go.mod @@ -1,6 +1,6 @@ module github.com/aws/karpenter-provider-aws/tools/kompat -go 1.20 +go 1.22 require ( github.com/Masterminds/semver/v3 v3.2.1 diff --git a/tools/kompat/pkg/kompat/kompat.go b/tools/kompat/pkg/kompat/kompat.go index 47492cabd828..75772ab3596c 100644 --- a/tools/kompat/pkg/kompat/kompat.go +++ b/tools/kompat/pkg/kompat/kompat.go @@ -193,9 +193,9 @@ func (k Kompat) Markdown(_ ...Options) string { data := []string{k.Name} for _, c := range k.Compatibility { if c.MaxK8sVersion == "" || c.MinK8sVersion == c.MaxK8sVersion { - headers = append(headers, fmt.Sprintf("%s+", c.MinK8sVersion)) + headers = append(headers, fmt.Sprintf("\\>= `%s`", c.MinK8sVersion)) } else { - headers = append(headers, fmt.Sprintf("%s - %s", c.MinK8sVersion, c.MaxK8sVersion)) + headers = append(headers, fmt.Sprintf("\\>= `%s` \\<= `%s`", c.MinK8sVersion, c.MaxK8sVersion)) } data = append(data, c.AppVersion) } @@ -365,10 +365,10 @@ func semverRange(semvers []string, allSemvers ...string) string { allSems := allSemvers sortSemvers(allSems) if allSems[len(allSems)-1] == semvers[len(semvers)-1] { - return fmt.Sprintf("%s+", semvers[0]) + return fmt.Sprintf("\\>= %s", strings.ReplaceAll(semvers[0], ".x", "")) } } - return fmt.Sprintf("%s - %s", semvers[0], semvers[len(semvers)-1]) + return fmt.Sprintf("\\>= %s \\<= %s", strings.ReplaceAll(semvers[0], ".x", ""), strings.ReplaceAll(semvers[len(semvers)-1], ".x", "")) } func sortSemvers(semvers []string) { diff --git a/website/content/en/_index.html b/website/content/en/_index.html index eb77316f4c5b..663c0328fd60 100644 --- a/website/content/en/_index.html +++ b/website/content/en/_index.html @@ -36,7 +36,7 @@

{{< /blocks/feature >}} {{< blocks/feature title="Minimize operational overhead" icon="fa-wrench" feature_class="feature" >}} -

Karpenter comes with a set of opinionated defaults in a single, declarative Provisioner resource which can easily be customized.

+

Karpenter comes with a set of opinionated defaults in a single, declarative NodePool resource which can easily be customized.

No additional configuration required!

{{< /blocks/feature >}} {{< /blocks/section >}} diff --git a/website/content/en/docs/concepts/disruption.md b/website/content/en/docs/concepts/disruption.md index a43d761915b8..48e0b479c5f3 100644 --- a/website/content/en/docs/concepts/disruption.md +++ b/website/content/en/docs/concepts/disruption.md @@ -13,11 +13,13 @@ The finalizer blocks deletion of the node object while the Termination Controlle ### Disruption Controller -Karpenter automatically discovers disruptable nodes and spins up replacements when needed. Karpenter disrupts nodes by executing one [automatic method](#automatic-methods) at a time, in order of Expiration, Drift, and then Consolidation. Each method varies slightly, but they all follow the standard disruption process: +Karpenter automatically discovers disruptable nodes and spins up replacements when needed. Karpenter disrupts nodes by executing one [automated method](#automated-methods) at a time, in order of Expiration, Drift, and then Consolidation. Each method varies slightly, but they all follow the standard disruption process. Karpenter uses [disruption budgets]({{}}) to control the speed of disruption. 1. Identify a list of prioritized candidates for the disruption method. * If there are [pods that cannot be evicted](#pod-eviction) on the node, Karpenter will ignore the node and try disrupting it later. * If there are no disruptable nodes, continue to the next disruption method. -2. For each disruptable node, execute a scheduling simulation with the pods on the node to find if any replacement nodes are needed. +2. For each disruptable node: + 1. Check if disrupting it would violate its NodePool's disruption budget. + 2. Execute a scheduling simulation with the pods on the node to find if any replacement nodes are needed. 3. Add the `karpenter.sh/disruption:NoSchedule` taint to the node(s) to prevent pods from scheduling to it. 4. Pre-spin any replacement nodes needed as calculated in Step (2), and wait for them to become ready. * If a replacement node fails to initialize, un-taint the node(s), and restart from Step (1), starting at the first disruption method again. @@ -27,8 +29,8 @@ Karpenter automatically discovers disruptable nodes and spins up replacements wh ### Termination Controller When a Karpenter node is deleted, the Karpenter finalizer will block deletion and the APIServer will set the `DeletionTimestamp` on the node, allowing Karpenter to gracefully shutdown the node, modeled after [Kubernetes Graceful Node Shutdown](https://kubernetes.io/docs/concepts/architecture/nodes/#graceful-node-shutdown). Karpenter's graceful shutdown process will: -1. Add the `karpenter.sh/disruption:NoSchedule` taint to the node to prevent pods from scheduling to it. -2. Begin evicting the pods on the node with the [Kubernetes Eviction API](https://kubernetes.io/docs/concepts/scheduling-eviction/api-eviction/) to respect PDBs, while ignoring all non-daemonset pods and [static pods](https://kubernetes.io/docs/tasks/configure-pod-container/static-pod/). Wait for the node to be fully drained before proceeding to Step (3). +1. Add the `karpenter.sh/disruption=disrupting:NoSchedule` taint to the node to prevent pods from scheduling to it. +2. Begin evicting the pods on the node with the [Kubernetes Eviction API](https://kubernetes.io/docs/concepts/scheduling-eviction/api-eviction/) to respect PDBs, while ignoring all [static pods](https://kubernetes.io/docs/tasks/configure-pod-container/static-pod/), pods tolerating the `karpenter.sh/disruption=disrupting:NoSchedule` taint, and succeeded/failed pods. Wait for the node to be fully drained before proceeding to Step (3). * While waiting, if the underlying NodeClaim for the node no longer exists, remove the finalizer to allow the APIServer to delete the node, completing termination. 3. Terminate the NodeClaim in the Cloud Provider. 4. Remove the finalizer from the node to allow the APIServer to delete the node, completing termination. @@ -61,11 +63,13 @@ When you run `kubectl delete node` on a node without a finalizer, the node is de ## Automated Methods +Automated methods can be rate limited through [NodePool Disruption Budgets]({{}}) + * **Expiration**: Karpenter will mark nodes as expired and disrupt them after they have lived a set number of seconds, based on the NodePool's `spec.disruption.expireAfter` value. You can use node expiry to periodically recycle nodes due to security concerns. * [**Consolidation**]({{}}): Karpenter works to actively reduce cluster cost by identifying when: * Nodes can be removed because the node is empty * Nodes can be removed as their workloads will run on other nodes in the cluster. - * Nodes can be replaced with cheaper variants due to a change in the workloads. + * Nodes can be replaced with lower priced variants due to a change in the workloads. * [**Drift**]({{}}): Karpenter will mark nodes as drifted and disrupt nodes that have drifted from their desired specification. See [Drift]({{}}) to see which fields are considered. * [**Interruption**]({{}}): Karpenter will watch for upcoming interruption events that could affect your nodes (health events, spot interruption, etc.) and will taint, drain, and terminate the node(s) ahead of the event to reduce workload disruption. @@ -84,12 +88,12 @@ spec: Karpenter has two mechanisms for cluster consolidation: 1. **Deletion** - A node is eligible for deletion if all of its pods can run on free capacity of other nodes in the cluster. -2. **Replace** - A node can be replaced if all of its pods can run on a combination of free capacity of other nodes in the cluster and a single cheaper replacement node. +2. **Replace** - A node can be replaced if all of its pods can run on a combination of free capacity of other nodes in the cluster and a single lower price replacement node. Consolidation has three mechanisms that are performed in order to attempt to identify a consolidation action: 1. **Empty Node Consolidation** - Delete any entirely empty nodes in parallel -2. **Multi Node Consolidation** - Try to delete two or more nodes in parallel, possibly launching a single replacement that is cheaper than the price of all nodes being removed -3. **Single Node Consolidation** - Try to delete any single node, possibly launching a single replacement that is cheaper than the price of that node +2. **Multi Node Consolidation** - Try to delete two or more nodes in parallel, possibly launching a single replacement whose price is lower than that of all nodes being removed +3. **Single Node Consolidation** - Try to delete any single node, possibly launching a single replacement whose price is lower than that of the node being removed It's impractical to examine all possible consolidation options for multi-node consolidation, so Karpenter uses a heuristic to identify a likely set of nodes that can be consolidated. For single-node consolidation we consider each node in the cluster individually. @@ -106,16 +110,24 @@ Events: Type Reason Age From Message ---- ------ ---- ---- ------- Normal Unconsolidatable 66s karpenter pdb default/inflate-pdb prevents pod evictions - Normal Unconsolidatable 33s (x3 over 30m) karpenter can't replace with a cheaper node + Normal Unconsolidatable 33s (x3 over 30m) karpenter can't replace with a lower-priced node ``` {{% alert title="Warning" color="warning" %}} Using preferred anti-affinity and topology spreads can reduce the effectiveness of consolidation. At node launch, Karpenter attempts to satisfy affinity and topology spread preferences. In order to reduce node churn, consolidation must also attempt to satisfy these constraints to avoid immediately consolidating nodes after they launch. This means that consolidation may not disrupt nodes in order to avoid violating preferences, even if kube-scheduler can fit the host pods elsewhere. Karpenter reports these pods via logging to bring awareness to the possible issues they can cause (e.g. `pod default/inflate-anti-self-55894c5d8b-522jd has a preferred Anti-Affinity which can prevent consolidation`). {{% /alert %}} -{{% alert title="Note" color="primary" %}} -For spot nodes, Karpenter only uses the deletion consolidation mechanism. It will not replace a spot node with a cheaper spot node. Spot instance types are selected with the `price-capacity-optimized` strategy and often the cheapest spot instance type is not launched due to the likelihood of interruption. Consolidation would then replace the spot instance with a cheaper instance negating the `price-capacity-optimized` strategy entirely and increasing interruption rate. -{{% /alert %}} +#### Spot consolidation +For spot nodes, Karpenter has deletion consolidation enabled by default. If you would like to enable replacement with spot consolidation, you need to enable the feature through the [`SpotToSpotConsolidation` feature flag]({{}}). + +Lower priced spot instance types are selected with the [`price-capacity-optimized` strategy](https://aws.amazon.com/blogs/compute/introducing-price-capacity-optimized-allocation-strategy-for-ec2-spot-instances/). Sometimes, the lowest priced spot instance type is not launched due to the likelihood of interruption. As a result, Karpenter uses the number of available instance type options with a price lower than the currently launched spot instance as a heuristic for evaluating whether it should launch a replacement for the current spot node. + +We refer to the number of instances that Karpenter has within its launch decision as a launch's "instance type flexibility." When Karpenter is considering performing a spot-to-spot consolidation replacement, it will check whether replacing the instance type will lead to enough instance type flexibility in the subsequent launch request. As a result, we get the following properties when evaluating for consolidation: +1) We shouldn't continually consolidate down to the lowest priced spot instance which might have very high rates of interruption. +2) We launch with enough instance types that there’s high likelihood that our replacement instance has comparable availability to our current one. + +Karpenter requires a minimum instance type flexibility of 15 instance types when performing single node spot-to-spot consolidations (1 node to 1 node). It does not have the same instance type flexibility requirement for multi-node spot-to-spot consolidations (many nodes to 1 node) since doing so without requiring flexibility won't lead to "race to the bottom" scenarios. + ### Drift Drift handles changes to the NodePool/EC2NodeClass. For Drift, values in the NodePool/EC2NodeClass are reflected in the NodeClaimTemplateSpec/EC2NodeClassSpec in the same way that they’re set. A NodeClaim will be detected as drifted if the values in its owning NodePool/EC2NodeClass do not match the values in the NodeClaim. Similar to the upstream `deployment.spec.template` relationship to pods, Karpenter will annotate the owning NodePool and EC2NodeClass with a hash of the NodeClaimTemplateSpec to check for drift. Some special cases will be discovered either from Karpenter or through the CloudProvider interface, triggered by NodeClaim/Instance/NodePool/EC2NodeClass changes. @@ -139,11 +151,11 @@ In special cases, drift can correspond to multiple values and must be handled di Behavioral Fields are treated as over-arching settings on the NodePool to dictate how Karpenter behaves. These fields don’t correspond to settings on the NodeClaim or instance. They’re set by the user to control Karpenter’s Provisioning and disruption logic. Since these don’t map to a desired state of NodeClaims, __behavioral fields are not considered for Drift__. ##### NodePool -| Fields | -|----------------| -| spec.weight | -| spec.limits | -| spec.disruption.* | +| Fields | +|---------------------| +| spec.weight | +| spec.limits | +| spec.disruption.* | Read the [Drift Design](https://github.com/aws/karpenter-core/blob/main/designs/drift.md) for more. @@ -178,6 +190,64 @@ To enable interruption handling, configure the `--interruption-queue-name` CLI a ## Controls +### Disruption Budgets + +You can rate limit Karpenter's disruption through the NodePool's `spec.disruption.budgets`. If undefined, Karpenter will default to one budget with `nodes: 10%`. Budgets will consider nodes that are actively being deleted for any reason, and will only block Karpenter from disrupting nodes voluntarily through expiration, drift, emptiness, and consolidation. + +#### Nodes +When calculating if a budget will block nodes from disruption, Karpenter lists the total number of nodes owned by a NodePool, subtracting out the nodes owned by that NodePool that are currently being deleted and nodes that are NotReady. If the number of nodes being deleted by Karpenter or any other processes is greater than the number of allowed disruptions, disruption for this node will not proceed. + +If the budget is configured with a percentage value, such as `20%`, Karpenter will calculate the number of allowed disruptions as `allowed_disruptions = roundup(total * percentage) - total_deleting - total_notready`. If otherwise defined as a non-percentage value, Karpenter will simply subtract the number of nodes from the total `(total - non_percentage_value) - total_deleting - total_notready`. For multiple budgets in a NodePool, Karpenter will take the minimum value (most restrictive) of each of the budgets. + +For example, the following NodePool with three budgets defines the following requirements: +- The first budget will only allow 20% of nodes owned by that NodePool to be disrupted. For instance, if there were 19 nodes owned by the NodePool, 4 disruptions would be allowed, rounding up from `19 * .2 = 3.8`. +- The second budget acts as a ceiling to the previous budget, only allowing 5 disruptions when there are more than 25 nodes. +- The last budget only blocks disruptions during the first 10 minutes of the day, where 0 disruptions are allowed. + +```yaml +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: default +spec: + disruption: + consolidationPolicy: WhenUnderutilized + expireAfter: 720h # 30 * 24h = 720h + budgets: + - nodes: "20%" + - nodes: "5" + - nodes: "0" + schedule: "@daily" + duration: 10m +``` + +#### Schedule +Schedule is a cronjob schedule. Generally, the cron syntax is five space-delimited values with options below, with additional special macros like `@yearly`, `@monthly`, `@weekly`, `@daily`, `@hourly`. +Follow the [Kubernetes documentation](https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/#writing-a-cronjob-spec) for more information on how to follow the cron syntax. + +```bash +# ┌───────────── minute (0 - 59) +# │ ┌───────────── hour (0 - 23) +# │ │ ┌───────────── day of the month (1 - 31) +# │ │ │ ┌───────────── month (1 - 12) +# │ │ │ │ ┌───────────── day of the week (0 - 6) (Sunday to Saturday; +# │ │ │ │ │ 7 is also Sunday on some systems) +# │ │ │ │ │ OR sun, mon, tue, wed, thu, fri, sat +# │ │ │ │ │ +# * * * * * +``` + +{{% alert title="Note" color="primary" %}} +Timezones are not supported. Most images default to UTC, but it is best practice to ensure this is the case when considering how to define your budgets. +{{% /alert %}} + +#### Duration +Duration allows compound durations with minutes and hours values such as `10h5m` or `30m` or `160h`. Since cron syntax does not accept denominations smaller than minutes, users can only define minutes or hours. + +{{% alert title="Note" color="primary" %}} +Duration and Schedule must be defined together. When omitted, the budget is always active. When defined, the schedule determines a starting point where the budget will begin being enforced, and the duration determines how long from that starting point the budget will be enforced. +{{% /alert %}} + ### Pod-Level Controls You can block Karpenter from voluntarily choosing to disrupt certain pods by setting the `karpenter.sh/do-not-disrupt: "true"` annotation on the pod. This is useful for pods that you want to run from start to finish without disruption. By opting pods out of this disruption, you are telling Karpenter that it should not voluntarily remove a node containing this pod. @@ -195,7 +265,7 @@ spec: ``` {{% alert title="Note" color="primary" %}} -This annotation will be ignored for [terminating pods](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase), [terminal pods](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase) (Failed/Succeeded), [DaemonSet pods](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/), or [static pods](https://kubernetes.io/docs/tasks/configure-pod-container/static-pod/). +This annotation will be ignored for [terminating pods](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase) and [terminal pods](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase) (Failed/Succeeded). {{% /alert %}} Examples of voluntary node removal that will be prevented by this annotation include: @@ -209,7 +279,7 @@ Voluntary node removal does not include [Interruption]({{} ### Node-Level Controls -Nodes can be opted out of consolidation disruption by setting the annotation `karpenter.sh/do-not-disrupt: "true"` on the node. +You can block Karpenter from voluntarily choosing to disrupt certain nodes by setting the `karpenter.sh/do-not-disrupt: "true"` annotation on the node. This will prevent disruption actions on the node. ```yaml apiVersion: v1 @@ -221,7 +291,7 @@ metadata: #### Example: Disable Disruption on a NodePool -NodePool `.spec.annotations` allow you to set annotations that will be applied to all nodes launched by this NodePool. By setting the annotation `karpenter.sh/do-not-disrupt: "true"` on the NodePool, you will selectively prevent all nodes launched by this NodePool from being considered in consolidation calculations. +NodePool `.spec.annotations` allow you to set annotations that will be applied to all nodes launched by this NodePool. By setting the annotation `karpenter.sh/do-not-disrupt: "true"` on the NodePool, you will selectively prevent all nodes launched by this NodePool from being considered in disruption actions. ```yaml apiVersion: karpenter.sh/v1beta1 diff --git a/website/content/en/docs/concepts/nodeclasses.md b/website/content/en/docs/concepts/nodeclasses.md index 12c4c60a373f..d426f579e7e6 100644 --- a/website/content/en/docs/concepts/nodeclasses.md +++ b/website/content/en/docs/concepts/nodeclasses.md @@ -1,4 +1,4 @@ ---- + --- title: "NodeClasses" linkTitle: "NodeClasses" weight: 2 @@ -35,7 +35,7 @@ spec: # Each term in the array of subnetSelectorTerms is ORed together # Within a single term, all conditions are ANDed subnetSelectorTerms: - # Select on any subnet that has the "karpenter.sh/discovery: ${CLUSTER_NAME}" + # Select on any subnet that has the "karpenter.sh/discovery: ${CLUSTER_NAME}" # AND the "environment: test" tag OR any subnet with ID "subnet-09fa4a0a8f233a921" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" @@ -46,8 +46,8 @@ spec: # Each term in the array of securityGroupSelectorTerms is ORed together # Within a single term, all conditions are ANDed securityGroupSelectorTerms: - # Select on any security group that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag - # AND the "environment: test" tag OR any security group with the "my-security-group" name + # Select on any security group that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any security group with the "my-security-group" name # OR any security group with ID "sg-063d7acfb4b06c82c" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" @@ -70,8 +70,8 @@ spec: # Each term in the array of amiSelectorTerms is ORed together # Within a single term, all conditions are ANDed amiSelectorTerms: - # Select on any AMI that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag - # AND the "environment: test" tag OR any AMI with the "my-ami" name + # Select on any AMI that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any AMI with the "my-ami" name # OR any AMI with ID "ami-123" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" @@ -79,6 +79,9 @@ spec: - name: my-ami - id: ami-123 + # Optional, use instance-store volumes for node ephemeral-storage + instanceStorePolicy: RAID0 + # Optional, overrides autogenerated userdata with a merge semantic userData: | echo "Hello world" @@ -110,6 +113,10 @@ spec: # Optional, configures detailed monitoring for the instance detailedMonitoring: true + + # Optional, configures if the instance should be launched with an associated public IP address. + # If not specified, the default value depends on the subnet's public IP auto-assign setting. + associatePublicIPAddress: true status: # Resolved subnets subnets: @@ -157,7 +164,7 @@ Refer to the [NodePool docs]({{}}) for settings applicable t ## spec.amiFamily -AMIFamily is a required field, dictating both the default bootstrapping logic for nodes provisioned through this `EC2NodeClass` but also selecting a group of recommended, latest AMIs by default. Currently, Karpenter supports `amiFamily` values `AL2`, `Bottlerocket`, `Ubuntu`, `Windows2019`, `Windows2022` and `Custom`. GPUs are only supported by default with `AL2` and `Bottlerocket`. The `AL2` amiFamily does not support ARM64 GPU instance types unless you specify custom [`amiSelectorTerms`]({{}}). Default bootstrapping logic is shown below for each of the supported families. +AMIFamily is a required field, dictating both the default bootstrapping logic for nodes provisioned through this `EC2NodeClass` but also selecting a group of recommended, latest AMIs by default. Currently, Karpenter supports `amiFamily` values `AL2`, `AL2023`, `Bottlerocket`, `Ubuntu`, `Windows2019`, `Windows2022` and `Custom`. GPUs are only supported by default with `AL2` and `Bottlerocket`. The `AL2` amiFamily does not support ARM64 GPU instance types unless you specify custom [`amiSelectorTerms`]({{}}). Default bootstrapping logic is shown below for each of the supported families. ### AL2 @@ -173,11 +180,38 @@ exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 /etc/eks/bootstrap.sh 'test-cluster' --apiserver-endpoint 'https://test-cluster' --b64-cluster-ca 'ca-bundle' \ --dns-cluster-ip '10.100.0.10' \ --use-max-pods false \ ---container-runtime containerd \ --kubelet-extra-args '--node-labels=karpenter.sh/capacity-type=on-demand,karpenter.sh/nodepool=test --max-pods=110' --//-- ``` +### AL2023 + +```text +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="//" + +--// +Content-Type: application/node.eks.aws + +# Karpenter Generated NodeConfig +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + cluster: + name: test-cluster + apiServerEndpoint: https://example.com + certificateAuthority: ca-bundle + cidr: 10.100.0.0/16 + kubelet: + config: + maxPods: 110 + flags: + - --node-labels=karpenter.sh/capacity-type=on-demand,karpenter.sh/nodepool=test + +--//-- + +``` + ### Bottlerocket ```toml @@ -246,7 +280,7 @@ This selection logic is modeled as terms, where each term contains multiple cond ```yaml subnetSelectorTerms: - # Select on any subnet that has the "karpenter.sh/discovery: ${CLUSTER_NAME}" + # Select on any subnet that has the "karpenter.sh/discovery: ${CLUSTER_NAME}" # AND the "environment: test" tag OR any subnet with ID "subnet-09fa4a0a8f233a921" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" @@ -304,6 +338,7 @@ spec: - id: "subnet-0471ca205b8a129ae" ``` + ## spec.securityGroupSelectorTerms Security Group Selector Terms allow you to specify selection logic for all security groups that will be attached to an instance launched from the `EC2NodeClass`. The security group of an instance is comparable to a set of firewall rules. @@ -313,8 +348,8 @@ This selection logic is modeled as terms, where each term contains multiple cond ```yaml securityGroupSelectorTerms: - # Select on any security group that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag - # AND the "environment: test" tag OR any security group with the "my-security-group" name + # Select on any security group that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any security group with the "my-security-group" name # OR any security group with ID "sg-063d7acfb4b06c82c" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" @@ -400,8 +435,8 @@ This selection logic is modeled as terms, where each term contains multiple cond ```yaml amiSelectorTerms: - # Select on any AMI that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag - # AND the "environment: test" tag OR any AMI with the "my-ami" name + # Select on any AMI that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any AMI with the "my-ami" name # OR any AMI with ID "ami-123" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" @@ -422,6 +457,7 @@ AMIs may be specified by any AWS tag, including `Name`. Selecting by tag or by n If `amiSelectorTerms` match more than one AMI, Karpenter will automatically determine which AMI best fits the workloads on the launched worker node under the following constraints: * When launching nodes, Karpenter automatically determines which architecture a custom AMI is compatible with and will use images that match an instanceType's requirements. + * Note that Karpenter **cannot** detect any requirement other than architecture. If you need to specify different AMIs for different kind of nodes (e.g. accelerated GPU AMIs), you should use a separate `EC2NodeClass`. * If multiple AMIs are found that can be used, Karpenter will choose the latest one. * If no AMIs are found that can be used, then no nodes will be provisioned. {{% /alert %}} @@ -481,7 +517,7 @@ Specify using ids: ## spec.role -`Role` is an optional field and is necessary to tell Karpenter which identity nodes from this `EC2NodeClass` should assume. You must specify one of `role` or `instanceProfile` when creating a Karpenter `EC2NodeClass`. If using the [Karpenter Getting Started Guide]({{}}) to deploy Karpenter, you can use the `KarpenterNodeRole-$CLUSTER_NAME` role provisioned by that process. +`Role` is an optional field and tells Karpenter which IAM identity nodes should assume. You must specify one of `role` or `instanceProfile` when creating a Karpenter `EC2NodeClass`. If using the [Karpenter Getting Started Guide]({{}}) to deploy Karpenter, you can use the `KarpenterNodeRole-$CLUSTER_NAME` role provisioned by that process. ```yaml spec: @@ -490,7 +526,9 @@ spec: ## spec.instanceProfile -`InstanceProfile` is an optional field and is necessary to tell Karpenter which identity nodes from this `EC2NodeClass` should assume. You must specify one of `role` or `instanceProfile` when creating a Karpenter `EC2NodeClasss`. If you use the `instanceProfile` field instead of `role`, Karpenter will not manage the InstanceProfile on your behalf. +`InstanceProfile` is an optional field and tells Karpenter which IAM identity nodes should assume. You must specify one of `role` or `instanceProfile` when creating a Karpenter `EC2NodeClass`. If you use the `instanceProfile` field instead of `role`, Karpenter will not manage the InstanceProfile on your behalf; instead, it expects that you have pre-provisioned an IAM instance profile and assigned it a role. + +You can provision and assign a role to an IAM instance profile using [CloudFormation](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iam-instanceprofile.html) or by using the [`aws iam create-instance-profile`](https://docs.aws.amazon.com/cli/latest/reference/iam/create-instance-profile.html) and [`aws iam add-role-to-instance-profile`](https://docs.aws.amazon.com/cli/latest/reference/iam/add-role-to-instance-profile.html) commands in the CLI. {{% alert title="Note" color="primary" %}} @@ -506,6 +544,7 @@ Karpenter adds tags to all resources it creates, including EC2 Instances, EBS vo Name: karpenter.sh/nodeclaim: karpenter.sh/nodepool: +karpenter.k8s.aws/ec2nodeclass: kubernetes.io/cluster/: owned ``` @@ -571,6 +610,17 @@ spec: encrypted: true ``` +### AL2023 +```yaml +spec: + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 20Gi + volumeType: gp3 + encrypted: true +``` + ### Bottlerocket ```yaml spec: @@ -615,6 +665,39 @@ spec: The `Custom` AMIFamily ships without any default `blockDeviceMappings`. +## spec.instanceStorePolicy + +The `instanceStorePolicy` field controls how [instance-store](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/InstanceStorage.html) volumes are handled. By default, Karpenter and Kubernetes will simply ignore them. + +### RAID0 + +If you intend to use these volumes for faster node ephemeral-storage, set `instanceStorePolicy` to `RAID0`: + +```yaml +spec: + instanceStorePolicy: RAID0 +``` + +This will set the allocatable ephemeral-storage of each node to the total size of the instance-store volume(s). + +The disks must be formatted & mounted in a RAID0 and be the underlying filesystem for the Kubelet & Containerd. Instructions for each AMI family are listed below: + +#### AL2 + +On AL2, Karpenter automatically configures the disks through an additional boostrap argument (`--local-disks raid0`). The device name is `/dev/md/0` and its mount point is `/mnt/k8s-disks/0`. You should ensure any additional disk setup does not interfere with these. + +#### AL2023 + +On AL2023, Karpenter automatically configures the disks via the generated `NodeConfig` object. Like AL2, the device name is `/dev/md/0` and its mount point is `/mnt/k8s-disks/0`. You should ensure any additional disk setup does not interfere with these. + +#### Others + +For all other AMI families, you must configure the disks yourself. Check out the [`setup-local-disks`](https://github.com/awslabs/amazon-eks-ami/blob/master/files/bin/setup-local-disks) script in [amazon-eks-ami](https://github.com/awslabs/amazon-eks-ami) to see how this is done for AL2. + +{{% alert title="Tip" color="secondary" %}} +Since the Kubelet & Containerd will be using the instance-store filesystem, you may consider using a more minimal root volume size. +{{% /alert %}} + ## spec.userData You can control the UserData that is applied to your worker nodes via this field. This allows you to run custom scripts or pass-through custom configuration to Karpenter instances on start-up. @@ -678,7 +761,30 @@ Consider the following example to understand how your custom UserData will be me ```bash #!/bin/bash -echo "Running custom user data script" +echo "Running custom user data script (bash)" +``` + +#### Merged UserData (bash) + +```bash +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="//" + +--// +Content-Type: text/x-shellscript; charset="us-ascii" + +#!/bin/bash +echo "Running custom user data script (bash)" + +--// +Content-Type: text/x-shellscript; charset="us-ascii" + +#!/bin/bash -xe +exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 +/etc/eks/bootstrap.sh 'test-cluster' --apiserver-endpoint 'https://test-cluster' --b64-cluster-ca 'ca-bundle' \ +--use-max-pods false \ +--kubelet-extra-args '--node-labels=karpenter.sh/capacity-type=on-demand,karpenter.sh/nodepool=test --max-pods=110' +--//-- ``` #### Passed-in UserData (MIME) @@ -691,12 +797,12 @@ Content-Type: multipart/mixed; boundary="BOUNDARY" Content-Type: text/x-shellscript; charset="us-ascii" #!/bin/bash -echo "Running custom user data script" +echo "Running custom user data script (mime)" --BOUNDARY-- ``` -#### Merged UserData +#### Merged UserData (MIME) ```bash MIME-Version: 1.0 @@ -706,7 +812,7 @@ Content-Type: multipart/mixed; boundary="//" Content-Type: text/x-shellscript; charset="us-ascii" #!/bin/bash -echo "Running custom user data script" +echo "Running custom user data script (mime)" --// Content-Type: text/x-shellscript; charset="us-ascii" @@ -715,7 +821,6 @@ Content-Type: text/x-shellscript; charset="us-ascii" exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 /etc/eks/bootstrap.sh 'test-cluster' --apiserver-endpoint 'https://test-cluster' --b64-cluster-ca 'ca-bundle' \ --use-max-pods false \ ---container-runtime containerd \ --kubelet-extra-args '--node-labels=karpenter.sh/capacity-type=on-demand,karpenter.sh/nodepool=test --max-pods=110' --//-- ``` @@ -737,6 +842,169 @@ spec: ``` {{% /alert %}} +### AL2023 + +* Your UserData may be in one of three formats: a [MIME multi part archive](https://cloudinit.readthedocs.io/en/latest/topics/format.html#mime-multi-part-archive), a NodeConfig YAML / JSON string, or a shell script. +* Karpenter will transform your custom UserData into a MIME part, if necessary, and then create a MIME multi-part archive. This archive will consist of a generated NodeConfig, containing Karpenter's default values, followed by the transformed custom UserData. For more information on the NodeConfig spec, refer to the [AL2023 EKS Optimized AMI docs](https://awslabs.github.io/amazon-eks-ami/nodeadm/doc/examples/). +* If a value is specified both in the Karpenter generated NodeConfig and the same value is specified in the custom user data, the value in the custom user data will take precedence. + +#### Passed-in UserData (NodeConfig) + +```yaml +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + kubelet: + config: + maxPods: 42 +``` + +#### Merged UserData (NodeConfig) + +```text +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="//" + +--// +# Karpenter Generated NodeConfig +Content-Type: application/node.eks.aws + +# Karpenter Generated NodeConfig +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + cluster: + apiServerEndpoint: https://test-cluster + certificateAuthority: cluster-ca + cidr: 10.100.0.0/16 + name: test-cluster + kubelet: + config: + clusterDNS: + - 10.100.0.10 + maxPods: 118 + flags: + - --node-labels="karpenter.sh/capacity-type=on-demand,karpenter.sh/nodepool=default" + +--// +Content-Type: application/node.eks.aws + +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + kubelet: + config: + maxPods: 42 +--//-- +``` + +#### Passed-in UserData (bash) + +```shell +#!/bin/bash +echo "Hello, AL2023!" +``` + +#### Merged UserData (bash) + +```text +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="//" + +--// +Content-Type: application/node.eks.aws + +# Karpenter Generated NodeConfig +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + cluster: + apiServerEndpoint: https://test-cluster + certificateAuthority: cluster-ca + cidr: 10.100.0.0/16 + name: test-cluster + kubelet: + config: + clusterDNS: + - 10.100.0.10 + maxPods: 118 + flags: + - --node-labels="karpenter.sh/capacity-type=on-demand,karpenter.sh/nodepool=default" + +--// +Content-Type: text/x-shellscript; charset="us-ascii" + +#!/bin/bash +echo "Hello, AL2023!" +--//-- +``` + +#### Passed-in UserData (MIME) + +```text +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="//" + +--// +Content-Type: application/node.eks.aws + +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + kubelet: + config: + maxPods: 42 +--// +Content-Type: text/x-shellscript; charset="us-ascii" + +#!/bin/bash +echo "Hello, AL2023!" +--// +``` + +#### Merged UserData (MIME) + +```text +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="//" + +--// +Content-Type: application/node.eks.aws + +# Karpenter Generated NodeConfig +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + cluster: + apiServerEndpoint: https://test-cluster + certificateAuthority: cluster-ca + cidr: 10.100.0.0/16 + name: test-cluster + kubelet: + config: + clusterDNS: + - 10.100.0.10 + maxPods: 118 + flags: + - --node-labels="karpenter.sh/capacity-type=on-demand,karpenter.sh/nodepool=default" + +--// +Content-Type: application/node.eks.aws + +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + kubelet: + config: + maxPods: 42 +--// +Content-Type: text/x-shellscript; charset="us-ascii" + +#!/bin/bash +echo "Hello, AL2023!" +--//-- +``` + ### Bottlerocket * Your UserData must be valid TOML. @@ -829,6 +1097,15 @@ spec: detailedMonitoring: true ``` +## spec.associatePublicIPAddress + +A boolean field that controls whether instances created by Karpenter for this EC2NodeClass will have an associated public IP address. This overrides the `MapPublicIpOnLaunch` setting applied to the subnet the node is launched in. If this field is not set, the `MapPublicIpOnLaunch` field will be respected. + +{{% alert title="Note" color="warning" %}} +If a `NodeClaim` requests `vpc.amazonaws.com/efa` resources, `spec.associatePublicIPAddress` is respected. However, if this `NodeClaim` requests **multiple** EFA resources and the value for `spec.associatePublicIPAddress` is true, the instance will fail to launch. This is due to an EC2 restriction which +requires that the field is only set to true when configuring an instance with a single ENI at launch. When using this field, it is advised that users segregate their EFA workload to use a separate `NodePool` / `EC2NodeClass` pair. +{{% /alert %}} + ## status.subnets [`status.subnets`]({{< ref "#statussubnets" >}}) contains the resolved `id` and `zone` of the subnets that were selected by the [`spec.subnetSelectorTerms`]({{< ref "#specsubnetselectorterms" >}}) for the node class. The subnets will be sorted by the available IP address count in decreasing order. diff --git a/website/content/en/docs/concepts/nodepools.md b/website/content/en/docs/concepts/nodepools.md index 7ff7f1f64402..d679bb5f4373 100644 --- a/website/content/en/docs/concepts/nodepools.md +++ b/website/content/en/docs/concepts/nodepools.md @@ -46,6 +46,8 @@ spec: spec: # References the Cloud Provider's NodeClass resource, see your cloud provider specific documentation nodeClassRef: + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass name: default # Provisioned nodes will have these taints @@ -69,6 +71,13 @@ spec: - key: "karpenter.k8s.aws/instance-category" operator: In values: ["c", "m", "r"] + # minValues here enforces the scheduler to consider at least that number of unique instance-category to schedule the pods. + # This field is ALPHA and can be dropped or replaced at any time + minValues: 2 + - key: "karpenter.k8s.aws/instance-family" + operator: In + values: ["m5","m5d","c5","c5d","c4","r4"] + minValues: 5 - key: "karpenter.k8s.aws/instance-cpu" operator: In values: ["4", "8", "16", "32"] @@ -127,17 +136,27 @@ spec: # If using 'WhenUnderutilized', Karpenter will consider all nodes for consolidation and attempt to remove or replace Nodes when it discovers that the Node is underutilized and could be changed to reduce cost # If using `WhenEmpty`, Karpenter will only consider nodes for consolidation that contain no workload pods consolidationPolicy: WhenUnderutilized | WhenEmpty - + # The amount of time Karpenter should wait after discovering a consolidation decision # This value can currently only be set when the consolidationPolicy is 'WhenEmpty' # You can choose to disable consolidation entirely by setting the string value 'Never' here consolidateAfter: 30s - + # The amount of time a Node can live on the cluster before being removed # Avoiding long-running Nodes helps to reduce security vulnerabilities as well as to reduce the chance of issues that can plague Nodes with long uptimes such as file fragmentation or memory leaks from system processes # You can choose to disable expiration entirely by setting the string value 'Never' here expireAfter: 720h + # Budgets control the speed Karpenter can scale down nodes. + # Karpenter will respect the minimum of the currently active budgets, and will round up + # when considering percentages. Duration and Schedule must be set together. + budgets: + - nodes: 10% + # On Weekdays during business hours, don't do any deprovisioning. + - schedule: "0 9 * * mon-fri" + duration: 8h + nodes: "0" + # Resource limits constrain the total size of the cluster. # Limits prevent Karpenter from creating new instances once the limit is exceeded. limits: @@ -160,7 +179,9 @@ These well-known labels may be specified at the NodePool level, or in a workload For example, an instance type may be specified using a nodeSelector in a pod spec. If the instance type requested is not included in the NodePool list and the NodePool has instance type requirements, Karpenter will not create a node or schedule the pod. -### Instance Types +### Well-Known Labels + +#### Instance Types - key: `node.kubernetes.io/instance-type` - key: `karpenter.k8s.aws/instance-family` @@ -171,7 +192,7 @@ Generally, instance types should be a list and not a single value. Leaving these Review [AWS instance types](../instance-types). Most instance types are supported with the exclusion of [non-HVM](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/virtualization_types.html). -### Availability Zones +#### Availability Zones - key: `topology.kubernetes.io/zone` - value example: `us-east-1c` @@ -182,7 +203,7 @@ Karpenter can be configured to create nodes in a particular zone. Note that the [Learn more about Availability Zone IDs.](https://docs.aws.amazon.com/ram/latest/userguide/working-with-az-ids.html) -### Architecture +#### Architecture - key: `kubernetes.io/arch` - values @@ -191,7 +212,7 @@ IDs.](https://docs.aws.amazon.com/ram/latest/userguide/working-with-az-ids.html) Karpenter supports `amd64` nodes, and `arm64` nodes. -### Operating System +#### Operating System - key: `kubernetes.io/os` - values - `linux` @@ -199,7 +220,7 @@ Karpenter supports `amd64` nodes, and `arm64` nodes. Karpenter supports `linux` and `windows` operating systems. -### Capacity Type +#### Capacity Type - key: `karpenter.sh/capacity-type` - values @@ -212,6 +233,71 @@ Karpenter prioritizes Spot offerings if the NodePool allows Spot and on-demand i Karpenter also allows `karpenter.sh/capacity-type` to be used as a topology key for enforcing topology-spread. +### Min Values + +Along with the combination of [key,operator,values] in the requirements, Karpenter also supports `minValues` in the NodePool requirements block, allowing the scheduler to be aware of user-specified flexibility minimums while scheduling pods to a cluster. If Karpenter cannot meet this minimum flexibility for each key when scheduling a pod, it will fail the scheduling loop for that NodePool, either falling back to another NodePool which meets the pod requirements or failing scheduling the pod altogether. + +For example, the below spec will use spot instance type for all provisioned instances and enforces `minValues` to various keys where it is defined +i.e at least 2 unique instance families from [c,m,r], 5 unique instance families [eg: "m5","m5d","r4","c5","c5d","c4" etc], 10 unique instance types [eg: "c5.2xlarge","c4.xlarge" etc] is required for scheduling the pods. + +```yaml +spec: + template: + spec: + requirements: + - key: kubernetes.io/arch + operator: In + values: ["amd64"] + - key: kubernetes.io/os + operator: In + values: ["linux"] + - key: karpenter.k8s.aws/instance-category + operator: In + values: ["c", "m", "r"] + minValues: 2 + - key: karpenter.k8s.aws/instance-family + operator: Exists + minValues: 5 + - key: node.kubernetes.io/instance-type + operator: Exists + minValues: 10 + - key: karpenter.k8s.aws/instance-generation + operator: Gt + values: ["2"] +``` + +Note that `minValues` can be used with multiple operators and multiple requirements. And if the `minValues` are defined with multiple operators for the same requirement key, scheduler considers the max of all the `minValues` for that requirement. For example, the below spec requires scheduler to consider at least 5 instance-family to schedule the pods. + +```yaml +spec: + template: + spec: + requirements: + - key: kubernetes.io/arch + operator: In + values: ["amd64"] + - key: kubernetes.io/os + operator: In + values: ["linux"] + - key: karpenter.k8s.aws/instance-category + operator: In + values: ["c", "m", "r"] + minValues: 2 + - key: karpenter.k8s.aws/instance-family + operator: Exists + minValues: 5 + - key: karpenter.k8s.aws/instance-family + operator: In + values: ["m5","m5d","c5","c5d","c4","r4"] + minValues: 3 + - key: node.kubernetes.io/instance-type + operator: Exists + minValues: 10 + - key: karpenter.k8s.aws/instance-generation + operator: Gt + values: ["2"] +``` + {{% alert title="Recommended" color="primary" %}} Karpenter allows you to be extremely flexible with your NodePools by only constraining your instance types in ways that are absolutely necessary for your cluster. By default, Karpenter will enforce that you specify the `spec.template.spec.requirements` field, but will not enforce that you specify any requirements within the field. If you choose to specify `requirements: []`, this means that you will completely flexible to _all_ instance types that your cloud provider supports. @@ -285,7 +371,12 @@ kubelet: Karpenter will automatically configure the system and kube reserved resource requests on the fly on your behalf. These requests are used to configure your node and to make scheduling decisions for your pods. If you have specific requirements or know that you will have additional capacity requirements, you can optionally override the `--system-reserved` configuration defaults with the `.spec.template.spec.kubelet.systemReserved` values and the `--kube-reserved` configuration defaults with the `.spec.template.spec.kubelet.kubeReserved` values. -For more information on the default `--system-reserved` and `--kube-reserved` configuration refer to the [Kubelet Docs](https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/#kube-reserved) +{{% alert title="Note" color="primary" %}} +Karpenter considers these reserved resources when computing the allocatable ephemeral storage on a given instance type. +If `kubeReserved` is not specified, Karpenter will compute the default reserved [CPU](https://github.com/awslabs/amazon-eks-ami/blob/db28da15d2b696bc08ac3aacc9675694f4a69933/files/bootstrap.sh#L251) and [memory](https://github.com/awslabs/amazon-eks-ami/blob/db28da15d2b696bc08ac3aacc9675694f4a69933/files/bootstrap.sh#L235) resources for the purpose of ephemeral storage computation. +These defaults are based on the defaults on Karpenter's supported AMI families, which are not the same as the kubelet defaults. +You should be aware of the CPU and memory default calculation when using Custom AMI Families. If they don't align, there may be a difference in Karpenter's computed allocatable ephemeral storage and the actually ephemeral storage available on the node. +{{% /alert %}} ### Eviction Thresholds @@ -431,7 +522,7 @@ Review the [Kubernetes core API](https://github.com/kubernetes/api/blob/37748cca Karpenter allows you to describe NodePool preferences through a `weight` mechanism similar to how weight is described with [pod and node affinities](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity). -For more information on weighting NodePools, see the [Weighting NodePools section]({{}}) in the scheduling details. +For more information on weighting NodePools, see the [Weighted NodePools section]({{}}) in the scheduling docs. ## Examples diff --git a/website/content/en/docs/concepts/scheduling.md b/website/content/en/docs/concepts/scheduling.md index 90339674f24c..5352f0706fb9 100755 --- a/website/content/en/docs/concepts/scheduling.md +++ b/website/content/en/docs/concepts/scheduling.md @@ -214,7 +214,7 @@ All examples below assume that the NodePool doesn't have constraints to prevent - matchExpressions: - key: "topology.kubernetes.io/zone" operator: "In" - values: ["us-west-2a, us-west-2b"] + values: ["us-west-2a", "us-west-2b"] - key: "topology.kubernetes.io/zone" operator: "In" values: ["us-west-2b"] @@ -225,7 +225,7 @@ Changing the second operator to `NotIn` would allow the pod to run in `us-west-2 ```yaml - key: "topology.kubernetes.io/zone" operator: "In" - values: ["us-west-2a, us-west-2b"] + values: ["us-west-2a", "us-west-2b"] - key: "topology.kubernetes.io/zone" operator: "NotIn" values: ["us-west-2b"] @@ -243,7 +243,7 @@ Here, if `us-west-2a` is not available, the second term will cause the pod to ru - matchExpressions: # OR - key: "topology.kubernetes.io/zone" # AND operator: "In" - values: ["us-west-2a, us-west-2b"] + values: ["us-west-2a", "us-west-2b"] - key: "topology.kubernetes.io/zone" # AND operator: "NotIn" values: ["us-west-2b"] @@ -349,9 +349,12 @@ The three supported `topologyKey` values that Karpenter supports are: - `kubernetes.io/hostname` - `karpenter.sh/capacity-type` - See [Pod Topology Spread Constraints](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/) for details. +{{% alert title="Note" color="primary" %}} +NodePools do not attempt to balance or rebalance the availability zones for their nodes. Availability zone balancing may be achieved by defining zonal Topology Spread Constraints for Pods that require multi-zone durability, and NodePools will respect these constraints while optimizing for compute costs. +{{% /alert %}} + ## Pod affinity/anti-affinity By using the `podAffinity` and `podAntiAffinity` configuration on a pod spec, you can inform the Karpenter scheduler of your desire for pods to schedule together or apart with respect to different topology domains. For example: diff --git a/website/content/en/docs/contributing/development-guide.md b/website/content/en/docs/contributing/development-guide.md index e803169d6c5a..2afbb4d2ead7 100644 --- a/website/content/en/docs/contributing/development-guide.md +++ b/website/content/en/docs/contributing/development-guide.md @@ -73,7 +73,8 @@ make test # E2E correctness tests ### Change Log Level -By default, `make apply` will set the log level to debug. You can change the log level by setting the log level in your helm values. +By default, `make apply` will set the log level to debug. You can change the log level by setting the log level in your Helm values. + ```bash --set logLevel=debug ``` diff --git a/website/content/en/docs/faq.md b/website/content/en/docs/faq.md index bf3102323d98..a825e34056f6 100644 --- a/website/content/en/docs/faq.md +++ b/website/content/en/docs/faq.md @@ -14,10 +14,10 @@ See [Configuring NodePools]({{< ref "./concepts/#configuring-nodepools" >}}) for AWS is the first cloud provider supported by Karpenter, although it is designed to be used with other cloud providers as well. ### Can I write my own cloud provider for Karpenter? -Yes, but there is no documentation yet for it. Start with Karpenter's GitHub [cloudprovider](https://github.com/aws/karpenter-core/tree/v0.33.1/pkg/cloudprovider) documentation to see how the AWS provider is built, but there are other sections of the code that will require changes too. +Yes, but there is no documentation yet for it. Start with Karpenter's GitHub [cloudprovider](https://github.com/aws/karpenter-core/tree/v0.35.4/pkg/cloudprovider) documentation to see how the AWS provider is built, but there are other sections of the code that will require changes too. ### What operating system nodes does Karpenter deploy? -Karpenter uses the OS defined by the [AMI Family in your EC2NodeClass]({{< ref "./concepts/nodeclasses#specamifamily" >}}). +Karpenter uses the OS defined by the [AMI Family in your EC2NodeClass]({{< ref "./concepts/nodeclasses#specamifamily" >}}). ### Can I provide my own custom operating system images? Karpenter has multiple mechanisms for configuring the [operating system]({{< ref "./concepts/nodeclasses/#specamiselectorterms" >}}) for your nodes. @@ -26,7 +26,7 @@ Karpenter has multiple mechanisms for configuring the [operating system]({{< ref Karpenter is flexible to multi-architecture configurations using [well known labels]({{< ref "./concepts/scheduling/#supported-labels">}}). ### What RBAC access is required? -All the required RBAC rules can be found in the helm chart template. See [clusterrole-core.yaml](https://github.com/aws/karpenter/blob/v0.33.1/charts/karpenter/templates/clusterrole-core.yaml), [clusterrole.yaml](https://github.com/aws/karpenter/blob/v0.33.1/charts/karpenter/templates/clusterrole.yaml), [rolebinding.yaml](https://github.com/aws/karpenter/blob/v0.33.1/charts/karpenter/templates/rolebinding.yaml), and [role.yaml](https://github.com/aws/karpenter/blob/v0.33.1/charts/karpenter/templates/role.yaml) files for details. +All the required RBAC rules can be found in the Helm chart template. See [clusterrole-core.yaml](https://github.com/aws/karpenter/blob/v0.35.4/charts/karpenter/templates/clusterrole-core.yaml), [clusterrole.yaml](https://github.com/aws/karpenter/blob/v0.35.4/charts/karpenter/templates/clusterrole.yaml), [rolebinding.yaml](https://github.com/aws/karpenter/blob/v0.35.4/charts/karpenter/templates/rolebinding.yaml), and [role.yaml](https://github.com/aws/karpenter/blob/v0.35.4/charts/karpenter/templates/role.yaml) files for details. ### Can I run Karpenter outside of a Kubernetes cluster? Yes, as long as the controller has network and IAM/RBAC access to the Kubernetes API and your provider API. @@ -179,10 +179,10 @@ Yes, see the [KubeletConfiguration Section in the NodePool docs]({{}} by configuring an `amiSelector` that references the AMI name. +You can specify the [Amazon EKS optimized AMI](https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-windows-ami.html) with Windows Server 2022 Full for Kubernetes 1.29 by configuring an `amiSelector` that references the AMI name. ```yaml amiSelectorTerms: - - name: Windows_Server-2022-English-Full-EKS_Optimized-{{< param "latest_k8s_version" >}}* + - name: Windows_Server-2022-English-Full-EKS_Optimized-1.29* ``` ### Can I use Karpenter to scale my workload's pods? @@ -202,7 +202,7 @@ Use your existing upgrade mechanisms to upgrade your core add-ons in Kubernetes Karpenter requires proper permissions in the `KarpenterNode IAM Role` and the `KarpenterController IAM Role`. To upgrade Karpenter to version `$VERSION`, make sure that the `KarpenterNode IAM Role` and the `KarpenterController IAM Role` have the right permission described in `https://karpenter.sh/$VERSION/getting-started/getting-started-with-karpenter/cloudformation.yaml`. -Next, locate `KarpenterController IAM Role` ARN (i.e., ARN of the resource created in [Create the KarpenterController IAM Role](../getting-started/getting-started-with-karpenter/#create-the-karpentercontroller-iam-role)) and pass them to the helm upgrade command. +Next, locate `KarpenterController IAM Role` ARN (i.e., ARN of the resource created in [Create the KarpenterController IAM Role](../getting-started/getting-started-with-karpenter/#create-the-karpentercontroller-iam-role)) and pass them to the Helm upgrade command. {{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step08-apply-helm-chart.sh" language="bash"%}} For information on upgrading Karpenter, see the [Upgrade Guide]({{< ref "./upgrading/upgrade-guide/" >}}). @@ -211,7 +211,7 @@ For information on upgrading Karpenter, see the [Upgrade Guide]({{< ref "./upgra ### How do I upgrade an EKS Cluster with Karpenter? -When upgrading an Amazon EKS cluster, [Karpenter's Drift feature]({{}}) can automatically upgrade the Karpenter-provisioned nodes to stay in-sync with the EKS control plane. Karpenter Drift is enabled by default starting v0.33.x. +When upgrading an Amazon EKS cluster, [Karpenter's Drift feature]({{}}) can automatically upgrade the Karpenter-provisioned nodes to stay in-sync with the EKS control plane. Karpenter Drift is enabled by default starting `0.33.0`. {{% alert title="Note" color="primary" %}} Karpenter's default [EC2NodeClass `amiFamily` configuration]({{}}) uses the latest EKS Optimized AL2 AMI for the same major and minor version as the EKS cluster's control plane, meaning that an upgrade of the control plane will cause Karpenter to auto-discover the new AMIs for that version. diff --git a/website/content/en/docs/getting-started/_index.md b/website/content/en/docs/getting-started/_index.md index a70f6bbc422b..cebdf66a99fd 100644 --- a/website/content/en/docs/getting-started/_index.md +++ b/website/content/en/docs/getting-started/_index.md @@ -21,6 +21,6 @@ Learn more about Karpenter and how to get started below. * [Karpenter EKS Best Practices](https://aws.github.io/aws-eks-best-practices/karpenter/) guide * [EC2 Spot Workshop for Karpenter](https://ec2spotworkshops.com/karpenter.html) * [EKS Karpenter Workshop](https://www.eksworkshop.com/docs/autoscaling/compute/karpenter/) -* [Advanced EKS Immersion Karpenter Workshop](https://catalog.workshops.aws/eks-advanced/karpenter/) +* [Advanced EKS Immersion Karpenter Workshop](https://catalog.us-east-1.prod.workshops.aws/workshops/76a5dd80-3249-4101-8726-9be3eeee09b2/en-US/autoscaling/karpenter) * [Karpenter Blueprints](https://github.com/aws-samples/karpenter-blueprints) * [Tutorial: Run Kubernetes Clusters for Less with Amazon EC2 Spot and Karpenter](https://community.aws/tutorials/run-kubernetes-clusters-for-less-with-amazon-ec2-spot-and-karpenter#step-6-optional-simulate-spot-interruption) diff --git a/website/content/en/docs/getting-started/getting-started-with-karpenter/_index.md b/website/content/en/docs/getting-started/getting-started-with-karpenter/_index.md index 0a1bb9085440..fa34d319bd24 100644 --- a/website/content/en/docs/getting-started/getting-started-with-karpenter/_index.md +++ b/website/content/en/docs/getting-started/getting-started-with-karpenter/_index.md @@ -32,7 +32,7 @@ Install these tools before proceeding: 1. [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2-linux.html) 2. `kubectl` - [the Kubernetes CLI](https://kubernetes.io/docs/tasks/tools/install-kubectl-linux/) -3. `eksctl` (>= v0.165.0) - [the CLI for AWS EKS](https://docs.aws.amazon.com/eks/latest/userguide/eksctl.html) +3. `eksctl` (>= v0.169.0) - [the CLI for AWS EKS](https://docs.aws.amazon.com/eks/latest/userguide/eksctl.html) 4. `helm` - [the package manager for Kubernetes](https://helm.sh/docs/intro/install/) [Configure the AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-quickstart.html) @@ -44,9 +44,9 @@ authenticate properly by running `aws sts get-caller-identity`. After setting up the tools, set the Karpenter and Kubernetes version: ```bash -export KARPENTER_NAMESPACE=kube-system -export KARPENTER_VERSION=v0.33.1 -export K8S_VERSION={{< param "latest_k8s_version" >}} +export KARPENTER_NAMESPACE="kube-system" +export KARPENTER_VERSION="0.35.4" +export K8S_VERSION="1.29" ``` Then set the following environment variable: @@ -58,7 +58,7 @@ If you open a new shell to run steps in this procedure, you need to set some or To remind yourself of these values, type: ```bash -echo $KARPENTER_NAMESPACE $KARPENTER_VERSION $K8S_VERSION $CLUSTER_NAME $AWS_DEFAULT_REGION $AWS_ACCOUNT_ID $TEMPOUT +echo "${KARPENTER_NAMESPACE}" "${KARPENTER_VERSION}" "${K8S_VERSION}" "${CLUSTER_NAME}" "${AWS_DEFAULT_REGION}" "${AWS_ACCOUNT_ID}" "${TEMPOUT}" "${ARM_AMI_ID}" "${AMD_AMI_ID}" "${GPU_AMI_ID}" ``` {{% /alert %}} @@ -75,7 +75,7 @@ The following cluster configuration will: * Use [AWS EKS managed node groups](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html) for the kube-system and karpenter namespaces. Uncomment fargateProfiles settings (and comment out managedNodeGroups settings) to use Fargate for both namespaces instead. * Set KARPENTER_IAM_ROLE_ARN variables. * Create a role to allow spot instances. -* Run helm to install karpenter +* Run Helm to install Karpenter {{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step02-create-cluster.sh" language="bash"%}} @@ -90,12 +90,18 @@ See [Enabling Windows support](https://docs.aws.amazon.com/eks/latest/userguide/ {{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step08-apply-helm-chart.sh" language="bash"%}} +{{% alert title="DNS Policy Notice" color="warning" %}} +Karpenter uses the `ClusterFirst` pod DNS policy by default. This is the Kubernetes cluster default and this ensures that Karpetner can reach-out to internal Kubernetes services during its lifetime. There may be cases where you do not have the DNS service that you are using on your cluster up-and-running before Karpenter starts up. The most common case of this is you want Karpenter to manage the node capacity where your DNS service pods are running. + +If you need Karpenter to manage the DNS service pods' capacity, this means that DNS won't be running when Karpenter starts-up. In this case, you will need to set the pod DNS policy to `Default` with `--set dnsPolicy=Default`. This will tell Karpenter to use the host's DNS resolution instead of the internal DNS resolution, ensuring that you don't have a dependency on the DNS service pods to run. More details on this issue can be found in the following Github issues: [#2186](https://github.com/aws/karpenter-provider-aws/issues/2186) and [#4947](https://github.com/aws/karpenter-provider-aws/issues/4947). +{{% /alert %}} + {{% alert title="Common Expression Language/Webhooks Notice" color="warning" %}} -Karpenter supports using [Kubernetes Common Expression Language](https://kubernetes.io/docs/reference/using-api/cel/) for validating its Custom Resource Definitions out-of-the-box; however, this feature is not supported on versions of Kubernetes < 1.25. If you are running an earlier version of Kubernetes, you will need to use the Karpenter admission webhooks for validation instead. You can enable these webhooks with `--set webhook.enabled=true` when applying the Karpenter helm chart. +Karpenter supports using [Kubernetes Common Expression Language](https://kubernetes.io/docs/reference/using-api/cel/) for validating its Custom Resource Definitions out-of-the-box; however, this feature is not supported on versions of Kubernetes < 1.25. If you are running an earlier version of Kubernetes, you will need to use the Karpenter admission webhooks for validation instead. You can enable these webhooks with `--set webhook.enabled=true` when applying the Karpenter Helm chart. {{% /alert %}} -{{% alert title="Pod Identity Support Notice" color="warning" %}} -Karpenter now supports using [Pod Identity](https://docs.aws.amazon.com/eks/latest/userguide/pod-identities.html) to authenticate AWS SDK to make API requests to AWS services using AWS Identity and Access Management (IAM) permissions. This feature not supported on versions of Kubernetes < 1.24. If you are running an earlier version of Kubernetes, you will need to use the [IAM Roles for Service Accounts(IRSA)](https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/setting-up-enable-IAM.html) for pod authentication instead. You can enable these IRSA with `--set "serviceAccount.annotations.eks\.amazonaws\.com/role-arn=${KARPENTER_IAM_ROLE_ARN}"` when applying the Karpenter helm chart. +{{% alert title="Pod Identity Supports Notice" color="warning" %}} +Karpenter now supports using [Pod Identity](https://docs.aws.amazon.com/eks/latest/userguide/pod-identities.html) to authenticate AWS SDK to make API requests to AWS services using AWS Identity and Access Management (IAM) permissions. This feature not supported on versions of Kubernetes < 1.24. If you are running an earlier version of Kubernetes, you will need to use the [IAM Roles for Service Accounts(IRSA)](https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/setting-up-enable-IAM.html) for pod authentication instead. You can enable these IRSA with `--set "serviceAccount.annotations.eks\.amazonaws\.com/role-arn=${KARPENTER_IAM_ROLE_ARN}"` when applying the Karpenter Helm chart. {{% /alert %}} {{% alert title="Warning" color="warning" %}} @@ -171,7 +177,7 @@ The section below covers advanced installation techniques for installing Karpent ### Private Clusters -You can optionally install Karpenter on a [private cluster](https://docs.aws.amazon.com/eks/latest/userguide/private-clusters.html#private-cluster-requirements) using the `eksctl` installation by setting `privateCluster.enabled` to true in your [ClusterConfig](https://eksctl.io/usage/eks-private-cluster/#eks-fully-private-cluster) and by setting `--set settings.isolatedVPC=true` when installing the `karpenter` helm chart. +You can optionally install Karpenter on a [private cluster](https://docs.aws.amazon.com/eks/latest/userguide/private-clusters.html#private-cluster-requirements) using the `eksctl` installation by setting `privateCluster.enabled` to true in your [ClusterConfig](https://eksctl.io/usage/eks-private-cluster/#eks-fully-private-cluster) and by setting `--set settings.isolatedVPC=true` when installing the `karpenter` Helm chart. ```bash privateCluster: @@ -188,6 +194,7 @@ com.amazonaws..s3 – For pulling container images com.amazonaws..sts – For IAM roles for service accounts com.amazonaws..ssm - For resolving default AMIs com.amazonaws..sqs - For accessing SQS if using interruption handling +com.amazonaws..eks - For Karpenter to discover the cluster endpoint ``` If you do not currently have these endpoints surfaced in your VPC, you can add the endpoints by running @@ -233,7 +240,7 @@ caused by: Post "https://api.pricing.us-east-1.amazonaws.com/": dial tcp 52.94.2 Kubernetes uses [FlowSchemas](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/#flowschema) and [PriorityLevelConfigurations](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/#prioritylevelconfiguration) to map calls to the API server into buckets which determine each user agent's throttling limits. -By default, Karpenter is installed into the `kube-system` namespace, which leverages the `system-leader-election` and `kube-system-service-accounts` [FlowSchemas] to map calls from the `kube-system` namespace to the `leader-election` and `workload-high` PriorityLevelConfigurations respectively. By putting Karpenter in these PriorityLevelConfigurations, we ensure that Karpenter and other critical cluster components are able to run even if other components on the cluster are throttled in other PriorityLevelConfigurations. +By default, Karpenter is installed into the `kube-system` namespace, which leverages the `system-leader-election` and `kube-system-service-accounts` [FlowSchemas](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/#flowschema) to map calls from the `kube-system` namespace to the `leader-election` and `workload-high` PriorityLevelConfigurations respectively. By putting Karpenter in these PriorityLevelConfigurations, we ensure that Karpenter and other critical cluster components are able to run even if other components on the cluster are throttled in other PriorityLevelConfigurations. If you install Karpenter in a different namespace than the default `kube-system` namespace, Karpenter will not be put into these higher-priority FlowSchemas by default. Instead, you will need to create custom FlowSchemas for the namespace and service account where Karpenter is installed to ensure that requests are put into this higher PriorityLevelConfiguration. diff --git a/website/content/en/docs/getting-started/getting-started-with-karpenter/cloudformation.yaml b/website/content/en/docs/getting-started/getting-started-with-karpenter/cloudformation.yaml index 94c233b7c6c8..207761007d2d 100644 --- a/website/content/en/docs/getting-started/getting-started-with-karpenter/cloudformation.yaml +++ b/website/content/en/docs/getting-started/getting-started-with-karpenter/cloudformation.yaml @@ -35,20 +35,36 @@ Resources: "Version": "2012-10-17", "Statement": [ { - "Sid": "AllowScopedEC2InstanceActions", + "Sid": "AllowScopedEC2InstanceAccessActions", "Effect": "Allow", "Resource": [ "arn:${AWS::Partition}:ec2:${AWS::Region}::image/*", "arn:${AWS::Partition}:ec2:${AWS::Region}::snapshot/*", "arn:${AWS::Partition}:ec2:${AWS::Region}:*:security-group/*", - "arn:${AWS::Partition}:ec2:${AWS::Region}:*:subnet/*", - "arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*" + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:subnet/*" ], "Action": [ "ec2:RunInstances", "ec2:CreateFleet" ] }, + { + "Sid": "AllowScopedEC2LaunchTemplateAccessActions", + "Effect": "Allow", + "Resource": "arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*", + "Action": [ + "ec2:RunInstances", + "ec2:CreateFleet" + ], + "Condition": { + "StringEquals": { + "aws:ResourceTag/kubernetes.io/cluster/${ClusterName}": "owned" + }, + "StringLike": { + "aws:ResourceTag/karpenter.sh/nodepool": "*" + } + } + }, { "Sid": "AllowScopedEC2InstanceActionsWithTags", "Effect": "Allow", diff --git a/website/content/en/docs/getting-started/getting-started-with-karpenter/grafana-values.yaml b/website/content/en/docs/getting-started/getting-started-with-karpenter/grafana-values.yaml index 55f55bf8cb41..67d28f71217c 100644 --- a/website/content/en/docs/getting-started/getting-started-with-karpenter/grafana-values.yaml +++ b/website/content/en/docs/getting-started/getting-started-with-karpenter/grafana-values.yaml @@ -22,6 +22,6 @@ dashboardProviders: dashboards: default: capacity-dashboard: - url: https://karpenter.sh/v0.33/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json + url: https://karpenter.sh/v0.35/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json performance-dashboard: - url: https://karpenter.sh/v0.33/getting-started/getting-started-with-karpenter/karpenter-performance-dashboard.json + url: https://karpenter.sh/v0.35/getting-started/getting-started-with-karpenter/karpenter-performance-dashboard.json diff --git a/website/content/en/docs/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json b/website/content/en/docs/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json index 9d9cb6ad14a8..d474d01f4e16 100644 --- a/website/content/en/docs/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json +++ b/website/content/en/docs/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json @@ -484,7 +484,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by ($distribution_filter)(\n karpenter_pods_state{arch=~\"$arch\", capacity_type=~\"$capacity_type\", instance_type=~\"$instance_type\", provisioner=~\"$provisioner\"}\n)", + "expr": "sum by ($distribution_filter)(\n karpenter_pods_state{arch=~\"$arch\", capacity_type=~\"$capacity_type\", instance_type=~\"$instance_type\", nodepool=~\"$nodepool\"}\n)", "legendFormat": "{{label_name}}", "range": true, "refId": "A" @@ -588,7 +588,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "karpenter_provisioner_usage{resource_type=\"cpu\"} / karpenter_provisioner_limit{resource_type=\"cpu\"}", + "expr": "karpenter_nodepool_usage{resource_type=\"cpu\"} / karpenter_nodepool_limit{resource_type=\"cpu\"}", "format": "table", "instant": true, "legendFormat": "CPU Limit Utilization", @@ -602,7 +602,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "count by (provisioner)(karpenter_nodes_allocatable{provisioner!=\"N/A\",resource_type=\"cpu\"}) # Selects a single resource type to get node count", + "expr": "count by (nodepool)(karpenter_nodes_allocatable{nodepool!=\"N/A\",resource_type=\"cpu\"}) # Selects a single resource type to get node count", "format": "table", "hide": false, "instant": true, @@ -616,7 +616,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "karpenter_provisioner_usage{resource_type=\"memory\"} / karpenter_provisioner_limit{resource_type=\"memory\"}", + "expr": "karpenter_nodepool_usage{resource_type=\"memory\"} / karpenter_nodepool_limit{resource_type=\"memory\"}", "format": "table", "hide": false, "instant": true, @@ -631,7 +631,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum by (provisioner)(karpenter_nodes_allocatable{provisioner!=\"N/A\",resource_type=\"cpu\"})", + "expr": "sum by (nodepool)(karpenter_nodes_allocatable{nodepool!=\"N/A\",resource_type=\"cpu\"})", "format": "table", "hide": false, "instant": true, @@ -645,7 +645,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum by (provisioner)(karpenter_nodes_allocatable{provisioner!=\"N/A\",resource_type=\"memory\"})", + "expr": "sum by (nodepool)(karpenter_nodes_allocatable{nodepool!=\"N/A\",resource_type=\"memory\"})", "format": "table", "hide": false, "instant": true, @@ -653,12 +653,12 @@ "refId": "Memory Capacity" } ], - "title": "Provisioner Summary", + "title": "Nodepool Summary", "transformations": [ { "id": "seriesToColumns", "options": { - "byField": "provisioner" + "byField": "nodepool" } }, { @@ -697,7 +697,7 @@ "instance 2": 12, "job 1": 9, "job 2": 13, - "provisioner": 0, + "nodepool": 0, "resource_type 1": 10, "resource_type 2": 14 }, @@ -714,7 +714,7 @@ "instance": "", "instance 1": "", "job": "", - "provisioner": "Provisioner" + "nodepool": "Nodepool" } } } @@ -804,7 +804,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "(count(karpenter_nodes_allocatable{arch=~\"$arch\",capacity_type=\"spot\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"}) or vector(0)) / count(karpenter_nodes_allocatable{arch=~\"$arch\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"})", + "expr": "(count(karpenter_nodes_allocatable{arch=~\"$arch\",capacity_type=\"spot\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}) or vector(0)) / count(karpenter_nodes_allocatable{arch=~\"$arch\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"})", "legendFormat": "Percentage", "range": true, "refId": "A" @@ -946,7 +946,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "((karpenter_nodes_total_daemon_requests{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0) + \n(karpenter_nodes_total_pod_requests{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0)) / \nkarpenter_nodes_allocatable{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"}", + "expr": "((karpenter_nodes_total_daemon_requests{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0) + \n(karpenter_nodes_total_pod_requests{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0)) / \nkarpenter_nodes_allocatable{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}", "format": "table", "hide": false, "instant": true, @@ -961,7 +961,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "((karpenter_nodes_total_daemon_requests{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0) + \n(karpenter_nodes_total_pod_requests{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0)) / \nkarpenter_nodes_allocatable{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"}", + "expr": "((karpenter_nodes_total_daemon_requests{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0) + \n(karpenter_nodes_total_pod_requests{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0)) / \nkarpenter_nodes_allocatable{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}", "format": "table", "hide": false, "instant": true, @@ -976,7 +976,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "karpenter_nodes_total_daemon_requests{resource_type=\"pods\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"} + \nkarpenter_nodes_total_pod_requests{resource_type=\"pods\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"}", + "expr": "karpenter_nodes_total_daemon_requests{resource_type=\"pods\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} + \nkarpenter_nodes_total_pod_requests{resource_type=\"pods\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}", "format": "table", "hide": false, "instant": true, @@ -1091,9 +1091,9 @@ "os 1": true, "os 2": true, "os 3": true, - "provisioner 1": false, - "provisioner 2": true, - "provisioner 3": true, + "nodepool 1": false, + "nodepool 2": true, + "nodepool 3": true, "resource_type": true, "resource_type 1": true, "resource_type 2": true, @@ -1161,9 +1161,9 @@ "os 1": 23, "os 2": 41, "os 3": 61, - "provisioner 1": 2, - "provisioner 2": 42, - "provisioner 3": 62, + "nodepool 1": 2, + "nodepool 2": 42, + "nodepool 3": 62, "resource_type 1": 24, "resource_type 2": 43, "resource_type 3": 63, @@ -1190,7 +1190,7 @@ "instance_type": "Instance Type", "instance_type 1": "Instance Type", "node_name": "Node Name", - "provisioner 1": "Provisioner", + "nodepool 1": "Nodepool", "zone 1": "Zone" } } @@ -1237,14 +1237,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "definition": "label_values(karpenter_nodes_allocatable, provisioner)", + "definition": "label_values(karpenter_nodes_allocatable, nodepool)", "hide": 0, "includeAll": true, "multi": true, - "name": "provisioner", + "name": "nodepool", "options": [], "query": { - "query": "label_values(karpenter_nodes_allocatable, provisioner)", + "query": "label_values(karpenter_nodes_allocatable, nodepool)", "refId": "StandardVariableQuery" }, "refresh": 2, @@ -1376,8 +1376,8 @@ { "current": { "selected": true, - "text": "provisioner", - "value": "provisioner" + "text": "nodepool", + "value": "nodepool" }, "hide": 0, "includeAll": false, @@ -1411,8 +1411,8 @@ }, { "selected": true, - "text": "provisioner", - "value": "provisioner" + "text": "nodepool", + "value": "nodepool" }, { "selected": false, @@ -1420,7 +1420,7 @@ "value": "zone" } ], - "query": "arch,capacity_type,instance_type,namespace,node,provisioner,zone", + "query": "arch,capacity_type,instance_type,namespace,node,nodepool,zone", "queryValue": "", "skipUrlSync": false, "type": "custom" diff --git a/website/content/en/docs/getting-started/getting-started-with-karpenter/prometheus-values.yaml b/website/content/en/docs/getting-started/getting-started-with-karpenter/prometheus-values.yaml index f7c34fd90dea..588762a8c9dc 100644 --- a/website/content/en/docs/getting-started/getting-started-with-karpenter/prometheus-values.yaml +++ b/website/content/en/docs/getting-started/getting-started-with-karpenter/prometheus-values.yaml @@ -13,8 +13,10 @@ extraScrapeConfigs: | - role: endpoints namespaces: names: - - kube-system + - $KARPENTER_NAMESPACE relabel_configs: - - source_labels: [__meta_kubernetes_endpoint_port_name] - regex: http-metrics + - source_labels: + - __meta_kubernetes_endpoints_name + - __meta_kubernetes_endpoint_port_name action: keep + regex: karpenter;http-metrics diff --git a/website/content/en/docs/getting-started/getting-started-with-karpenter/scripts/add-monitoring.sh b/website/content/en/docs/getting-started/getting-started-with-karpenter/scripts/add-monitoring.sh deleted file mode 100755 index 6d2332b06be2..000000000000 --- a/website/content/en/docs/getting-started/getting-started-with-karpenter/scripts/add-monitoring.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -set -euo pipefail #fail if one step fails - -if [ "$#" -ne 1 ] -then - echo "Missing required Karpenter version. Usage: add-monitoring.sh v0.0.1" - exit 1 -fi - -export KARPENTER_VERSION=$1 - -declare -a steps=( - step01-config.sh - step09-add-prometheus-grafana.sh - step10-add-grafana-port-forward.sh - step11-grafana-get-password.sh -) - -for step in "${steps[@]}"; do - echo "$step" - source $step -done diff --git a/website/content/en/docs/getting-started/getting-started-with-karpenter/scripts/add-provisioner.sh b/website/content/en/docs/getting-started/getting-started-with-karpenter/scripts/add-provisioner.sh deleted file mode 100755 index 51f286de0c3c..000000000000 --- a/website/content/en/docs/getting-started/getting-started-with-karpenter/scripts/add-provisioner.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -set -euo pipefail #fail if one step fails - -declare -a steps=( - step01-config.sh - step12-add-provisioner.sh - step13-automatic-node-provisioning.sh - step14-automatic-node-termination.sh -) - -for step in "${steps[@]}"; do - echo "$step" - source $step -done diff --git a/website/content/en/docs/getting-started/getting-started-with-karpenter/scripts/add-roles.sh b/website/content/en/docs/getting-started/getting-started-with-karpenter/scripts/add-roles.sh deleted file mode 100755 index cf0656f3db3d..000000000000 --- a/website/content/en/docs/getting-started/getting-started-with-karpenter/scripts/add-roles.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -set -euo pipefail #fail if one step fails - -if [ "$#" -ne 1 ] -then - echo "Missing required Karpenter version. Usage: setup-roles.sh v0.0.1" - exit 1 -fi - -export KARPENTER_VERSION=$1 -SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) - -declare -a steps=( - step03-iam-cloud-formation.sh - step04-grant-access.sh - step05-controller-iam.sh - step06-add-spot-role.sh -) - -for step in "${steps[@]}"; do - echo "$step" - source "$SCRIPT_DIR/$step" -done diff --git a/website/content/en/docs/getting-started/getting-started-with-karpenter/scripts/cleanup.sh b/website/content/en/docs/getting-started/getting-started-with-karpenter/scripts/cleanup.sh deleted file mode 100755 index 52851e7f975a..000000000000 --- a/website/content/en/docs/getting-started/getting-started-with-karpenter/scripts/cleanup.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -declare -a steps=( - step01-config.sh - step16-cleanup.sh -) - -for step in "${steps[@]}"; do - echo "$step" - source $step -done diff --git a/website/content/en/docs/getting-started/getting-started-with-karpenter/scripts/install.sh b/website/content/en/docs/getting-started/getting-started-with-karpenter/scripts/install.sh deleted file mode 100755 index e0ebfb32e511..000000000000 --- a/website/content/en/docs/getting-started/getting-started-with-karpenter/scripts/install.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -set -euo pipefail #fail if one step fails - -if [ "$#" -ne 1 ] -then - echo "Missing required Karpenter version. Usage: install.sh v0.0.1" - exit 1 -fi - -export KARPENTER_VERSION=$1 - -declare -a steps=( - step01-config.sh - step02-create-cluster.sh - step03-iam-cloud-formation.sh - step04-grant-access.sh - step05-controller-iam.sh - step06-add-spot-role.sh - step08-apply-helm-chart.sh -) - -for step in "${steps[@]}"; do - echo "$step" - source $step -done diff --git a/website/content/en/docs/getting-started/getting-started-with-karpenter/scripts/step01-config.sh b/website/content/en/docs/getting-started/getting-started-with-karpenter/scripts/step01-config.sh index a3af512d02ac..b8209b641640 100755 --- a/website/content/en/docs/getting-started/getting-started-with-karpenter/scripts/step01-config.sh +++ b/website/content/en/docs/getting-started/getting-started-with-karpenter/scripts/step01-config.sh @@ -2,4 +2,7 @@ export AWS_PARTITION="aws" # if you are not using standard partitions, you may n export CLUSTER_NAME="${USER}-karpenter-demo" export AWS_DEFAULT_REGION="us-west-2" export AWS_ACCOUNT_ID="$(aws sts get-caller-identity --query Account --output text)" -export TEMPOUT=$(mktemp) +export TEMPOUT="$(mktemp)" +export ARM_AMI_ID="$(aws ssm get-parameter --name /aws/service/eks/optimized-ami/${K8S_VERSION}/amazon-linux-2-arm64/recommended/image_id --query Parameter.Value --output text)" +export AMD_AMI_ID="$(aws ssm get-parameter --name /aws/service/eks/optimized-ami/${K8S_VERSION}/amazon-linux-2/recommended/image_id --query Parameter.Value --output text)" +export GPU_AMI_ID="$(aws ssm get-parameter --name /aws/service/eks/optimized-ami/${K8S_VERSION}/amazon-linux-2-gpu/recommended/image_id --query Parameter.Value --output text)" diff --git a/website/content/en/docs/getting-started/getting-started-with-karpenter/scripts/step02-create-cluster.sh b/website/content/en/docs/getting-started/getting-started-with-karpenter/scripts/step02-create-cluster.sh index 105affa1d517..cdad96eeca23 100755 --- a/website/content/en/docs/getting-started/getting-started-with-karpenter/scripts/step02-create-cluster.sh +++ b/website/content/en/docs/getting-started/getting-started-with-karpenter/scripts/step02-create-cluster.sh @@ -1,4 +1,4 @@ -curl -fsSL https://raw.githubusercontent.com/aws/karpenter-provider-aws/"${KARPENTER_VERSION}"/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml > $TEMPOUT \ +curl -fsSL https://raw.githubusercontent.com/aws/karpenter-provider-aws/v"${KARPENTER_VERSION}"/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml > "${TEMPOUT}" \ && aws cloudformation deploy \ --stack-name "Karpenter-${CLUSTER_NAME}" \ --template-file "${TEMPOUT}" \ @@ -19,14 +19,14 @@ metadata: iam: withOIDC: true podIdentityAssociations: - - namespace: "${KARPENTER_NAMESPACE}" - serviceAccountName: karpenter - roleName: ${CLUSTER_NAME}-karpenter - permissionPolicyARNs: - - arn:${AWS_PARTITION}:iam::${AWS_ACCOUNT_ID}:policy/KarpenterControllerPolicy-${CLUSTER_NAME} + - namespace: "${KARPENTER_NAMESPACE}" + serviceAccountName: karpenter + roleName: ${CLUSTER_NAME}-karpenter + permissionPolicyARNs: + - arn:${AWS_PARTITION}:iam::${AWS_ACCOUNT_ID}:policy/KarpenterControllerPolicy-${CLUSTER_NAME} ## Optionally run on fargate or on k8s 1.23 -# Pod Identity is not available on fargate +# Pod Identity is not available on fargate # https://docs.aws.amazon.com/eks/latest/userguide/pod-identities.html # iam: # withOIDC: true @@ -58,7 +58,7 @@ managedNodeGroups: maxSize: 10 addons: - - name: eks-pod-identity-agent +- name: eks-pod-identity-agent ## Optionally run on fargate # fargateProfiles: @@ -67,7 +67,7 @@ addons: # - namespace: "${KARPENTER_NAMESPACE}" EOF -export CLUSTER_ENDPOINT="$(aws eks describe-cluster --name ${CLUSTER_NAME} --query "cluster.endpoint" --output text)" +export CLUSTER_ENDPOINT="$(aws eks describe-cluster --name "${CLUSTER_NAME}" --query "cluster.endpoint" --output text)" export KARPENTER_IAM_ROLE_ARN="arn:${AWS_PARTITION}:iam::${AWS_ACCOUNT_ID}:role/${CLUSTER_NAME}-karpenter" -echo $CLUSTER_ENDPOINT $KARPENTER_IAM_ROLE_ARN +echo "${CLUSTER_ENDPOINT} ${KARPENTER_IAM_ROLE_ARN}" diff --git a/website/content/en/docs/getting-started/getting-started-with-karpenter/scripts/step03-iam-cloud-formation.sh b/website/content/en/docs/getting-started/getting-started-with-karpenter/scripts/step03-iam-cloud-formation.sh index b8e610f7bee6..54e826db269b 100755 --- a/website/content/en/docs/getting-started/getting-started-with-karpenter/scripts/step03-iam-cloud-formation.sh +++ b/website/content/en/docs/getting-started/getting-started-with-karpenter/scripts/step03-iam-cloud-formation.sh @@ -1,6 +1,6 @@ -TEMPOUT=$(mktemp) +TEMPOUT="$(mktemp)" -curl -fsSL https://raw.githubusercontent.com/aws/karpenter-provider-aws/"${KARPENTER_VERSION}"/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml > $TEMPOUT \ +curl -fsSL https://raw.githubusercontent.com/aws/karpenter-provider-aws/v"${KARPENTER_VERSION}"/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml > "${TEMPOUT}" \ && aws cloudformation deploy \ --stack-name "Karpenter-${CLUSTER_NAME}" \ --template-file "${TEMPOUT}" \ diff --git a/website/content/en/docs/getting-started/getting-started-with-karpenter/scripts/step09-add-prometheus-grafana.sh b/website/content/en/docs/getting-started/getting-started-with-karpenter/scripts/step09-add-prometheus-grafana.sh index a539f7491e10..1107c2ec1d24 100755 --- a/website/content/en/docs/getting-started/getting-started-with-karpenter/scripts/step09-add-prometheus-grafana.sh +++ b/website/content/en/docs/getting-started/getting-started-with-karpenter/scripts/step09-add-prometheus-grafana.sh @@ -4,8 +4,8 @@ helm repo update kubectl create namespace monitoring -curl -fsSL https://raw.githubusercontent.com/aws/karpenter-provider-aws/"${KARPENTER_VERSION}"/website/content/en/preview/getting-started/getting-started-with-karpenter/prometheus-values.yaml | tee prometheus-values.yaml +curl -fsSL https://raw.githubusercontent.com/aws/karpenter-provider-aws/v"${KARPENTER_VERSION}"/website/content/en/preview/getting-started/getting-started-with-karpenter/prometheus-values.yaml | envsubst | tee prometheus-values.yaml helm install --namespace monitoring prometheus prometheus-community/prometheus --values prometheus-values.yaml -curl -fsSL https://raw.githubusercontent.com/aws/karpenter-provider-aws/"${KARPENTER_VERSION}"/website/content/en/preview/getting-started/getting-started-with-karpenter/grafana-values.yaml | tee grafana-values.yaml +curl -fsSL https://raw.githubusercontent.com/aws/karpenter-provider-aws/v"${KARPENTER_VERSION}"/website/content/en/preview/getting-started/getting-started-with-karpenter/grafana-values.yaml | tee grafana-values.yaml helm install --namespace monitoring grafana grafana-charts/grafana --values grafana-values.yaml diff --git a/website/content/en/docs/getting-started/getting-started-with-karpenter/scripts/step12-add-nodepool.sh b/website/content/en/docs/getting-started/getting-started-with-karpenter/scripts/step12-add-nodepool.sh index 8c518e9e74b8..33f1cb553b1b 100755 --- a/website/content/en/docs/getting-started/getting-started-with-karpenter/scripts/step12-add-nodepool.sh +++ b/website/content/en/docs/getting-started/getting-started-with-karpenter/scripts/step12-add-nodepool.sh @@ -23,6 +23,8 @@ spec: operator: Gt values: ["2"] nodeClassRef: + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass name: default limits: cpu: 1000 @@ -43,4 +45,9 @@ spec: securityGroupSelectorTerms: - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + amiSelectorTerms: + - id: "${ARM_AMI_ID}" + - id: "${AMD_AMI_ID}" +# - id: "${GPU_AMI_ID}" # <- GPU Optimized AMD AMI +# - name: "amazon-eks-node-${K8S_VERSION}-*" # <- automatically upgrade when a new AL2 EKS Optimized AMI is released. This is unsafe for production workloads. Validate AMIs in lower environments before deploying them to production. EOF diff --git a/website/content/en/docs/getting-started/getting-started-with-karpenter/scripts/step16-delete-node.sh b/website/content/en/docs/getting-started/getting-started-with-karpenter/scripts/step16-delete-node.sh index 9d431160dda0..93b1ac792ade 100755 --- a/website/content/en/docs/getting-started/getting-started-with-karpenter/scripts/step16-delete-node.sh +++ b/website/content/en/docs/getting-started/getting-started-with-karpenter/scripts/step16-delete-node.sh @@ -1 +1 @@ -kubectl delete node $NODE_NAME +kubectl delete node "${NODE_NAME}" diff --git a/website/content/en/docs/getting-started/getting-started-with-karpenter/scripts/step17-cleanup.sh b/website/content/en/docs/getting-started/getting-started-with-karpenter/scripts/step17-cleanup.sh index dc84b673389f..f694ac3797f4 100755 --- a/website/content/en/docs/getting-started/getting-started-with-karpenter/scripts/step17-cleanup.sh +++ b/website/content/en/docs/getting-started/getting-started-with-karpenter/scripts/step17-cleanup.sh @@ -1,6 +1,6 @@ helm uninstall karpenter --namespace "${KARPENTER_NAMESPACE}" aws cloudformation delete-stack --stack-name "Karpenter-${CLUSTER_NAME}" -aws ec2 describe-launch-templates --filters Name=tag:karpenter.k8s.aws/cluster,Values=${CLUSTER_NAME} | +aws ec2 describe-launch-templates --filters "Name=tag:karpenter.k8s.aws/cluster,Values=${CLUSTER_NAME}" | jq -r ".LaunchTemplates[].LaunchTemplateName" | xargs -I{} aws ec2 delete-launch-template --launch-template-name {} eksctl delete cluster --name "${CLUSTER_NAME}" diff --git a/website/content/en/docs/getting-started/migrating-from-cas/_index.md b/website/content/en/docs/getting-started/migrating-from-cas/_index.md index 52489ae67a5f..398bf41e3eb8 100644 --- a/website/content/en/docs/getting-started/migrating-from-cas/_index.md +++ b/website/content/en/docs/getting-started/migrating-from-cas/_index.md @@ -92,10 +92,10 @@ One for your Karpenter node role and one for your existing node group. First set the Karpenter release you want to deploy. ```bash -export KARPENTER_VERSION=v0.33.1 +export KARPENTER_VERSION="0.35.4" ``` -We can now generate a full Karpenter deployment yaml from the helm chart. +We can now generate a full Karpenter deployment yaml from the Helm chart. {{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step08-generate-chart.sh" language="bash" %}} @@ -133,7 +133,7 @@ Now that our deployment is ready we can create the karpenter namespace, create t ## Create default NodePool -We need to create a default NodePool so Karpenter knows what types of nodes we want for unscheduled workloads. You can refer to some of the [example NodePool](https://github.com/aws/karpenter/tree/v0.33.1/examples/v1beta1) for specific needs. +We need to create a default NodePool so Karpenter knows what types of nodes we want for unscheduled workloads. You can refer to some of the [example NodePool](https://github.com/aws/karpenter/tree/v0.35.4/examples/v1beta1) for specific needs. {{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step10-create-nodepool.sh" language="bash" %}} diff --git a/website/content/en/docs/getting-started/migrating-from-cas/scripts/step01-env.sh b/website/content/en/docs/getting-started/migrating-from-cas/scripts/step01-env.sh index 20645685137b..b3ee0fa3d0ef 100644 --- a/website/content/en/docs/getting-started/migrating-from-cas/scripts/step01-env.sh +++ b/website/content/en/docs/getting-started/migrating-from-cas/scripts/step01-env.sh @@ -1,6 +1,10 @@ AWS_PARTITION="aws" # if you are not using standard partitions, you may need to configure to aws-cn / aws-us-gov AWS_REGION="$(aws configure list | grep region | tr -s " " | cut -d" " -f3)" -OIDC_ENDPOINT="$(aws eks describe-cluster --name ${CLUSTER_NAME} \ +OIDC_ENDPOINT="$(aws eks describe-cluster --name "${CLUSTER_NAME}" \ --query "cluster.identity.oidc.issuer" --output text)" AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query 'Account' \ --output text) +K8S_VERSION=1.28 +ARM_AMI_ID="$(aws ssm get-parameter --name /aws/service/eks/optimized-ami/${K8S_VERSION}/amazon-linux-2-arm64/recommended/image_id --query Parameter.Value --output text)" +AMD_AMI_ID="$(aws ssm get-parameter --name /aws/service/eks/optimized-ami/${K8S_VERSION}/amazon-linux-2/recommended/image_id --query Parameter.Value --output text)" +GPU_AMI_ID="$(aws ssm get-parameter --name /aws/service/eks/optimized-ami/${K8S_VERSION}/amazon-linux-2-gpu/recommended/image_id --query Parameter.Value --output text)" diff --git a/website/content/en/docs/getting-started/migrating-from-cas/scripts/step03-node-policies.sh b/website/content/en/docs/getting-started/migrating-from-cas/scripts/step03-node-policies.sh index d57f79039d04..fbc5455e541b 100644 --- a/website/content/en/docs/getting-started/migrating-from-cas/scripts/step03-node-policies.sh +++ b/website/content/en/docs/getting-started/migrating-from-cas/scripts/step03-node-policies.sh @@ -1,11 +1,11 @@ aws iam attach-role-policy --role-name "KarpenterNodeRole-${CLUSTER_NAME}" \ - --policy-arn arn:${AWS_PARTITION}:iam::aws:policy/AmazonEKSWorkerNodePolicy + --policy-arn "arn:${AWS_PARTITION}:iam::aws:policy/AmazonEKSWorkerNodePolicy" aws iam attach-role-policy --role-name "KarpenterNodeRole-${CLUSTER_NAME}" \ - --policy-arn arn:${AWS_PARTITION}:iam::aws:policy/AmazonEKS_CNI_Policy + --policy-arn "arn:${AWS_PARTITION}:iam::aws:policy/AmazonEKS_CNI_Policy" aws iam attach-role-policy --role-name "KarpenterNodeRole-${CLUSTER_NAME}" \ - --policy-arn arn:${AWS_PARTITION}:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly + --policy-arn "arn:${AWS_PARTITION}:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" aws iam attach-role-policy --role-name "KarpenterNodeRole-${CLUSTER_NAME}" \ - --policy-arn arn:${AWS_PARTITION}:iam::aws:policy/AmazonSSMManagedInstanceCore + --policy-arn "arn:${AWS_PARTITION}:iam::aws:policy/AmazonSSMManagedInstanceCore" diff --git a/website/content/en/docs/getting-started/migrating-from-cas/scripts/step04-controller-iam.sh b/website/content/en/docs/getting-started/migrating-from-cas/scripts/step04-controller-iam.sh index 70c59e826393..cc3d7f929986 100644 --- a/website/content/en/docs/getting-started/migrating-from-cas/scripts/step04-controller-iam.sh +++ b/website/content/en/docs/getting-started/migrating-from-cas/scripts/step04-controller-iam.sh @@ -19,7 +19,7 @@ cat << EOF > controller-trust-policy.json } EOF -aws iam create-role --role-name KarpenterControllerRole-${CLUSTER_NAME} \ +aws iam create-role --role-name "KarpenterControllerRole-${CLUSTER_NAME}" \ --assume-role-policy-document file://controller-trust-policy.json cat << EOF > controller-policy.json @@ -138,6 +138,6 @@ cat << EOF > controller-policy.json } EOF -aws iam put-role-policy --role-name KarpenterControllerRole-${CLUSTER_NAME} \ - --policy-name KarpenterControllerPolicy-${CLUSTER_NAME} \ +aws iam put-role-policy --role-name "KarpenterControllerRole-${CLUSTER_NAME}" \ + --policy-name "KarpenterControllerPolicy-${CLUSTER_NAME}" \ --policy-document file://controller-policy.json diff --git a/website/content/en/docs/getting-started/migrating-from-cas/scripts/step05-tag-subnets.sh b/website/content/en/docs/getting-started/migrating-from-cas/scripts/step05-tag-subnets.sh index de972ea2bddd..47df188dc87d 100644 --- a/website/content/en/docs/getting-started/migrating-from-cas/scripts/step05-tag-subnets.sh +++ b/website/content/en/docs/getting-started/migrating-from-cas/scripts/step05-tag-subnets.sh @@ -1,6 +1,6 @@ -for NODEGROUP in $(aws eks list-nodegroups --cluster-name ${CLUSTER_NAME} \ - --query 'nodegroups' --output text); do aws ec2 create-tags \ +for NODEGROUP in $(aws eks list-nodegroups --cluster-name "${CLUSTER_NAME}" --query 'nodegroups' --output text); do + aws ec2 create-tags \ --tags "Key=karpenter.sh/discovery,Value=${CLUSTER_NAME}" \ - --resources $(aws eks describe-nodegroup --cluster-name ${CLUSTER_NAME} \ - --nodegroup-name $NODEGROUP --query 'nodegroup.subnets' --output text ) + --resources "$(aws eks describe-nodegroup --cluster-name "${CLUSTER_NAME}" \ + --nodegroup-name "${NODEGROUP}" --query 'nodegroup.subnets' --output text )" done diff --git a/website/content/en/docs/getting-started/migrating-from-cas/scripts/step06-tag-security-groups.sh b/website/content/en/docs/getting-started/migrating-from-cas/scripts/step06-tag-security-groups.sh index 397e40904cee..c63bde3b78dc 100644 --- a/website/content/en/docs/getting-started/migrating-from-cas/scripts/step06-tag-security-groups.sh +++ b/website/content/en/docs/getting-started/migrating-from-cas/scripts/step06-tag-security-groups.sh @@ -1,22 +1,22 @@ -NODEGROUP=$(aws eks list-nodegroups --cluster-name ${CLUSTER_NAME} \ +NODEGROUP=$(aws eks list-nodegroups --cluster-name "${CLUSTER_NAME}" \ --query 'nodegroups[0]' --output text) -LAUNCH_TEMPLATE=$(aws eks describe-nodegroup --cluster-name ${CLUSTER_NAME} \ - --nodegroup-name ${NODEGROUP} --query 'nodegroup.launchTemplate.{id:id,version:version}' \ +LAUNCH_TEMPLATE=$(aws eks describe-nodegroup --cluster-name "${CLUSTER_NAME}" \ + --nodegroup-name "${NODEGROUP}" --query 'nodegroup.launchTemplate.{id:id,version:version}' \ --output text | tr -s "\t" ",") # If your EKS setup is configured to use only Cluster security group, then please execute - SECURITY_GROUPS=$(aws eks describe-cluster \ - --name ${CLUSTER_NAME} --query "cluster.resourcesVpcConfig.clusterSecurityGroupId" --output text) + --name "${CLUSTER_NAME}" --query "cluster.resourcesVpcConfig.clusterSecurityGroupId" --output text) # If your setup uses the security groups in the Launch template of a managed node group, then : -SECURITY_GROUPS=$(aws ec2 describe-launch-template-versions \ - --launch-template-id ${LAUNCH_TEMPLATE%,*} --versions ${LAUNCH_TEMPLATE#*,} \ +SECURITY_GROUPS="$(aws ec2 describe-launch-template-versions \ + --launch-template-id "${LAUNCH_TEMPLATE%,*}" --versions "${LAUNCH_TEMPLATE#*,}" \ --query 'LaunchTemplateVersions[0].LaunchTemplateData.[NetworkInterfaces[0].Groups||SecurityGroupIds]' \ - --output text) + --output text)" aws ec2 create-tags \ --tags "Key=karpenter.sh/discovery,Value=${CLUSTER_NAME}" \ - --resources ${SECURITY_GROUPS} + --resources "${SECURITY_GROUPS}" diff --git a/website/content/en/docs/getting-started/migrating-from-cas/scripts/step09-deploy.sh b/website/content/en/docs/getting-started/migrating-from-cas/scripts/step09-deploy.sh index 51714d78f6dd..e46742fd22ea 100644 --- a/website/content/en/docs/getting-started/migrating-from-cas/scripts/step09-deploy.sh +++ b/website/content/en/docs/getting-started/migrating-from-cas/scripts/step09-deploy.sh @@ -1,8 +1,8 @@ kubectl create namespace "${KARPENTER_NAMESPACE}" || true kubectl create -f \ - https://raw.githubusercontent.com/aws/karpenter-provider-aws/${KARPENTER_VERSION}/pkg/apis/crds/karpenter.sh_nodepools.yaml + "https://raw.githubusercontent.com/aws/karpenter-provider-aws/v${KARPENTER_VERSION}/pkg/apis/crds/karpenter.sh_nodepools.yaml" kubectl create -f \ - https://raw.githubusercontent.com/aws/karpenter-provider-aws/${KARPENTER_VERSION}/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml + "https://raw.githubusercontent.com/aws/karpenter-provider-aws/v${KARPENTER_VERSION}/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml" kubectl create -f \ - https://raw.githubusercontent.com/aws/karpenter-provider-aws/${KARPENTER_VERSION}/pkg/apis/crds/karpenter.sh_nodeclaims.yaml + "https://raw.githubusercontent.com/aws/karpenter-provider-aws/v${KARPENTER_VERSION}/pkg/apis/crds/karpenter.sh_nodeclaims.yaml" kubectl apply -f karpenter.yaml diff --git a/website/content/en/docs/getting-started/migrating-from-cas/scripts/step10-create-nodepool.sh b/website/content/en/docs/getting-started/migrating-from-cas/scripts/step10-create-nodepool.sh index 8c518e9e74b8..33f1cb553b1b 100644 --- a/website/content/en/docs/getting-started/migrating-from-cas/scripts/step10-create-nodepool.sh +++ b/website/content/en/docs/getting-started/migrating-from-cas/scripts/step10-create-nodepool.sh @@ -23,6 +23,8 @@ spec: operator: Gt values: ["2"] nodeClassRef: + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass name: default limits: cpu: 1000 @@ -43,4 +45,9 @@ spec: securityGroupSelectorTerms: - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + amiSelectorTerms: + - id: "${ARM_AMI_ID}" + - id: "${AMD_AMI_ID}" +# - id: "${GPU_AMI_ID}" # <- GPU Optimized AMD AMI +# - name: "amazon-eks-node-${K8S_VERSION}-*" # <- automatically upgrade when a new AL2 EKS Optimized AMI is released. This is unsafe for production workloads. Validate AMIs in lower environments before deploying them to production. EOF diff --git a/website/content/en/docs/getting-started/migrating-from-cas/scripts/step12-scale-multiple-ng.sh b/website/content/en/docs/getting-started/migrating-from-cas/scripts/step12-scale-multiple-ng.sh index d88a6d5c7236..a28c9759bcdb 100644 --- a/website/content/en/docs/getting-started/migrating-from-cas/scripts/step12-scale-multiple-ng.sh +++ b/website/content/en/docs/getting-started/migrating-from-cas/scripts/step12-scale-multiple-ng.sh @@ -1,5 +1,5 @@ -for NODEGROUP in $(aws eks list-nodegroups --cluster-name ${CLUSTER_NAME} \ - --query 'nodegroups' --output text); do aws eks update-nodegroup-config --cluster-name ${CLUSTER_NAME} \ - --nodegroup-name ${NODEGROUP} \ +for NODEGROUP in $(aws eks list-nodegroups --cluster-name "${CLUSTER_NAME}" \ + --query 'nodegroups' --output text); do aws eks update-nodegroup-config --cluster-name "${CLUSTER_NAME}" \ + --nodegroup-name "${NODEGROUP}" \ --scaling-config "minSize=1,maxSize=1,desiredSize=1" done diff --git a/website/content/en/docs/getting-started/migrating-from-cas/scripts/step12-scale-single-ng.sh b/website/content/en/docs/getting-started/migrating-from-cas/scripts/step12-scale-single-ng.sh index 51ad964c28a7..efce68394779 100644 --- a/website/content/en/docs/getting-started/migrating-from-cas/scripts/step12-scale-single-ng.sh +++ b/website/content/en/docs/getting-started/migrating-from-cas/scripts/step12-scale-single-ng.sh @@ -1,3 +1,3 @@ -aws eks update-nodegroup-config --cluster-name ${CLUSTER_NAME} \ - --nodegroup-name ${NODEGROUP} \ +aws eks update-nodegroup-config --cluster-name "${CLUSTER_NAME}" \ + --nodegroup-name "${NODEGROUP}" \ --scaling-config "minSize=2,maxSize=2,desiredSize=2" diff --git a/website/content/en/docs/reference/cloudformation.md b/website/content/en/docs/reference/cloudformation.md index d469ceb2dbac..e764749ecbd1 100644 --- a/website/content/en/docs/reference/cloudformation.md +++ b/website/content/en/docs/reference/cloudformation.md @@ -17,8 +17,8 @@ These descriptions should allow you to understand: To download a particular version of `cloudformation.yaml`, set the version and use `curl` to pull the file to your local system: ```bash -export KARPENTER_VERSION=v0.33.1 -curl https://raw.githubusercontent.com/aws/karpenter-provider-aws/"${KARPENTER_VERSION}"/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml > cloudformation.yaml +export KARPENTER_VERSION="0.35.4" +curl https://raw.githubusercontent.com/aws/karpenter-provider-aws/v"${KARPENTER_VERSION}"/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml > cloudformation.yaml ``` Following some header information, the rest of the `cloudformation.yaml` file describes the resources that CloudFormation deploys. @@ -87,7 +87,7 @@ The resources defined in this section are associated with: * KarpenterControllerPolicy -Because the scope of the KarpenterControllerPolicy is an AWS region, the cluster's AWS region is included in the `AllowScopedEC2InstanceActions`. +Because the scope of the KarpenterControllerPolicy is an AWS region, the cluster's AWS region is included in the `AllowScopedEC2InstanceAccessActions`. ### KarpenterControllerPolicy @@ -109,22 +109,21 @@ KarpenterControllerPolicy: Someone wanting to add Karpenter to an existing cluster, instead of using `cloudformation.yaml`, would need to create the IAM policy directly and assign that policy to the role leveraged by the service account using IRSA. -#### AllowScopedEC2InstanceActions +#### AllowScopedEC2InstanceAccessActions -The AllowScopedEC2InstanceActions statement ID (Sid) identifies a set of EC2 resources that are allowed to be accessed with +The AllowScopedEC2InstanceAccessActions statement ID (Sid) identifies a set of EC2 resources that are allowed to be accessed with [RunInstances](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_RunInstances.html) and [CreateFleet](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html) actions. For `RunInstances` and `CreateFleet` actions, the Karpenter controller can read (but not create) `image`, `snapshot`, `security-group`, `subnet` and `launch-template` EC2 resources, scoped for the particular AWS partition and region. ```json { - "Sid": "AllowScopedEC2InstanceActions", + "Sid": "AllowScopedEC2InstanceAccessActions", "Effect": "Allow", "Resource": [ "arn:${AWS::Partition}:ec2:${AWS::Region}::image/*", "arn:${AWS::Partition}:ec2:${AWS::Region}::snapshot/*", "arn:${AWS::Partition}:ec2:${AWS::Region}:*:security-group/*", - "arn:${AWS::Partition}:ec2:${AWS::Region}:*:subnet/*", - "arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*" + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:subnet/*" ], "Action": [ "ec2:RunInstances", @@ -133,6 +132,32 @@ For `RunInstances` and `CreateFleet` actions, the Karpenter controller can read } ``` +#### AllowScopedEC2LaunchTemplateAccessActions + +The AllowScopedEC2InstanceAccessActions statement ID (Sid) identifies launch templates that are allowed to be accessed with +[RunInstances](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_RunInstances.html) and [CreateFleet](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html) actions. +For `RunInstances` and `CreateFleet` actions, the Karpenter controller can read (but not create) `launch-template` EC2 resources that have the `kubernetes.io/cluster/${ClusterName}` tag be set to `owned` and a `karpenter.sh/nodepool` tag, scoped for the particular AWS partition and region. This ensures that an instance launch can't access launch templates that weren't provisioned by Karpenter. + +```json +{ + "Sid": "AllowScopedEC2LaunchTemplateAccessActions", + "Effect": "Allow", + "Resource": "arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*", + "Action": [ + "ec2:RunInstances", + "ec2:CreateFleet" + ], + "Condition": { + "StringEquals": { + "aws:ResourceTag/kubernetes.io/cluster/${ClusterName}": "owned" + }, + "StringLike": { + "aws:ResourceTag/karpenter.sh/nodepool": "*" + } + } +} +``` + #### AllowScopedEC2InstanceActionsWithTags The AllowScopedEC2InstanceActionsWithTags Sid allows the diff --git a/website/content/en/docs/reference/instance-types.md b/website/content/en/docs/reference/instance-types.md index 44bc7b15b0f6..6978c20ef258 100644 --- a/website/content/en/docs/reference/instance-types.md +++ b/website/content/en/docs/reference/instance-types.md @@ -2253,7 +2253,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|6| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| - |karpenter.k8s.aws/instance-network-bandwidth|15000| + |karpenter.k8s.aws/instance-network-bandwidth|25000| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3090,6 +3090,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|2048| + |karpenter.k8s.aws/instance-network-bandwidth|390| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3101,6 +3102,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|1451Mi| |pods|8| + |vpc.amazonaws.com/pod-eni|4| ### `c7a.large` #### Labels | Label | Value | @@ -3112,6 +3114,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|4096| + |karpenter.k8s.aws/instance-network-bandwidth|781| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3123,6 +3126,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|3114Mi| |pods|29| + |vpc.amazonaws.com/pod-eni|9| ### `c7a.xlarge` #### Labels | Label | Value | @@ -3134,6 +3138,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| + |karpenter.k8s.aws/instance-network-bandwidth|1562| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3145,6 +3150,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|6584Mi| |pods|58| + |vpc.amazonaws.com/pod-eni|18| ### `c7a.2xlarge` #### Labels | Label | Value | @@ -3156,6 +3162,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| + |karpenter.k8s.aws/instance-network-bandwidth|3125| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3167,6 +3174,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|14162Mi| |pods|58| + |vpc.amazonaws.com/pod-eni|38| ### `c7a.4xlarge` #### Labels | Label | Value | @@ -3178,6 +3186,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| + |karpenter.k8s.aws/instance-network-bandwidth|6250| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3189,6 +3198,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|27381Mi| |pods|234| + |vpc.amazonaws.com/pod-eni|54| ### `c7a.8xlarge` #### Labels | Label | Value | @@ -3200,6 +3210,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| + |karpenter.k8s.aws/instance-network-bandwidth|12500| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3211,6 +3222,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|57691Mi| |pods|234| + |vpc.amazonaws.com/pod-eni|84| ### `c7a.12xlarge` #### Labels | Label | Value | @@ -3222,6 +3234,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|98304| + |karpenter.k8s.aws/instance-network-bandwidth|18750| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3233,6 +3246,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|88002Mi| |pods|234| + |vpc.amazonaws.com/pod-eni|114| ### `c7a.16xlarge` #### Labels | Label | Value | @@ -3244,6 +3258,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| + |karpenter.k8s.aws/instance-network-bandwidth|25000| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3255,6 +3270,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|112779Mi| |pods|737| + |vpc.amazonaws.com/pod-eni|107| ### `c7a.24xlarge` #### Labels | Label | Value | @@ -3266,6 +3282,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|196608| + |karpenter.k8s.aws/instance-network-bandwidth|37500| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3277,6 +3294,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|173400Mi| |pods|737| + |vpc.amazonaws.com/pod-eni|107| ### `c7a.32xlarge` #### Labels | Label | Value | @@ -3288,6 +3306,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| + |karpenter.k8s.aws/instance-network-bandwidth|50000| |karpenter.k8s.aws/instance-size|32xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3299,6 +3318,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|234021Mi| |pods|737| + |vpc.amazonaws.com/pod-eni|107| ### `c7a.48xlarge` #### Labels | Label | Value | @@ -3310,6 +3330,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|393216| + |karpenter.k8s.aws/instance-network-bandwidth|50000| |karpenter.k8s.aws/instance-size|48xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3322,6 +3343,7 @@ below are the resources available with some assumptions and after the instance o |memory|355262Mi| |pods|737| |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| ### `c7a.metal-48xl` #### Labels | Label | Value | @@ -3333,6 +3355,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|393216| + |karpenter.k8s.aws/instance-network-bandwidth|50000| |karpenter.k8s.aws/instance-size|metal-48xl| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3345,6 +3368,7 @@ below are the resources available with some assumptions and after the instance o |memory|355262Mi| |pods|737| |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| ## c7g Family ### `c7g.medium` #### Labels @@ -3984,6 +4008,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|3114Mi| |pods|29| + |vpc.amazonaws.com/pod-eni|9| ### `c7i.xlarge` #### Labels | Label | Value | @@ -4007,6 +4032,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|6584Mi| |pods|58| + |vpc.amazonaws.com/pod-eni|18| ### `c7i.2xlarge` #### Labels | Label | Value | @@ -4030,6 +4056,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|14162Mi| |pods|58| + |vpc.amazonaws.com/pod-eni|38| ### `c7i.4xlarge` #### Labels | Label | Value | @@ -4053,6 +4080,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|27381Mi| |pods|234| + |vpc.amazonaws.com/pod-eni|54| ### `c7i.8xlarge` #### Labels | Label | Value | @@ -4076,6 +4104,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|57691Mi| |pods|234| + |vpc.amazonaws.com/pod-eni|84| ### `c7i.12xlarge` #### Labels | Label | Value | @@ -4099,6 +4128,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|88002Mi| |pods|234| + |vpc.amazonaws.com/pod-eni|114| ### `c7i.16xlarge` #### Labels | Label | Value | @@ -4122,6 +4152,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|112779Mi| |pods|737| + |vpc.amazonaws.com/pod-eni|107| ### `c7i.24xlarge` #### Labels | Label | Value | @@ -4145,6 +4176,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|173400Mi| |pods|737| + |vpc.amazonaws.com/pod-eni|107| ### `c7i.metal-24xl` #### Labels | Label | Value | @@ -4156,6 +4188,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|196608| + |karpenter.k8s.aws/instance-network-bandwidth|37500| |karpenter.k8s.aws/instance-size|metal-24xl| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4167,6 +4200,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|173400Mi| |pods|737| + |vpc.amazonaws.com/pod-eni|107| ### `c7i.48xlarge` #### Labels | Label | Value | @@ -4191,6 +4225,7 @@ below are the resources available with some assumptions and after the instance o |memory|355262Mi| |pods|737| |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| ### `c7i.metal-48xl` #### Labels | Label | Value | @@ -4202,6 +4237,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|393216| + |karpenter.k8s.aws/instance-network-bandwidth|50000| |karpenter.k8s.aws/instance-size|metal-48xl| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4214,6 +4250,7 @@ below are the resources available with some assumptions and after the instance o |memory|355262Mi| |pods|737| |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| ## d2 Family ### `d2.xlarge` #### Labels @@ -5652,8 +5689,8 @@ below are the resources available with some assumptions and after the instance o |--|--| |cpu|63770m| |ephemeral-storage|17Gi| - |memory|234021Mi| - |pods|737| + |memory|237794Mi| + |pods|394| ## hpc7g Family ### `hpc7g.4xlarge` #### Labels @@ -6480,6 +6517,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-local-nvme|11250| |karpenter.k8s.aws/instance-memory|393216| + |karpenter.k8s.aws/instance-network-bandwidth|28120| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6491,6 +6529,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|360795Mi| |pods|234| + |vpc.amazonaws.com/pod-eni|112| ### `i4i.16xlarge` #### Labels | Label | Value | @@ -6528,6 +6567,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-local-nvme|22500| |karpenter.k8s.aws/instance-memory|786432| + |karpenter.k8s.aws/instance-network-bandwidth|56250| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6539,6 +6579,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|722287Mi| |pods|437| + |vpc.amazonaws.com/pod-eni|105| ### `i4i.32xlarge` #### Labels | Label | Value | @@ -11370,6 +11411,30 @@ below are the resources available with some assumptions and after the instance o |pods|737| |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| +### `m7gd.metal` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|m| + |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|m7gd| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|| + |karpenter.k8s.aws/instance-local-nvme|3800| + |karpenter.k8s.aws/instance-memory|262144| + |karpenter.k8s.aws/instance-size|metal| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|m7gd.metal| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|63770m| + |ephemeral-storage|17Gi| + |memory|233962Mi| + |pods|737| + |vpc.amazonaws.com/efa|1| ## m7i Family ### `m7i.large` #### Labels @@ -11574,6 +11639,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|393216| + |karpenter.k8s.aws/instance-network-bandwidth|37500| |karpenter.k8s.aws/instance-size|metal-24xl| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11585,6 +11651,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|355262Mi| |pods|737| + |vpc.amazonaws.com/pod-eni|107| ### `m7i.48xlarge` #### Labels | Label | Value | @@ -11621,6 +11688,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|786432| + |karpenter.k8s.aws/instance-network-bandwidth|50000| |karpenter.k8s.aws/instance-size|metal-48xl| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11633,6 +11701,7 @@ below are the resources available with some assumptions and after the instance o |memory|718987Mi| |pods|737| |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| ## m7i-flex Family ### `m7i-flex.large` #### Labels @@ -11866,7 +11935,6 @@ below are the resources available with some assumptions and after the instance o |memory|56786Mi| |nvidia.com/gpu|1| |pods|58| - |vpc.amazonaws.com/pod-eni|38| ### `p3.8xlarge` #### Labels | Label | Value | @@ -11895,7 +11963,6 @@ below are the resources available with some assumptions and after the instance o |memory|228187Mi| |nvidia.com/gpu|4| |pods|234| - |vpc.amazonaws.com/pod-eni|54| ### `p3.16xlarge` #### Labels | Label | Value | @@ -11924,7 +11991,6 @@ below are the resources available with some assumptions and after the instance o |memory|459304Mi| |nvidia.com/gpu|8| |pods|234| - |vpc.amazonaws.com/pod-eni|114| ## p3dn Family ### `p3dn.24xlarge` #### Labels @@ -15499,6 +15565,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|7134Mi| |pods|8| + |vpc.amazonaws.com/pod-eni|4| ### `r7a.large` #### Labels | Label | Value | @@ -15522,6 +15589,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|14481Mi| |pods|29| + |vpc.amazonaws.com/pod-eni|9| ### `r7a.xlarge` #### Labels | Label | Value | @@ -15545,6 +15613,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|29317Mi| |pods|58| + |vpc.amazonaws.com/pod-eni|18| ### `r7a.2xlarge` #### Labels | Label | Value | @@ -15568,6 +15637,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|59627Mi| |pods|58| + |vpc.amazonaws.com/pod-eni|38| ### `r7a.4xlarge` #### Labels | Label | Value | @@ -15591,6 +15661,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|118312Mi| |pods|234| + |vpc.amazonaws.com/pod-eni|54| ### `r7a.8xlarge` #### Labels | Label | Value | @@ -15614,6 +15685,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|239554Mi| |pods|234| + |vpc.amazonaws.com/pod-eni|84| ### `r7a.12xlarge` #### Labels | Label | Value | @@ -15637,6 +15709,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|360795Mi| |pods|234| + |vpc.amazonaws.com/pod-eni|114| ### `r7a.16xlarge` #### Labels | Label | Value | @@ -15660,6 +15733,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|476504Mi| |pods|737| + |vpc.amazonaws.com/pod-eni|107| ### `r7a.24xlarge` #### Labels | Label | Value | @@ -15683,6 +15757,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|718987Mi| |pods|737| + |vpc.amazonaws.com/pod-eni|107| ### `r7a.32xlarge` #### Labels | Label | Value | @@ -15706,6 +15781,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|961470Mi| |pods|737| + |vpc.amazonaws.com/pod-eni|107| ### `r7a.48xlarge` #### Labels | Label | Value | @@ -15730,6 +15806,7 @@ below are the resources available with some assumptions and after the instance o |memory|1446437Mi| |pods|737| |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| ### `r7a.metal-48xl` #### Labels | Label | Value | @@ -15741,6 +15818,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|1572864| + |karpenter.k8s.aws/instance-network-bandwidth|50000| |karpenter.k8s.aws/instance-size|metal-48xl| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15753,6 +15831,7 @@ below are the resources available with some assumptions and after the instance o |memory|1446437Mi| |pods|737| |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| ## r7g Family ### `r7g.medium` #### Labels @@ -16109,7 +16188,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-local-nvme|1900| - |karpenter.k8s.aws/instance-memory|196608| + |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|15000| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|arm64| @@ -16120,7 +16199,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |cpu|31850m| |ephemeral-storage|17Gi| - |memory|178874Mi| + |memory|239495Mi| |pods|234| |vpc.amazonaws.com/pod-eni|54| ### `r7gd.12xlarge` @@ -16134,7 +16213,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-local-nvme|2850| - |karpenter.k8s.aws/instance-memory|262144| + |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|22500| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|arm64| @@ -16145,7 +16224,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |cpu|47810m| |ephemeral-storage|17Gi| - |memory|239495Mi| + |memory|360736Mi| |pods|234| |vpc.amazonaws.com/pod-eni|54| ### `r7gd.16xlarge` @@ -16186,6 +16265,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| + |karpenter.k8s.aws/instance-network-bandwidth|781| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16197,6 +16277,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|14481Mi| |pods|29| + |vpc.amazonaws.com/pod-eni|9| ### `r7i.xlarge` #### Labels | Label | Value | @@ -16208,6 +16289,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| + |karpenter.k8s.aws/instance-network-bandwidth|1562| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16219,6 +16301,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|29317Mi| |pods|58| + |vpc.amazonaws.com/pod-eni|18| ### `r7i.2xlarge` #### Labels | Label | Value | @@ -16230,6 +16313,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| + |karpenter.k8s.aws/instance-network-bandwidth|3125| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16241,6 +16325,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|59627Mi| |pods|58| + |vpc.amazonaws.com/pod-eni|38| ### `r7i.4xlarge` #### Labels | Label | Value | @@ -16252,6 +16337,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| + |karpenter.k8s.aws/instance-network-bandwidth|6250| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16263,6 +16349,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|118312Mi| |pods|234| + |vpc.amazonaws.com/pod-eni|54| ### `r7i.8xlarge` #### Labels | Label | Value | @@ -16274,6 +16361,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| + |karpenter.k8s.aws/instance-network-bandwidth|12500| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16285,6 +16373,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|239554Mi| |pods|234| + |vpc.amazonaws.com/pod-eni|84| ### `r7i.12xlarge` #### Labels | Label | Value | @@ -16296,6 +16385,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|393216| + |karpenter.k8s.aws/instance-network-bandwidth|18750| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16307,6 +16397,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|360795Mi| |pods|234| + |vpc.amazonaws.com/pod-eni|114| ### `r7i.16xlarge` #### Labels | Label | Value | @@ -16318,6 +16409,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|524288| + |karpenter.k8s.aws/instance-network-bandwidth|25000| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16329,6 +16421,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|476504Mi| |pods|737| + |vpc.amazonaws.com/pod-eni|107| ### `r7i.24xlarge` #### Labels | Label | Value | @@ -16340,6 +16433,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|786432| + |karpenter.k8s.aws/instance-network-bandwidth|37500| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16351,6 +16445,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|718987Mi| |pods|737| + |vpc.amazonaws.com/pod-eni|107| ### `r7i.metal-24xl` #### Labels | Label | Value | @@ -16362,6 +16457,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|786432| + |karpenter.k8s.aws/instance-network-bandwidth|37500| |karpenter.k8s.aws/instance-size|metal-24xl| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16373,6 +16469,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|718987Mi| |pods|737| + |vpc.amazonaws.com/pod-eni|107| ### `r7i.48xlarge` #### Labels | Label | Value | @@ -16384,6 +16481,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|1572864| + |karpenter.k8s.aws/instance-network-bandwidth|50000| |karpenter.k8s.aws/instance-size|48xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16396,6 +16494,7 @@ below are the resources available with some assumptions and after the instance o |memory|1446437Mi| |pods|737| |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| ### `r7i.metal-48xl` #### Labels | Label | Value | @@ -16407,6 +16506,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|1572864| + |karpenter.k8s.aws/instance-network-bandwidth|50000| |karpenter.k8s.aws/instance-size|metal-48xl| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16419,6 +16519,7 @@ below are the resources available with some assumptions and after the instance o |memory|1446437Mi| |pods|737| |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| ## r7iz Family ### `r7iz.large` #### Labels @@ -16443,6 +16544,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|14481Mi| |pods|29| + |vpc.amazonaws.com/pod-eni|9| ### `r7iz.xlarge` #### Labels | Label | Value | @@ -16466,6 +16568,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|29317Mi| |pods|58| + |vpc.amazonaws.com/pod-eni|18| ### `r7iz.2xlarge` #### Labels | Label | Value | @@ -16489,6 +16592,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|59627Mi| |pods|58| + |vpc.amazonaws.com/pod-eni|38| ### `r7iz.4xlarge` #### Labels | Label | Value | @@ -16512,6 +16616,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|118312Mi| |pods|234| + |vpc.amazonaws.com/pod-eni|54| ### `r7iz.8xlarge` #### Labels | Label | Value | @@ -16535,6 +16640,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|239554Mi| |pods|234| + |vpc.amazonaws.com/pod-eni|84| ### `r7iz.12xlarge` #### Labels | Label | Value | @@ -16558,6 +16664,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|360795Mi| |pods|234| + |vpc.amazonaws.com/pod-eni|114| ### `r7iz.16xlarge` #### Labels | Label | Value | @@ -16581,6 +16688,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|476504Mi| |pods|737| + |vpc.amazonaws.com/pod-eni|107| ### `r7iz.metal-16xl` #### Labels | Label | Value | @@ -16592,6 +16700,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|524288| + |karpenter.k8s.aws/instance-network-bandwidth|25000| |karpenter.k8s.aws/instance-size|metal-16xl| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16603,6 +16712,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|476504Mi| |pods|737| + |vpc.amazonaws.com/pod-eni|107| ### `r7iz.32xlarge` #### Labels | Label | Value | @@ -16627,6 +16737,7 @@ below are the resources available with some assumptions and after the instance o |memory|961470Mi| |pods|737| |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| ### `r7iz.metal-32xl` #### Labels | Label | Value | @@ -16638,6 +16749,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|1048576| + |karpenter.k8s.aws/instance-network-bandwidth|50000| |karpenter.k8s.aws/instance-size|metal-32xl| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16650,6 +16762,7 @@ below are the resources available with some assumptions and after the instance o |memory|961470Mi| |pods|737| |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| ## t1 Family ### `t1.micro` #### Labels diff --git a/website/content/en/docs/reference/metrics.md b/website/content/en/docs/reference/metrics.md index cdb7e7fff63c..b7cf1366f50f 100644 --- a/website/content/en/docs/reference/metrics.md +++ b/website/content/en/docs/reference/metrics.md @@ -8,166 +8,186 @@ description: > --- Karpenter makes several metrics available in Prometheus format to allow monitoring cluster provisioning status. These metrics are available by default at `karpenter.karpenter.svc.cluster.local:8000/metrics` configurable via the `METRICS_PORT` environment variable documented [here](../settings) -## Controller Runtime Metrics +### `karpenter_build_info` +A metric with a constant '1' value labeled by version from which karpenter was built. -### `controller_runtime_active_workers` -Number of currently used workers per controller +## Nodepool Metrics -### `controller_runtime_max_concurrent_reconciles` -Maximum number of concurrent reconciles per controller +### `karpenter_nodepool_usage` +The nodepool usage is the amount of resources that have been provisioned by a particular nodepool. Labeled by nodepool name and resource type. -### `controller_runtime_reconcile_errors_total` -Total number of reconciliation errors per controller +### `karpenter_nodepool_limit` +The nodepool limits are the limits specified on the nodepool that restrict the quantity of resources provisioned. Labeled by nodepool name and resource type. -### `controller_runtime_reconcile_time_seconds` -Length of time per reconciliation per controller +## Nodes Metrics -### `controller_runtime_reconcile_total` -Total number of reconciliations per controller +### `karpenter_nodes_total_pod_requests` +Node total pod requests are the resources requested by non-DaemonSet pods bound to nodes. -## Consistency Metrics +### `karpenter_nodes_total_pod_limits` +Node total pod limits are the resources specified by non-DaemonSet pod limits. -### `karpenter_consistency_errors` -Number of consistency checks that have failed. +### `karpenter_nodes_total_daemon_requests` +Node total daemon requests are the resource requested by DaemonSet pods bound to nodes. -## Disruption Metrics +### `karpenter_nodes_total_daemon_limits` +Node total daemon limits are the resources specified by DaemonSet pod limits. -### `karpenter_disruption_actions_performed_total` -Number of disruption actions performed. Labeled by disruption method. +### `karpenter_nodes_termination_time_seconds` +The time taken between a node's deletion request and the removal of its finalizer -### `karpenter_disruption_consolidation_timeouts_total` -Number of times the Consolidation algorithm has reached a timeout. Labeled by consolidation type. +### `karpenter_nodes_terminated` +Number of nodes terminated in total by Karpenter. Labeled by owning nodepool. -### `karpenter_disruption_eligible_nodes` -Number of nodes eligible for disruption by Karpenter. Labeled by disruption method. +### `karpenter_nodes_system_overhead` +Node system daemon overhead are the resources reserved for system overhead, the difference between the node's capacity and allocatable values are reported by the status. -### `karpenter_disruption_evaluation_duration_seconds` -Duration of the disruption evaluation process in seconds. +### `karpenter_nodes_leases_deleted` +Number of deleted leaked leases. -### `karpenter_disruption_queue_depth` -The number of commands currently being waited on in the disruption orchestration queue. +### `karpenter_nodes_created` +Number of nodes created in total by Karpenter. Labeled by owning nodepool. -### `karpenter_disruption_replacement_nodeclaim_failures_total` -The number of times that Karpenter failed to launch a replacement node for disruption. Labeled by disruption method. +### `karpenter_nodes_allocatable` +Node allocatable are the resources allocatable by nodes. -### `karpenter_disruption_replacement_nodeclaim_initialized_seconds` -Amount of time required for a replacement nodeclaim to become initialized. +## Pods Metrics -## Interruption Metrics +### `karpenter_pods_state` +Pod state is the current state of pods. This metric can be used several ways as it is labeled by the pod name, namespace, owner, node, nodepool name, zone, architecture, capacity type, instance type and pod phase. -### `karpenter_interruption_actions_performed` -Number of notification actions performed. Labeled by action +### `karpenter_pods_startup_time_seconds` +The time from pod creation until the pod is running. -### `karpenter_interruption_deleted_messages` -Count of messages deleted from the SQS queue. +## Provisioner Metrics -### `karpenter_interruption_message_latency_time_seconds` -Length of time between message creation in queue and an action taken on the message by the controller. +### `karpenter_provisioner_scheduling_simulation_duration_seconds` +Duration of scheduling simulations used for deprovisioning and provisioning in seconds. -### `karpenter_interruption_received_messages` -Count of messages received from the SQS queue. Broken down by message type and whether the message was actionable. +### `karpenter_provisioner_scheduling_duration_seconds` +Duration of scheduling process in seconds. ## Nodeclaims Metrics -### `karpenter_nodeclaims_created` -Number of nodeclaims created in total by Karpenter. Labeled by reason the nodeclaim was created and the owning nodepool. +### `karpenter_nodeclaims_terminated` +Number of nodeclaims terminated in total by Karpenter. Labeled by reason the nodeclaim was terminated and the owning nodepool. -### `karpenter_nodeclaims_disrupted` -Number of nodeclaims disrupted in total by Karpenter. Labeled by disruption type of the nodeclaim and the owning nodepool. +### `karpenter_nodeclaims_registered` +Number of nodeclaims registered in total by Karpenter. Labeled by the owning nodepool. -### `karpenter_nodeclaims_drifted` -Number of nodeclaims drifted reasons in total by Karpenter. Labeled by drift type of the nodeclaim and the owning nodepool. +### `karpenter_nodeclaims_launched` +Number of nodeclaims launched in total by Karpenter. Labeled by the owning nodepool. ### `karpenter_nodeclaims_initialized` Number of nodeclaims initialized in total by Karpenter. Labeled by the owning nodepool. -### `karpenter_nodeclaims_launched` -Number of nodeclaims launched in total by Karpenter. Labeled by the owning nodepool. +### `karpenter_nodeclaims_drifted` +Number of nodeclaims drifted reasons in total by Karpenter. Labeled by drift type of the nodeclaim and the owning nodepool. -### `karpenter_nodeclaims_registered` -Number of nodeclaims registered in total by Karpenter. Labeled by the owning nodepool. +### `karpenter_nodeclaims_disrupted` +Number of nodeclaims disrupted in total by Karpenter. Labeled by disruption type of the nodeclaim and the owning nodepool. -### `karpenter_nodeclaims_terminated` -Number of nodeclaims terminated in total by Karpenter. Labeled by reason the nodeclaim was terminated and the owning nodepool. +### `karpenter_nodeclaims_created` +Number of nodeclaims created in total by Karpenter. Labeled by reason the nodeclaim was created and the owning nodepool. -## Provisioner Metrics +## Interruption Metrics -### `karpenter_provisioner_scheduling_duration_seconds` -Duration of scheduling process in seconds. +### `karpenter_interruption_received_messages` +Count of messages received from the SQS queue. Broken down by message type and whether the message was actionable. -### `karpenter_provisioner_scheduling_simulation_duration_seconds` -Duration of scheduling simulations used for deprovisioning and provisioning in seconds. +### `karpenter_interruption_message_latency_time_seconds` +Length of time between message creation in queue and an action taken on the message by the controller. -## Nodepool Metrics +### `karpenter_interruption_deleted_messages` +Count of messages deleted from the SQS queue. -### `karpenter_nodepool_limit` -The nodepool limits are the limits specified on the nodepool that restrict the quantity of resources provisioned. Labeled by nodepool name and resource type. +### `karpenter_interruption_actions_performed` +Number of notification actions performed. Labeled by action -### `karpenter_nodepool_usage` -The nodepool usage is the amount of resources that have been provisioned by a particular nodepool. Labeled by nodepool name and resource type. +## Disruption Metrics -## Nodes Metrics +### `karpenter_disruption_replacement_nodeclaim_initialized_seconds` +Amount of time required for a replacement nodeclaim to become initialized. -### `karpenter_nodes_allocatable` -Node allocatable are the resources allocatable by nodes. +### `karpenter_disruption_replacement_nodeclaim_failures_total` +The number of times that Karpenter failed to launch a replacement node for disruption. Labeled by disruption method. -### `karpenter_nodes_created` -Number of nodes created in total by Karpenter. Labeled by owning nodepool. +### `karpenter_disruption_queue_depth` +The number of commands currently being waited on in the disruption orchestration queue. -### `karpenter_nodes_leases_deleted` -Number of deleted leaked leases. +### `karpenter_disruption_pods_disrupted_total` +Total number of reschedulable pods disrupted on nodes. Labeled by NodePool, disruption action, method, and consolidation type. -### `karpenter_nodes_system_overhead` -Node system daemon overhead are the resources reserved for system overhead, the difference between the node's capacity and allocatable values are reported by the status. +### `karpenter_disruption_nodes_disrupted_total` +Total number of nodes disrupted. Labeled by NodePool, disruption action, method, and consolidation type. -### `karpenter_nodes_terminated` -Number of nodes terminated in total by Karpenter. Labeled by owning nodepool. +### `karpenter_disruption_evaluation_duration_seconds` +Duration of the disruption evaluation process in seconds. Labeled by method and consolidation type. -### `karpenter_nodes_termination_time_seconds` -The time taken between a node's deletion request and the removal of its finalizer +### `karpenter_disruption_eligible_nodes` +Number of nodes eligible for disruption by Karpenter. Labeled by disruption method and consolidation type. -### `karpenter_nodes_total_daemon_limits` -Node total daemon limits are the resources specified by DaemonSet pod limits. +### `karpenter_disruption_consolidation_timeouts_total` +Number of times the Consolidation algorithm has reached a timeout. Labeled by consolidation type. -### `karpenter_nodes_total_daemon_requests` -Node total daemon requests are the resource requested by DaemonSet pods bound to nodes. +### `karpenter_disruption_budgets_allowed_disruptions` +The number of nodes for a given NodePool that can be disrupted at a point in time. Labeled by NodePool. Note that allowed disruptions can change very rapidly, as new nodes may be created and others may be deleted at any point. -### `karpenter_nodes_total_pod_limits` -Node total pod limits are the resources specified by non-DaemonSet pod limits. +### `karpenter_disruption_actions_performed_total` +Number of disruption actions performed. Labeled by disruption action, method, and consolidation type. -### `karpenter_nodes_total_pod_requests` -Node total pod requests are the resources requested by non-DaemonSet pods bound to nodes. +## Consistency Metrics -## Pods Metrics +### `karpenter_consistency_errors` +Number of consistency checks that have failed. -### `karpenter_pods_startup_time_seconds` -The time from pod creation until the pod is running. +## Cluster State Metrics -### `karpenter_pods_state` -Pod state is the current state of pods. This metric can be used several ways as it is labeled by the pod name, namespace, owner, node, nodepool name, zone, architecture, capacity type, instance type and pod phase. +### `karpenter_cluster_state_synced` +Returns 1 if cluster state is synced and 0 otherwise. Synced checks that nodeclaims and nodes that are stored in the APIServer have the same representation as Karpenter's cluster state + +### `karpenter_cluster_state_node_count` +Current count of nodes in cluster state ## Cloudprovider Metrics -### `karpenter_cloudprovider_duration_seconds` -Duration of cloud provider method calls. Labeled by the controller, method name and provider. +### `karpenter_cloudprovider_instance_type_price_estimate` +Estimated hourly price used when making informed decisions on node cost calculation. This is updated once on startup and then every 12 hours. -### `karpenter_cloudprovider_errors_total` -Total number of errors returned from CloudProvider calls. +### `karpenter_cloudprovider_instance_type_memory_bytes` +Memory, in bytes, for a given instance type. ### `karpenter_cloudprovider_instance_type_cpu_cores` VCPUs cores for a given instance type. -### `karpenter_cloudprovider_instance_type_memory_bytes` -Memory, in bytes, for a given instance type. +### `karpenter_cloudprovider_errors_total` +Total number of errors returned from CloudProvider calls. -### `karpenter_cloudprovider_instance_type_price_estimate` -Estimated hourly price used when making informed decisions on node cost calculation. This is updated once on startup and then every 12 hours. +### `karpenter_cloudprovider_duration_seconds` +Duration of cloud provider method calls. Labeled by the controller, method name and provider. ## Cloudprovider Batcher Metrics +### `karpenter_cloudprovider_batcher_batch_time_seconds` +Duration of the batching window per batcher + ### `karpenter_cloudprovider_batcher_batch_size` Size of the request batch per batcher -### `karpenter_cloudprovider_batcher_batch_time_seconds` -Duration of the batching window per batcher +## Controller Runtime Metrics + +### `controller_runtime_reconcile_total` +Total number of reconciliations per controller + +### `controller_runtime_reconcile_time_seconds` +Length of time per reconciliation per controller + +### `controller_runtime_reconcile_errors_total` +Total number of reconciliation errors per controller + +### `controller_runtime_max_concurrent_reconciles` +Maximum number of concurrent reconciles per controller + +### `controller_runtime_active_workers` +Number of currently used workers per controller diff --git a/website/content/en/docs/reference/settings.md b/website/content/en/docs/reference/settings.md index 438c24244d26..9377841178e5 100644 --- a/website/content/en/docs/reference/settings.md +++ b/website/content/en/docs/reference/settings.md @@ -10,43 +10,44 @@ Karpenter surfaces environment variables and CLI parameters to allow you to conf [comment]: <> (the content below is generated from hack/docs/configuration_gen_docs.go) -| Environment Variable | CLI Flag | Description | -|--|--|--| -| ASSUME_ROLE_ARN | \-\-assume-role-arn | Role to assume for calling AWS services.| -| ASSUME_ROLE_DURATION | \-\-assume-role-duration | Duration of assumed credentials in minutes. Default value is 15 minutes. Not used unless aws.assumeRole set. (default = 15m0s)| -| BATCH_IDLE_DURATION | \-\-batch-idle-duration | The maximum amount of time with no new pending pods that if exceeded ends the current batching window. If pods arrive faster than this time, the batching window will be extended up to the maxDuration. If they arrive slower, the pods will be batched separately. (default = 1s)| -| BATCH_MAX_DURATION | \-\-batch-max-duration | The maximum length of a batch window. The longer this is, the more pods we can consider for provisioning at one time which usually results in fewer but larger nodes. (default = 10s)| -| CLUSTER_CA_BUNDLE | \-\-cluster-ca-bundle | Cluster CA bundle for nodes to use for TLS connections with the API server. If not set, this is taken from the controller's TLS configuration.| -| CLUSTER_ENDPOINT | \-\-cluster-endpoint | The external kubernetes cluster endpoint for new nodes to connect with. If not specified, will discover the cluster endpoint using DescribeCluster API.| -| CLUSTER_NAME | \-\-cluster-name | [REQUIRED] The kubernetes cluster name for resource discovery.| -| DISABLE_WEBHOOK | \-\-disable-webhook | Disable the admission and validation webhooks| -| ENABLE_PROFILING | \-\-enable-profiling | Enable the profiling on the metric endpoint| -| FEATURE_GATES | \-\-feature-gates | Optional features can be enabled / disabled using feature gates. Current options are: Drift (default = Drift=true)| -| HEALTH_PROBE_PORT | \-\-health-probe-port | The port the health probe endpoint binds to for reporting controller health (default = 8081)| -| INTERRUPTION_QUEUE | \-\-interruption-queue | Interruption queue is disabled if not specified. Enabling interruption handling may require additional permissions on the controller service account. Additional permissions are outlined in the docs.| -| ISOLATED_VPC | \-\-isolated-vpc | If true, then assume we can't reach AWS services which don't have a VPC endpoint. This also has the effect of disabling look-ups to the AWS pricing endpoint.| -| KARPENTER_SERVICE | \-\-karpenter-service | The Karpenter Service name for the dynamic webhook certificate| -| KUBE_CLIENT_BURST | \-\-kube-client-burst | The maximum allowed burst of queries to the kube-apiserver (default = 300)| -| KUBE_CLIENT_QPS | \-\-kube-client-qps | The smoothed rate of qps to kube-apiserver (default = 200)| -| LEADER_ELECT | \-\-leader-elect | Start leader election client and gain leadership before executing the main loop. Enable this when running replicated components for high availability.| -| LOG_LEVEL | \-\-log-level | Log verbosity level. Can be one of 'debug', 'info', or 'error' (default = info)| -| MEMORY_LIMIT | \-\-memory-limit | Memory limit on the container running the controller. The GC soft memory limit is set to 90% of this value. (default = -1)| -| METRICS_PORT | \-\-metrics-port | The port the metric endpoint binds to for operating metrics about the controller itself (default = 8000)| -| RESERVED_ENIS | \-\-reserved-enis | Reserved ENIs are not included in the calculations for max-pods or kube-reserved. This is most often used in the VPC CNI custom networking setup https://docs.aws.amazon.com/eks/latest/userguide/cni-custom-network.html. (default = 0)| -| VM_MEMORY_OVERHEAD_PERCENT | \-\-vm-memory-overhead-percent | The VM memory overhead as a percent that will be subtracted from the total memory for all instance types. (default = 0.075)| -| WEBHOOK_METRICS_PORT | \-\-webhook-metrics-port | The port the webhook metric endpoing binds to for operating metrics about the webhook (default = 8001)| -| WEBHOOK_PORT | \-\-webhook-port | The port the webhook endpoint binds to for validation and mutation of resources (default = 8443)| +| Environment Variable | CLI Flag | Description | +|--|--|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| ASSUME_ROLE_ARN | \-\-assume-role-arn | Role to assume for calling AWS services. | +| ASSUME_ROLE_DURATION | \-\-assume-role-duration | Duration of assumed credentials in minutes. Default value is 15 minutes. Not used unless aws.assumeRole set. (default = 15m0s) | +| BATCH_IDLE_DURATION | \-\-batch-idle-duration | The maximum amount of time with no new pending pods that if exceeded ends the current batching window. If pods arrive faster than this time, the batching window will be extended up to the maxDuration. If they arrive slower, the pods will be batched separately. (default = 1s) | +| BATCH_MAX_DURATION | \-\-batch-max-duration | The maximum length of a batch window. The longer this is, the more pods we can consider for provisioning at one time which usually results in fewer but larger nodes. (default = 10s) | +| CLUSTER_CA_BUNDLE | \-\-cluster-ca-bundle | Cluster CA bundle for nodes to use for TLS connections with the API server. If not set, this is taken from the controller's TLS configuration. | +| CLUSTER_ENDPOINT | \-\-cluster-endpoint | The external kubernetes cluster endpoint for new nodes to connect with. If not specified, will discover the cluster endpoint using DescribeCluster API. | +| CLUSTER_NAME | \-\-cluster-name | [REQUIRED] The kubernetes cluster name for resource discovery. | +| DISABLE_WEBHOOK | \-\-disable-webhook | Disable the admission and validation webhooks | +| ENABLE_PROFILING | \-\-enable-profiling | Enable the profiling on the metric endpoint | +| FEATURE_GATES | \-\-feature-gates | Optional features can be enabled / disabled using feature gates. Current options are: Drift,SpotToSpotConsolidation (default = Drift=true,SpotToSpotConsolidation=false) | +| HEALTH_PROBE_PORT | \-\-health-probe-port | The port the health probe endpoint binds to for reporting controller health (default = 8081) | +| INTERRUPTION_QUEUE | \-\-interruption-queue | Interruption queue is disabled if not specified. Enabling interruption handling may require additional permissions on the controller service account. Additional permissions are outlined in the docs. | +| ISOLATED_VPC | \-\-isolated-vpc | If true, then assume we can't reach AWS services which don't have a VPC endpoint. This also has the effect of disabling look-ups to the AWS pricing endpoint. | +| KARPENTER_SERVICE | \-\-karpenter-service | The Karpenter Service name for the dynamic webhook certificate | +| KUBE_CLIENT_BURST | \-\-kube-client-burst | The maximum allowed burst of queries to the kube-apiserver (default = 300) | +| KUBE_CLIENT_QPS | \-\-kube-client-qps | The smoothed rate of qps to kube-apiserver (default = 200) | +| LEADER_ELECT | \-\-leader-elect | Start leader election client and gain leadership before executing the main loop. Enable this when running replicated components for high availability. | +| LOG_LEVEL | \-\-log-level | Log verbosity level. Can be one of 'debug', 'info', or 'error' (default = info) | +| MEMORY_LIMIT | \-\-memory-limit | Memory limit on the container running the controller. The GC soft memory limit is set to 90% of this value. (default = -1) | +| METRICS_PORT | \-\-metrics-port | The port the metric endpoint binds to for operating metrics about the controller itself (default = 8000) | +| RESERVED_ENIS | \-\-reserved-enis | Reserved ENIs are not included in the calculations for max-pods or kube-reserved. This is most often used in the VPC CNI custom networking setup https://docs.aws.amazon.com/eks/latest/userguide/cni-custom-network.html. (default = 0) | +| VM_MEMORY_OVERHEAD_PERCENT | \-\-vm-memory-overhead-percent | The VM memory overhead as a percent that will be subtracted from the total memory for all instance types. (default = 0.075) | +| WEBHOOK_METRICS_PORT | \-\-webhook-metrics-port | The port the webhook metric endpoing binds to for operating metrics about the webhook (default = 8001) | +| WEBHOOK_PORT | \-\-webhook-port | The port the webhook endpoint binds to for validation and mutation of resources (default = 8443) | [comment]: <> (end docs generated content from hack/docs/configuration_gen_docs.go) ### Feature Gates -Karpenter uses [feature gates](https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/#feature-gates-for-alpha-or-beta-features) You can enable the feature gates through the `--feature-gates` CLI environment variable or the `FEATURE_GATES` environment variable in the Karpenter deployment. For example, you can configure drift by setting the following CLI argument: `--feature-gates Drift=true`. +Karpenter uses [feature gates](https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/#feature-gates-for-alpha-or-beta-features) You can enable the feature gates through the `--feature-gates` CLI environment variable or the `FEATURE_GATES` environment variable in the Karpenter deployment. For example, you can configure drift, spotToSpotConsolidation by setting the CLI argument: `--feature-gates Drift=true,SpotToSpotConsolidation=true`. -| Feature | Default | Stage | Since | Until | -|---------|---------|-------|---------|---------| -| Drift | false | Alpha | v0.21.x | v0.32.x | -| Drift | true | Beta | v0.33.x | | +| Feature | Default | Stage | Since | Until | +|-------------------------|---------|-------|---------|---------| +| Drift | false | Alpha | v0.21.x | v0.32.x | +| Drift | true | Beta | v0.33.x | | +| SpotToSpotConsolidation | false | Alpha | v0.34.x | | ### Batching Parameters diff --git a/website/content/en/docs/reference/threat-model.md b/website/content/en/docs/reference/threat-model.md index 778af396a826..88f735fb3215 100644 --- a/website/content/en/docs/reference/threat-model.md +++ b/website/content/en/docs/reference/threat-model.md @@ -31,11 +31,11 @@ A Cluster Developer has the ability to create pods via `Deployments`, `ReplicaSe Karpenter has permissions to create and manage cloud instances. Karpenter has Kubernetes API permissions to create, update, and remove nodes, as well as evict pods. For a full list of the permissions, see the RBAC rules in the helm chart template. Karpenter also has AWS IAM permissions to create instances with IAM roles. -* [aggregate-clusterrole.yaml](https://github.com/aws/karpenter/blob/v0.33.1/charts/karpenter/templates/aggregate-clusterrole.yaml) -* [clusterrole-core.yaml](https://github.com/aws/karpenter/blob/v0.33.1/charts/karpenter/templates/clusterrole-core.yaml) -* [clusterrole.yaml](https://github.com/aws/karpenter/blob/v0.33.1/charts/karpenter/templates/clusterrole.yaml) -* [rolebinding.yaml](https://github.com/aws/karpenter/blob/v0.33.1/charts/karpenter/templates/rolebinding.yaml) -* [role.yaml](https://github.com/aws/karpenter/blob/v0.33.1/charts/karpenter/templates/role.yaml) +* [aggregate-clusterrole.yaml](https://github.com/aws/karpenter/blob/v0.35.4/charts/karpenter/templates/aggregate-clusterrole.yaml) +* [clusterrole-core.yaml](https://github.com/aws/karpenter/blob/v0.35.4/charts/karpenter/templates/clusterrole-core.yaml) +* [clusterrole.yaml](https://github.com/aws/karpenter/blob/v0.35.4/charts/karpenter/templates/clusterrole.yaml) +* [rolebinding.yaml](https://github.com/aws/karpenter/blob/v0.35.4/charts/karpenter/templates/rolebinding.yaml) +* [role.yaml](https://github.com/aws/karpenter/blob/v0.35.4/charts/karpenter/templates/role.yaml) ## Assumptions @@ -63,16 +63,16 @@ Karpenter has permissions to create and manage cloud instances. Karpenter has Ku ### Threat: Using EC2 CreateTag/DeleteTag Permissions to Orchestrate Instance Creation/Deletion -**Background**: As of v0.28.0, Karpenter creates a mapping between CloudProvider instances and CustomResources in the cluster for capacity tracking. To ensure this mapping is consistent, Karpenter utilizes the following tag keys: +**Background**: As of `0.28.0`, Karpenter creates a mapping between CloudProvider instances and CustomResources in the cluster for capacity tracking. To ensure this mapping is consistent, Karpenter utilizes the following tag keys: * `karpenter.sh/managed-by` * `karpenter.sh/nodepool` * `kubernetes.io/cluster/${CLUSTER_NAME}` -* `karpenter.sh/provisioner-name` (prior to `v0.32.0`) +* `karpenter.sh/provisioner-name` (prior to `0.32.0`) Any user that has the ability to Create/Delete tags on CloudProvider instances will have the ability to orchestrate Karpenter to Create/Delete CloudProvider instances as a side effect. -In addition, as of v0.29.0, Karpenter will Drift on Security Groups and Subnets. If a user has the Create/Delete tags permission for either of resources, they can orchestrate Karpenter to Create/Delete CloudProvider instances as a side effect. +In addition, as of `0.29.0`, Karpenter will Drift on Security Groups and Subnets. If a user has the Create/Delete tags permission for either of resources, they can orchestrate Karpenter to Create/Delete CloudProvider instances as a side effect. **Threat:** A Cluster Operator attempts to create or delete a tag on a resource discovered by Karpenter. If it has the ability to create a tag it can effectively create or delete CloudProvider instances associated with the tagged resources. diff --git a/website/content/en/docs/tasks/_index.md b/website/content/en/docs/tasks/_index.md new file mode 100644 index 000000000000..7d4ac8605f4e --- /dev/null +++ b/website/content/en/docs/tasks/_index.md @@ -0,0 +1,10 @@ +--- +title: "Tasks" +linkTitle: "Tasks" +weight: 25 +description: > + Tasks to run with Karpenter +cascade: + type: docs +--- + diff --git a/website/content/en/docs/tasks/managing-amis.md b/website/content/en/docs/tasks/managing-amis.md new file mode 100644 index 000000000000..1ef31f141e08 --- /dev/null +++ b/website/content/en/docs/tasks/managing-amis.md @@ -0,0 +1,153 @@ +--- +title: "Managing AMIs" +linkTitle: "Managing AMIs" +weight: 10 +description: > + Task for managing AMIs in Karpenter +--- + +Understanding how Karpenter assigns AMIs to nodes can help ensure that your workloads will run successfully on those nodes and continue to run if the nodes are upgraded to newer AMIs. +Below we describe how Karpenter assigns AMIs to nodes when they are first deployed and how newer AMIs are assigned later when nodes are spun up to replace old ones. +Later, it describes the options you have to assert control over how AMIs are used by Karpenter for your clusters. + +Features for managing AMIs described here should be considered as part of the larger upgrade policies that you have for your clusters. +See [How do I upgrade an EKS Cluster with Karpenter]({{< relref "../faq/#how-do-i-upgrade-an-eks-cluster-with-karpenter" >}}) for details on this process. + +## How Karpenter assigns AMIs to nodes + +Here is how Karpenter assigns AMIs nodes: + +* When you create an `EC2NodeClass`, you are required to set the family of AMIs to use. For example, for the AL2 family, you would set `amiFamily: AL2`. +* With that `amiFamily` set, any time Karpenter spins up a new node, it uses the latest [Amazon EKS optimized Amazon Linux 2 AMIs](https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami.html) release. +* Later, if an existing node needs to be replaced, Karpenter checks to see if a newer AMI in the AL2 family is available and automatically uses the new AMI instead to spin up the new node. In other words, you may automatically get an AMI that you have not tested with your workloads. + +You can manually delete a node managed by Karpenter, which will cause the default behavior just described to take effect. +However, there are situations that will cause node replacements with newer AMIs to happen automatically. +These include: Expiration (if node expiry is set, the node is marked for deletion at a certain time after the node is created), [**Consolidation**]({{< relref "../concepts/disruption/#consolidation" >}}) (if a node is empty of workloads, or deemed to be inefficiently running workloads, nodes can be deleted and more appropriately featured nodes are brought up to consolidate workloads), [Drift]({{< relref "../concepts/disruption/#drift" >}}) (nodes are set for deletion when they drift from the desired state of the `NodeClaims` and new nodes are brought up to replace them), and [Interruption]({{< relref "../concepts/disruption/#interruption" >}}) (nodes are sometimes involuntarily disrupted by things like Spot interruption, health changes, and instance events, requiring new nodes to be deployed). + +See [**Automated Methods**]({{< relref "../concepts/disruption/#automated-methods" >}}) for details on how Karpenter uses these automated actions to replace nodes. + +With these types of automated updates in place, there is some risk that the new AMI being used when replacing instances will introduce some regressions or bugs that cause your workloads to be degraded or fail altogether. +The options described below tell you how to take more control over the ways in which Karpenter selects AMIs for your nodes. + +{{% alert title="Important" color="warning" %}} +If you are new to Karpenter, you should know that the behavior described here is different than you get with Managed Node Groups (MNG). MNG will always use the assigned AMI when it creates a new node and will never automatically upgrade to a new AMI when a new node is required. See [Updating a Managed Node Group](https://docs.aws.amazon.com/eks/latest/userguide/update-managed-node-group.html) to see how you would manually update MNG to use new AMIs. +{{% /alert %}} + +## Choosing AMI options +One of Karpenter's greatest assets is its ability to provide the right node at the right time, with little intervention from the person managing the cluster. +Its default behavior of using a later AMI if one becomes available in the selected family means you automatically get the latest security fixes and features. +However, with this comes the risk that the new AMI could break or degrade your workloads. + +As the Karpenter team looks for new ways to manage AMIs, the options below offer some means of reducing these risks, based on your own security and ease-of-use requirements. +Here are the advantages and challenges of each of the options described below: + +* [Option 1]({{< relref "#option-1-manage-how-amis-are-tested-and-rolled-out" >}}) (Test AMIs): The safest way, and the one we recommend, for ensuring that a new AMI doesn't break your workloads is to test it before putting it into production. This takes the most effort on your part, but most effectively models how your workloads will run in production, allowing you to catch issues ahead of time. Note that you can sometimes get different results from your test environment when you roll a new AMI into production, since issues like scale and other factors can elevate problems you might not see in test. So combining this with other options, that do things like slow rollouts, can allow you to catch problems before they impact your whole cluster. +* [Option 2]({{< relref "#option-2-lock-down-which-amis-are-selected" >}}) (Lock down AMIs): If workloads require a particluar AMI, this option can make sure that it is the only AMI used by Karpenter. This can be used in combination with Option 1, where you lock down the AMI in production, but allow the newest AMIs in a test cluster while you test your workloads before upgrading production. Keep in mind that this makes upgrades a manual process for you. +* [Option 3]({{< relref "#option-3-control-the-pace-of-node-disruptions" >}}) ([Disruption budgets]({{< relref "../concepts/disruption/" >}})): This option can be used as a way of mitigating the scope of impact if a new AMI causes problems with your workloads. With Disruption budgets you can slow the pace of upgrades to nodes with new AMIs or make sure that upgrades only happen during selected dates and times (using `schedule`). This doesn't prevent a bad AMI from being deployed, but it allows you to control when nodes are upgraded, and gives you more time respond to rollout issues. + +## Options + +The following lays out the options you have to impact Karpenter’s behavior as it relates to how nodes are created and AMIs are consumed. + +### Option 1: Manage how AMIs are tested and rolled out + +Instead of just avoiding AMI upgrades, you can set up test clusters where you can try out new AMI releases before they are put into production. +For example, you could have: + +* **Test clusters**: On lower environment clusters, you can run the latest AMIs for your workloads in a safe environment. The `EC2NodeClass` for these clusters could be set with a chosen `amiFamily`, but no `amiSelectorTerms` set. For example, the `NodePool` and `EC2NodeClass` could begin with the following: + + ```yaml + apiVersion: karpenter.sh/v1beta1 + kind: NodePool + metadata: + name: default + spec: + template: + spec: + nodeClassRef: + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass + name: default + --- + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass + metadata: + name: default + spec: + # The latest AMI in this family will be used + amiFamily: AL2 + ``` +* **Production clusters**: After you've confirmed that the AMI works in your lower environments, you can pin the latest AMIs to be deployed in your production clusters to roll out the AMI. One way to do that is to use `amiSelectorTerms` to set the tested AMI to be used in your production cluster. Refer to Option 2 for how to choose a particular AMI by `name` or `id`. Remember that it is still best practice to gradually roll new AMIs into your cluster, even if they have been tested. So consider implementing that for your production clusters as described in Option 3. + +### Option 2: Lock down which AMIs are selected + +Instead of letting Karpenter always run the latest AMI, you can change Karpenter’s default behavior. +When you configure the [**EC2NodeClass**]({{< relref "../concepts/nodeclasses" >}}), you can set a specific AMI that you want Karpenter to always choose, using the `amiSelectorTerms` field. +This prevents a new and potentially untested AMI from replacing existing nodes when those nodes are terminated. + +With the `amiSelectorTerms` field in an `EC2NodeClass`, you can set a specific AMI for Karpenter to use, based on AMI name or id (only one is required). +These examples show two different ways to identify the same AMI: + +```yaml +amiSelectorTerms: +- tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" + environment: prod +- name: al2023-ami-2023.3.20240219.0-kernel-6.1-x86_64 +``` + +or + +```yaml +amiSelectorTerms: +- tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" + environment: prod +- id: ami-052c9ea013e6e3567 +``` + +See the [**spec.amiSelectorTerms**]({{< relref "../concepts/nodeclasses/#specamiselectorterms" >}}) section of the NodeClasses page for details. +Keep in mind, that this could prevent you from getting critical security patches when new AMIs are available, but it does give you control over exactly which AMI is running. + + +### Option 3: Control the pace of node disruptions + +To reduce the risk of entire workloads being immediately degraded when a new AMI is deployed, you can enable Karpenter [**Disruption Budgets**]({{< relref "../concepts/disruption/#disruption-budgets " >}}). +Disruption Budgets limit when and to what extent nodes can be disrupted. +You can prevent disruption based on nodes (a percentage or number of nodes that can be disrupted at a time) and schedule (excluding certain times from disrupting nodes). +You can set Disruption Budgets in a `NodePool` spec. +Here is an example: + +```yaml +disruption: + consolidationPolicy: WhenEmpty + expireAfter: 1440h + budgets: + - nodes: 15% + - nodes: "3" + - nodes: "0" + schedule: "0 7 * * sat-sun" + duration: 12h +``` + +The `disruption` settings define a few fields that indicate the state of a node that should be disrupted. +The `consolidationPolicy` field indicates that a node should be disrupted if the node is either underutilized (`WhenUnderutilized`) or not running any pods (`WhenEmpty`). +With `expireAfter` set to `1440` hours, the node expires after 60 days. +Extending those values causes longer times without disruption. + +Settings for budgets in the above example include the following: + +* **Percentage of nodes**: From the first `nodes` setting, only `15%` of the NodePool’s nodes can be disrupted at a time. +* **Number of nodes**: The second `nodes` setting limits the number of nodes that can be disrupted at a time to `3`. +* **Schedule**: The third `nodes` setting uses schedule to say that zero disruptions (`0`) are allowed starting at 7am on Saturday and Sunday and continues for 12 hours. +The format of the schedule follows the `crontab` format for identifying dates and times. +See the [crontab](https://man7.org/linux/man-pages/man5/crontab.5.html) page for information on the supported values for these fields. + +As with all disruption settings, keep in mind that avoiding updated AMIs for your nodes can result in not getting fixes for known security risks and bugs. +You need to balance that with your desire to not risk breaking the workloads on your cluster. + +## Follow-up + +The Karpenter project continues to add features to give you greater control over AMI upgrades on your clusters. +If you have opinions about features you would like to see to manage AMIs with Karpenter, feel free to enter a Karpenter [New Issue](https://github.com/aws/karpenter-provider-aws/issues/new/choose). diff --git a/website/content/en/docs/troubleshooting.md b/website/content/en/docs/troubleshooting.md index 41e23bc964f7..6dc784007b75 100644 --- a/website/content/en/docs/troubleshooting.md +++ b/website/content/en/docs/troubleshooting.md @@ -29,7 +29,7 @@ Update the zap-logger-config "level" and restart the Karpenter pod(s) to enable #### Debug logging via Helm -You can enable debug logging during installation with helm by setting the option `logLevel`. +You can enable debug logging during installation with Helm by setting the option `logLevel`. ``` helm upgrade --install karpenter oci://public.ecr.aws/karpenter/karpenter \ @@ -53,6 +53,18 @@ This can be resolved by creating the [Service Linked Role](https://docs.aws.amaz aws iam create-service-linked-role --aws-service-name spot.amazonaws.com ``` +### Failed Resolving STS Credentials with I/O Timeout + +```bash +Checking EC2 API connectivity, WebIdentityErr: failed to retrieve credentials\ncaused by: RequestError: send request failed\ncaused by: Post \"https://sts.us-east-1.amazonaws.com/\": dial tcp: lookup sts.us-east-1.amazonaws.com: i/o timeout +``` + +If you see the error above when you attempt to install Karpenter, this indicates that Karpenter is unable to reach out to the STS endpoint due to failed DNS resolution. This can happen when Karpenter is running with `dnsPolicy: ClusterFirst` and your in-cluster DNS service is not yet running. + +You have two mitigations to resolve this error: +1. Let Karpenter manage your in-cluster DNS service - You can let Karpenter manage your DNS application pods' capacity by changing Karpenter's `dnsPolicy` to be `Default` (run `--set dnsPolicy=Default` with a Helm installation). This ensures that Karpenter reaches out to the [VPC DNS service](https://docs.aws.amazon.com/vpc/latest/userguide/vpc-dns.html) when running its controllers, allowing Karpenter to start-up without the DNS application pods running, enabling Karpenter to manage the capacity for these pods. +2. Let MNG/Fargate manage your in-cluster DNS service - If running a cluster with MNG, ensure that your group has enough capacity to support the DNS application pods and ensure that the application has the correct tolerations to schedule against the capacity. If running a cluster with Fargate, ensure that you have a [fargate profile](https://docs.aws.amazon.com/eks/latest/userguide/fargate-profile.html) that selects against your DNS application pods. + ### Karpenter Role names exceeding 64-character limit If you use a tool such as AWS CDK to generate your Kubernetes cluster name, when you add Karpenter to your cluster you could end up with a cluster name that is too long to incorporate into your KarpenterNodeRole name (which is limited to 64 characters). @@ -77,7 +89,7 @@ Info on whether there has been a change to the CRD between versions of Karpenter ### Unable to schedule pod due to insufficient node group instances -v0.16.0 changed the default replicas from 1 to 2. +`0.16.0` changed the default replicas from 1 to 2. Karpenter won't launch capacity to run itself (log related to the `karpenter.sh/provisioner-name DoesNotExist requirement`) so it can't provision for the second Karpenter pod. @@ -89,30 +101,30 @@ To do so on AWS increase the `minimum` and `desired` parameters on the node grou ### Helm Error When Pulling the Chart -If Helm is showing an error when trying to install Karpenter helm charts: +If Helm is showing an error when trying to install Karpenter Helm charts: -- Ensure you are using a newer Helm version, Helm started supporting OCI images since v3.8.0. -- Helm does not have an `helm repo add` concept in OCI, so to install Karpenter you no longer need this +- Ensure you are using a newer Helm version, Helm started supporting OCI images since `3.8.0`. +- Helm does not have an `helm repo add` concept in OCI, so to install Karpenter you no longer need this. +- If you get an error like `Error: public.ecr.aws/karpenter/karpenter:0.34.0: not found` make sure you're adding a `v` prefix for Karpenter versions between `0.17.0` & `0.34.x`. - Verify that the image you are trying to pull actually exists in [gallery.ecr.aws/karpenter](https://gallery.ecr.aws/karpenter/karpenter) -- Sometimes Helm generates a generic error, you can add the --debug switch to any of the helm commands in this doc for more verbose error messages -- If you are getting a 403 forbidden error, you can try `docker logout public.ecr.aws` as explained [here](https://docs.aws.amazon.com/AmazonECR/latest/public/public-troubleshooting.html) -- If you are receiving this error: `Error: failed to download "oci://public.ecr.aws/karpenter/karpenter" at version "0.17.0"`, then you need to prepend a `v` to the version number: `v0.17.0`. Before Karpenter moved to OCI helm charts (pre-v0.17.0), both `v0.16.0` and `0.16.0` would work, but OCI charts require an exact version match. +- Sometimes Helm generates a generic error, you can add the --debug switch to any of the Helm commands in this doc for more verbose error messages +- If you are getting a 403 forbidden error, you can try `docker logout public.ecr.aws` as explained [here](https://docs.aws.amazon.com/AmazonECR/latest/public/public-troubleshooting.html). ### Helm Error when installing the `karpenter-crd` chart -Karpenter v0.26.1+ introduced the `karpenter-crd` helm chart. When installing this chart on your cluster, if you have previously added the Karpenter CRDs to your cluster through the `karpenter` controller chart or through `kubectl replace`, Helm will reject the install of the chart due to `invalid ownership metadata`. +Karpenter `0.26.1` introduced the `karpenter-crd` Helm chart. When installing this chart on your cluster, if you have previously added the Karpenter CRDs to your cluster through the `karpenter` controller chart or through `kubectl replace`, Helm will reject the install of the chart due to `invalid ownership metadata`. - In the case of `invalid ownership metadata; label validation error: missing key "app.kubernetes.io/managed-by": must be set to "Helm"` run: ```shell -kubectl label crd awsnodetemplates.karpenter.k8s.aws provisioners.karpenter.sh machines.karpenter.sh app.kubernetes.io/managed-by=Helm --overwrite +kubectl label crd ec2nodeclasses.karpenter.k8s.aws nodepools.karpenter.sh nodeclaims.karpenter.sh app.kubernetes.io/managed-by=Helm --overwrite ``` - In the case of `annotation validation error: missing key "meta.helm.sh/release-namespace": must be set to "karpenter"` run: ```shell -kubectl annotate crd awsnodetemplates.karpenter.k8s.aws provisioners.karpenter.sh machines.karpenter.sh meta.helm.sh/release-name=karpenter-crd --overwrite -kubectl annotate crd awsnodetemplates.karpenter.k8s.aws provisioners.karpenter.sh machines.karpenter.sh meta.helm.sh/release-namespace=karpenter --overwrite +kubectl annotate crd ec2nodeclasses.karpenter.k8s.aws nodepools.karpenter.sh nodeclaims.karpenter.sh meta.helm.sh/release-name=karpenter-crd --overwrite +kubectl annotate crd ec2nodeclasses.karpenter.k8s.aws nodepools.karpenter.sh nodeclaims.karpenter.sh meta.helm.sh/release-namespace=karpenter --overwrite ``` ## Uninstallation @@ -137,7 +149,7 @@ kubectl get nodes -ojsonpath='{range .items[*].metadata}{@.name}:{@.finalizers}{ If you are not able to create a provisioner due to `Internal error occurred: failed calling webhook "validation.webhook.provisioners.karpenter.sh":` -Webhooks were renamed in `v0.19.0`. There's a bug in ArgoCD's upgrade workflow where webhooks are leaked. This results in Provisioner's failing to be validated, since the validation server no longer corresponds to the webhook definition. +Webhooks were renamed in `0.19.0`. There's a bug in ArgoCD's upgrade workflow where webhooks are leaked. This results in Provisioner's failing to be validated, since the validation server no longer corresponds to the webhook definition. Delete the stale webhooks. @@ -148,7 +160,7 @@ kubectl delete validatingwebhookconfiguration validation.webhook.provisioners.ka ### Failed calling webhook "defaulting.webhook.karpenter.sh" -The `defaulting.webhook.karpenter.sh` mutating webhook was removed in `v0.27.3`. If you are coming from an older version of Karpenter where this webhook existed and the webhook was not managed by Helm, you may need to delete the stale webhook. +The `defaulting.webhook.karpenter.sh` mutating webhook was removed in `0.27.3`. If you are coming from an older version of Karpenter where this webhook existed and the webhook was not managed by Helm, you may need to delete the stale webhook. ```text kubectl delete mutatingwebhookconfigurations defaulting.webhook.karpenter.sh @@ -192,11 +204,11 @@ Disabling swap will allow kubelet to join the cluster successfully, however user ### DaemonSets can result in deployment failures -For Karpenter versions 0.5.3 and earlier, DaemonSets were not properly considered when provisioning nodes. +For Karpenter versions `0.5.3` and earlier, DaemonSets were not properly considered when provisioning nodes. This sometimes caused nodes to be deployed that could not meet the needs of the requested DaemonSets and workloads. -This issue no longer occurs after Karpenter version 0.5.3 (see [PR #1155](https://github.com/aws/karpenter/pull/1155)). +This issue no longer occurs after Karpenter version `0.5.3` (see [PR #1155](https://github.com/aws/karpenter/pull/1155)). -If you are using a pre-0.5.3 version of Karpenter, one workaround is to set your provisioner to only use larger instance types that you know will be big enough for the DaemonSet and the workload. +If you are using a pre `0.5.3` version of Karpenter, one workaround is to set your provisioner to only use larger instance types that you know will be big enough for the DaemonSet and the workload. For more information, see [Issue #1084](https://github.com/aws/karpenter/issues/1084). Examples of this behavior are included in [Issue #1180](https://github.com/aws/karpenter/issues/1180). @@ -213,7 +225,7 @@ See the Karpenter [Best Practices Guide](https://aws.github.io/aws-eks-best-prac ### Missing subnetSelector and securityGroupSelector tags causes provisioning failures -Starting with Karpenter v0.5.5, if you are using Karpenter-generated launch template, provisioners require that [subnetSelector]({{}}) and [securityGroupSelector]({{}}) tags be set to match your cluster. +Starting with Karpenter `0.5.5`, if you are using Karpenter-generated launch template, provisioners require that [subnetSelector]({{}}) and [securityGroupSelector]({{}}) tags be set to match your cluster. The [Provisioner]({{}}) section in the Karpenter Getting Started Guide uses the following example: ```text @@ -284,7 +296,23 @@ Karpenter does not support [in-tree storage plugins](https://kubernetes.io/blog/ #### Pods were scheduled due to a race condition in Kubernetes -Due to [this race condition in Kubernetes](https://github.com/kubernetes/kubernetes/issues/95911), it's possible that the scheduler and the CSINode can race during node registration such that the scheduler assumes that a node can mount more volumes than the node attachments support. There is currently no solve for this problem other than enforcing `toplogySpreadConstraints` and `podAntiAffinity` on your workloads that use PVCs such that you attempt to reduce the number of PVCs that schedule to a given node. +Due to [this race condition in Kubernetes](https://github.com/kubernetes/kubernetes/issues/95911), it's possible that the scheduler and the CSINode can race during node registration such that the scheduler assumes that a node can mount more volumes than the node attachments support. There is currently no universal solve for this problem other than enforcing `toplogySpreadConstraints` and `podAntiAffinity` on your workloads that use PVCs such that you attempt to reduce the number of PVCs that schedule to a given node. + +The following is a list of known CSI drivers which support a startupTaint to eliminate this issue: +- [aws-ebs-csi-driver](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/master/docs/install.md#configure-node-startup-taint) +- [aws-efs-csi-driver](https://github.com/kubernetes-sigs/aws-efs-csi-driver/tree/master/docs#configure-node-startup-taint) + +These taints should be configured via `startupTaints` on your `NodePool`. For example, to enable this for EBS, add the following to your `NodePool`: +```yaml +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +spec: + template: + spec: + startupTaints: + - key: ebs.csi.aws.com/agent-not-ready + effect: NoExecute +``` ### CNI is unable to allocate IPs to pods @@ -642,7 +670,7 @@ Karpenter [doesn't currently support draining and terminating on spot rebalance These two components do not share information between each other, meaning if you have drain and terminate functionality enabled on NTH, NTH may remove a node for a spot rebalance recommendation. Karpenter will replace the node to fulfill the pod capacity that was being fulfilled by the old node; however, Karpenter won't be aware of the reason that that node was terminated. This means that Karpenter may launch the same instance type that was just deprovisioned, causing a spot rebalance recommendation to be sent again. This can result in very short-lived instances where NTH continually removes nodes and Karpeneter re-launches the same instance type over and over again. -Karpenter doesn't recommend reacting to spot rebalance recommendations when running Karpenter with spot nodes; however, if you absolutely require this functionality, note that the above scenario is possible. +Karpenter doesn't recommend reacting to spot rebalance recommendations when running Karpenter with spot nodes; however, if you absolutely require this functionality, note that the above scenario is possible. Spot instances are time limited and, therefore, interruptible. When a signal is sent by AWS, it triggers actions from NTH and Karpenter, where the former signals a shutdown and the later provisions, creating a recursive situation. This can be mitigated by either completely removing NTH or by setting the following values: @@ -652,7 +680,6 @@ enableSpotInterruptionDraining: false * enableRebalanceDrainin: If true, drain nodes when the rebalance recommendation notice is received. Only used in IMDS mode. enableRebalanceDraining: false - ## Pricing ### Stale pricing data on isolated subnet diff --git a/website/content/en/docs/upgrading/compatibility.md b/website/content/en/docs/upgrading/compatibility.md index dccc4caf6aa1..0f2c8d6cab94 100644 --- a/website/content/en/docs/upgrading/compatibility.md +++ b/website/content/en/docs/upgrading/compatibility.md @@ -15,9 +15,9 @@ Before you begin upgrading Karpenter, consider Karpenter compatibility issues re [comment]: <> (the content below is generated from hack/docs/compataiblitymetrix_gen_docs.go) -| KUBERNETES | 1.23 | 1.24 | 1.25 | 1.26 | 1.27 | 1.28 | -|------------|---------|---------|---------|---------|---------|---------| -| karpenter | 0.21.x+ | 0.21.x+ | 0.25.x+ | 0.28.x+ | 0.28.x+ | 0.31.x+ | +| KUBERNETES | 1.23 | 1.24 | 1.25 | 1.26 | 1.27 | 1.28 | 1.29 | +|------------|----------|----------|----------|----------|----------|----------|------------| +| karpenter | \>= 0.21 | \>= 0.21 | \>= 0.25 | \>= 0.28 | \>= 0.28 | \>= 0.31 | \>= 0.34.0 | [comment]: <> (end docs generated content from hack/docs/compataiblitymetrix_gen_docs.go) @@ -31,7 +31,7 @@ For more information on Karpenter's support for these keys, view [this tracking {{% /alert %}} {{% alert title="Note" color="warning" %}} -Karpenter supports using [Kubernetes Common Expression Language](https://kubernetes.io/docs/reference/using-api/cel/) for validating its Custom Resource Definitions out-of-the-box; however, this feature is not supported on versions of Kubernetes < 1.25. If you are running an earlier version of Kubernetes, you will need to use the Karpenter admission webhooks for validation instead. You can enable these webhooks with `--set webhook.enabled=true` when applying the Karpenter helm chart. +Karpenter supports using [Kubernetes Common Expression Language](https://kubernetes.io/docs/reference/using-api/cel/) for validating its Custom Resource Definitions out-of-the-box; however, this feature is not supported on versions of Kubernetes < 1.25. If you are running an earlier version of Kubernetes, you will need to use the Karpenter admission webhooks for validation instead. You can enable these webhooks with `--set webhook.enabled=true` when applying the Karpenter Helm chart. {{% /alert %}} ## Compatibility issues @@ -39,7 +39,7 @@ Karpenter supports using [Kubernetes Common Expression Language](https://kuberne When we introduce a breaking change, we do so only as described in this document. Karpenter follows [Semantic Versioning 2.0.0](https://semver.org/) in its stable release versions, while in -major version zero (v0.y.z) [anything may change at any time](https://semver.org/#spec-item-4). +major version zero (`0.y.z`) [anything may change at any time](https://semver.org/#spec-item-4). However, to further protect users during this phase we will only introduce breaking changes in minor releases (releases that increment y in x.y.z). Note this does not mean every minor upgrade has a breaking change as we will also increment the minor version when we release a new feature. @@ -51,7 +51,7 @@ Users should therefore check to see if there is a breaking change every time the When there is a breaking change we will: * Increment the minor version when in major version 0 -* Add a permanent separate section named `upgrading to vx.y.z+` under [release upgrade notes](#release-upgrade-notes) +* Add a permanent separate section named `upgrading to x.y.z+` under [release upgrade notes](#release-upgrade-notes) clearly explaining the breaking change and what needs to be done on the user side to ensure a safe upgrade * Add the sentence “This is a breaking change, please refer to the above link for upgrade instructions” to the top of the release notes and in all our announcements @@ -77,7 +77,7 @@ Karpenter offers three types of releases. This section explains the purpose of e Stable releases are the most reliable releases that are released with weekly cadence. Stable releases are our only recommended versions for production environments. Sometimes we skip a stable release because we find instability or problems that need to be fixed before having a stable release. -Stable releases are tagged with Semantic Versioning. For example `v0.13.0`. +Stable releases are tagged with a semantic version prefixed by a `v`. For example `v0.13.0`. ### Release Candidates @@ -89,7 +89,7 @@ By adopting this practice we allow our users who are early adopters to test out We release a snapshot release for every commit that gets merged into [`aws/karpenter-provider-aws`](https://www.github.com/aws/karpenter-provider-aws). This enables users to immediately try a new feature or fix right after it's merged rather than waiting days or weeks for release. Snapshot releases are not made available in the same public ECR repository as other release types, they are instead published to a separate private ECR repository. -Helm charts are published to `oci://{{< param "snapshot_repo.account_id" >}}.dkr.ecr.{{< param "snapshot_repo.region" >}}.amazonaws.com/karpenter/snapshot/karpenter` and are tagged with the git commit hash prefixed by the Karpenter major version (e.g. `v0-fc17bfc89ebb30a3b102a86012b3e3992ec08adf`). +Helm charts are published to `oci://{{< param "snapshot_repo.account_id" >}}.dkr.ecr.{{< param "snapshot_repo.region" >}}.amazonaws.com/karpenter/snapshot/karpenter` and are tagged with the git commit hash prefixed by the Karpenter major version (e.g. `0-fc17bfc89ebb30a3b102a86012b3e3992ec08adf`). Anyone with an AWS account can pull from this repository, but must first authenticate: ```bash diff --git a/website/content/en/docs/upgrading/upgrade-guide.md b/website/content/en/docs/upgrading/upgrade-guide.md index d111bf20c2be..bb03be392e08 100644 --- a/website/content/en/docs/upgrading/upgrade-guide.md +++ b/website/content/en/docs/upgrading/upgrade-guide.md @@ -13,92 +13,123 @@ This guide contains information needed to upgrade to the latest release of Karpe ### CRD Upgrades Karpenter ships with a few Custom Resource Definitions (CRDs). These CRDs are published: -* As an independent helm chart [karpenter-crd](https://gallery.ecr.aws/karpenter/karpenter-crd) - [source](https://github.com/aws/karpenter/blob/main/charts/karpenter-crd) that can be used by Helm to manage the lifecycle of these CRDs. To upgrade or install `karpenter-crd` run: +* As an independent Helm chart [karpenter-crd](https://gallery.ecr.aws/karpenter/karpenter-crd) - [source](https://github.com/aws/karpenter/blob/main/charts/karpenter-crd) that can be used by Helm to manage the lifecycle of these CRDs. To upgrade or install `karpenter-crd` run: ```bash KARPENTER_NAMESPACE=kube-system - helm upgrade --install karpenter-crd oci://public.ecr.aws/karpenter/karpenter-crd --version vx.y.z --namespace "${KARPENTER_NAMESPACE}" --create-namespace + helm upgrade --install karpenter-crd oci://public.ecr.aws/karpenter/karpenter-crd --version x.y.z --namespace "${KARPENTER_NAMESPACE}" --create-namespace ``` {{% alert title="Note" color="warning" %}} If you get the error `invalid ownership metadata; label validation error:` while installing the `karpenter-crd` chart from an older version of Karpenter, follow the [Troubleshooting Guide]({{}}) for details on how to resolve these errors. {{% /alert %}} -* As part of the helm chart [karpenter](https://gallery.ecr.aws/karpenter/karpenter) - [source](https://github.com/aws/karpenter/blob/main/charts/karpenter/crds). Helm [does not manage the lifecycle of CRDs using this method](https://helm.sh/docs/chart_best_practices/custom_resource_definitions/), the tool will only install the CRD during the first installation of the helm chart. Subsequent chart upgrades will not add or remove CRDs, even if the CRDs have changed. When CRDs are changed, we will make a note in the version's upgrade guide. +* As part of the helm chart [karpenter](https://gallery.ecr.aws/karpenter/karpenter) - [source](https://github.com/aws/karpenter/blob/main/charts/karpenter/crds). Helm [does not manage the lifecycle of CRDs using this method](https://helm.sh/docs/chart_best_practices/custom_resource_definitions/), the tool will only install the CRD during the first installation of the Helm chart. Subsequent chart upgrades will not add or remove CRDs, even if the CRDs have changed. When CRDs are changed, we will make a note in the version's upgrade guide. -In general, you can reapply the CRDs in the `crds` directory of the Karpenter helm chart: +In general, you can reapply the CRDs in the `crds` directory of the Karpenter Helm chart: ```shell -kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.33.1/pkg/apis/crds/karpenter.sh_nodepools.yaml -kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.33.1/pkg/apis/crds/karpenter.sh_nodeclaims.yaml -kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.33.1/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml +kubectl apply -f https://raw.githubusercontent.com/aws/karpenter/v0.35.4/pkg/apis/crds/karpenter.sh_nodepools.yaml +kubectl apply -f https://raw.githubusercontent.com/aws/karpenter/v0.35.4/pkg/apis/crds/karpenter.sh_nodeclaims.yaml +kubectl apply -f https://raw.githubusercontent.com/aws/karpenter/v0.35.4/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml ``` -### Upgrading to v0.33.0+ + -[comment]: <> (WHEN CREATING A NEW SECTION OF THE UPGRADE GUIDANCE FOR NEWER VERSIONS, ENSURE THAT YOU COPY THE ALERT SECTION BELOW TO PROPERLY WARN USERS OF THE RISK OF UPGRADING WITHOUT GOING TO v0.32 FIRST) +### Upgrading to `0.35.0`+ {{% alert title="Warning" color="warning" %}} -v0.33.0+ _only_ supports Karpenter v1beta1 APIs and will not work with existing Provisioner, AWSNodeTemplate or Machine alpha APIs. **Do not** upgrade to v0.33.0+ without first [upgrading to v0.32.x]({{}}). This version supports both the alpha and beta APIs, allowing you to migrate all of your existing APIs to beta APIs without experiencing downtime. +`0.33.0`+ _only_ supports Karpenter v1beta1 APIs and will not work with existing Provisioner, AWSNodeTemplate or Machine alpha APIs. Do not upgrade to `0.35.0`+ without first [upgrading to `0.32.x`]({{}}). This version supports both the alpha and beta APIs, allowing you to migrate all of your existing APIs to beta APIs without experiencing downtime. {{% /alert %}} +* Karpenter OCI tags and Helm chart version are now valid semantic versions, meaning that the `v` prefix from the git tag has been removed and they now follow the `x.y.z` pattern. + +### Upgrading to `0.34.0`+ + +{{% alert title="Warning" color="warning" %}} +`0.33.0`+ _only_ supports Karpenter v1beta1 APIs and will not work with existing Provisioner, AWSNodeTemplate or Machine alpha APIs. Do not upgrade to `0.34.0`+ without first [upgrading to `0.32.x`]({{}}). This version supports both the alpha and beta APIs, allowing you to migrate all of your existing APIs to beta APIs without experiencing downtime. +{{% /alert %}} + +{{% alert title="Warning" color="warning" %}} +The Ubuntu EKS optimized AMI has moved from 20.04 to 22.04 for Kubernetes 1.29+. This new AMI version is __not currently__ supported for users relying on AMI auto-discovery with the Ubuntu AMI family. More details can be found in this [GitHub issue](https://github.com/aws/karpenter-provider-aws/issues/5572). Please review this issue before upgrading to Kubernetes 1.29 if you are using the Ubuntu AMI family. Upgrading to 1.29 without making any changes to your EC2NodeClass will result in Karpenter being unable to create new nodes. +{{% /alert %}} + +* Karpenter now supports `nodepool.spec.disruption.budgets`, which allows users to control the speed of disruption in the cluster. Since this requires an update to the Custom Resource, before upgrading, you should re-apply the new updates to the CRDs. Check out [Disruption Budgets]({{}}) for more. +* With Disruption Budgets, Karpenter will disrupt multiple batches of nodes simultaneously, which can result in overall quicker scale-down of your cluster. Before `0.34.0`, Karpenter had a hard-coded parallelism limit for each type of disruption. In `0.34.0`+, Karpenter will now disrupt at most 10% of nodes for a given NodePool. There is no setting that will be perfectly equivalent with the behavior prior to `0.34.0`. When considering how to configure your budgets, please refer to the following limits for versions prior to `0.34.0`: + * `Empty Expiration / Empty Drift / Empty Consolidation`: infinite parallelism + * `Non-Empty Expiration / Non-Empty Drift / Single-Node Consolidation`: one node at a time + * `Multi-Node Consolidation`: max 100 nodes +* To support Disruption Budgets, `0.34.0`+ includes critical changes to Karpenter's core controllers, which allows Karpenter to consider multiple batches of disrupting nodes simultaneously. This increases Karpenter's performance with the potential downside of higher CPU and memory utilization from the Karpenter pod. While the magnitude of this difference varies on a case-by-case basis, when upgrading to Karpenter `0.34.0`+, please note that you may need to increase the resources allocated to the Karpenter controller pods. +* Karpenter now adds a default `podSecurityContext` that configures the `fsgroup: 65536` of volumes in the pod. If you are using sidecar containers, you should review if this configuration is compatible for them. You can disable this default `podSecurityContext` through helm by performing `--set podSecurityContext=null` when installing/upgrading the chart. +* The `dnsPolicy` for the Karpenter controller pod has been changed back to the Kubernetes cluster default of `ClusterFirst`. Setting our `dnsPolicy` to `Default` (confusingly, this is not the Kubernetes cluster default) caused more confusion for any users running IPv6 clusters with dual-stack nodes or anyone running Karpenter with dependencies on cluster services (like clusters running service meshes). This change may be breaking for any users on Fargate or MNG who were using Karpenter to manage their in-cluster DNS service (`core-dns` on most clusters). If you still want the old behavior here, you can change the `dnsPolicy` to point to use `Default` by setting the helm value on install/upgrade with `--set dnsPolicy=Default`. More details on this issue can be found in the following Github issues: [#2186](https://github.com/aws/karpenter-provider-aws/issues/2186) and [#4947](https://github.com/aws/karpenter-provider-aws/issues/4947). +* Karpenter now disallows `nodepool.spec.template.spec.resources` to be set. The webhook validation never allowed `nodepool.spec.template.spec.resources`. We are now ensuring that CEL validation also disallows `nodepool.spec.template.spec.resources` to be set. If you were previously setting the resources field on your NodePool, ensure that you remove this field before upgrading to the newest version of Karpenter or else updates to the resource may fail on the new version. + +### Upgrading to `0.33.0`+ + +{{% alert title="Warning" color="warning" %}} +`0.33.0`+ _only_ supports Karpenter v1beta1 APIs and will not work with existing Provisioner, AWSNodeTemplate or Machine alpha APIs. **Do not** upgrade to `0.33.0`+ without first [upgrading to `0.32.x`]({{}}). This version supports both the alpha and beta APIs, allowing you to migrate all of your existing APIs to beta APIs without experiencing downtime. +{{% /alert %}} + +* Karpenter no longer supports using the `karpenter.sh/provisioner-name` label in NodePool labels and requirements or in application node selectors, affinities, or topologySpreadConstraints. If you were previously using this label to target applications to specific Provisioners, you should update your applications to use the `karpenter.sh/nodepool` label instead before upgrading. If you upgrade without changing these labels, you may begin to see pod scheduling failures for these applications. * Karpenter now tags `spot-instances-request` with the same tags that it tags instances, volumes, and primary ENIs. This means that you will now need to add `ec2:CreateTags` permission for `spot-instances-request`. You can also further scope your controller policy for the `ec2:RunInstances` action to require that it launches the `spot-instances-request` with these specific tags. You can view an example of scoping these actions in the [Getting Started Guide's default CloudFormation controller policy](https://github.com/aws/karpenter/blob/v0.33.0/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml#L61). * We now recommend that you set the installation namespace for your Karpenter controllers to `kube-system` to denote Karpenter as a critical cluster component. This ensures that requests from the Karpenter controllers are treated with higher priority by assigning them to a different [PriorityLevelConfiguration](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/#prioritylevelconfiguration) than generic requests from other namespaces. For more details on API Priority and Fairness, read the [Kubernetes API Priority and Fairness Conceptual Docs](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/). Note: Changing the namespace for your Karpenter release will cause the service account namespace to change. If you are using IRSA for authentication with AWS, you will need to change scoping set in the controller's trust policy from `karpenter:karpenter` to `kube-system:karpenter`. -* `v0.33.x` disables mutating and validating webhooks by default in favor of using [Common Expression Language for CRD validation](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#validation). The Common Expression Language Validation Feature [is enabled by default on EKS 1.25](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#validation-rules). If you are using Kubernetes version >= 1.25, no further action is required. If you are using a Kubernetes version below 1.25, you now need to set `DISABLE_WEBHOOK=false` in your container environment variables or `--set webhook.enabled=true` if using Helm. View the [Webhook Support Deprecated in Favor of CEL Section of the v1beta1 Migration Guide]({{}}). -* `v0.33.x` drops support for passing settings through the `karpenter-global-settings` ConfigMap. You should pass settings through the container environment variables in the Karpenter deployment manifest. View the [Global Settings Section of the v1beta1 Migration Guide]({{}}) for more details. -* `v0.33.x` enables `Drift=true` by default in the `FEATURE_GATES`. If you previously didn't enable the feature gate, Karpenter will now check if there is a difference between the desired state of your nodes declared in your NodePool and the actual state of your nodes. View the [Drift Section of Disruption Conceptual Docs]({{}}) for more details. -* `v0.33.x` drops looking up the `zap-logger-config` through ConfigMap discovery. Instead, Karpenter now expects the logging config to be mounted on the filesystem if you are using this to configure Zap logging. This is not enabled by default, but can be enabled through `--set logConfig.enabled=true` in the helm values. If you are setting any values in the `logConfig` from the `v0.32.x` upgrade, such as `logConfig.logEncoding`, note that you will have to explicitly set `logConfig.enabled=true` alongside it. Also, note that setting the Zap logging config is a deprecated feature in beta and is planned to be dropped at v1. View the [Logging Configuration Section of the v1beta1 Migration Guide]({{}}) for more details. -* `v0.33.x` change the default `LOG_LEVEL` from `debug` to `info` by default. If you are still enabling logging configuration through the `zap-logger-config`, no action is required. -* `v0.33.x` drops support for comma delimited lists on tags for `SubnetSelectorTerm`, `SecurityGroupsSelectorTerm`, and `AMISelectorTerm`. Karpenter now supports multiple terms for each of the selectors which means that we can specify a more explicit OR-based constraint through separate terms rather than a comma-delimited list of values. +* `0.33.0` disables mutating and validating webhooks by default in favor of using [Common Expression Language for CRD validation](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#validation). The Common Expression Language Validation Feature [is enabled by default on EKS 1.25](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#validation-rules). If you are using Kubernetes version >= 1.25, no further action is required. If you are using a Kubernetes version below 1.25, you now need to set `DISABLE_WEBHOOK=false` in your container environment variables or `--set webhook.enabled=true` if using Helm. View the [Webhook Support Deprecated in Favor of CEL Section of the v1beta1 Migration Guide]({{}}). +* `0.33.0` drops support for passing settings through the `karpenter-global-settings` ConfigMap. You should pass settings through the container environment variables in the Karpenter deployment manifest. View the [Global Settings Section of the v1beta1 Migration Guide]({{}}) for more details. +* `0.33.0` enables `Drift=true` by default in the `FEATURE_GATES`. If you previously didn't enable the feature gate, Karpenter will now check if there is a difference between the desired state of your nodes declared in your NodePool and the actual state of your nodes. View the [Drift Section of Disruption Conceptual Docs]({{}}) for more details. +* `0.33.0` drops looking up the `zap-logger-config` through ConfigMap discovery. Instead, Karpenter now expects the logging config to be mounted on the filesystem if you are using this to configure Zap logging. This is not enabled by default, but can be enabled through `--set logConfig.enabled=true` in the Helm values. If you are setting any values in the `logConfig` from the `0.32.x` upgrade, such as `logConfig.logEncoding`, note that you will have to explicitly set `logConfig.enabled=true` alongside it. Also, note that setting the Zap logging config is a deprecated feature in beta and is planned to be dropped at v1. View the [Logging Configuration Section of the v1beta1 Migration Guide]({{}}) for more details. +* `0.33.0` change the default `LOG_LEVEL` from `debug` to `info` by default. If you are still enabling logging configuration through the `zap-logger-config`, no action is required. +* `0.33.0` drops support for comma delimited lists on tags for `SubnetSelectorTerm`, `SecurityGroupsSelectorTerm`, and `AMISelectorTerm`. Karpenter now supports multiple terms for each of the selectors which means that we can specify a more explicit OR-based constraint through separate terms rather than a comma-delimited list of values. -### Upgrading to v0.32.0+ +### Upgrading to `0.32.0`+ {{% alert title="Warning" color="warning" %}} -Karpenter v0.32.0 introduces v1beta1 APIs, including _significant_ changes to the API and installation procedures for the Karpenter controllers. Do not upgrade to v0.32.0+ without referencing the [v1beta1 Migration Upgrade Procedure]({{}}). +Karpenter `0.32.0` introduces v1beta1 APIs, including _significant_ changes to the API and installation procedures for the Karpenter controllers. **Do not** upgrade to `0.32.0`+ without referencing the [v1beta1 Migration Upgrade Procedure]({{}}). This version includes **dual support** for both alpha and beta APIs to ensure that you can slowly migrate your existing Provisioner, AWSNodeTemplate, and Machine alpha APIs to the newer NodePool, EC2NodeClass, and NodeClaim beta APIs. -Note that if you are rolling back after upgrading to v0.32.0, note that v0.31.2 and v0.31.3 are the only versions that support handling rollback after you have deployed the v1beta1 APIs to your cluster. +Note that if you are rolling back after upgrading to `0.32.0`, note that `0.31.4` is the only version that supports handling rollback after you have deployed the v1beta1 APIs to your cluster. {{% /alert %}} * Karpenter now serves the webhook prometheus metrics server on port `8001`. If this port is already in-use on the pod or you are running in `hostNetworking` mode, you may need to change this port value. You can configure this port value through the `WEBHOOK_METRICS_PORT` environment variable or the `webhook.metrics.port` value if installing via Helm. * Karpenter now exposes the ability to disable webhooks through the `webhook.enabled=false` value. This value will disable the webhook server and will prevent any permissions, mutating or validating webhook configurations from being deployed to the cluster. * Karpenter now moves all logging configuration for the Zap logger into the `logConfig` values block. Configuring Karpenter logging with this mechanism _is_ deprecated and will be dropped at v1. Karpenter now only surfaces logLevel through the `logLevel` helm value. If you need more advanced configuration due to log parsing constraints, we recommend configuring your log parser to handle Karpenter's Zap JSON logging. -* The default log encoding changed from `console` to `json`. If you were previously not setting the type of log encoding, this default will change with the helm chart. If you were setting the value through `logEncoding`, this value will continue to work until v0.33.x but it is deprecated in favor of `logConfig.logEncoding` +* The default log encoding changed from `console` to `json`. If you were previously not setting the type of log encoding, this default will change with the Helm chart. If you were setting the value through `logEncoding`, this value will continue to work until `0.33.x` but it is deprecated in favor of `logConfig.logEncoding` * Karpenter now uses the `karpenter.sh/disruption:NoSchedule=disrupting` taint instead of the upstream `node.kubernetes.io/unschedulable` taint for nodes spawned with a NodePool to prevent pods from scheduling to nodes being disrupted. Pods that previously tolerated the `node.kubernetes.io/unschedulable` taint that previously weren't evicted during termination will now be evicted. This most notably affects DaemonSets, which have the `node.kubernetes.io/unschedulable` toleration by default, where Karpenter will now remove these pods during termination. If you want your specific pods to not be evicted when nodes are scaled down, you should add a toleration to the pods with the following: `Key=karpenter.sh/disruption, Effect=NoSchedule, Operator=Equals, Values=disrupting`. * Note: Karpenter will continue to use the old `node.kubernetes.io/unschedulable` taint for nodes spawned with a Provisioner. -### Upgrading to v0.31.0+ +### Upgrading to `0.31.0`+ * Karpenter moved its `securityContext` constraints from pod-wide to only applying to the Karpenter container exclusively. If you were previously relying on the pod-wide `securityContext` for your sidecar containers, you will now need to set these values explicitly in your sidecar container configuration. -### Upgrading to v0.30.0+ +### Upgrading to `0.30.0`+ * Karpenter will now [statically drift]({{}}) on both Provisioner and AWSNodeTemplate Fields. For Provisioner Static Drift, the `karpenter.sh/provisioner-hash` annotation must be present on both the Provisioner and Machine. For AWSNodeTemplate drift, the `karpenter.k8s.aws/nodetemplate-hash` annotation must be present on the AWSNodeTemplate and Machine. Karpenter will not add these annotations to pre-existing nodes, so each of these nodes will need to be recycled one time for the annotations to be added. * Karpenter will now fail validation on AWSNodeTemplates and Provisioner `spec.provider` that have `amiSelectors`, `subnetSelectors`, or `securityGroupSelectors` set with a combination of id selectors (`aws-ids`, `aws::ids`) and other selectors. -* Karpenter now statically sets the `securityContext` at both the pod and container-levels and doesn't allow override values to be passed through the helm chart. This change was made to adhere to [Restricted Pod Security Standard](https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted), which follows pod hardening best practices. +* Karpenter now statically sets the `securityContext` at both the pod and container-levels and doesn't allow override values to be passed through the Helm chart. This change was made to adhere to [Restricted Pod Security Standard](https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted), which follows pod hardening best practices. {{% alert title="Note" color="primary" %}} If you have sidecar containers configured to run alongside Karpenter that cannot tolerate the [pod-wide `securityContext` constraints](https://github.com/aws/karpenter/blob/v0.30.0/charts/karpenter/templates/deployment.yaml#L40), you will need to specify overrides to the sidecar `securityContext` in your deployment. {{% /alert %}} -### Upgrading to v0.29.0+ +### Upgrading to `0.29.0`+ {{% alert title="Warning" color="warning" %}} -Karpenter `v0.29.1` contains a [file descriptor and memory leak bug](https://github.com/aws/karpenter/issues/4296) that leads to Karpenter getting OOMKilled and restarting at the point that it hits its memory or file descriptor limit. Karpenter `>v0.29.2` fixes this leak. +Karpenter `0.29.1` contains a [file descriptor and memory leak bug](https://github.com/aws/karpenter/issues/4296) that leads to Karpenter getting OOMKilled and restarting at the point that it hits its memory or file descriptor limit. Karpenter `0.29.2`+ fixes this leak. {{% /alert %}} -* Karpenter has changed the default metrics service port from 8080 to 8000 and the default webhook service port from 443 to 8443. In `v0.28.0`, the Karpenter pod port was changed to 8000, but referenced the service by name, allowing users to scrape the service at port 8080 for metrics. `v0.29.0` aligns the two ports so that service and pod metrics ports are the same. These ports are set by the `controller.metrics.port` and `webhook.port` helm chart values, so if you have previously set these to non-default values, you may need to update your Prometheus scraper to match these new values. +* Karpenter has changed the default metrics service port from 8080 to 8000 and the default webhook service port from 443 to 8443. In `0.28.0`, the Karpenter pod port was changed to 8000, but referenced the service by name, allowing users to scrape the service at port 8080 for metrics. `0.29.0` aligns the two ports so that service and pod metrics ports are the same. These ports are set by the `controller.metrics.port` and `webhook.port` Helm chart values, so if you have previously set these to non-default values, you may need to update your Prometheus scraper to match these new values. * Karpenter will now reconcile nodes that are drifted due to their Security Groups or their Subnets. If your AWSNodeTemplate's Security Groups differ from the Security Groups used for an instance, Karpenter will consider it drifted. If the Subnet used by an instance is not contained in the allowed list of Subnets for an AWSNodeTemplate, Karpenter will also consider it drifted. * Since Karpenter uses tags for discovery of Subnets and SecurityGroups, check the [Threat Model]({{}}) to see how to manage this IAM Permission. -### Upgrading to v0.28.0+ +### Upgrading to `0.28.0`+ {{% alert title="Warning" color="warning" %}} -Karpenter `v0.28.0` is incompatible with Kubernetes version 1.26+, which can result in additional node scale outs when using `--cloudprovider=external`, which is the default for the EKS Optimized AMI. See: https://github.com/aws/karpenter-core/pull/375. Karpenter `>v0.28.1` fixes this issue and is compatible with Kubernetes version 1.26+. +Karpenter `0.28.0` is incompatible with Kubernetes version 1.26+, which can result in additional node scale outs when using `--cloudprovider=external`, which is the default for the EKS Optimized AMI. See: https://github.com/aws/karpenter-core/pull/375. Karpenter `0.28.1`+ fixes this issue and is compatible with Kubernetes version 1.26+. {{% /alert %}} -* The `extraObjects` value is now removed from the Helm chart. Having this value in the chart proved to not work in the majority of Karpenter installs and often led to anti-patterns, where the Karpenter resources installed to manage Karpenter's capacity were directly tied to the install of the Karpenter controller deployments. The Karpenter team recommends that, if you want to install Karpenter manifests alongside the Karpenter helm chart, to do so by creating a separate chart for the manifests, creating a dependency on the controller chart. +* The `extraObjects` value is now removed from the Helm chart. Having this value in the chart proved to not work in the majority of Karpenter installs and often led to anti-patterns, where the Karpenter resources installed to manage Karpenter's capacity were directly tied to the install of the Karpenter controller deployments. The Karpenter team recommends that, if you want to install Karpenter manifests alongside the Karpenter Helm chart, to do so by creating a separate chart for the manifests, creating a dependency on the controller chart. * The `aws.nodeNameConvention` setting is now removed from the [`karpenter-global-settings`]({{}}) ConfigMap. Because Karpenter is now driving its orchestration of capacity through Machines, it no longer needs to know the node name, making this setting obsolete. Karpenter ignores configuration that it doesn't recognize in the [`karpenter-global-settings`]({{}}) ConfigMap, so leaving the `aws.nodeNameConvention` in the ConfigMap will simply cause this setting to be ignored. * Karpenter now defines a set of "restricted tags" which can't be overridden with custom tagging in the AWSNodeTemplate or in the [`karpenter-global-settings`]({{}}) ConfigMap. If you are currently using any of these tag overrides when tagging your instances, webhook validation will now fail. These tags include: @@ -110,7 +141,7 @@ Karpenter `v0.28.0` is incompatible with Kubernetes version 1.26+, which can res * `karpenter_nodes_terminated`: Use `karpenter_machines_terminated` if you are interested in the reason why a Karpenter machine was deleted. `karpenter_nodes_terminated` now only tracks the count of terminated nodes without any additional labels. * `karpenter_nodes_created`: Use `karpenter_machines_created` if you are interested in the reason why a Karpenter machine was created. `karpenter_nodes_created` now only tracks the count of created nodes without any additional labels. * `karpenter_deprovisioning_replacement_node_initialized_seconds`: This metric has been replaced in favor of `karpenter_deprovisioning_replacement_machine_initialized_seconds`. -* `v0.28.0` introduces the Machine CustomResource into the `karpenter.sh` API Group and requires this CustomResourceDefinition to run properly. Karpenter now orchestrates its CloudProvider capacity through these in-cluster Machine CustomResources. When performing a scheduling decision, Karpenter will create a Machine, resulting in launching CloudProvider capacity. The kubelet running on the new capacity will then register the node to the cluster shortly after launch. +* `0.28.0` introduces the Machine CustomResource into the `karpenter.sh` API Group and requires this CustomResourceDefinition to run properly. Karpenter now orchestrates its CloudProvider capacity through these in-cluster Machine CustomResources. When performing a scheduling decision, Karpenter will create a Machine, resulting in launching CloudProvider capacity. The kubelet running on the new capacity will then register the node to the cluster shortly after launch. * If you are using Helm to upgrade between versions of Karpenter, note that [Helm does not automate the process of upgrading or install the new CRDs into your cluster](https://helm.sh/docs/chart_best_practices/custom_resource_definitions/#some-caveats-and-explanations). To install or upgrade the existing CRDs, follow the guidance under the [Custom Resource Definition (CRD) Upgrades]({{< relref "#custom-resource-definition-crd-upgrades" >}}) section of the upgrade guide. * Karpenter will hydrate Machines on startup for existing capacity managed by Karpenter into the cluster. Existing capacity launched by an older version of Karpenter is discovered by finding CloudProvider capacity with the `karpenter.sh/provisioner-name` tag or the `karpenter.sh/provisioner-name` label on nodes. * The metrics port for the Karpenter deployment was changed from 8080 to 8000. Users who scrape the pod directly for metrics rather than the service will need to adjust the commands they use to reference port 8000. Any users who scrape metrics from the service should be unaffected. @@ -126,19 +157,21 @@ Because Karpenter takes this dependency, any user that has the ability to Create {{% /alert %}} {{% alert title="Rolling Back" color="warning" %}} -If, after upgrading to `v0.28.0+`, a rollback to an older version of Karpenter needs to be performed, Karpenter will continue to function normally, though you will still have the Machine CustomResources on your cluster. You will need to manually delete the Machines and patch out the finalizers to fully complete the rollback. +If, after upgrading to `0.28.0`+, a rollback to an older version of Karpenter needs to be performed, Karpenter will continue to function normally, though you will still have the Machine CustomResources on your cluster. You will need to manually delete the Machines and patch out the finalizers to fully complete the rollback. -Karpenter marks CloudProvider capacity as "managed by" a Machine using the `karpenter-sh/managed-by` tag on the CloudProvider machine. It uses this tag to ensure that the Machine CustomResources in the cluster match the CloudProvider capacity managed by Karpenter. If these states don't match, Karpenter will garbage collect the capacity. Because of this, if performing an upgrade, followed by a rollback, followed by another upgrade to `v0.28.0+`, ensure you remove the `karpenter.sh/managed-by` tags from existing capacity; otherwise, Karpenter will deprovision the capacity without a Machine CR counterpart. +Karpenter marks CloudProvider capacity as "managed by" a Machine using the `karpenter-sh/managed-by` tag on the CloudProvider machine. It uses this tag to ensure that the Machine CustomResources in the cluster match the CloudProvider capacity managed by Karpenter. If these states don't match, Karpenter will garbage collect the capacity. Because of this, if performing an upgrade, followed by a rollback, followed by another upgrade to `0.28.0`+, ensure you remove the `karpenter.sh/managed-by` tags from existing capacity; otherwise, Karpenter will deprovision the capacity without a Machine CR counterpart. {{% /alert %}} -### Upgrading to v0.27.3+ -* The `defaulting.webhook.karpenter.sh` mutating webhook was removed in `v0.27.3`. If you are coming from an older version of Karpenter where this webhook existed and the webhook was not managed by Helm, you may need to delete the stale webhook. +### Upgrading to `0.27.3`+ + +* The `defaulting.webhook.karpenter.sh` mutating webhook was removed in `0.27.3`. If you are coming from an older version of Karpenter where this webhook existed and the webhook was not managed by Helm, you may need to delete the stale webhook. ```bash kubectl delete mutatingwebhookconfigurations defaulting.webhook.karpenter.sh ``` -### Upgrading to v0.27.0+ +### Upgrading to `0.27.0`+ + * The Karpenter controller pods now deploy with `kubernetes.io/hostname` self anti-affinity by default. If you are running Karpenter in HA (high-availability) mode and you do not have enough nodes to match the number of pod replicas you are deploying with, you will need to scale-out your nodes for Karpenter. * The following controller metrics changed and moved under the `controller_runtime` metrics namespace: * `karpenter_metricscraper_...` @@ -155,27 +188,33 @@ kubectl delete mutatingwebhookconfigurations defaulting.webhook.karpenter.sh * `provisioner-state` -> `provisioner_state` * The `karpenter_allocation_controller_scheduling_duration_seconds` metric name changed to `karpenter_provisioner_scheduling_duration_seconds` -### Upgrading to v0.26.0+ +### Upgrading to `0.26.0`+ + * The `karpenter.sh/do-not-evict` annotation no longer blocks node termination when running `kubectl delete node`. This annotation on pods will only block automatic deprovisioning that is considered "voluntary," that is, disruptions that can be avoided. Disruptions that Karpenter deems as "involuntary" and will ignore the `karpenter.sh/do-not-evict` annotation include spot interruption and manual deletion of the node. See [Disabling Deprovisioning]({{}}) for more details. -* Default resources `requests` and `limits` are removed from the Karpenter's controller deployment through the Helm chart. If you have not set custom resource `requests` or `limits` in your helm values and are using Karpenter's defaults, you will now need to set these values in your helm chart deployment. -* The `controller.image` value in the helm chart has been broken out to a map consisting of `controller.image.repository`, `controller.image.tag`, and `controller.image.digest`. If manually overriding the `controller.image`, you will need to update your values to the new design. +* Default resources `requests` and `limits` are removed from the Karpenter's controller deployment through the Helm chart. If you have not set custom resource `requests` or `limits` in your Helm values and are using Karpenter's defaults, you will now need to set these values in your Helm chart deployment. +* The `controller.image` value in the Helm chart has been broken out to a map consisting of `controller.image.repository`, `controller.image.tag`, and `controller.image.digest`. If manually overriding the `controller.image`, you will need to update your values to the new design. + +### Upgrading to `0.25.0`+ -### Upgrading to v0.25.0+ * Cluster Endpoint can now be automatically discovered. If you are using Amazon Elastic Kubernetes Service (EKS), you can now omit the `clusterEndpoint` field in your configuration. In order to allow the resolving, you have to add the permission `eks:DescribeCluster` to the Karpenter Controller IAM role. -### Upgrading to v0.24.0+ +### Upgrading to `0.24.0`+ + * Settings are no longer updated dynamically while Karpenter is running. If you manually make a change to the [`karpenter-global-settings`]({{}}) ConfigMap, you will need to reload the containers by restarting the deployment with `kubectl rollout restart -n karpenter deploy/karpenter` * Karpenter no longer filters out instance types internally. Previously, `g2` (not supported by the NVIDIA device plugin) and FPGA instance types were filtered. The only way to filter instance types now is to set requirements on your provisioner or pods using well-known node labels described [here]({{}}). If you are currently using overly broad requirements that allows all of the `g` instance-category, you will want to tighten the requirement, or add an instance-generation requirement. * `aws.tags` in [`karpenter-global-settings`]({{}}) ConfigMap is now a top-level field and expects the value associated with this key to be a JSON object of string to string. This is change from previous versions where keys were given implicitly by providing the key-value pair `aws.tags.: value` in the ConfigMap. -### Upgrading to v0.22.0+ +### Upgrading to `0.22.0`+ + * Do not upgrade to this version unless you are on Kubernetes >= v1.21. Karpenter no longer supports Kubernetes v1.20, but now supports Kubernetes v1.25. This change is due to the v1 PDB API, which was introduced in K8s v1.20 and subsequent removal of the v1beta1 API in K8s v1.25. -### Upgrading to v0.20.0+ -* Prior to v0.20.0, Karpenter would prioritize certain instance type categories absent of any requirements in the Provisioner. v0.20.0+ removes prioritizing these instance type categories ("m", "c", "r", "a", "t", "i") in code. Bare Metal and GPU instance types are still deprioritized and only used if no other instance types are compatible with the node requirements. Since Karpenter does not prioritize any instance types, if you do not want exotic instance types and are not using the runtime Provisioner defaults, you will need to specify this in the Provisioner. +### Upgrading to `0.20.0`+ -### Upgrading to v0.19.0+ -* The karpenter webhook and controller containers are combined into a single binary, which requires changes to the helm chart. If your Karpenter installation (helm or otherwise) currently customizes the karpenter webhook, your deployment tooling may require minor changes. +* Prior to `0.20.0`, Karpenter would prioritize certain instance type categories absent of any requirements in the Provisioner. `0.20.0`+ removes prioritizing these instance type categories ("m", "c", "r", "a", "t", "i") in code. Bare Metal and GPU instance types are still deprioritized and only used if no other instance types are compatible with the node requirements. Since Karpenter does not prioritize any instance types, if you do not want exotic instance types and are not using the runtime Provisioner defaults, you will need to specify this in the Provisioner. + +### Upgrading to `0.19.0`+ + +* The karpenter webhook and controller containers are combined into a single binary, which requires changes to the Helm chart. If your Karpenter installation (Helm or otherwise) currently customizes the karpenter webhook, your deployment tooling may require minor changes. * Karpenter now supports native interruption handling. If you were previously using Node Termination Handler for spot interruption handling and health events, you will need to remove the component from your cluster before enabling `aws.interruptionQueueName`. For more details on Karpenter's interruption handling, see the [Interruption Handling Docs]({{< ref "../concepts/disruption/#interruption" >}}). * Instance category defaults are now explicitly persisted in the Provisioner, rather than handled implicitly in memory. By default, Provisioners will limit instance category to c,m,r. If any instance type constraints are applied, it will override this default. If you have created Provisioners in the past with unconstrained instance type, family, or category, Karpenter will now more flexibly use instance types than before. If you would like to apply these constraints, they must be included in the Provisioner CRD. * Karpenter CRD raw YAML URLs have migrated from `https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.19.3/charts/karpenter/crds/...` to `https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.19.3/pkg/apis/crds/...`. If you reference static Karpenter CRDs or rely on `kubectl replace -f` to apply these CRDs from their remote location, you will need to migrate to the new location. @@ -191,38 +230,44 @@ kubectl delete mutatingwebhookconfigurations defaulting.webhook.karpenter.sh * `AWS_NODE_NAME_CONVENTION` -> `settings.aws.nodeNameConvention` * `VM_MEMORY_OVERHEAD` -> `settings.aws.vmMemoryOverheadPercent` -### Upgrading to v0.18.0+ -* v0.18.0 removes the `karpenter_consolidation_nodes_created` and `karpenter_consolidation_nodes_terminated` prometheus metrics in favor of the more generic `karpenter_nodes_created` and `karpenter_nodes_terminated` metrics. You can still see nodes created and terminated by consolidation by checking the `reason` label on the metrics. Check out all the metrics published by Karpenter [here]({{}}). +### Upgrading to `0.18.0`+ + +* `0.18.0` removes the `karpenter_consolidation_nodes_created` and `karpenter_consolidation_nodes_terminated` prometheus metrics in favor of the more generic `karpenter_nodes_created` and `karpenter_nodes_terminated` metrics. You can still see nodes created and terminated by consolidation by checking the `reason` label on the metrics. Check out all the metrics published by Karpenter [here]({{}}). + +### Upgrading to `0.17.0`+ -### Upgrading to v0.17.0+ Karpenter's Helm chart package is now stored in [Karpenter's OCI (Open Container Initiative) registry](https://gallery.ecr.aws/karpenter/karpenter). The Helm CLI supports the new format since [v3.8.0+](https://helm.sh/docs/topics/registries/). -With this change [charts.karpenter.sh](https://charts.karpenter.sh/) is no longer updated but preserved to allow using older Karpenter versions. For examples on working with the Karpenter helm charts look at [Install Karpenter Helm Chart]({{< ref "../getting-started/getting-started-with-karpenter/#install-karpenter-helm-chart" >}}). +With this change [charts.karpenter.sh](https://charts.karpenter.sh/) is no longer updated but preserved to allow using older Karpenter versions. For examples on working with the Karpenter Helm charts look at [Install Karpenter Helm Chart]({{< ref "../getting-started/getting-started-with-karpenter/#install-karpenter-helm-chart" >}}). Users who have scripted the installation or upgrading of Karpenter need to adjust their scripts with the following changes: -1. There is no longer a need to add the Karpenter helm repo to helm -2. The full URL of the Helm chart needs to be present when using the helm commands -3. If you were not prepending a `v` to the version (i.e. `0.17.0`), you will need to do so with the OCI chart, `v0.17.0`. +1. There is no longer a need to add the Karpenter Helm repo with `helm repo add` +2. The full URL of the Helm chart needs to be present when using the `helm` CLI +3. If you were not prepending a `v` to the version (i.e. `0.17.0`), you will need to do so with the OCI chart (i.e `v0.17.0`). + +### Upgrading to `0.16.2`+ -### Upgrading to v0.16.2+ -* v0.16.2 adds new kubeletConfiguration fields to the `provisioners.karpenter.sh` v1alpha5 CRD. The CRD will need to be updated to use the new parameters: +* `0.16.2` adds new kubeletConfiguration fields to the `provisioners.karpenter.sh` v1alpha5 CRD. The CRD will need to be updated to use the new parameters: ```bash kubectl replace -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.16.2/charts/karpenter/crds/karpenter.sh_provisioners.yaml ``` -### Upgrading to v0.16.0+ -* v0.16.0 adds a new weight field to the `provisioners.karpenter.sh` v1alpha5 CRD. The CRD will need to be updated to use the new parameters: +### Upgrading to `0.16.0`+ + +* `0.16.0` adds a new weight field to the `provisioners.karpenter.sh` v1alpha5 CRD. The CRD will need to be updated to use the new parameters: ```bash kubectl replace -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.16.0/charts/karpenter/crds/karpenter.sh_provisioners.yaml ``` -### Upgrading to v0.15.0+ -* v0.15.0 adds a new consolidation field to the `provisioners.karpenter.sh` v1alpha5 CRD. The CRD will need to be updated to use the new parameters: +### Upgrading to `0.15.0`+ + +* `0.15.0` adds a new consolidation field to the `provisioners.karpenter.sh` v1alpha5 CRD. The CRD will need to be updated to use the new parameters: ```bash kubectl replace -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.15.0/charts/karpenter/crds/karpenter.sh_provisioners.yaml ``` -### Upgrading to v0.14.0+ -* v0.14.0 adds new fields to the `provisioners.karpenter.sh` v1alpha5 and `awsnodetemplates.karpenter.k8s.aws` v1alpha1 CRDs. The CRDs will need to be updated to use the new parameters: +### Upgrading to `0.14.0`+ + +* `0.14.0` adds new fields to the `provisioners.karpenter.sh` v1alpha5 and `awsnodetemplates.karpenter.k8s.aws` v1alpha1 CRDs. The CRDs will need to be updated to use the new parameters: ```bash kubectl replace -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.14.0/charts/karpenter/crds/karpenter.sh_provisioners.yaml @@ -230,7 +275,7 @@ kubectl replace -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/ kubectl replace -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.14.0/charts/karpenter/crds/karpenter.k8s.aws_awsnodetemplates.yaml ``` -* v0.14.0 changes the way Karpenter discovers its dynamically generated AWS launch templates to use a tag rather than a Name scheme. The previous name scheme was `Karpenter-${CLUSTER_NAME}-*` which could collide with user created launch templates that Karpenter should not manage. The new scheme uses a tag on the launch template `karpenter.k8s.aws/cluster: ${CLUSTER_NAME}`. As a result, Karpenter will not clean-up dynamically generated launch templates using the old name scheme. You can manually clean these up with the following commands: +* `0.14.0` changes the way Karpenter discovers its dynamically generated AWS launch templates to use a tag rather than a Name scheme. The previous name scheme was `Karpenter-${CLUSTER_NAME}-*` which could collide with user created launch templates that Karpenter should not manage. The new scheme uses a tag on the launch template `karpenter.k8s.aws/cluster: ${CLUSTER_NAME}`. As a result, Karpenter will not clean-up dynamically generated launch templates using the old name scheme. You can manually clean these up with the following commands: ```bash ## Find launch templates that match the naming pattern and you do not want to keep @@ -240,52 +285,54 @@ aws ec2 describe-launch-templates --filters="Name=launch-template-name,Values=Ka aws ec2 delete-launch-template --launch-template-id ``` -* v0.14.0 introduces additional instance type filtering if there are no `node.kubernetes.io/instance-type` or `karpenter.k8s.aws/instance-family` or `karpenter.k8s.aws/instance-category` requirements that restrict instance types specified on the provisioner. This prevents Karpenter from launching bare metal and some older non-current generation instance types unless the provisioner has been explicitly configured to allow them. If you specify an instance type or family requirement that supplies a list of instance-types or families, that list will be used regardless of filtering. The filtering can also be completely eliminated by adding an `Exists` requirement for instance type or family. +* `0.14.0` introduces additional instance type filtering if there are no `node.kubernetes.io/instance-type` or `karpenter.k8s.aws/instance-family` or `karpenter.k8s.aws/instance-category` requirements that restrict instance types specified on the provisioner. This prevents Karpenter from launching bare metal and some older non-current generation instance types unless the provisioner has been explicitly configured to allow them. If you specify an instance type or family requirement that supplies a list of instance-types or families, that list will be used regardless of filtering. The filtering can also be completely eliminated by adding an `Exists` requirement for instance type or family. ```yaml - key: node.kubernetes.io/instance-type operator: Exists ``` -* v0.14.0 introduces support for custom AMIs without the need for an entire launch template. You must add the `ec2:DescribeImages` permission to the Karpenter Controller Role for this feature to work. This permission is needed for Karpenter to discover custom images specified. Read the [Custom AMI documentation here]({{}}) to get started -* v0.14.0 adds an an additional default toleration (CriticalAddonOnly=Exists) to the Karpenter helm chart. This may cause Karpenter to run on nodes with that use this Taint which previously would not have been schedulable. This can be overridden by using `--set tolerations[0]=null`. +* `0.14.0` introduces support for custom AMIs without the need for an entire launch template. You must add the `ec2:DescribeImages` permission to the Karpenter Controller Role for this feature to work. This permission is needed for Karpenter to discover custom images specified. Read the [Custom AMI documentation here]({{}}) to get started +* `0.14.0` adds an an additional default toleration (CriticalAddonOnly=Exists) to the Karpenter Helm chart. This may cause Karpenter to run on nodes with that use this Taint which previously would not have been schedulable. This can be overridden by using `--set tolerations[0]=null`. + +* `0.14.0` deprecates the `AWS_ENI_LIMITED_POD_DENSITY` environment variable in-favor of specifying `spec.kubeletConfiguration.maxPods` on the Provisioner. `AWS_ENI_LIMITED_POD_DENSITY` will continue to work when `maxPods` is not set on the Provisioner. If `maxPods` is set, it will override `AWS_ENI_LIMITED_POD_DENSITY` on that specific Provisioner. -* v0.14.0 deprecates the `AWS_ENI_LIMITED_POD_DENSITY` environment variable in-favor of specifying `spec.kubeletConfiguration.maxPods` on the Provisioner. `AWS_ENI_LIMITED_POD_DENSITY` will continue to work when `maxPods` is not set on the Provisioner. If `maxPods` is set, it will override `AWS_ENI_LIMITED_POD_DENSITY` on that specific Provisioner. +### Upgrading to `0.13.0`+ -### Upgrading to v0.13.0+ -* v0.13.0 introduces a new CRD named `AWSNodeTemplate` which can be used to specify AWS Cloud Provider parameters. Everything that was previously specified under `spec.provider` in the Provisioner resource, can now be specified in the spec of the new resource. The use of `spec.provider` is deprecated but will continue to function to maintain backwards compatibility for the current API version (v1alpha5) of the Provisioner resource. v0.13.0 also introduces support for custom user data that doesn't require the use of a custom launch template. The user data can be specified in-line in the AWSNodeTemplate resource. +* `0.13.0` introduces a new CRD named `AWSNodeTemplate` which can be used to specify AWS Cloud Provider parameters. Everything that was previously specified under `spec.provider` in the Provisioner resource, can now be specified in the spec of the new resource. The use of `spec.provider` is deprecated but will continue to function to maintain backwards compatibility for the current API version (v1alpha5) of the Provisioner resource. `0.13.0` also introduces support for custom user data that doesn't require the use of a custom launch template. The user data can be specified in-line in the AWSNodeTemplate resource. - If you are upgrading from v0.10.1 - v0.11.1, a new CRD `awsnodetemplate` was added. In v0.12.0, this crd was renamed to `awsnodetemplates`. Since helm does not manage the lifecycle of CRDs, you will need to perform a few manual steps for this CRD upgrade: + If you are upgrading from `0.10.1` - `0.11.1`, a new CRD `awsnodetemplate` was added. In `0.12.0`, this crd was renamed to `awsnodetemplates`. Since Helm does not manage the lifecycle of CRDs, you will need to perform a few manual steps for this CRD upgrade: 1. Make sure any `awsnodetemplate` manifests are saved somewhere so that they can be reapplied to the cluster. 2. `kubectl delete crd awsnodetemplate` 3. `kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.13.2/charts/karpenter/crds/karpenter.k8s.aws_awsnodetemplates.yaml` - 4. Perform the Karpenter upgrade to v0.13.x, which will install the new `awsnodetemplates` CRD. + 4. Perform the Karpenter upgrade to `0.13.0`+, which will install the new `awsnodetemplates` CRD. 5. Reapply the `awsnodetemplate` manifests you saved from step 1, if applicable. -* v0.13.0 also adds EC2/spot price fetching to Karpenter to allow making more accurate decisions regarding node deployments. Our [getting started guide]({{< ref "../getting-started/getting-started-with-karpenter" >}}) documents this, but if you are upgrading Karpenter you will need to modify your Karpenter controller policy to add the `pricing:GetProducts` and `ec2:DescribeSpotPriceHistory` permissions. +* `0.13.0` also adds EC2/spot price fetching to Karpenter to allow making more accurate decisions regarding node deployments. Our [getting started guide]({{< ref "../getting-started/getting-started-with-karpenter" >}}) documents this, but if you are upgrading Karpenter you will need to modify your Karpenter controller policy to add the `pricing:GetProducts` and `ec2:DescribeSpotPriceHistory` permissions. + +### Upgrading to `0.12.0`+ -### Upgrading to v0.12.0+ -* v0.12.0 adds an OwnerReference to each Node created by a provisioner. Previously, deleting a provisioner would orphan nodes. Now, deleting a provisioner will cause Kubernetes [cascading delete](https://kubernetes.io/docs/concepts/architecture/garbage-collection/#cascading-deletion) logic to gracefully terminate the nodes using the Karpenter node finalizer. You may still orphan nodes by removing the owner reference. -* If you are upgrading from v0.10.1 - v0.11.1, a new CRD `awsnodetemplate` was added. In v0.12.0, this crd was renamed to `awsnodetemplates`. Since helm does not manage the lifecycle of CRDs, you will need to perform a few manual steps for this CRD upgrade: +* `0.12.0` adds an OwnerReference to each Node created by a provisioner. Previously, deleting a provisioner would orphan nodes. Now, deleting a provisioner will cause Kubernetes [cascading delete](https://kubernetes.io/docs/concepts/architecture/garbage-collection/#cascading-deletion) logic to gracefully terminate the nodes using the Karpenter node finalizer. You may still orphan nodes by removing the owner reference. +* If you are upgrading from `0.10.1` - `0.11.1`, a new CRD `awsnodetemplate` was added. In `0.12.0`, this crd was renamed to `awsnodetemplates`. Since Helm does not manage the lifecycle of CRDs, you will need to perform a few manual steps for this CRD upgrade: 1. Make sure any `awsnodetemplate` manifests are saved somewhere so that they can be reapplied to the cluster. 2. `kubectl delete crd awsnodetemplate` 3. `kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.12.1/charts/karpenter/crds/karpenter.k8s.aws_awsnodetemplates.yaml` - 4. Perform the Karpenter upgrade to v0.12.x, which will install the new `awsnodetemplates` CRD. + 4. Perform the Karpenter upgrade to `0.12.0`+, which will install the new `awsnodetemplates` CRD. 5. Reapply the `awsnodetemplate` manifests you saved from step 1, if applicable. -### Upgrading to v0.11.0+ +### Upgrading to `0.11.0`+ -v0.11.0 changes the way that the `vpc.amazonaws.com/pod-eni` resource is reported. Instead of being reported for all nodes that could support the resources regardless of if the cluster is configured to support it, it is now controlled by a command line flag or environment variable. The parameter defaults to false and must be set if your cluster uses [security groups for pods](https://docs.aws.amazon.com/eks/latest/userguide/security-groups-for-pods.html). This can be enabled by setting the environment variable `AWS_ENABLE_POD_ENI` to true via the helm value `controller.env`. +`0.11.0` changes the way that the `vpc.amazonaws.com/pod-eni` resource is reported. Instead of being reported for all nodes that could support the resources regardless of if the cluster is configured to support it, it is now controlled by a command line flag or environment variable. The parameter defaults to false and must be set if your cluster uses [security groups for pods](https://docs.aws.amazon.com/eks/latest/userguide/security-groups-for-pods.html). This can be enabled by setting the environment variable `AWS_ENABLE_POD_ENI` to true via the helm value `controller.env`. Other extended resources must be registered on nodes by their respective device plugins which are typically installed as DaemonSets (e.g. the `nvidia.com/gpu` resource will be registered by the [NVIDIA device plugin](https://github.com/NVIDIA/k8s-device-plugin). Previously, Karpenter would register these resources on nodes at creation and they would be zeroed out by `kubelet` at startup. By allowing the device plugins to register the resources, pods will not bind to the nodes before any device plugin initialization has occurred. -v0.11.0 adds a `providerRef` field in the Provisioner CRD. To use this new field you will need to replace the Provisioner CRD manually: +`0.11.0` adds a `providerRef` field in the Provisioner CRD. To use this new field you will need to replace the Provisioner CRD manually: ```shell kubectl replace -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.11.0/charts/karpenter/crds/karpenter.sh_provisioners.yaml ``` -### Upgrading to v0.10.0+ +### Upgrading to `0.10.0`+ -v0.10.0 adds a new field, `startupTaints` to the provisioner spec. Standard Helm upgrades [do not upgrade CRDs](https://helm.sh/docs/chart_best_practices/custom_resource_definitions/#some-caveats-and-explanations) so the field will not be available unless the CRD is manually updated. This can be performed prior to the standard upgrade by applying the new CRD manually: +`0.10.0` adds a new field, `startupTaints` to the provisioner spec. Standard Helm upgrades [do not upgrade CRDs](https://helm.sh/docs/chart_best_practices/custom_resource_definitions/#some-caveats-and-explanations) so the field will not be available unless the CRD is manually updated. This can be performed prior to the standard upgrade by applying the new CRD manually: ```shell kubectl replace -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.10.0/charts/karpenter/crds/karpenter.sh_provisioners.yaml @@ -293,7 +340,7 @@ kubectl replace -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/ 📝 If you don't perform this manual CRD update, Karpenter will work correctly except for rejecting the creation/update of provisioners that use `startupTaints`. -### Upgrading to v0.6.2+ +### Upgrading to `0.6.2`+ If using Helm, the variable names have changed for the cluster's name and endpoint. You may need to update any configuration that sets the old variable names. diff --git a/website/content/en/preview/concepts/disruption.md b/website/content/en/preview/concepts/disruption.md index 6710a5f099a1..48e0b479c5f3 100644 --- a/website/content/en/preview/concepts/disruption.md +++ b/website/content/en/preview/concepts/disruption.md @@ -13,7 +13,7 @@ The finalizer blocks deletion of the node object while the Termination Controlle ### Disruption Controller -Karpenter automatically discovers disruptable nodes and spins up replacements when needed. Karpenter disrupts nodes by executing one [automatic method](#automatic-methods) at a time, in order of Expiration, Drift, and then Consolidation. Each method varies slightly, but they all follow the standard disruption process. Karpenter uses [disruption budgets]({{}}) to control the speed of disruption. +Karpenter automatically discovers disruptable nodes and spins up replacements when needed. Karpenter disrupts nodes by executing one [automated method](#automated-methods) at a time, in order of Expiration, Drift, and then Consolidation. Each method varies slightly, but they all follow the standard disruption process. Karpenter uses [disruption budgets]({{}}) to control the speed of disruption. 1. Identify a list of prioritized candidates for the disruption method. * If there are [pods that cannot be evicted](#pod-eviction) on the node, Karpenter will ignore the node and try disrupting it later. * If there are no disruptable nodes, continue to the next disruption method. @@ -29,8 +29,8 @@ Karpenter automatically discovers disruptable nodes and spins up replacements wh ### Termination Controller When a Karpenter node is deleted, the Karpenter finalizer will block deletion and the APIServer will set the `DeletionTimestamp` on the node, allowing Karpenter to gracefully shutdown the node, modeled after [Kubernetes Graceful Node Shutdown](https://kubernetes.io/docs/concepts/architecture/nodes/#graceful-node-shutdown). Karpenter's graceful shutdown process will: -1. Add the `karpenter.sh/disruption:NoSchedule` taint to the node to prevent pods from scheduling to it. -2. Begin evicting the pods on the node with the [Kubernetes Eviction API](https://kubernetes.io/docs/concepts/scheduling-eviction/api-eviction/) to respect PDBs, while ignoring all non-daemonset pods and [static pods](https://kubernetes.io/docs/tasks/configure-pod-container/static-pod/). Wait for the node to be fully drained before proceeding to Step (3). +1. Add the `karpenter.sh/disruption=disrupting:NoSchedule` taint to the node to prevent pods from scheduling to it. +2. Begin evicting the pods on the node with the [Kubernetes Eviction API](https://kubernetes.io/docs/concepts/scheduling-eviction/api-eviction/) to respect PDBs, while ignoring all [static pods](https://kubernetes.io/docs/tasks/configure-pod-container/static-pod/), pods tolerating the `karpenter.sh/disruption=disrupting:NoSchedule` taint, and succeeded/failed pods. Wait for the node to be fully drained before proceeding to Step (3). * While waiting, if the underlying NodeClaim for the node no longer exists, remove the finalizer to allow the APIServer to delete the node, completing termination. 3. Terminate the NodeClaim in the Cloud Provider. 4. Remove the finalizer from the node to allow the APIServer to delete the node, completing termination. @@ -69,7 +69,7 @@ Automated methods can be rate limited through [NodePool Disruption Budgets]({{}}): Karpenter works to actively reduce cluster cost by identifying when: * Nodes can be removed because the node is empty * Nodes can be removed as their workloads will run on other nodes in the cluster. - * Nodes can be replaced with cheaper variants due to a change in the workloads. + * Nodes can be replaced with lower priced variants due to a change in the workloads. * [**Drift**]({{}}): Karpenter will mark nodes as drifted and disrupt nodes that have drifted from their desired specification. See [Drift]({{}}) to see which fields are considered. * [**Interruption**]({{}}): Karpenter will watch for upcoming interruption events that could affect your nodes (health events, spot interruption, etc.) and will taint, drain, and terminate the node(s) ahead of the event to reduce workload disruption. @@ -88,12 +88,12 @@ spec: Karpenter has two mechanisms for cluster consolidation: 1. **Deletion** - A node is eligible for deletion if all of its pods can run on free capacity of other nodes in the cluster. -2. **Replace** - A node can be replaced if all of its pods can run on a combination of free capacity of other nodes in the cluster and a single cheaper replacement node. +2. **Replace** - A node can be replaced if all of its pods can run on a combination of free capacity of other nodes in the cluster and a single lower price replacement node. Consolidation has three mechanisms that are performed in order to attempt to identify a consolidation action: 1. **Empty Node Consolidation** - Delete any entirely empty nodes in parallel -2. **Multi Node Consolidation** - Try to delete two or more nodes in parallel, possibly launching a single replacement that is cheaper than the price of all nodes being removed -3. **Single Node Consolidation** - Try to delete any single node, possibly launching a single replacement that is cheaper than the price of that node +2. **Multi Node Consolidation** - Try to delete two or more nodes in parallel, possibly launching a single replacement whose price is lower than that of all nodes being removed +3. **Single Node Consolidation** - Try to delete any single node, possibly launching a single replacement whose price is lower than that of the node being removed It's impractical to examine all possible consolidation options for multi-node consolidation, so Karpenter uses a heuristic to identify a likely set of nodes that can be consolidated. For single-node consolidation we consider each node in the cluster individually. @@ -110,7 +110,7 @@ Events: Type Reason Age From Message ---- ------ ---- ---- ------- Normal Unconsolidatable 66s karpenter pdb default/inflate-pdb prevents pod evictions - Normal Unconsolidatable 33s (x3 over 30m) karpenter can't replace with a cheaper node + Normal Unconsolidatable 33s (x3 over 30m) karpenter can't replace with a lower-priced node ``` {{% alert title="Warning" color="warning" %}} @@ -120,10 +120,10 @@ Using preferred anti-affinity and topology spreads can reduce the effectiveness #### Spot consolidation For spot nodes, Karpenter has deletion consolidation enabled by default. If you would like to enable replacement with spot consolidation, you need to enable the feature through the [`SpotToSpotConsolidation` feature flag]({{}}). -Cheaper spot instance types are selected with the [`price-capacity-optimized` strategy](https://aws.amazon.com/blogs/compute/introducing-price-capacity-optimized-allocation-strategy-for-ec2-spot-instances/). Often, the cheapest spot instance type is not launched due to the likelihood of interruption. As a result, Karpenter uses the number of available instance type options cheaper than the currently launched spot instance as a heuristic for evaluating whether it should launch a replacement for the current spot node. +Lower priced spot instance types are selected with the [`price-capacity-optimized` strategy](https://aws.amazon.com/blogs/compute/introducing-price-capacity-optimized-allocation-strategy-for-ec2-spot-instances/). Sometimes, the lowest priced spot instance type is not launched due to the likelihood of interruption. As a result, Karpenter uses the number of available instance type options with a price lower than the currently launched spot instance as a heuristic for evaluating whether it should launch a replacement for the current spot node. We refer to the number of instances that Karpenter has within its launch decision as a launch's "instance type flexibility." When Karpenter is considering performing a spot-to-spot consolidation replacement, it will check whether replacing the instance type will lead to enough instance type flexibility in the subsequent launch request. As a result, we get the following properties when evaluating for consolidation: -1) We shouldn't continually consolidate down to the cheapest spot instance which might have very high rates of interruption. +1) We shouldn't continually consolidate down to the lowest priced spot instance which might have very high rates of interruption. 2) We launch with enough instance types that there’s high likelihood that our replacement instance has comparable availability to our current one. Karpenter requires a minimum instance type flexibility of 15 instance types when performing single node spot-to-spot consolidations (1 node to 1 node). It does not have the same instance type flexibility requirement for multi-node spot-to-spot consolidations (many nodes to 1 node) since doing so without requiring flexibility won't lead to "race to the bottom" scenarios. @@ -195,9 +195,9 @@ To enable interruption handling, configure the `--interruption-queue-name` CLI a You can rate limit Karpenter's disruption through the NodePool's `spec.disruption.budgets`. If undefined, Karpenter will default to one budget with `nodes: 10%`. Budgets will consider nodes that are actively being deleted for any reason, and will only block Karpenter from disrupting nodes voluntarily through expiration, drift, emptiness, and consolidation. #### Nodes -When calculating if a budget will block nodes from disruption, Karpenter lists the total number of nodes owned by a NodePool, subtracting out the nodes owned by that NodePool that are currently being deleted. If the number of nodes being deleted by Karpenter or any other processes is greater than the number of allowed disruptions, disruption for this node will not proceed. +When calculating if a budget will block nodes from disruption, Karpenter lists the total number of nodes owned by a NodePool, subtracting out the nodes owned by that NodePool that are currently being deleted and nodes that are NotReady. If the number of nodes being deleted by Karpenter or any other processes is greater than the number of allowed disruptions, disruption for this node will not proceed. -If the budget is configured with a percentage value, such as `20%`, Karpenter will calculate the number of allowed disruptions as `allowed_disruptions = roundup(total * percentage) - total_deleting`. If otherwise defined as a non-percentage value, Karpenter will simply subtract the number of nodes from the total `(total * percentage) - total_deleting`. For multiple budgets in a NodePool, Karpenter will take the minimum value (most restrictive) of each of the budgets. +If the budget is configured with a percentage value, such as `20%`, Karpenter will calculate the number of allowed disruptions as `allowed_disruptions = roundup(total * percentage) - total_deleting - total_notready`. If otherwise defined as a non-percentage value, Karpenter will simply subtract the number of nodes from the total `(total - non_percentage_value) - total_deleting - total_notready`. For multiple budgets in a NodePool, Karpenter will take the minimum value (most restrictive) of each of the budgets. For example, the following NodePool with three budgets defines the following requirements: - The first budget will only allow 20% of nodes owned by that NodePool to be disrupted. For instance, if there were 19 nodes owned by the NodePool, 4 disruptions would be allowed, rounding up from `19 * .2 = 3.8`. @@ -213,7 +213,7 @@ spec: disruption: consolidationPolicy: WhenUnderutilized expireAfter: 720h # 30 * 24h = 720h - budgets: + budgets: - nodes: "20%" - nodes: "5" - nodes: "0" @@ -222,8 +222,8 @@ spec: ``` #### Schedule -Schedule is a cronjob schedule. Generally, the cron syntax is five space-delimited values with options below, with additional special macros like `@yearly`, `monthly`, `@weekly`, `@daily`, `@hourly`. -Follow the [Kubernetes documentation](https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/#writing-a-cronjob-spec) for more information on how to follow the cron syntax. +Schedule is a cronjob schedule. Generally, the cron syntax is five space-delimited values with options below, with additional special macros like `@yearly`, `@monthly`, `@weekly`, `@daily`, `@hourly`. +Follow the [Kubernetes documentation](https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/#writing-a-cronjob-spec) for more information on how to follow the cron syntax. ```bash # ┌───────────── minute (0 - 59) @@ -238,14 +238,14 @@ Follow the [Kubernetes documentation](https://kubernetes.io/docs/concepts/worklo ``` {{% alert title="Note" color="primary" %}} -Timezones are not supported. Most images default to UTC, but it is best practice to ensure this is the case when considering how to define your budgets. +Timezones are not supported. Most images default to UTC, but it is best practice to ensure this is the case when considering how to define your budgets. {{% /alert %}} #### Duration Duration allows compound durations with minutes and hours values such as `10h5m` or `30m` or `160h`. Since cron syntax does not accept denominations smaller than minutes, users can only define minutes or hours. {{% alert title="Note" color="primary" %}} -Duration and Schedule must be defined together. When omitted, the budget is always active. When defined, the schedule determines a starting point where the budget will begin being enforced, and the duration determines how long from that starting point the budget will be enforced. +Duration and Schedule must be defined together. When omitted, the budget is always active. When defined, the schedule determines a starting point where the budget will begin being enforced, and the duration determines how long from that starting point the budget will be enforced. {{% /alert %}} ### Pod-Level Controls @@ -265,7 +265,7 @@ spec: ``` {{% alert title="Note" color="primary" %}} -This annotation will be ignored for [terminating pods](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase), [terminal pods](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase) (Failed/Succeeded), [DaemonSet pods](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/), or [static pods](https://kubernetes.io/docs/tasks/configure-pod-container/static-pod/). +This annotation will be ignored for [terminating pods](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase) and [terminal pods](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase) (Failed/Succeeded). {{% /alert %}} Examples of voluntary node removal that will be prevented by this annotation include: @@ -279,7 +279,7 @@ Voluntary node removal does not include [Interruption]({{} ### Node-Level Controls -Nodes can be opted out of consolidation disruption by setting the annotation `karpenter.sh/do-not-disrupt: "true"` on the node. +You can block Karpenter from voluntarily choosing to disrupt certain nodes by setting the `karpenter.sh/do-not-disrupt: "true"` annotation on the node. This will prevent disruption actions on the node. ```yaml apiVersion: v1 @@ -291,7 +291,7 @@ metadata: #### Example: Disable Disruption on a NodePool -NodePool `.spec.annotations` allow you to set annotations that will be applied to all nodes launched by this NodePool. By setting the annotation `karpenter.sh/do-not-disrupt: "true"` on the NodePool, you will selectively prevent all nodes launched by this NodePool from being considered in consolidation calculations. +NodePool `.spec.annotations` allow you to set annotations that will be applied to all nodes launched by this NodePool. By setting the annotation `karpenter.sh/do-not-disrupt: "true"` on the NodePool, you will selectively prevent all nodes launched by this NodePool from being considered in disruption actions. ```yaml apiVersion: karpenter.sh/v1beta1 diff --git a/website/content/en/preview/concepts/nodeclasses.md b/website/content/en/preview/concepts/nodeclasses.md index 164566b452c5..d426f579e7e6 100644 --- a/website/content/en/preview/concepts/nodeclasses.md +++ b/website/content/en/preview/concepts/nodeclasses.md @@ -1,4 +1,4 @@ ---- + --- title: "NodeClasses" linkTitle: "NodeClasses" weight: 2 @@ -35,7 +35,7 @@ spec: # Each term in the array of subnetSelectorTerms is ORed together # Within a single term, all conditions are ANDed subnetSelectorTerms: - # Select on any subnet that has the "karpenter.sh/discovery: ${CLUSTER_NAME}" + # Select on any subnet that has the "karpenter.sh/discovery: ${CLUSTER_NAME}" # AND the "environment: test" tag OR any subnet with ID "subnet-09fa4a0a8f233a921" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" @@ -46,8 +46,8 @@ spec: # Each term in the array of securityGroupSelectorTerms is ORed together # Within a single term, all conditions are ANDed securityGroupSelectorTerms: - # Select on any security group that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag - # AND the "environment: test" tag OR any security group with the "my-security-group" name + # Select on any security group that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any security group with the "my-security-group" name # OR any security group with ID "sg-063d7acfb4b06c82c" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" @@ -70,15 +70,15 @@ spec: # Each term in the array of amiSelectorTerms is ORed together # Within a single term, all conditions are ANDed amiSelectorTerms: - # Select on any AMI that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag - # AND the "environment: test" tag OR any AMI with the "my-ami" name + # Select on any AMI that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any AMI with the "my-ami" name # OR any AMI with ID "ami-123" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" environment: test - name: my-ami - id: ami-123 - + # Optional, use instance-store volumes for node ephemeral-storage instanceStorePolicy: RAID0 @@ -113,6 +113,10 @@ spec: # Optional, configures detailed monitoring for the instance detailedMonitoring: true + + # Optional, configures if the instance should be launched with an associated public IP address. + # If not specified, the default value depends on the subnet's public IP auto-assign setting. + associatePublicIPAddress: true status: # Resolved subnets subnets: @@ -160,7 +164,7 @@ Refer to the [NodePool docs]({{}}) for settings applicable t ## spec.amiFamily -AMIFamily is a required field, dictating both the default bootstrapping logic for nodes provisioned through this `EC2NodeClass` but also selecting a group of recommended, latest AMIs by default. Currently, Karpenter supports `amiFamily` values `AL2`, `Bottlerocket`, `Ubuntu`, `Windows2019`, `Windows2022` and `Custom`. GPUs are only supported by default with `AL2` and `Bottlerocket`. The `AL2` amiFamily does not support ARM64 GPU instance types unless you specify custom [`amiSelectorTerms`]({{}}). Default bootstrapping logic is shown below for each of the supported families. +AMIFamily is a required field, dictating both the default bootstrapping logic for nodes provisioned through this `EC2NodeClass` but also selecting a group of recommended, latest AMIs by default. Currently, Karpenter supports `amiFamily` values `AL2`, `AL2023`, `Bottlerocket`, `Ubuntu`, `Windows2019`, `Windows2022` and `Custom`. GPUs are only supported by default with `AL2` and `Bottlerocket`. The `AL2` amiFamily does not support ARM64 GPU instance types unless you specify custom [`amiSelectorTerms`]({{}}). Default bootstrapping logic is shown below for each of the supported families. ### AL2 @@ -180,6 +184,34 @@ exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 --//-- ``` +### AL2023 + +```text +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="//" + +--// +Content-Type: application/node.eks.aws + +# Karpenter Generated NodeConfig +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + cluster: + name: test-cluster + apiServerEndpoint: https://example.com + certificateAuthority: ca-bundle + cidr: 10.100.0.0/16 + kubelet: + config: + maxPods: 110 + flags: + - --node-labels=karpenter.sh/capacity-type=on-demand,karpenter.sh/nodepool=test + +--//-- + +``` + ### Bottlerocket ```toml @@ -248,7 +280,7 @@ This selection logic is modeled as terms, where each term contains multiple cond ```yaml subnetSelectorTerms: - # Select on any subnet that has the "karpenter.sh/discovery: ${CLUSTER_NAME}" + # Select on any subnet that has the "karpenter.sh/discovery: ${CLUSTER_NAME}" # AND the "environment: test" tag OR any subnet with ID "subnet-09fa4a0a8f233a921" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" @@ -306,6 +338,7 @@ spec: - id: "subnet-0471ca205b8a129ae" ``` + ## spec.securityGroupSelectorTerms Security Group Selector Terms allow you to specify selection logic for all security groups that will be attached to an instance launched from the `EC2NodeClass`. The security group of an instance is comparable to a set of firewall rules. @@ -315,8 +348,8 @@ This selection logic is modeled as terms, where each term contains multiple cond ```yaml securityGroupSelectorTerms: - # Select on any security group that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag - # AND the "environment: test" tag OR any security group with the "my-security-group" name + # Select on any security group that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any security group with the "my-security-group" name # OR any security group with ID "sg-063d7acfb4b06c82c" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" @@ -402,8 +435,8 @@ This selection logic is modeled as terms, where each term contains multiple cond ```yaml amiSelectorTerms: - # Select on any AMI that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag - # AND the "environment: test" tag OR any AMI with the "my-ami" name + # Select on any AMI that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any AMI with the "my-ami" name # OR any AMI with ID "ami-123" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" @@ -424,6 +457,7 @@ AMIs may be specified by any AWS tag, including `Name`. Selecting by tag or by n If `amiSelectorTerms` match more than one AMI, Karpenter will automatically determine which AMI best fits the workloads on the launched worker node under the following constraints: * When launching nodes, Karpenter automatically determines which architecture a custom AMI is compatible with and will use images that match an instanceType's requirements. + * Note that Karpenter **cannot** detect any requirement other than architecture. If you need to specify different AMIs for different kind of nodes (e.g. accelerated GPU AMIs), you should use a separate `EC2NodeClass`. * If multiple AMIs are found that can be used, Karpenter will choose the latest one. * If no AMIs are found that can be used, then no nodes will be provisioned. {{% /alert %}} @@ -483,7 +517,7 @@ Specify using ids: ## spec.role -`Role` is an optional field and is necessary to tell Karpenter which identity nodes from this `EC2NodeClass` should assume. You must specify one of `role` or `instanceProfile` when creating a Karpenter `EC2NodeClass`. If using the [Karpenter Getting Started Guide]({{}}) to deploy Karpenter, you can use the `KarpenterNodeRole-$CLUSTER_NAME` role provisioned by that process. +`Role` is an optional field and tells Karpenter which IAM identity nodes should assume. You must specify one of `role` or `instanceProfile` when creating a Karpenter `EC2NodeClass`. If using the [Karpenter Getting Started Guide]({{}}) to deploy Karpenter, you can use the `KarpenterNodeRole-$CLUSTER_NAME` role provisioned by that process. ```yaml spec: @@ -492,7 +526,9 @@ spec: ## spec.instanceProfile -`InstanceProfile` is an optional field and is necessary to tell Karpenter which identity nodes from this `EC2NodeClass` should assume. You must specify one of `role` or `instanceProfile` when creating a Karpenter `EC2NodeClasss`. If you use the `instanceProfile` field instead of `role`, Karpenter will not manage the InstanceProfile on your behalf. +`InstanceProfile` is an optional field and tells Karpenter which IAM identity nodes should assume. You must specify one of `role` or `instanceProfile` when creating a Karpenter `EC2NodeClass`. If you use the `instanceProfile` field instead of `role`, Karpenter will not manage the InstanceProfile on your behalf; instead, it expects that you have pre-provisioned an IAM instance profile and assigned it a role. + +You can provision and assign a role to an IAM instance profile using [CloudFormation](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iam-instanceprofile.html) or by using the [`aws iam create-instance-profile`](https://docs.aws.amazon.com/cli/latest/reference/iam/create-instance-profile.html) and [`aws iam add-role-to-instance-profile`](https://docs.aws.amazon.com/cli/latest/reference/iam/add-role-to-instance-profile.html) commands in the CLI. {{% alert title="Note" color="primary" %}} @@ -508,6 +544,7 @@ Karpenter adds tags to all resources it creates, including EC2 Instances, EBS vo Name: karpenter.sh/nodeclaim: karpenter.sh/nodepool: +karpenter.k8s.aws/ec2nodeclass: kubernetes.io/cluster/: owned ``` @@ -573,6 +610,17 @@ spec: encrypted: true ``` +### AL2023 +```yaml +spec: + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 20Gi + volumeType: gp3 + encrypted: true +``` + ### Bottlerocket ```yaml spec: @@ -625,25 +673,29 @@ The `instanceStorePolicy` field controls how [instance-store](https://docs.aws.a If you intend to use these volumes for faster node ephemeral-storage, set `instanceStorePolicy` to `RAID0`: -```yaml -spec: +```yaml +spec: instanceStorePolicy: RAID0 ``` -This will set the allocatable ephemeral-storage of each node to the total size of the instance-store volume(s). +This will set the allocatable ephemeral-storage of each node to the total size of the instance-store volume(s). -The disks must be formatted & mounted in a RAID0 and be the underlying filesystem for the Kubelet & Containerd. Instructions for each AMI family are listed below: +The disks must be formatted & mounted in a RAID0 and be the underlying filesystem for the Kubelet & Containerd. Instructions for each AMI family are listed below: -#### AL2 +#### AL2 -On AL2, Karpenter automatically configures the disks through an additional boostrap argument (`--local-disks raid0`). The device name is `/dev/md/0` and its mount point is `/mnt/k8s-disks/0`. You should ensure any additional disk setup does not interfere with these. +On AL2, Karpenter automatically configures the disks through an additional boostrap argument (`--local-disks raid0`). The device name is `/dev/md/0` and its mount point is `/mnt/k8s-disks/0`. You should ensure any additional disk setup does not interfere with these. -#### Others +#### AL2023 -For all other AMI families, you must configure the disks yourself. Check out the [`setup-local-disks`](https://github.com/awslabs/amazon-eks-ami/blob/master/files/bin/setup-local-disks) script in [amazon-eks-ami](https://github.com/awslabs/amazon-eks-ami) to see how this is done for AL2. +On AL2023, Karpenter automatically configures the disks via the generated `NodeConfig` object. Like AL2, the device name is `/dev/md/0` and its mount point is `/mnt/k8s-disks/0`. You should ensure any additional disk setup does not interfere with these. -{{% alert title="Tip" color="secondary" %}} -Since the Kubelet & Containerd will be using the instance-store filesystem, you may consider using a more minimal root volume size. +#### Others + +For all other AMI families, you must configure the disks yourself. Check out the [`setup-local-disks`](https://github.com/awslabs/amazon-eks-ami/blob/master/files/bin/setup-local-disks) script in [amazon-eks-ami](https://github.com/awslabs/amazon-eks-ami) to see how this is done for AL2. + +{{% alert title="Tip" color="secondary" %}} +Since the Kubelet & Containerd will be using the instance-store filesystem, you may consider using a more minimal root volume size. {{% /alert %}} ## spec.userData @@ -709,7 +761,30 @@ Consider the following example to understand how your custom UserData will be me ```bash #!/bin/bash -echo "Running custom user data script" +echo "Running custom user data script (bash)" +``` + +#### Merged UserData (bash) + +```bash +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="//" + +--// +Content-Type: text/x-shellscript; charset="us-ascii" + +#!/bin/bash +echo "Running custom user data script (bash)" + +--// +Content-Type: text/x-shellscript; charset="us-ascii" + +#!/bin/bash -xe +exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 +/etc/eks/bootstrap.sh 'test-cluster' --apiserver-endpoint 'https://test-cluster' --b64-cluster-ca 'ca-bundle' \ +--use-max-pods false \ +--kubelet-extra-args '--node-labels=karpenter.sh/capacity-type=on-demand,karpenter.sh/nodepool=test --max-pods=110' +--//-- ``` #### Passed-in UserData (MIME) @@ -722,12 +797,12 @@ Content-Type: multipart/mixed; boundary="BOUNDARY" Content-Type: text/x-shellscript; charset="us-ascii" #!/bin/bash -echo "Running custom user data script" +echo "Running custom user data script (mime)" --BOUNDARY-- ``` -#### Merged UserData +#### Merged UserData (MIME) ```bash MIME-Version: 1.0 @@ -737,7 +812,7 @@ Content-Type: multipart/mixed; boundary="//" Content-Type: text/x-shellscript; charset="us-ascii" #!/bin/bash -echo "Running custom user data script" +echo "Running custom user data script (mime)" --// Content-Type: text/x-shellscript; charset="us-ascii" @@ -767,6 +842,169 @@ spec: ``` {{% /alert %}} +### AL2023 + +* Your UserData may be in one of three formats: a [MIME multi part archive](https://cloudinit.readthedocs.io/en/latest/topics/format.html#mime-multi-part-archive), a NodeConfig YAML / JSON string, or a shell script. +* Karpenter will transform your custom UserData into a MIME part, if necessary, and then create a MIME multi-part archive. This archive will consist of a generated NodeConfig, containing Karpenter's default values, followed by the transformed custom UserData. For more information on the NodeConfig spec, refer to the [AL2023 EKS Optimized AMI docs](https://awslabs.github.io/amazon-eks-ami/nodeadm/doc/examples/). +* If a value is specified both in the Karpenter generated NodeConfig and the same value is specified in the custom user data, the value in the custom user data will take precedence. + +#### Passed-in UserData (NodeConfig) + +```yaml +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + kubelet: + config: + maxPods: 42 +``` + +#### Merged UserData (NodeConfig) + +```text +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="//" + +--// +# Karpenter Generated NodeConfig +Content-Type: application/node.eks.aws + +# Karpenter Generated NodeConfig +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + cluster: + apiServerEndpoint: https://test-cluster + certificateAuthority: cluster-ca + cidr: 10.100.0.0/16 + name: test-cluster + kubelet: + config: + clusterDNS: + - 10.100.0.10 + maxPods: 118 + flags: + - --node-labels="karpenter.sh/capacity-type=on-demand,karpenter.sh/nodepool=default" + +--// +Content-Type: application/node.eks.aws + +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + kubelet: + config: + maxPods: 42 +--//-- +``` + +#### Passed-in UserData (bash) + +```shell +#!/bin/bash +echo "Hello, AL2023!" +``` + +#### Merged UserData (bash) + +```text +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="//" + +--// +Content-Type: application/node.eks.aws + +# Karpenter Generated NodeConfig +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + cluster: + apiServerEndpoint: https://test-cluster + certificateAuthority: cluster-ca + cidr: 10.100.0.0/16 + name: test-cluster + kubelet: + config: + clusterDNS: + - 10.100.0.10 + maxPods: 118 + flags: + - --node-labels="karpenter.sh/capacity-type=on-demand,karpenter.sh/nodepool=default" + +--// +Content-Type: text/x-shellscript; charset="us-ascii" + +#!/bin/bash +echo "Hello, AL2023!" +--//-- +``` + +#### Passed-in UserData (MIME) + +```text +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="//" + +--// +Content-Type: application/node.eks.aws + +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + kubelet: + config: + maxPods: 42 +--// +Content-Type: text/x-shellscript; charset="us-ascii" + +#!/bin/bash +echo "Hello, AL2023!" +--// +``` + +#### Merged UserData (MIME) + +```text +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="//" + +--// +Content-Type: application/node.eks.aws + +# Karpenter Generated NodeConfig +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + cluster: + apiServerEndpoint: https://test-cluster + certificateAuthority: cluster-ca + cidr: 10.100.0.0/16 + name: test-cluster + kubelet: + config: + clusterDNS: + - 10.100.0.10 + maxPods: 118 + flags: + - --node-labels="karpenter.sh/capacity-type=on-demand,karpenter.sh/nodepool=default" + +--// +Content-Type: application/node.eks.aws + +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + kubelet: + config: + maxPods: 42 +--// +Content-Type: text/x-shellscript; charset="us-ascii" + +#!/bin/bash +echo "Hello, AL2023!" +--//-- +``` + ### Bottlerocket * Your UserData must be valid TOML. @@ -859,6 +1097,15 @@ spec: detailedMonitoring: true ``` +## spec.associatePublicIPAddress + +A boolean field that controls whether instances created by Karpenter for this EC2NodeClass will have an associated public IP address. This overrides the `MapPublicIpOnLaunch` setting applied to the subnet the node is launched in. If this field is not set, the `MapPublicIpOnLaunch` field will be respected. + +{{% alert title="Note" color="warning" %}} +If a `NodeClaim` requests `vpc.amazonaws.com/efa` resources, `spec.associatePublicIPAddress` is respected. However, if this `NodeClaim` requests **multiple** EFA resources and the value for `spec.associatePublicIPAddress` is true, the instance will fail to launch. This is due to an EC2 restriction which +requires that the field is only set to true when configuring an instance with a single ENI at launch. When using this field, it is advised that users segregate their EFA workload to use a separate `NodePool` / `EC2NodeClass` pair. +{{% /alert %}} + ## status.subnets [`status.subnets`]({{< ref "#statussubnets" >}}) contains the resolved `id` and `zone` of the subnets that were selected by the [`spec.subnetSelectorTerms`]({{< ref "#specsubnetselectorterms" >}}) for the node class. The subnets will be sorted by the available IP address count in decreasing order. diff --git a/website/content/en/preview/concepts/nodepools.md b/website/content/en/preview/concepts/nodepools.md index f9ee114f47fe..d679bb5f4373 100644 --- a/website/content/en/preview/concepts/nodepools.md +++ b/website/content/en/preview/concepts/nodepools.md @@ -46,6 +46,8 @@ spec: spec: # References the Cloud Provider's NodeClass resource, see your cloud provider specific documentation nodeClassRef: + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass name: default # Provisioned nodes will have these taints @@ -69,6 +71,13 @@ spec: - key: "karpenter.k8s.aws/instance-category" operator: In values: ["c", "m", "r"] + # minValues here enforces the scheduler to consider at least that number of unique instance-category to schedule the pods. + # This field is ALPHA and can be dropped or replaced at any time + minValues: 2 + - key: "karpenter.k8s.aws/instance-family" + operator: In + values: ["m5","m5d","c5","c5d","c4","r4"] + minValues: 5 - key: "karpenter.k8s.aws/instance-cpu" operator: In values: ["4", "8", "16", "32"] @@ -127,12 +136,12 @@ spec: # If using 'WhenUnderutilized', Karpenter will consider all nodes for consolidation and attempt to remove or replace Nodes when it discovers that the Node is underutilized and could be changed to reduce cost # If using `WhenEmpty`, Karpenter will only consider nodes for consolidation that contain no workload pods consolidationPolicy: WhenUnderutilized | WhenEmpty - + # The amount of time Karpenter should wait after discovering a consolidation decision # This value can currently only be set when the consolidationPolicy is 'WhenEmpty' # You can choose to disable consolidation entirely by setting the string value 'Never' here consolidateAfter: 30s - + # The amount of time a Node can live on the cluster before being removed # Avoiding long-running Nodes helps to reduce security vulnerabilities as well as to reduce the chance of issues that can plague Nodes with long uptimes such as file fragmentation or memory leaks from system processes # You can choose to disable expiration entirely by setting the string value 'Never' here @@ -140,8 +149,8 @@ spec: # Budgets control the speed Karpenter can scale down nodes. # Karpenter will respect the minimum of the currently active budgets, and will round up - # when considering percentages. Duration and Schedule must be set together. - budgets: + # when considering percentages. Duration and Schedule must be set together. + budgets: - nodes: 10% # On Weekdays during business hours, don't do any deprovisioning. - schedule: "0 9 * * mon-fri" @@ -170,7 +179,9 @@ These well-known labels may be specified at the NodePool level, or in a workload For example, an instance type may be specified using a nodeSelector in a pod spec. If the instance type requested is not included in the NodePool list and the NodePool has instance type requirements, Karpenter will not create a node or schedule the pod. -### Instance Types +### Well-Known Labels + +#### Instance Types - key: `node.kubernetes.io/instance-type` - key: `karpenter.k8s.aws/instance-family` @@ -181,7 +192,7 @@ Generally, instance types should be a list and not a single value. Leaving these Review [AWS instance types](../instance-types). Most instance types are supported with the exclusion of [non-HVM](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/virtualization_types.html). -### Availability Zones +#### Availability Zones - key: `topology.kubernetes.io/zone` - value example: `us-east-1c` @@ -192,7 +203,7 @@ Karpenter can be configured to create nodes in a particular zone. Note that the [Learn more about Availability Zone IDs.](https://docs.aws.amazon.com/ram/latest/userguide/working-with-az-ids.html) -### Architecture +#### Architecture - key: `kubernetes.io/arch` - values @@ -201,7 +212,7 @@ IDs.](https://docs.aws.amazon.com/ram/latest/userguide/working-with-az-ids.html) Karpenter supports `amd64` nodes, and `arm64` nodes. -### Operating System +#### Operating System - key: `kubernetes.io/os` - values - `linux` @@ -209,7 +220,7 @@ Karpenter supports `amd64` nodes, and `arm64` nodes. Karpenter supports `linux` and `windows` operating systems. -### Capacity Type +#### Capacity Type - key: `karpenter.sh/capacity-type` - values @@ -222,6 +233,71 @@ Karpenter prioritizes Spot offerings if the NodePool allows Spot and on-demand i Karpenter also allows `karpenter.sh/capacity-type` to be used as a topology key for enforcing topology-spread. +### Min Values + +Along with the combination of [key,operator,values] in the requirements, Karpenter also supports `minValues` in the NodePool requirements block, allowing the scheduler to be aware of user-specified flexibility minimums while scheduling pods to a cluster. If Karpenter cannot meet this minimum flexibility for each key when scheduling a pod, it will fail the scheduling loop for that NodePool, either falling back to another NodePool which meets the pod requirements or failing scheduling the pod altogether. + +For example, the below spec will use spot instance type for all provisioned instances and enforces `minValues` to various keys where it is defined +i.e at least 2 unique instance families from [c,m,r], 5 unique instance families [eg: "m5","m5d","r4","c5","c5d","c4" etc], 10 unique instance types [eg: "c5.2xlarge","c4.xlarge" etc] is required for scheduling the pods. + +```yaml +spec: + template: + spec: + requirements: + - key: kubernetes.io/arch + operator: In + values: ["amd64"] + - key: kubernetes.io/os + operator: In + values: ["linux"] + - key: karpenter.k8s.aws/instance-category + operator: In + values: ["c", "m", "r"] + minValues: 2 + - key: karpenter.k8s.aws/instance-family + operator: Exists + minValues: 5 + - key: node.kubernetes.io/instance-type + operator: Exists + minValues: 10 + - key: karpenter.k8s.aws/instance-generation + operator: Gt + values: ["2"] +``` + +Note that `minValues` can be used with multiple operators and multiple requirements. And if the `minValues` are defined with multiple operators for the same requirement key, scheduler considers the max of all the `minValues` for that requirement. For example, the below spec requires scheduler to consider at least 5 instance-family to schedule the pods. + +```yaml +spec: + template: + spec: + requirements: + - key: kubernetes.io/arch + operator: In + values: ["amd64"] + - key: kubernetes.io/os + operator: In + values: ["linux"] + - key: karpenter.k8s.aws/instance-category + operator: In + values: ["c", "m", "r"] + minValues: 2 + - key: karpenter.k8s.aws/instance-family + operator: Exists + minValues: 5 + - key: karpenter.k8s.aws/instance-family + operator: In + values: ["m5","m5d","c5","c5d","c4","r4"] + minValues: 3 + - key: node.kubernetes.io/instance-type + operator: Exists + minValues: 10 + - key: karpenter.k8s.aws/instance-generation + operator: Gt + values: ["2"] +``` + {{% alert title="Recommended" color="primary" %}} Karpenter allows you to be extremely flexible with your NodePools by only constraining your instance types in ways that are absolutely necessary for your cluster. By default, Karpenter will enforce that you specify the `spec.template.spec.requirements` field, but will not enforce that you specify any requirements within the field. If you choose to specify `requirements: []`, this means that you will completely flexible to _all_ instance types that your cloud provider supports. @@ -295,7 +371,12 @@ kubelet: Karpenter will automatically configure the system and kube reserved resource requests on the fly on your behalf. These requests are used to configure your node and to make scheduling decisions for your pods. If you have specific requirements or know that you will have additional capacity requirements, you can optionally override the `--system-reserved` configuration defaults with the `.spec.template.spec.kubelet.systemReserved` values and the `--kube-reserved` configuration defaults with the `.spec.template.spec.kubelet.kubeReserved` values. -For more information on the default `--system-reserved` and `--kube-reserved` configuration refer to the [Kubelet Docs](https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/#kube-reserved) +{{% alert title="Note" color="primary" %}} +Karpenter considers these reserved resources when computing the allocatable ephemeral storage on a given instance type. +If `kubeReserved` is not specified, Karpenter will compute the default reserved [CPU](https://github.com/awslabs/amazon-eks-ami/blob/db28da15d2b696bc08ac3aacc9675694f4a69933/files/bootstrap.sh#L251) and [memory](https://github.com/awslabs/amazon-eks-ami/blob/db28da15d2b696bc08ac3aacc9675694f4a69933/files/bootstrap.sh#L235) resources for the purpose of ephemeral storage computation. +These defaults are based on the defaults on Karpenter's supported AMI families, which are not the same as the kubelet defaults. +You should be aware of the CPU and memory default calculation when using Custom AMI Families. If they don't align, there may be a difference in Karpenter's computed allocatable ephemeral storage and the actually ephemeral storage available on the node. +{{% /alert %}} ### Eviction Thresholds @@ -441,7 +522,7 @@ Review the [Kubernetes core API](https://github.com/kubernetes/api/blob/37748cca Karpenter allows you to describe NodePool preferences through a `weight` mechanism similar to how weight is described with [pod and node affinities](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity). -For more information on weighting NodePools, see the [Weighting NodePools section]({{}}) in the scheduling details. +For more information on weighting NodePools, see the [Weighted NodePools section]({{}}) in the scheduling docs. ## Examples diff --git a/website/content/en/preview/concepts/scheduling.md b/website/content/en/preview/concepts/scheduling.md index 90339674f24c..738bc9aa9d8d 100755 --- a/website/content/en/preview/concepts/scheduling.md +++ b/website/content/en/preview/concepts/scheduling.md @@ -152,6 +152,7 @@ Take care to ensure the label domains are correct. A well known label like `karp | karpenter.k8s.aws/instance-family | g4dn | [AWS Specific] Instance types of similar properties but different resource quantities | | karpenter.k8s.aws/instance-size | 8xlarge | [AWS Specific] Instance types of similar resource quantities but different properties | | karpenter.k8s.aws/instance-cpu | 32 | [AWS Specific] Number of CPUs on the instance | +| karpenter.k8s.aws/instance-cpu-manufacturer | aws | [AWS Specific] Name of the CPU manufacturer | | karpenter.k8s.aws/instance-memory | 131072 | [AWS Specific] Number of mebibytes of memory on the instance | | karpenter.k8s.aws/instance-network-bandwidth | 131072 | [AWS Specific] Number of [baseline megabits](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-network-bandwidth.html) available on the instance | | karpenter.k8s.aws/instance-pods | 110 | [AWS Specific] Number of pods the instance supports | @@ -214,7 +215,7 @@ All examples below assume that the NodePool doesn't have constraints to prevent - matchExpressions: - key: "topology.kubernetes.io/zone" operator: "In" - values: ["us-west-2a, us-west-2b"] + values: ["us-west-2a", "us-west-2b"] - key: "topology.kubernetes.io/zone" operator: "In" values: ["us-west-2b"] @@ -225,7 +226,7 @@ Changing the second operator to `NotIn` would allow the pod to run in `us-west-2 ```yaml - key: "topology.kubernetes.io/zone" operator: "In" - values: ["us-west-2a, us-west-2b"] + values: ["us-west-2a", "us-west-2b"] - key: "topology.kubernetes.io/zone" operator: "NotIn" values: ["us-west-2b"] @@ -243,7 +244,7 @@ Here, if `us-west-2a` is not available, the second term will cause the pod to ru - matchExpressions: # OR - key: "topology.kubernetes.io/zone" # AND operator: "In" - values: ["us-west-2a, us-west-2b"] + values: ["us-west-2a", "us-west-2b"] - key: "topology.kubernetes.io/zone" # AND operator: "NotIn" values: ["us-west-2b"] @@ -349,9 +350,12 @@ The three supported `topologyKey` values that Karpenter supports are: - `kubernetes.io/hostname` - `karpenter.sh/capacity-type` - See [Pod Topology Spread Constraints](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/) for details. +{{% alert title="Note" color="primary" %}} +NodePools do not attempt to balance or rebalance the availability zones for their nodes. Availability zone balancing may be achieved by defining zonal Topology Spread Constraints for Pods that require multi-zone durability, and NodePools will respect these constraints while optimizing for compute costs. +{{% /alert %}} + ## Pod affinity/anti-affinity By using the `podAffinity` and `podAntiAffinity` configuration on a pod spec, you can inform the Karpenter scheduler of your desire for pods to schedule together or apart with respect to different topology domains. For example: diff --git a/website/content/en/preview/contributing/development-guide.md b/website/content/en/preview/contributing/development-guide.md index e803169d6c5a..2afbb4d2ead7 100644 --- a/website/content/en/preview/contributing/development-guide.md +++ b/website/content/en/preview/contributing/development-guide.md @@ -73,7 +73,8 @@ make test # E2E correctness tests ### Change Log Level -By default, `make apply` will set the log level to debug. You can change the log level by setting the log level in your helm values. +By default, `make apply` will set the log level to debug. You can change the log level by setting the log level in your Helm values. + ```bash --set logLevel=debug ``` diff --git a/website/content/en/preview/faq.md b/website/content/en/preview/faq.md index 23cb4f03aa84..4549a26bef94 100644 --- a/website/content/en/preview/faq.md +++ b/website/content/en/preview/faq.md @@ -26,7 +26,7 @@ Karpenter has multiple mechanisms for configuring the [operating system]({{< ref Karpenter is flexible to multi-architecture configurations using [well known labels]({{< ref "./concepts/scheduling/#supported-labels">}}). ### What RBAC access is required? -All the required RBAC rules can be found in the helm chart template. See [clusterrole-core.yaml](https://github.com/aws/karpenter/blob{{< githubRelRef >}}charts/karpenter/templates/clusterrole-core.yaml), [clusterrole.yaml](https://github.com/aws/karpenter/blob{{< githubRelRef >}}charts/karpenter/templates/clusterrole.yaml), [rolebinding.yaml](https://github.com/aws/karpenter/blob{{< githubRelRef >}}charts/karpenter/templates/rolebinding.yaml), and [role.yaml](https://github.com/aws/karpenter/blob{{< githubRelRef >}}charts/karpenter/templates/role.yaml) files for details. +All the required RBAC rules can be found in the Helm chart template. See [clusterrole-core.yaml](https://github.com/aws/karpenter/blob{{< githubRelRef >}}charts/karpenter/templates/clusterrole-core.yaml), [clusterrole.yaml](https://github.com/aws/karpenter/blob{{< githubRelRef >}}charts/karpenter/templates/clusterrole.yaml), [rolebinding.yaml](https://github.com/aws/karpenter/blob{{< githubRelRef >}}charts/karpenter/templates/rolebinding.yaml), and [role.yaml](https://github.com/aws/karpenter/blob{{< githubRelRef >}}charts/karpenter/templates/role.yaml) files for details. ### Can I run Karpenter outside of a Kubernetes cluster? Yes, as long as the controller has network and IAM/RBAC access to the Kubernetes API and your provider API. @@ -202,7 +202,7 @@ Use your existing upgrade mechanisms to upgrade your core add-ons in Kubernetes Karpenter requires proper permissions in the `KarpenterNode IAM Role` and the `KarpenterController IAM Role`. To upgrade Karpenter to version `$VERSION`, make sure that the `KarpenterNode IAM Role` and the `KarpenterController IAM Role` have the right permission described in `https://karpenter.sh/$VERSION/getting-started/getting-started-with-karpenter/cloudformation.yaml`. -Next, locate `KarpenterController IAM Role` ARN (i.e., ARN of the resource created in [Create the KarpenterController IAM Role](../getting-started/getting-started-with-karpenter/#create-the-karpentercontroller-iam-role)) and pass them to the helm upgrade command. +Next, locate `KarpenterController IAM Role` ARN (i.e., ARN of the resource created in [Create the KarpenterController IAM Role](../getting-started/getting-started-with-karpenter/#create-the-karpentercontroller-iam-role)) and pass them to the Helm upgrade command. {{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step08-apply-helm-chart.sh" language="bash"%}} For information on upgrading Karpenter, see the [Upgrade Guide]({{< ref "./upgrading/upgrade-guide/" >}}). @@ -211,7 +211,7 @@ For information on upgrading Karpenter, see the [Upgrade Guide]({{< ref "./upgra ### How do I upgrade an EKS Cluster with Karpenter? -When upgrading an Amazon EKS cluster, [Karpenter's Drift feature]({{}}) can automatically upgrade the Karpenter-provisioned nodes to stay in-sync with the EKS control plane. Karpenter Drift is enabled by default starting v0.33.x. +When upgrading an Amazon EKS cluster, [Karpenter's Drift feature]({{}}) can automatically upgrade the Karpenter-provisioned nodes to stay in-sync with the EKS control plane. Karpenter Drift is enabled by default starting `0.33.0`. {{% alert title="Note" color="primary" %}} Karpenter's default [EC2NodeClass `amiFamily` configuration]({{}}) uses the latest EKS Optimized AL2 AMI for the same major and minor version as the EKS cluster's control plane, meaning that an upgrade of the control plane will cause Karpenter to auto-discover the new AMIs for that version. diff --git a/website/content/en/preview/getting-started/_index.md b/website/content/en/preview/getting-started/_index.md index a70f6bbc422b..cebdf66a99fd 100644 --- a/website/content/en/preview/getting-started/_index.md +++ b/website/content/en/preview/getting-started/_index.md @@ -21,6 +21,6 @@ Learn more about Karpenter and how to get started below. * [Karpenter EKS Best Practices](https://aws.github.io/aws-eks-best-practices/karpenter/) guide * [EC2 Spot Workshop for Karpenter](https://ec2spotworkshops.com/karpenter.html) * [EKS Karpenter Workshop](https://www.eksworkshop.com/docs/autoscaling/compute/karpenter/) -* [Advanced EKS Immersion Karpenter Workshop](https://catalog.workshops.aws/eks-advanced/karpenter/) +* [Advanced EKS Immersion Karpenter Workshop](https://catalog.us-east-1.prod.workshops.aws/workshops/76a5dd80-3249-4101-8726-9be3eeee09b2/en-US/autoscaling/karpenter) * [Karpenter Blueprints](https://github.com/aws-samples/karpenter-blueprints) * [Tutorial: Run Kubernetes Clusters for Less with Amazon EC2 Spot and Karpenter](https://community.aws/tutorials/run-kubernetes-clusters-for-less-with-amazon-ec2-spot-and-karpenter#step-6-optional-simulate-spot-interruption) diff --git a/website/content/en/preview/getting-started/getting-started-with-karpenter/_index.md b/website/content/en/preview/getting-started/getting-started-with-karpenter/_index.md index e1a52bd4a6cd..bf8689367cde 100644 --- a/website/content/en/preview/getting-started/getting-started-with-karpenter/_index.md +++ b/website/content/en/preview/getting-started/getting-started-with-karpenter/_index.md @@ -32,7 +32,7 @@ Install these tools before proceeding: 1. [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2-linux.html) 2. `kubectl` - [the Kubernetes CLI](https://kubernetes.io/docs/tasks/tools/install-kubectl-linux/) -3. `eksctl` (>= v0.165.0) - [the CLI for AWS EKS](https://docs.aws.amazon.com/eks/latest/userguide/eksctl.html) +3. `eksctl` (>= v0.169.0) - [the CLI for AWS EKS](https://docs.aws.amazon.com/eks/latest/userguide/eksctl.html) 4. `helm` - [the package manager for Kubernetes](https://helm.sh/docs/intro/install/) [Configure the AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-quickstart.html) @@ -44,9 +44,9 @@ authenticate properly by running `aws sts get-caller-identity`. After setting up the tools, set the Karpenter and Kubernetes version: ```bash -export KARPENTER_NAMESPACE=kube-system -export KARPENTER_VERSION=v0.33.1 -export K8S_VERSION={{< param "latest_k8s_version" >}} +export KARPENTER_NAMESPACE="kube-system" +export KARPENTER_VERSION="{{< param "latest_release_version" >}}" +export K8S_VERSION="{{< param "latest_k8s_version" >}}" ``` Then set the following environment variable: @@ -58,7 +58,7 @@ If you open a new shell to run steps in this procedure, you need to set some or To remind yourself of these values, type: ```bash -echo $KARPENTER_NAMESPACE $KARPENTER_VERSION $K8S_VERSION $CLUSTER_NAME $AWS_DEFAULT_REGION $AWS_ACCOUNT_ID $TEMPOUT +echo "${KARPENTER_NAMESPACE}" "${KARPENTER_VERSION}" "${K8S_VERSION}" "${CLUSTER_NAME}" "${AWS_DEFAULT_REGION}" "${AWS_ACCOUNT_ID}" "${TEMPOUT}" "${ARM_AMI_ID}" "${AMD_AMI_ID}" "${GPU_AMI_ID}" ``` {{% /alert %}} @@ -75,7 +75,7 @@ The following cluster configuration will: * Use [AWS EKS managed node groups](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html) for the kube-system and karpenter namespaces. Uncomment fargateProfiles settings (and comment out managedNodeGroups settings) to use Fargate for both namespaces instead. * Set KARPENTER_IAM_ROLE_ARN variables. * Create a role to allow spot instances. -* Run helm to install karpenter +* Run Helm to install Karpenter {{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step02-create-cluster.sh" language="bash"%}} @@ -90,6 +90,18 @@ See [Enabling Windows support](https://docs.aws.amazon.com/eks/latest/userguide/ {{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step08-apply-helm-chart.sh" language="bash"%}} +As the OCI Helm chart is signed by [Cosign](https://github.com/sigstore/cosign) as part of the release process you can verify the chart before installing it by running the following command. + +```bash +cosign verify public.ecr.aws/karpenter/karpenter:{{< param "latest_release_version" >}} \ + --certificate-oidc-issuer=https://token.actions.githubusercontent.com \ + --certificate-identity-regexp='https://github\.com/aws/karpenter-provider-aws/\.github/workflows/release\.yaml@.+' \ + --certificate-github-workflow-repository=aws/karpenter-provider-aws \ + --certificate-github-workflow-name=Release \ + --certificate-github-workflow-ref=refs/tags/v{{< param "latest_release_version" >}} \ + --annotations version={{< param "latest_release_version" >}} +``` + {{% alert title="DNS Policy Notice" color="warning" %}} Karpenter uses the `ClusterFirst` pod DNS policy by default. This is the Kubernetes cluster default and this ensures that Karpetner can reach-out to internal Kubernetes services during its lifetime. There may be cases where you do not have the DNS service that you are using on your cluster up-and-running before Karpenter starts up. The most common case of this is you want Karpenter to manage the node capacity where your DNS service pods are running. @@ -97,11 +109,11 @@ If you need Karpenter to manage the DNS service pods' capacity, this means that {{% /alert %}} {{% alert title="Common Expression Language/Webhooks Notice" color="warning" %}} -Karpenter supports using [Kubernetes Common Expression Language](https://kubernetes.io/docs/reference/using-api/cel/) for validating its Custom Resource Definitions out-of-the-box; however, this feature is not supported on versions of Kubernetes < 1.25. If you are running an earlier version of Kubernetes, you will need to use the Karpenter admission webhooks for validation instead. You can enable these webhooks with `--set webhook.enabled=true` when applying the Karpenter helm chart. +Karpenter supports using [Kubernetes Common Expression Language](https://kubernetes.io/docs/reference/using-api/cel/) for validating its Custom Resource Definitions out-of-the-box; however, this feature is not supported on versions of Kubernetes < 1.25. If you are running an earlier version of Kubernetes, you will need to use the Karpenter admission webhooks for validation instead. You can enable these webhooks with `--set webhook.enabled=true` when applying the Karpenter Helm chart. {{% /alert %}} {{% alert title="Pod Identity Supports Notice" color="warning" %}} -Karpenter now supports using [Pod Identity](https://docs.aws.amazon.com/eks/latest/userguide/pod-identities.html) to authenticate AWS SDK to make API requests to AWS services using AWS Identity and Access Management (IAM) permissions. This feature not supported on versions of Kubernetes < 1.24. If you are running an earlier version of Kubernetes, you will need to use the [IAM Roles for Service Accounts(IRSA)](https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/setting-up-enable-IAM.html) for pod authentication instead. You can enable these IRSA with `--set "serviceAccount.annotations.eks\.amazonaws\.com/role-arn=${KARPENTER_IAM_ROLE_ARN}"` when applying the Karpenter helm chart. +Karpenter now supports using [Pod Identity](https://docs.aws.amazon.com/eks/latest/userguide/pod-identities.html) to authenticate AWS SDK to make API requests to AWS services using AWS Identity and Access Management (IAM) permissions. This feature not supported on versions of Kubernetes < 1.24. If you are running an earlier version of Kubernetes, you will need to use the [IAM Roles for Service Accounts(IRSA)](https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/setting-up-enable-IAM.html) for pod authentication instead. You can enable these IRSA with `--set "serviceAccount.annotations.eks\.amazonaws\.com/role-arn=${KARPENTER_IAM_ROLE_ARN}"` when applying the Karpenter Helm chart. {{% /alert %}} {{% alert title="Warning" color="warning" %}} @@ -177,7 +189,7 @@ The section below covers advanced installation techniques for installing Karpent ### Private Clusters -You can optionally install Karpenter on a [private cluster](https://docs.aws.amazon.com/eks/latest/userguide/private-clusters.html#private-cluster-requirements) using the `eksctl` installation by setting `privateCluster.enabled` to true in your [ClusterConfig](https://eksctl.io/usage/eks-private-cluster/#eks-fully-private-cluster) and by setting `--set settings.isolatedVPC=true` when installing the `karpenter` helm chart. +You can optionally install Karpenter on a [private cluster](https://docs.aws.amazon.com/eks/latest/userguide/private-clusters.html#private-cluster-requirements) using the `eksctl` installation by setting `privateCluster.enabled` to true in your [ClusterConfig](https://eksctl.io/usage/eks-private-cluster/#eks-fully-private-cluster) and by setting `--set settings.isolatedVPC=true` when installing the `karpenter` Helm chart. ```bash privateCluster: @@ -194,6 +206,7 @@ com.amazonaws..s3 – For pulling container images com.amazonaws..sts – For IAM roles for service accounts com.amazonaws..ssm - For resolving default AMIs com.amazonaws..sqs - For accessing SQS if using interruption handling +com.amazonaws..eks - For Karpenter to discover the cluster endpoint ``` If you do not currently have these endpoints surfaced in your VPC, you can add the endpoints by running @@ -239,7 +252,7 @@ caused by: Post "https://api.pricing.us-east-1.amazonaws.com/": dial tcp 52.94.2 Kubernetes uses [FlowSchemas](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/#flowschema) and [PriorityLevelConfigurations](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/#prioritylevelconfiguration) to map calls to the API server into buckets which determine each user agent's throttling limits. -By default, Karpenter is installed into the `kube-system` namespace, which leverages the `system-leader-election` and `kube-system-service-accounts` [FlowSchemas] to map calls from the `kube-system` namespace to the `leader-election` and `workload-high` PriorityLevelConfigurations respectively. By putting Karpenter in these PriorityLevelConfigurations, we ensure that Karpenter and other critical cluster components are able to run even if other components on the cluster are throttled in other PriorityLevelConfigurations. +By default, Karpenter is installed into the `kube-system` namespace, which leverages the `system-leader-election` and `kube-system-service-accounts` [FlowSchemas](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/#flowschema) to map calls from the `kube-system` namespace to the `leader-election` and `workload-high` PriorityLevelConfigurations respectively. By putting Karpenter in these PriorityLevelConfigurations, we ensure that Karpenter and other critical cluster components are able to run even if other components on the cluster are throttled in other PriorityLevelConfigurations. If you install Karpenter in a different namespace than the default `kube-system` namespace, Karpenter will not be put into these higher-priority FlowSchemas by default. Instead, you will need to create custom FlowSchemas for the namespace and service account where Karpenter is installed to ensure that requests are put into this higher PriorityLevelConfiguration. diff --git a/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml b/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml index 94c233b7c6c8..207761007d2d 100644 --- a/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml +++ b/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml @@ -35,20 +35,36 @@ Resources: "Version": "2012-10-17", "Statement": [ { - "Sid": "AllowScopedEC2InstanceActions", + "Sid": "AllowScopedEC2InstanceAccessActions", "Effect": "Allow", "Resource": [ "arn:${AWS::Partition}:ec2:${AWS::Region}::image/*", "arn:${AWS::Partition}:ec2:${AWS::Region}::snapshot/*", "arn:${AWS::Partition}:ec2:${AWS::Region}:*:security-group/*", - "arn:${AWS::Partition}:ec2:${AWS::Region}:*:subnet/*", - "arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*" + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:subnet/*" ], "Action": [ "ec2:RunInstances", "ec2:CreateFleet" ] }, + { + "Sid": "AllowScopedEC2LaunchTemplateAccessActions", + "Effect": "Allow", + "Resource": "arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*", + "Action": [ + "ec2:RunInstances", + "ec2:CreateFleet" + ], + "Condition": { + "StringEquals": { + "aws:ResourceTag/kubernetes.io/cluster/${ClusterName}": "owned" + }, + "StringLike": { + "aws:ResourceTag/karpenter.sh/nodepool": "*" + } + } + }, { "Sid": "AllowScopedEC2InstanceActionsWithTags", "Effect": "Allow", diff --git a/website/content/en/preview/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json b/website/content/en/preview/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json index 9d9cb6ad14a8..d474d01f4e16 100644 --- a/website/content/en/preview/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json +++ b/website/content/en/preview/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json @@ -484,7 +484,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by ($distribution_filter)(\n karpenter_pods_state{arch=~\"$arch\", capacity_type=~\"$capacity_type\", instance_type=~\"$instance_type\", provisioner=~\"$provisioner\"}\n)", + "expr": "sum by ($distribution_filter)(\n karpenter_pods_state{arch=~\"$arch\", capacity_type=~\"$capacity_type\", instance_type=~\"$instance_type\", nodepool=~\"$nodepool\"}\n)", "legendFormat": "{{label_name}}", "range": true, "refId": "A" @@ -588,7 +588,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "karpenter_provisioner_usage{resource_type=\"cpu\"} / karpenter_provisioner_limit{resource_type=\"cpu\"}", + "expr": "karpenter_nodepool_usage{resource_type=\"cpu\"} / karpenter_nodepool_limit{resource_type=\"cpu\"}", "format": "table", "instant": true, "legendFormat": "CPU Limit Utilization", @@ -602,7 +602,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "count by (provisioner)(karpenter_nodes_allocatable{provisioner!=\"N/A\",resource_type=\"cpu\"}) # Selects a single resource type to get node count", + "expr": "count by (nodepool)(karpenter_nodes_allocatable{nodepool!=\"N/A\",resource_type=\"cpu\"}) # Selects a single resource type to get node count", "format": "table", "hide": false, "instant": true, @@ -616,7 +616,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "karpenter_provisioner_usage{resource_type=\"memory\"} / karpenter_provisioner_limit{resource_type=\"memory\"}", + "expr": "karpenter_nodepool_usage{resource_type=\"memory\"} / karpenter_nodepool_limit{resource_type=\"memory\"}", "format": "table", "hide": false, "instant": true, @@ -631,7 +631,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum by (provisioner)(karpenter_nodes_allocatable{provisioner!=\"N/A\",resource_type=\"cpu\"})", + "expr": "sum by (nodepool)(karpenter_nodes_allocatable{nodepool!=\"N/A\",resource_type=\"cpu\"})", "format": "table", "hide": false, "instant": true, @@ -645,7 +645,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum by (provisioner)(karpenter_nodes_allocatable{provisioner!=\"N/A\",resource_type=\"memory\"})", + "expr": "sum by (nodepool)(karpenter_nodes_allocatable{nodepool!=\"N/A\",resource_type=\"memory\"})", "format": "table", "hide": false, "instant": true, @@ -653,12 +653,12 @@ "refId": "Memory Capacity" } ], - "title": "Provisioner Summary", + "title": "Nodepool Summary", "transformations": [ { "id": "seriesToColumns", "options": { - "byField": "provisioner" + "byField": "nodepool" } }, { @@ -697,7 +697,7 @@ "instance 2": 12, "job 1": 9, "job 2": 13, - "provisioner": 0, + "nodepool": 0, "resource_type 1": 10, "resource_type 2": 14 }, @@ -714,7 +714,7 @@ "instance": "", "instance 1": "", "job": "", - "provisioner": "Provisioner" + "nodepool": "Nodepool" } } } @@ -804,7 +804,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "(count(karpenter_nodes_allocatable{arch=~\"$arch\",capacity_type=\"spot\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"}) or vector(0)) / count(karpenter_nodes_allocatable{arch=~\"$arch\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"})", + "expr": "(count(karpenter_nodes_allocatable{arch=~\"$arch\",capacity_type=\"spot\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}) or vector(0)) / count(karpenter_nodes_allocatable{arch=~\"$arch\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"})", "legendFormat": "Percentage", "range": true, "refId": "A" @@ -946,7 +946,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "((karpenter_nodes_total_daemon_requests{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0) + \n(karpenter_nodes_total_pod_requests{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0)) / \nkarpenter_nodes_allocatable{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"}", + "expr": "((karpenter_nodes_total_daemon_requests{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0) + \n(karpenter_nodes_total_pod_requests{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0)) / \nkarpenter_nodes_allocatable{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}", "format": "table", "hide": false, "instant": true, @@ -961,7 +961,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "((karpenter_nodes_total_daemon_requests{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0) + \n(karpenter_nodes_total_pod_requests{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0)) / \nkarpenter_nodes_allocatable{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"}", + "expr": "((karpenter_nodes_total_daemon_requests{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0) + \n(karpenter_nodes_total_pod_requests{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0)) / \nkarpenter_nodes_allocatable{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}", "format": "table", "hide": false, "instant": true, @@ -976,7 +976,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "karpenter_nodes_total_daemon_requests{resource_type=\"pods\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"} + \nkarpenter_nodes_total_pod_requests{resource_type=\"pods\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"}", + "expr": "karpenter_nodes_total_daemon_requests{resource_type=\"pods\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} + \nkarpenter_nodes_total_pod_requests{resource_type=\"pods\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}", "format": "table", "hide": false, "instant": true, @@ -1091,9 +1091,9 @@ "os 1": true, "os 2": true, "os 3": true, - "provisioner 1": false, - "provisioner 2": true, - "provisioner 3": true, + "nodepool 1": false, + "nodepool 2": true, + "nodepool 3": true, "resource_type": true, "resource_type 1": true, "resource_type 2": true, @@ -1161,9 +1161,9 @@ "os 1": 23, "os 2": 41, "os 3": 61, - "provisioner 1": 2, - "provisioner 2": 42, - "provisioner 3": 62, + "nodepool 1": 2, + "nodepool 2": 42, + "nodepool 3": 62, "resource_type 1": 24, "resource_type 2": 43, "resource_type 3": 63, @@ -1190,7 +1190,7 @@ "instance_type": "Instance Type", "instance_type 1": "Instance Type", "node_name": "Node Name", - "provisioner 1": "Provisioner", + "nodepool 1": "Nodepool", "zone 1": "Zone" } } @@ -1237,14 +1237,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "definition": "label_values(karpenter_nodes_allocatable, provisioner)", + "definition": "label_values(karpenter_nodes_allocatable, nodepool)", "hide": 0, "includeAll": true, "multi": true, - "name": "provisioner", + "name": "nodepool", "options": [], "query": { - "query": "label_values(karpenter_nodes_allocatable, provisioner)", + "query": "label_values(karpenter_nodes_allocatable, nodepool)", "refId": "StandardVariableQuery" }, "refresh": 2, @@ -1376,8 +1376,8 @@ { "current": { "selected": true, - "text": "provisioner", - "value": "provisioner" + "text": "nodepool", + "value": "nodepool" }, "hide": 0, "includeAll": false, @@ -1411,8 +1411,8 @@ }, { "selected": true, - "text": "provisioner", - "value": "provisioner" + "text": "nodepool", + "value": "nodepool" }, { "selected": false, @@ -1420,7 +1420,7 @@ "value": "zone" } ], - "query": "arch,capacity_type,instance_type,namespace,node,provisioner,zone", + "query": "arch,capacity_type,instance_type,namespace,node,nodepool,zone", "queryValue": "", "skipUrlSync": false, "type": "custom" diff --git a/website/content/en/preview/getting-started/getting-started-with-karpenter/prometheus-values.yaml b/website/content/en/preview/getting-started/getting-started-with-karpenter/prometheus-values.yaml index f7c34fd90dea..588762a8c9dc 100644 --- a/website/content/en/preview/getting-started/getting-started-with-karpenter/prometheus-values.yaml +++ b/website/content/en/preview/getting-started/getting-started-with-karpenter/prometheus-values.yaml @@ -13,8 +13,10 @@ extraScrapeConfigs: | - role: endpoints namespaces: names: - - kube-system + - $KARPENTER_NAMESPACE relabel_configs: - - source_labels: [__meta_kubernetes_endpoint_port_name] - regex: http-metrics + - source_labels: + - __meta_kubernetes_endpoints_name + - __meta_kubernetes_endpoint_port_name action: keep + regex: karpenter;http-metrics diff --git a/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/add-monitoring.sh b/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/add-monitoring.sh deleted file mode 100755 index 6d2332b06be2..000000000000 --- a/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/add-monitoring.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -set -euo pipefail #fail if one step fails - -if [ "$#" -ne 1 ] -then - echo "Missing required Karpenter version. Usage: add-monitoring.sh v0.0.1" - exit 1 -fi - -export KARPENTER_VERSION=$1 - -declare -a steps=( - step01-config.sh - step09-add-prometheus-grafana.sh - step10-add-grafana-port-forward.sh - step11-grafana-get-password.sh -) - -for step in "${steps[@]}"; do - echo "$step" - source $step -done diff --git a/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/add-provisioner.sh b/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/add-provisioner.sh deleted file mode 100755 index 51f286de0c3c..000000000000 --- a/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/add-provisioner.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -set -euo pipefail #fail if one step fails - -declare -a steps=( - step01-config.sh - step12-add-provisioner.sh - step13-automatic-node-provisioning.sh - step14-automatic-node-termination.sh -) - -for step in "${steps[@]}"; do - echo "$step" - source $step -done diff --git a/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/add-roles.sh b/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/add-roles.sh deleted file mode 100755 index cf0656f3db3d..000000000000 --- a/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/add-roles.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -set -euo pipefail #fail if one step fails - -if [ "$#" -ne 1 ] -then - echo "Missing required Karpenter version. Usage: setup-roles.sh v0.0.1" - exit 1 -fi - -export KARPENTER_VERSION=$1 -SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) - -declare -a steps=( - step03-iam-cloud-formation.sh - step04-grant-access.sh - step05-controller-iam.sh - step06-add-spot-role.sh -) - -for step in "${steps[@]}"; do - echo "$step" - source "$SCRIPT_DIR/$step" -done diff --git a/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/cleanup.sh b/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/cleanup.sh deleted file mode 100755 index 52851e7f975a..000000000000 --- a/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/cleanup.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -declare -a steps=( - step01-config.sh - step16-cleanup.sh -) - -for step in "${steps[@]}"; do - echo "$step" - source $step -done diff --git a/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/install.sh b/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/install.sh deleted file mode 100755 index e0ebfb32e511..000000000000 --- a/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/install.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -set -euo pipefail #fail if one step fails - -if [ "$#" -ne 1 ] -then - echo "Missing required Karpenter version. Usage: install.sh v0.0.1" - exit 1 -fi - -export KARPENTER_VERSION=$1 - -declare -a steps=( - step01-config.sh - step02-create-cluster.sh - step03-iam-cloud-formation.sh - step04-grant-access.sh - step05-controller-iam.sh - step06-add-spot-role.sh - step08-apply-helm-chart.sh -) - -for step in "${steps[@]}"; do - echo "$step" - source $step -done diff --git a/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step01-config.sh b/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step01-config.sh index a3af512d02ac..b8209b641640 100755 --- a/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step01-config.sh +++ b/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step01-config.sh @@ -2,4 +2,7 @@ export AWS_PARTITION="aws" # if you are not using standard partitions, you may n export CLUSTER_NAME="${USER}-karpenter-demo" export AWS_DEFAULT_REGION="us-west-2" export AWS_ACCOUNT_ID="$(aws sts get-caller-identity --query Account --output text)" -export TEMPOUT=$(mktemp) +export TEMPOUT="$(mktemp)" +export ARM_AMI_ID="$(aws ssm get-parameter --name /aws/service/eks/optimized-ami/${K8S_VERSION}/amazon-linux-2-arm64/recommended/image_id --query Parameter.Value --output text)" +export AMD_AMI_ID="$(aws ssm get-parameter --name /aws/service/eks/optimized-ami/${K8S_VERSION}/amazon-linux-2/recommended/image_id --query Parameter.Value --output text)" +export GPU_AMI_ID="$(aws ssm get-parameter --name /aws/service/eks/optimized-ami/${K8S_VERSION}/amazon-linux-2-gpu/recommended/image_id --query Parameter.Value --output text)" diff --git a/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step02-create-cluster.sh b/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step02-create-cluster.sh index 7a7b57c05a4c..cdad96eeca23 100755 --- a/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step02-create-cluster.sh +++ b/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step02-create-cluster.sh @@ -1,4 +1,4 @@ -curl -fsSL https://raw.githubusercontent.com/aws/karpenter-provider-aws/"${KARPENTER_VERSION}"/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml > $TEMPOUT \ +curl -fsSL https://raw.githubusercontent.com/aws/karpenter-provider-aws/v"${KARPENTER_VERSION}"/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml > "${TEMPOUT}" \ && aws cloudformation deploy \ --stack-name "Karpenter-${CLUSTER_NAME}" \ --template-file "${TEMPOUT}" \ @@ -26,7 +26,7 @@ iam: - arn:${AWS_PARTITION}:iam::${AWS_ACCOUNT_ID}:policy/KarpenterControllerPolicy-${CLUSTER_NAME} ## Optionally run on fargate or on k8s 1.23 -# Pod Identity is not available on fargate +# Pod Identity is not available on fargate # https://docs.aws.amazon.com/eks/latest/userguide/pod-identities.html # iam: # withOIDC: true @@ -67,7 +67,7 @@ addons: # - namespace: "${KARPENTER_NAMESPACE}" EOF -export CLUSTER_ENDPOINT="$(aws eks describe-cluster --name ${CLUSTER_NAME} --query "cluster.endpoint" --output text)" +export CLUSTER_ENDPOINT="$(aws eks describe-cluster --name "${CLUSTER_NAME}" --query "cluster.endpoint" --output text)" export KARPENTER_IAM_ROLE_ARN="arn:${AWS_PARTITION}:iam::${AWS_ACCOUNT_ID}:role/${CLUSTER_NAME}-karpenter" -echo $CLUSTER_ENDPOINT $KARPENTER_IAM_ROLE_ARN +echo "${CLUSTER_ENDPOINT} ${KARPENTER_IAM_ROLE_ARN}" diff --git a/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step03-iam-cloud-formation.sh b/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step03-iam-cloud-formation.sh index b8e610f7bee6..54e826db269b 100755 --- a/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step03-iam-cloud-formation.sh +++ b/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step03-iam-cloud-formation.sh @@ -1,6 +1,6 @@ -TEMPOUT=$(mktemp) +TEMPOUT="$(mktemp)" -curl -fsSL https://raw.githubusercontent.com/aws/karpenter-provider-aws/"${KARPENTER_VERSION}"/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml > $TEMPOUT \ +curl -fsSL https://raw.githubusercontent.com/aws/karpenter-provider-aws/v"${KARPENTER_VERSION}"/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml > "${TEMPOUT}" \ && aws cloudformation deploy \ --stack-name "Karpenter-${CLUSTER_NAME}" \ --template-file "${TEMPOUT}" \ diff --git a/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step09-add-prometheus-grafana.sh b/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step09-add-prometheus-grafana.sh index a539f7491e10..1107c2ec1d24 100755 --- a/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step09-add-prometheus-grafana.sh +++ b/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step09-add-prometheus-grafana.sh @@ -4,8 +4,8 @@ helm repo update kubectl create namespace monitoring -curl -fsSL https://raw.githubusercontent.com/aws/karpenter-provider-aws/"${KARPENTER_VERSION}"/website/content/en/preview/getting-started/getting-started-with-karpenter/prometheus-values.yaml | tee prometheus-values.yaml +curl -fsSL https://raw.githubusercontent.com/aws/karpenter-provider-aws/v"${KARPENTER_VERSION}"/website/content/en/preview/getting-started/getting-started-with-karpenter/prometheus-values.yaml | envsubst | tee prometheus-values.yaml helm install --namespace monitoring prometheus prometheus-community/prometheus --values prometheus-values.yaml -curl -fsSL https://raw.githubusercontent.com/aws/karpenter-provider-aws/"${KARPENTER_VERSION}"/website/content/en/preview/getting-started/getting-started-with-karpenter/grafana-values.yaml | tee grafana-values.yaml +curl -fsSL https://raw.githubusercontent.com/aws/karpenter-provider-aws/v"${KARPENTER_VERSION}"/website/content/en/preview/getting-started/getting-started-with-karpenter/grafana-values.yaml | tee grafana-values.yaml helm install --namespace monitoring grafana grafana-charts/grafana --values grafana-values.yaml diff --git a/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step12-add-nodepool.sh b/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step12-add-nodepool.sh index 8c518e9e74b8..33f1cb553b1b 100755 --- a/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step12-add-nodepool.sh +++ b/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step12-add-nodepool.sh @@ -23,6 +23,8 @@ spec: operator: Gt values: ["2"] nodeClassRef: + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass name: default limits: cpu: 1000 @@ -43,4 +45,9 @@ spec: securityGroupSelectorTerms: - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + amiSelectorTerms: + - id: "${ARM_AMI_ID}" + - id: "${AMD_AMI_ID}" +# - id: "${GPU_AMI_ID}" # <- GPU Optimized AMD AMI +# - name: "amazon-eks-node-${K8S_VERSION}-*" # <- automatically upgrade when a new AL2 EKS Optimized AMI is released. This is unsafe for production workloads. Validate AMIs in lower environments before deploying them to production. EOF diff --git a/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step16-delete-node.sh b/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step16-delete-node.sh index 9d431160dda0..93b1ac792ade 100755 --- a/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step16-delete-node.sh +++ b/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step16-delete-node.sh @@ -1 +1 @@ -kubectl delete node $NODE_NAME +kubectl delete node "${NODE_NAME}" diff --git a/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step17-cleanup.sh b/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step17-cleanup.sh index dc84b673389f..f694ac3797f4 100755 --- a/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step17-cleanup.sh +++ b/website/content/en/preview/getting-started/getting-started-with-karpenter/scripts/step17-cleanup.sh @@ -1,6 +1,6 @@ helm uninstall karpenter --namespace "${KARPENTER_NAMESPACE}" aws cloudformation delete-stack --stack-name "Karpenter-${CLUSTER_NAME}" -aws ec2 describe-launch-templates --filters Name=tag:karpenter.k8s.aws/cluster,Values=${CLUSTER_NAME} | +aws ec2 describe-launch-templates --filters "Name=tag:karpenter.k8s.aws/cluster,Values=${CLUSTER_NAME}" | jq -r ".LaunchTemplates[].LaunchTemplateName" | xargs -I{} aws ec2 delete-launch-template --launch-template-name {} eksctl delete cluster --name "${CLUSTER_NAME}" diff --git a/website/content/en/preview/getting-started/migrating-from-cas/_index.md b/website/content/en/preview/getting-started/migrating-from-cas/_index.md index cfd3b82fec51..24ab981ebbfc 100644 --- a/website/content/en/preview/getting-started/migrating-from-cas/_index.md +++ b/website/content/en/preview/getting-started/migrating-from-cas/_index.md @@ -92,10 +92,10 @@ One for your Karpenter node role and one for your existing node group. First set the Karpenter release you want to deploy. ```bash -export KARPENTER_VERSION={{< param "latest_release_version" >}} +export KARPENTER_VERSION="{{< param "latest_release_version" >}}" ``` -We can now generate a full Karpenter deployment yaml from the helm chart. +We can now generate a full Karpenter deployment yaml from the Helm chart. {{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step08-generate-chart.sh" language="bash" %}} diff --git a/website/content/en/preview/getting-started/migrating-from-cas/scripts/step01-env.sh b/website/content/en/preview/getting-started/migrating-from-cas/scripts/step01-env.sh index 20645685137b..b3ee0fa3d0ef 100644 --- a/website/content/en/preview/getting-started/migrating-from-cas/scripts/step01-env.sh +++ b/website/content/en/preview/getting-started/migrating-from-cas/scripts/step01-env.sh @@ -1,6 +1,10 @@ AWS_PARTITION="aws" # if you are not using standard partitions, you may need to configure to aws-cn / aws-us-gov AWS_REGION="$(aws configure list | grep region | tr -s " " | cut -d" " -f3)" -OIDC_ENDPOINT="$(aws eks describe-cluster --name ${CLUSTER_NAME} \ +OIDC_ENDPOINT="$(aws eks describe-cluster --name "${CLUSTER_NAME}" \ --query "cluster.identity.oidc.issuer" --output text)" AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query 'Account' \ --output text) +K8S_VERSION=1.28 +ARM_AMI_ID="$(aws ssm get-parameter --name /aws/service/eks/optimized-ami/${K8S_VERSION}/amazon-linux-2-arm64/recommended/image_id --query Parameter.Value --output text)" +AMD_AMI_ID="$(aws ssm get-parameter --name /aws/service/eks/optimized-ami/${K8S_VERSION}/amazon-linux-2/recommended/image_id --query Parameter.Value --output text)" +GPU_AMI_ID="$(aws ssm get-parameter --name /aws/service/eks/optimized-ami/${K8S_VERSION}/amazon-linux-2-gpu/recommended/image_id --query Parameter.Value --output text)" diff --git a/website/content/en/preview/getting-started/migrating-from-cas/scripts/step03-node-policies.sh b/website/content/en/preview/getting-started/migrating-from-cas/scripts/step03-node-policies.sh index d57f79039d04..fbc5455e541b 100644 --- a/website/content/en/preview/getting-started/migrating-from-cas/scripts/step03-node-policies.sh +++ b/website/content/en/preview/getting-started/migrating-from-cas/scripts/step03-node-policies.sh @@ -1,11 +1,11 @@ aws iam attach-role-policy --role-name "KarpenterNodeRole-${CLUSTER_NAME}" \ - --policy-arn arn:${AWS_PARTITION}:iam::aws:policy/AmazonEKSWorkerNodePolicy + --policy-arn "arn:${AWS_PARTITION}:iam::aws:policy/AmazonEKSWorkerNodePolicy" aws iam attach-role-policy --role-name "KarpenterNodeRole-${CLUSTER_NAME}" \ - --policy-arn arn:${AWS_PARTITION}:iam::aws:policy/AmazonEKS_CNI_Policy + --policy-arn "arn:${AWS_PARTITION}:iam::aws:policy/AmazonEKS_CNI_Policy" aws iam attach-role-policy --role-name "KarpenterNodeRole-${CLUSTER_NAME}" \ - --policy-arn arn:${AWS_PARTITION}:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly + --policy-arn "arn:${AWS_PARTITION}:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" aws iam attach-role-policy --role-name "KarpenterNodeRole-${CLUSTER_NAME}" \ - --policy-arn arn:${AWS_PARTITION}:iam::aws:policy/AmazonSSMManagedInstanceCore + --policy-arn "arn:${AWS_PARTITION}:iam::aws:policy/AmazonSSMManagedInstanceCore" diff --git a/website/content/en/preview/getting-started/migrating-from-cas/scripts/step04-controller-iam.sh b/website/content/en/preview/getting-started/migrating-from-cas/scripts/step04-controller-iam.sh index 70c59e826393..cc3d7f929986 100644 --- a/website/content/en/preview/getting-started/migrating-from-cas/scripts/step04-controller-iam.sh +++ b/website/content/en/preview/getting-started/migrating-from-cas/scripts/step04-controller-iam.sh @@ -19,7 +19,7 @@ cat << EOF > controller-trust-policy.json } EOF -aws iam create-role --role-name KarpenterControllerRole-${CLUSTER_NAME} \ +aws iam create-role --role-name "KarpenterControllerRole-${CLUSTER_NAME}" \ --assume-role-policy-document file://controller-trust-policy.json cat << EOF > controller-policy.json @@ -138,6 +138,6 @@ cat << EOF > controller-policy.json } EOF -aws iam put-role-policy --role-name KarpenterControllerRole-${CLUSTER_NAME} \ - --policy-name KarpenterControllerPolicy-${CLUSTER_NAME} \ +aws iam put-role-policy --role-name "KarpenterControllerRole-${CLUSTER_NAME}" \ + --policy-name "KarpenterControllerPolicy-${CLUSTER_NAME}" \ --policy-document file://controller-policy.json diff --git a/website/content/en/preview/getting-started/migrating-from-cas/scripts/step05-tag-subnets.sh b/website/content/en/preview/getting-started/migrating-from-cas/scripts/step05-tag-subnets.sh index de972ea2bddd..47df188dc87d 100644 --- a/website/content/en/preview/getting-started/migrating-from-cas/scripts/step05-tag-subnets.sh +++ b/website/content/en/preview/getting-started/migrating-from-cas/scripts/step05-tag-subnets.sh @@ -1,6 +1,6 @@ -for NODEGROUP in $(aws eks list-nodegroups --cluster-name ${CLUSTER_NAME} \ - --query 'nodegroups' --output text); do aws ec2 create-tags \ +for NODEGROUP in $(aws eks list-nodegroups --cluster-name "${CLUSTER_NAME}" --query 'nodegroups' --output text); do + aws ec2 create-tags \ --tags "Key=karpenter.sh/discovery,Value=${CLUSTER_NAME}" \ - --resources $(aws eks describe-nodegroup --cluster-name ${CLUSTER_NAME} \ - --nodegroup-name $NODEGROUP --query 'nodegroup.subnets' --output text ) + --resources "$(aws eks describe-nodegroup --cluster-name "${CLUSTER_NAME}" \ + --nodegroup-name "${NODEGROUP}" --query 'nodegroup.subnets' --output text )" done diff --git a/website/content/en/preview/getting-started/migrating-from-cas/scripts/step06-tag-security-groups.sh b/website/content/en/preview/getting-started/migrating-from-cas/scripts/step06-tag-security-groups.sh index 397e40904cee..c63bde3b78dc 100644 --- a/website/content/en/preview/getting-started/migrating-from-cas/scripts/step06-tag-security-groups.sh +++ b/website/content/en/preview/getting-started/migrating-from-cas/scripts/step06-tag-security-groups.sh @@ -1,22 +1,22 @@ -NODEGROUP=$(aws eks list-nodegroups --cluster-name ${CLUSTER_NAME} \ +NODEGROUP=$(aws eks list-nodegroups --cluster-name "${CLUSTER_NAME}" \ --query 'nodegroups[0]' --output text) -LAUNCH_TEMPLATE=$(aws eks describe-nodegroup --cluster-name ${CLUSTER_NAME} \ - --nodegroup-name ${NODEGROUP} --query 'nodegroup.launchTemplate.{id:id,version:version}' \ +LAUNCH_TEMPLATE=$(aws eks describe-nodegroup --cluster-name "${CLUSTER_NAME}" \ + --nodegroup-name "${NODEGROUP}" --query 'nodegroup.launchTemplate.{id:id,version:version}' \ --output text | tr -s "\t" ",") # If your EKS setup is configured to use only Cluster security group, then please execute - SECURITY_GROUPS=$(aws eks describe-cluster \ - --name ${CLUSTER_NAME} --query "cluster.resourcesVpcConfig.clusterSecurityGroupId" --output text) + --name "${CLUSTER_NAME}" --query "cluster.resourcesVpcConfig.clusterSecurityGroupId" --output text) # If your setup uses the security groups in the Launch template of a managed node group, then : -SECURITY_GROUPS=$(aws ec2 describe-launch-template-versions \ - --launch-template-id ${LAUNCH_TEMPLATE%,*} --versions ${LAUNCH_TEMPLATE#*,} \ +SECURITY_GROUPS="$(aws ec2 describe-launch-template-versions \ + --launch-template-id "${LAUNCH_TEMPLATE%,*}" --versions "${LAUNCH_TEMPLATE#*,}" \ --query 'LaunchTemplateVersions[0].LaunchTemplateData.[NetworkInterfaces[0].Groups||SecurityGroupIds]' \ - --output text) + --output text)" aws ec2 create-tags \ --tags "Key=karpenter.sh/discovery,Value=${CLUSTER_NAME}" \ - --resources ${SECURITY_GROUPS} + --resources "${SECURITY_GROUPS}" diff --git a/website/content/en/preview/getting-started/migrating-from-cas/scripts/step09-deploy.sh b/website/content/en/preview/getting-started/migrating-from-cas/scripts/step09-deploy.sh index 51714d78f6dd..e46742fd22ea 100644 --- a/website/content/en/preview/getting-started/migrating-from-cas/scripts/step09-deploy.sh +++ b/website/content/en/preview/getting-started/migrating-from-cas/scripts/step09-deploy.sh @@ -1,8 +1,8 @@ kubectl create namespace "${KARPENTER_NAMESPACE}" || true kubectl create -f \ - https://raw.githubusercontent.com/aws/karpenter-provider-aws/${KARPENTER_VERSION}/pkg/apis/crds/karpenter.sh_nodepools.yaml + "https://raw.githubusercontent.com/aws/karpenter-provider-aws/v${KARPENTER_VERSION}/pkg/apis/crds/karpenter.sh_nodepools.yaml" kubectl create -f \ - https://raw.githubusercontent.com/aws/karpenter-provider-aws/${KARPENTER_VERSION}/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml + "https://raw.githubusercontent.com/aws/karpenter-provider-aws/v${KARPENTER_VERSION}/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml" kubectl create -f \ - https://raw.githubusercontent.com/aws/karpenter-provider-aws/${KARPENTER_VERSION}/pkg/apis/crds/karpenter.sh_nodeclaims.yaml + "https://raw.githubusercontent.com/aws/karpenter-provider-aws/v${KARPENTER_VERSION}/pkg/apis/crds/karpenter.sh_nodeclaims.yaml" kubectl apply -f karpenter.yaml diff --git a/website/content/en/preview/getting-started/migrating-from-cas/scripts/step10-create-nodepool.sh b/website/content/en/preview/getting-started/migrating-from-cas/scripts/step10-create-nodepool.sh index 8c518e9e74b8..33f1cb553b1b 100644 --- a/website/content/en/preview/getting-started/migrating-from-cas/scripts/step10-create-nodepool.sh +++ b/website/content/en/preview/getting-started/migrating-from-cas/scripts/step10-create-nodepool.sh @@ -23,6 +23,8 @@ spec: operator: Gt values: ["2"] nodeClassRef: + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass name: default limits: cpu: 1000 @@ -43,4 +45,9 @@ spec: securityGroupSelectorTerms: - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + amiSelectorTerms: + - id: "${ARM_AMI_ID}" + - id: "${AMD_AMI_ID}" +# - id: "${GPU_AMI_ID}" # <- GPU Optimized AMD AMI +# - name: "amazon-eks-node-${K8S_VERSION}-*" # <- automatically upgrade when a new AL2 EKS Optimized AMI is released. This is unsafe for production workloads. Validate AMIs in lower environments before deploying them to production. EOF diff --git a/website/content/en/preview/getting-started/migrating-from-cas/scripts/step12-scale-multiple-ng.sh b/website/content/en/preview/getting-started/migrating-from-cas/scripts/step12-scale-multiple-ng.sh index d88a6d5c7236..a28c9759bcdb 100644 --- a/website/content/en/preview/getting-started/migrating-from-cas/scripts/step12-scale-multiple-ng.sh +++ b/website/content/en/preview/getting-started/migrating-from-cas/scripts/step12-scale-multiple-ng.sh @@ -1,5 +1,5 @@ -for NODEGROUP in $(aws eks list-nodegroups --cluster-name ${CLUSTER_NAME} \ - --query 'nodegroups' --output text); do aws eks update-nodegroup-config --cluster-name ${CLUSTER_NAME} \ - --nodegroup-name ${NODEGROUP} \ +for NODEGROUP in $(aws eks list-nodegroups --cluster-name "${CLUSTER_NAME}" \ + --query 'nodegroups' --output text); do aws eks update-nodegroup-config --cluster-name "${CLUSTER_NAME}" \ + --nodegroup-name "${NODEGROUP}" \ --scaling-config "minSize=1,maxSize=1,desiredSize=1" done diff --git a/website/content/en/preview/getting-started/migrating-from-cas/scripts/step12-scale-single-ng.sh b/website/content/en/preview/getting-started/migrating-from-cas/scripts/step12-scale-single-ng.sh index 51ad964c28a7..efce68394779 100644 --- a/website/content/en/preview/getting-started/migrating-from-cas/scripts/step12-scale-single-ng.sh +++ b/website/content/en/preview/getting-started/migrating-from-cas/scripts/step12-scale-single-ng.sh @@ -1,3 +1,3 @@ -aws eks update-nodegroup-config --cluster-name ${CLUSTER_NAME} \ - --nodegroup-name ${NODEGROUP} \ +aws eks update-nodegroup-config --cluster-name "${CLUSTER_NAME}" \ + --nodegroup-name "${NODEGROUP}" \ --scaling-config "minSize=2,maxSize=2,desiredSize=2" diff --git a/website/content/en/preview/reference/cloudformation.md b/website/content/en/preview/reference/cloudformation.md index 5dc95af3e482..1a984bbdb671 100644 --- a/website/content/en/preview/reference/cloudformation.md +++ b/website/content/en/preview/reference/cloudformation.md @@ -17,8 +17,8 @@ These descriptions should allow you to understand: To download a particular version of `cloudformation.yaml`, set the version and use `curl` to pull the file to your local system: ```bash -export KARPENTER_VERSION={{< param "latest_release_version" >}} -curl https://raw.githubusercontent.com/aws/karpenter-provider-aws/"${KARPENTER_VERSION}"/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml > cloudformation.yaml +export KARPENTER_VERSION="{{< param "latest_release_version" >}}" +curl https://raw.githubusercontent.com/aws/karpenter-provider-aws/v"${KARPENTER_VERSION}"/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml > cloudformation.yaml ``` Following some header information, the rest of the `cloudformation.yaml` file describes the resources that CloudFormation deploys. @@ -87,7 +87,7 @@ The resources defined in this section are associated with: * KarpenterControllerPolicy -Because the scope of the KarpenterControllerPolicy is an AWS region, the cluster's AWS region is included in the `AllowScopedEC2InstanceActions`. +Because the scope of the KarpenterControllerPolicy is an AWS region, the cluster's AWS region is included in the `AllowScopedEC2InstanceAccessActions`. ### KarpenterControllerPolicy @@ -109,22 +109,21 @@ KarpenterControllerPolicy: Someone wanting to add Karpenter to an existing cluster, instead of using `cloudformation.yaml`, would need to create the IAM policy directly and assign that policy to the role leveraged by the service account using IRSA. -#### AllowScopedEC2InstanceActions +#### AllowScopedEC2InstanceAccessActions -The AllowScopedEC2InstanceActions statement ID (Sid) identifies a set of EC2 resources that are allowed to be accessed with +The AllowScopedEC2InstanceAccessActions statement ID (Sid) identifies a set of EC2 resources that are allowed to be accessed with [RunInstances](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_RunInstances.html) and [CreateFleet](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html) actions. For `RunInstances` and `CreateFleet` actions, the Karpenter controller can read (but not create) `image`, `snapshot`, `security-group`, `subnet` and `launch-template` EC2 resources, scoped for the particular AWS partition and region. ```json { - "Sid": "AllowScopedEC2InstanceActions", + "Sid": "AllowScopedEC2InstanceAccessActions", "Effect": "Allow", "Resource": [ "arn:${AWS::Partition}:ec2:${AWS::Region}::image/*", "arn:${AWS::Partition}:ec2:${AWS::Region}::snapshot/*", "arn:${AWS::Partition}:ec2:${AWS::Region}:*:security-group/*", - "arn:${AWS::Partition}:ec2:${AWS::Region}:*:subnet/*", - "arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*" + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:subnet/*" ], "Action": [ "ec2:RunInstances", @@ -133,6 +132,32 @@ For `RunInstances` and `CreateFleet` actions, the Karpenter controller can read } ``` +#### AllowScopedEC2LaunchTemplateAccessActions + +The AllowScopedEC2InstanceAccessActions statement ID (Sid) identifies launch templates that are allowed to be accessed with +[RunInstances](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_RunInstances.html) and [CreateFleet](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html) actions. +For `RunInstances` and `CreateFleet` actions, the Karpenter controller can read (but not create) `launch-template` EC2 resources that have the `kubernetes.io/cluster/${ClusterName}` tag be set to `owned` and a `karpenter.sh/nodepool` tag, scoped for the particular AWS partition and region. This ensures that an instance launch can't access launch templates that weren't provisioned by Karpenter. + +```json +{ + "Sid": "AllowScopedEC2LaunchTemplateAccessActions", + "Effect": "Allow", + "Resource": "arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*", + "Action": [ + "ec2:RunInstances", + "ec2:CreateFleet" + ], + "Condition": { + "StringEquals": { + "aws:ResourceTag/kubernetes.io/cluster/${ClusterName}": "owned" + }, + "StringLike": { + "aws:ResourceTag/karpenter.sh/nodepool": "*" + } + } +} +``` + #### AllowScopedEC2InstanceActionsWithTags The AllowScopedEC2InstanceActionsWithTags Sid allows the diff --git a/website/content/en/preview/reference/instance-types.md b/website/content/en/preview/reference/instance-types.md index 44bc7b15b0f6..998458edd392 100644 --- a/website/content/en/preview/reference/instance-types.md +++ b/website/content/en/preview/reference/instance-types.md @@ -19,6 +19,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|a| |karpenter.k8s.aws/instance-cpu|1| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|a1| |karpenter.k8s.aws/instance-generation|1| @@ -43,6 +44,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|a| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|a1| |karpenter.k8s.aws/instance-generation|1| @@ -67,6 +69,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|a| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|a1| |karpenter.k8s.aws/instance-generation|1| @@ -91,6 +94,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|a| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|a1| |karpenter.k8s.aws/instance-generation|1| @@ -115,6 +119,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|a| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|a1| |karpenter.k8s.aws/instance-generation|1| @@ -139,6 +144,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|a| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|a1| |karpenter.k8s.aws/instance-generation|1| @@ -164,6 +170,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c1| |karpenter.k8s.aws/instance-generation|1| @@ -186,6 +193,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c1| |karpenter.k8s.aws/instance-generation|1| @@ -209,6 +217,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c3| |karpenter.k8s.aws/instance-generation|3| @@ -231,6 +240,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c3| |karpenter.k8s.aws/instance-generation|3| @@ -253,6 +263,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c3| |karpenter.k8s.aws/instance-generation|3| @@ -275,6 +286,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c3| |karpenter.k8s.aws/instance-generation|3| @@ -297,6 +309,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c3| |karpenter.k8s.aws/instance-generation|3| @@ -321,6 +334,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c4| |karpenter.k8s.aws/instance-generation|4| @@ -343,6 +357,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c4| |karpenter.k8s.aws/instance-generation|4| @@ -365,6 +380,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c4| |karpenter.k8s.aws/instance-generation|4| @@ -387,6 +403,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c4| |karpenter.k8s.aws/instance-generation|4| @@ -409,6 +426,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|36| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c4| |karpenter.k8s.aws/instance-generation|4| @@ -433,6 +451,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5| |karpenter.k8s.aws/instance-generation|5| @@ -457,6 +476,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5| |karpenter.k8s.aws/instance-generation|5| @@ -481,6 +501,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5| |karpenter.k8s.aws/instance-generation|5| @@ -505,6 +526,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5| |karpenter.k8s.aws/instance-generation|5| @@ -529,6 +551,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|36| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5| |karpenter.k8s.aws/instance-generation|5| @@ -553,6 +576,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5| |karpenter.k8s.aws/instance-generation|5| @@ -577,6 +601,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|72| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5| |karpenter.k8s.aws/instance-generation|5| @@ -601,6 +626,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5| |karpenter.k8s.aws/instance-generation|5| @@ -625,6 +651,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5| |karpenter.k8s.aws/instance-generation|5| @@ -650,6 +677,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5a| |karpenter.k8s.aws/instance-generation|5| @@ -674,6 +702,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5a| |karpenter.k8s.aws/instance-generation|5| @@ -698,6 +727,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5a| |karpenter.k8s.aws/instance-generation|5| @@ -722,6 +752,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5a| |karpenter.k8s.aws/instance-generation|5| @@ -746,6 +777,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5a| |karpenter.k8s.aws/instance-generation|5| @@ -770,6 +802,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5a| |karpenter.k8s.aws/instance-generation|5| @@ -794,6 +827,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5a| |karpenter.k8s.aws/instance-generation|5| @@ -818,6 +852,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5a| |karpenter.k8s.aws/instance-generation|5| @@ -843,6 +878,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5ad| |karpenter.k8s.aws/instance-generation|5| @@ -868,6 +904,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5ad| |karpenter.k8s.aws/instance-generation|5| @@ -893,6 +930,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5ad| |karpenter.k8s.aws/instance-generation|5| @@ -918,6 +956,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5ad| |karpenter.k8s.aws/instance-generation|5| @@ -943,6 +982,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5ad| |karpenter.k8s.aws/instance-generation|5| @@ -968,6 +1008,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5ad| |karpenter.k8s.aws/instance-generation|5| @@ -993,6 +1034,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5ad| |karpenter.k8s.aws/instance-generation|5| @@ -1018,6 +1060,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5ad| |karpenter.k8s.aws/instance-generation|5| @@ -1044,6 +1087,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5d| |karpenter.k8s.aws/instance-generation|5| @@ -1069,6 +1113,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5d| |karpenter.k8s.aws/instance-generation|5| @@ -1094,6 +1139,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5d| |karpenter.k8s.aws/instance-generation|5| @@ -1119,6 +1165,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5d| |karpenter.k8s.aws/instance-generation|5| @@ -1144,6 +1191,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|36| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5d| |karpenter.k8s.aws/instance-generation|5| @@ -1169,6 +1217,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5d| |karpenter.k8s.aws/instance-generation|5| @@ -1194,6 +1243,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|72| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5d| |karpenter.k8s.aws/instance-generation|5| @@ -1219,6 +1269,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5d| |karpenter.k8s.aws/instance-generation|5| @@ -1244,6 +1295,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c5d| |karpenter.k8s.aws/instance-generation|5| @@ -1270,6 +1322,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5n| |karpenter.k8s.aws/instance-generation|5| @@ -1294,6 +1347,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5n| |karpenter.k8s.aws/instance-generation|5| @@ -1318,6 +1372,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5n| |karpenter.k8s.aws/instance-generation|5| @@ -1342,6 +1397,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5n| |karpenter.k8s.aws/instance-generation|5| @@ -1366,6 +1422,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|36| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5n| |karpenter.k8s.aws/instance-generation|5| @@ -1391,6 +1448,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|72| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5n| |karpenter.k8s.aws/instance-generation|5| @@ -1416,6 +1474,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|72| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c5n| |karpenter.k8s.aws/instance-generation|5| @@ -1442,6 +1501,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6a| |karpenter.k8s.aws/instance-generation|6| @@ -1466,6 +1526,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6a| |karpenter.k8s.aws/instance-generation|6| @@ -1490,6 +1551,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6a| |karpenter.k8s.aws/instance-generation|6| @@ -1514,6 +1576,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6a| |karpenter.k8s.aws/instance-generation|6| @@ -1538,6 +1601,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6a| |karpenter.k8s.aws/instance-generation|6| @@ -1562,6 +1626,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6a| |karpenter.k8s.aws/instance-generation|6| @@ -1586,6 +1651,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6a| |karpenter.k8s.aws/instance-generation|6| @@ -1610,6 +1676,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6a| |karpenter.k8s.aws/instance-generation|6| @@ -1634,6 +1701,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6a| |karpenter.k8s.aws/instance-generation|6| @@ -1658,6 +1726,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|192| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6a| |karpenter.k8s.aws/instance-generation|6| @@ -1683,6 +1752,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|192| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6a| |karpenter.k8s.aws/instance-generation|6| @@ -1709,6 +1779,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|1| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6g| |karpenter.k8s.aws/instance-generation|6| @@ -1733,6 +1804,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6g| |karpenter.k8s.aws/instance-generation|6| @@ -1757,6 +1829,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6g| |karpenter.k8s.aws/instance-generation|6| @@ -1781,6 +1854,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6g| |karpenter.k8s.aws/instance-generation|6| @@ -1805,6 +1879,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6g| |karpenter.k8s.aws/instance-generation|6| @@ -1829,6 +1904,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6g| |karpenter.k8s.aws/instance-generation|6| @@ -1853,6 +1929,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6g| |karpenter.k8s.aws/instance-generation|6| @@ -1877,6 +1954,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6g| |karpenter.k8s.aws/instance-generation|6| @@ -1901,6 +1979,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6g| |karpenter.k8s.aws/instance-generation|6| @@ -1926,6 +2005,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|1| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6gd| |karpenter.k8s.aws/instance-generation|6| @@ -1951,6 +2031,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6gd| |karpenter.k8s.aws/instance-generation|6| @@ -1976,6 +2057,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6gd| |karpenter.k8s.aws/instance-generation|6| @@ -2001,6 +2083,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6gd| |karpenter.k8s.aws/instance-generation|6| @@ -2026,6 +2109,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6gd| |karpenter.k8s.aws/instance-generation|6| @@ -2051,6 +2135,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6gd| |karpenter.k8s.aws/instance-generation|6| @@ -2076,6 +2161,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6gd| |karpenter.k8s.aws/instance-generation|6| @@ -2101,6 +2187,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6gd| |karpenter.k8s.aws/instance-generation|6| @@ -2126,6 +2213,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|c6gd| |karpenter.k8s.aws/instance-generation|6| @@ -2152,6 +2240,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|1| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6gn| |karpenter.k8s.aws/instance-generation|6| @@ -2176,6 +2265,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6gn| |karpenter.k8s.aws/instance-generation|6| @@ -2200,6 +2290,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6gn| |karpenter.k8s.aws/instance-generation|6| @@ -2224,6 +2315,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6gn| |karpenter.k8s.aws/instance-generation|6| @@ -2248,12 +2340,13 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6gn| |karpenter.k8s.aws/instance-generation|6| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| - |karpenter.k8s.aws/instance-network-bandwidth|15000| + |karpenter.k8s.aws/instance-network-bandwidth|25000| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -2272,6 +2365,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6gn| |karpenter.k8s.aws/instance-generation|6| @@ -2296,6 +2390,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6gn| |karpenter.k8s.aws/instance-generation|6| @@ -2320,6 +2415,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6gn| |karpenter.k8s.aws/instance-generation|6| @@ -2346,6 +2442,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6i| |karpenter.k8s.aws/instance-generation|6| @@ -2370,6 +2467,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6i| |karpenter.k8s.aws/instance-generation|6| @@ -2394,6 +2492,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6i| |karpenter.k8s.aws/instance-generation|6| @@ -2418,6 +2517,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6i| |karpenter.k8s.aws/instance-generation|6| @@ -2442,6 +2542,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6i| |karpenter.k8s.aws/instance-generation|6| @@ -2466,6 +2567,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6i| |karpenter.k8s.aws/instance-generation|6| @@ -2490,6 +2592,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6i| |karpenter.k8s.aws/instance-generation|6| @@ -2514,6 +2617,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6i| |karpenter.k8s.aws/instance-generation|6| @@ -2538,6 +2642,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6i| |karpenter.k8s.aws/instance-generation|6| @@ -2563,6 +2668,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6i| |karpenter.k8s.aws/instance-generation|6| @@ -2589,6 +2695,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6id| |karpenter.k8s.aws/instance-generation|6| @@ -2614,6 +2721,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6id| |karpenter.k8s.aws/instance-generation|6| @@ -2639,6 +2747,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6id| |karpenter.k8s.aws/instance-generation|6| @@ -2664,6 +2773,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6id| |karpenter.k8s.aws/instance-generation|6| @@ -2689,6 +2799,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6id| |karpenter.k8s.aws/instance-generation|6| @@ -2714,6 +2825,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6id| |karpenter.k8s.aws/instance-generation|6| @@ -2739,6 +2851,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6id| |karpenter.k8s.aws/instance-generation|6| @@ -2764,6 +2877,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6id| |karpenter.k8s.aws/instance-generation|6| @@ -2789,6 +2903,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6id| |karpenter.k8s.aws/instance-generation|6| @@ -2815,6 +2930,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6id| |karpenter.k8s.aws/instance-generation|6| @@ -2842,6 +2958,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6in| |karpenter.k8s.aws/instance-generation|6| @@ -2866,6 +2983,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6in| |karpenter.k8s.aws/instance-generation|6| @@ -2890,6 +3008,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6in| |karpenter.k8s.aws/instance-generation|6| @@ -2914,6 +3033,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6in| |karpenter.k8s.aws/instance-generation|6| @@ -2938,6 +3058,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6in| |karpenter.k8s.aws/instance-generation|6| @@ -2962,6 +3083,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6in| |karpenter.k8s.aws/instance-generation|6| @@ -2986,6 +3108,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6in| |karpenter.k8s.aws/instance-generation|6| @@ -3010,6 +3133,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6in| |karpenter.k8s.aws/instance-generation|6| @@ -3034,6 +3158,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6in| |karpenter.k8s.aws/instance-generation|6| @@ -3059,6 +3184,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c6in| |karpenter.k8s.aws/instance-generation|6| @@ -3085,11 +3211,13 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|1| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7a| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|2048| + |karpenter.k8s.aws/instance-network-bandwidth|390| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3101,17 +3229,20 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|1451Mi| |pods|8| + |vpc.amazonaws.com/pod-eni|4| ### `c7a.large` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7a| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|4096| + |karpenter.k8s.aws/instance-network-bandwidth|781| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3123,17 +3254,20 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|3114Mi| |pods|29| + |vpc.amazonaws.com/pod-eni|9| ### `c7a.xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7a| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| + |karpenter.k8s.aws/instance-network-bandwidth|1562| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3145,17 +3279,20 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|6584Mi| |pods|58| + |vpc.amazonaws.com/pod-eni|18| ### `c7a.2xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7a| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| + |karpenter.k8s.aws/instance-network-bandwidth|3125| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3167,17 +3304,20 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|14162Mi| |pods|58| + |vpc.amazonaws.com/pod-eni|38| ### `c7a.4xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7a| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| + |karpenter.k8s.aws/instance-network-bandwidth|6250| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3189,17 +3329,20 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|27381Mi| |pods|234| + |vpc.amazonaws.com/pod-eni|54| ### `c7a.8xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7a| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| + |karpenter.k8s.aws/instance-network-bandwidth|12500| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3211,17 +3354,20 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|57691Mi| |pods|234| + |vpc.amazonaws.com/pod-eni|84| ### `c7a.12xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7a| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|98304| + |karpenter.k8s.aws/instance-network-bandwidth|18750| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3233,17 +3379,20 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|88002Mi| |pods|234| + |vpc.amazonaws.com/pod-eni|114| ### `c7a.16xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7a| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| + |karpenter.k8s.aws/instance-network-bandwidth|25000| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3255,17 +3404,20 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|112779Mi| |pods|737| + |vpc.amazonaws.com/pod-eni|107| ### `c7a.24xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7a| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|196608| + |karpenter.k8s.aws/instance-network-bandwidth|37500| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3277,17 +3429,20 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|173400Mi| |pods|737| + |vpc.amazonaws.com/pod-eni|107| ### `c7a.32xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7a| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| + |karpenter.k8s.aws/instance-network-bandwidth|50000| |karpenter.k8s.aws/instance-size|32xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3299,17 +3454,20 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|234021Mi| |pods|737| + |vpc.amazonaws.com/pod-eni|107| ### `c7a.48xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|192| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7a| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|393216| + |karpenter.k8s.aws/instance-network-bandwidth|50000| |karpenter.k8s.aws/instance-size|48xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3322,17 +3480,20 @@ below are the resources available with some assumptions and after the instance o |memory|355262Mi| |pods|737| |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| ### `c7a.metal-48xl` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|192| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7a| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|393216| + |karpenter.k8s.aws/instance-network-bandwidth|50000| |karpenter.k8s.aws/instance-size|metal-48xl| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3345,6 +3506,7 @@ below are the resources available with some assumptions and after the instance o |memory|355262Mi| |pods|737| |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| ## c7g Family ### `c7g.medium` #### Labels @@ -3352,6 +3514,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|1| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7g| |karpenter.k8s.aws/instance-generation|7| @@ -3376,6 +3539,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7g| |karpenter.k8s.aws/instance-generation|7| @@ -3400,6 +3564,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7g| |karpenter.k8s.aws/instance-generation|7| @@ -3424,6 +3589,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7g| |karpenter.k8s.aws/instance-generation|7| @@ -3448,6 +3614,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7g| |karpenter.k8s.aws/instance-generation|7| @@ -3472,6 +3639,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7g| |karpenter.k8s.aws/instance-generation|7| @@ -3496,6 +3664,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7g| |karpenter.k8s.aws/instance-generation|7| @@ -3520,6 +3689,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7g| |karpenter.k8s.aws/instance-generation|7| @@ -3545,6 +3715,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7g| |karpenter.k8s.aws/instance-generation|7| @@ -3571,6 +3742,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|1| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gd| |karpenter.k8s.aws/instance-generation|7| @@ -3596,6 +3768,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gd| |karpenter.k8s.aws/instance-generation|7| @@ -3621,6 +3794,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gd| |karpenter.k8s.aws/instance-generation|7| @@ -3646,6 +3820,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gd| |karpenter.k8s.aws/instance-generation|7| @@ -3671,6 +3846,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gd| |karpenter.k8s.aws/instance-generation|7| @@ -3696,6 +3872,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gd| |karpenter.k8s.aws/instance-generation|7| @@ -3721,6 +3898,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gd| |karpenter.k8s.aws/instance-generation|7| @@ -3746,6 +3924,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gd| |karpenter.k8s.aws/instance-generation|7| @@ -3766,6 +3945,32 @@ below are the resources available with some assumptions and after the instance o |pods|737| |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| +### `c7gd.metal` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7gd| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|| + |karpenter.k8s.aws/instance-local-nvme|3800| + |karpenter.k8s.aws/instance-memory|131072| + |karpenter.k8s.aws/instance-network-bandwidth|30000| + |karpenter.k8s.aws/instance-size|metal| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7gd.metal| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|63770m| + |ephemeral-storage|17Gi| + |memory|112720Mi| + |pods|737| + |vpc.amazonaws.com/efa|1| ## c7gn Family ### `c7gn.medium` #### Labels @@ -3773,6 +3978,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|1| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gn| |karpenter.k8s.aws/instance-generation|7| @@ -3797,6 +4003,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gn| |karpenter.k8s.aws/instance-generation|7| @@ -3821,6 +4028,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gn| |karpenter.k8s.aws/instance-generation|7| @@ -3845,6 +4053,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gn| |karpenter.k8s.aws/instance-generation|7| @@ -3869,6 +4078,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gn| |karpenter.k8s.aws/instance-generation|7| @@ -3893,6 +4103,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gn| |karpenter.k8s.aws/instance-generation|7| @@ -3917,6 +4128,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gn| |karpenter.k8s.aws/instance-generation|7| @@ -3941,6 +4153,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7gn| |karpenter.k8s.aws/instance-generation|7| @@ -3967,6 +4180,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7i| |karpenter.k8s.aws/instance-generation|7| @@ -3984,12 +4198,14 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|3114Mi| |pods|29| + |vpc.amazonaws.com/pod-eni|9| ### `c7i.xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7i| |karpenter.k8s.aws/instance-generation|7| @@ -4007,12 +4223,14 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|6584Mi| |pods|58| + |vpc.amazonaws.com/pod-eni|18| ### `c7i.2xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7i| |karpenter.k8s.aws/instance-generation|7| @@ -4030,12 +4248,14 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|14162Mi| |pods|58| + |vpc.amazonaws.com/pod-eni|38| ### `c7i.4xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7i| |karpenter.k8s.aws/instance-generation|7| @@ -4053,12 +4273,14 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|27381Mi| |pods|234| + |vpc.amazonaws.com/pod-eni|54| ### `c7i.8xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7i| |karpenter.k8s.aws/instance-generation|7| @@ -4076,12 +4298,14 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|57691Mi| |pods|234| + |vpc.amazonaws.com/pod-eni|84| ### `c7i.12xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7i| |karpenter.k8s.aws/instance-generation|7| @@ -4099,12 +4323,14 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|88002Mi| |pods|234| + |vpc.amazonaws.com/pod-eni|114| ### `c7i.16xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7i| |karpenter.k8s.aws/instance-generation|7| @@ -4122,12 +4348,14 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|112779Mi| |pods|737| + |vpc.amazonaws.com/pod-eni|107| ### `c7i.24xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7i| |karpenter.k8s.aws/instance-generation|7| @@ -4145,17 +4373,20 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|173400Mi| |pods|737| + |vpc.amazonaws.com/pod-eni|107| ### `c7i.metal-24xl` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7i| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|196608| + |karpenter.k8s.aws/instance-network-bandwidth|37500| |karpenter.k8s.aws/instance-size|metal-24xl| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4167,12 +4398,14 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|173400Mi| |pods|737| + |vpc.amazonaws.com/pod-eni|107| ### `c7i.48xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|192| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7i| |karpenter.k8s.aws/instance-generation|7| @@ -4191,17 +4424,20 @@ below are the resources available with some assumptions and after the instance o |memory|355262Mi| |pods|737| |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| ### `c7i.metal-48xl` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|c| |karpenter.k8s.aws/instance-cpu|192| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|c7i| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|393216| + |karpenter.k8s.aws/instance-network-bandwidth|50000| |karpenter.k8s.aws/instance-size|metal-48xl| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4214,6 +4450,7 @@ below are the resources available with some assumptions and after the instance o |memory|355262Mi| |pods|737| |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| ## d2 Family ### `d2.xlarge` #### Labels @@ -4221,6 +4458,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|d| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|d2| |karpenter.k8s.aws/instance-generation|2| @@ -4243,6 +4481,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|d| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|d2| |karpenter.k8s.aws/instance-generation|2| @@ -4265,6 +4504,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|d| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|d2| |karpenter.k8s.aws/instance-generation|2| @@ -4287,6 +4527,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|d| |karpenter.k8s.aws/instance-cpu|36| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|d2| |karpenter.k8s.aws/instance-generation|2| @@ -4311,6 +4552,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|d| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|d3| |karpenter.k8s.aws/instance-generation|3| @@ -4336,6 +4578,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|d| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|d3| |karpenter.k8s.aws/instance-generation|3| @@ -4361,6 +4604,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|d| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|d3| |karpenter.k8s.aws/instance-generation|3| @@ -4386,6 +4630,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|d| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|d3| |karpenter.k8s.aws/instance-generation|3| @@ -4412,6 +4657,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|d| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|d3en| |karpenter.k8s.aws/instance-generation|3| @@ -4437,6 +4683,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|d| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|d3en| |karpenter.k8s.aws/instance-generation|3| @@ -4462,6 +4709,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|d| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|d3en| |karpenter.k8s.aws/instance-generation|3| @@ -4487,6 +4735,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|d| |karpenter.k8s.aws/instance-cpu|24| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|d3en| |karpenter.k8s.aws/instance-generation|3| @@ -4512,6 +4761,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|d| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|d3en| |karpenter.k8s.aws/instance-generation|3| @@ -4537,6 +4787,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|d| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|d3en| |karpenter.k8s.aws/instance-generation|3| @@ -4563,6 +4814,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|dl| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|dl1| |karpenter.k8s.aws/instance-generation|1| @@ -4595,6 +4847,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|f| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|f1| |karpenter.k8s.aws/instance-generation|1| @@ -4619,6 +4872,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|f| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|f1| |karpenter.k8s.aws/instance-generation|1| @@ -4643,6 +4897,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|f| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|f1| |karpenter.k8s.aws/instance-generation|1| @@ -4668,6 +4923,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|g3| |karpenter.k8s.aws/instance-generation|3| @@ -4696,6 +4952,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|g3| |karpenter.k8s.aws/instance-generation|3| @@ -4724,6 +4981,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|g3| |karpenter.k8s.aws/instance-generation|3| @@ -4753,6 +5011,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|g3s| |karpenter.k8s.aws/instance-generation|3| @@ -4781,6 +5040,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g4ad| |karpenter.k8s.aws/instance-generation|4| @@ -4811,6 +5071,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g4ad| |karpenter.k8s.aws/instance-generation|4| @@ -4841,6 +5102,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g4ad| |karpenter.k8s.aws/instance-generation|4| @@ -4871,6 +5133,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g4ad| |karpenter.k8s.aws/instance-generation|4| @@ -4901,6 +5164,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g4ad| |karpenter.k8s.aws/instance-generation|4| @@ -4932,6 +5196,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g4dn| |karpenter.k8s.aws/instance-generation|4| @@ -4962,6 +5227,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g4dn| |karpenter.k8s.aws/instance-generation|4| @@ -4992,6 +5258,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g4dn| |karpenter.k8s.aws/instance-generation|4| @@ -5022,6 +5289,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g4dn| |karpenter.k8s.aws/instance-generation|4| @@ -5053,6 +5321,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g4dn| |karpenter.k8s.aws/instance-generation|4| @@ -5084,6 +5353,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g4dn| |karpenter.k8s.aws/instance-generation|4| @@ -5115,6 +5385,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g4dn| |karpenter.k8s.aws/instance-generation|4| @@ -5147,6 +5418,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g5| |karpenter.k8s.aws/instance-generation|5| @@ -5177,6 +5449,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g5| |karpenter.k8s.aws/instance-generation|5| @@ -5207,6 +5480,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g5| |karpenter.k8s.aws/instance-generation|5| @@ -5237,6 +5511,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g5| |karpenter.k8s.aws/instance-generation|5| @@ -5268,6 +5543,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g5| |karpenter.k8s.aws/instance-generation|5| @@ -5299,6 +5575,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g5| |karpenter.k8s.aws/instance-generation|5| @@ -5330,6 +5607,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g5| |karpenter.k8s.aws/instance-generation|5| @@ -5361,6 +5639,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|192| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|g5| |karpenter.k8s.aws/instance-generation|5| @@ -5393,6 +5672,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|g5g| |karpenter.k8s.aws/instance-generation|5| @@ -5422,6 +5702,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|g5g| |karpenter.k8s.aws/instance-generation|5| @@ -5451,6 +5732,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|g5g| |karpenter.k8s.aws/instance-generation|5| @@ -5480,6 +5762,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|g5g| |karpenter.k8s.aws/instance-generation|5| @@ -5509,6 +5792,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|g5g| |karpenter.k8s.aws/instance-generation|5| @@ -5538,6 +5822,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|g| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|g5g| |karpenter.k8s.aws/instance-generation|5| @@ -5568,6 +5853,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|h| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|h1| |karpenter.k8s.aws/instance-generation|1| @@ -5591,6 +5877,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|h| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|h1| |karpenter.k8s.aws/instance-generation|1| @@ -5614,6 +5901,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|h| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|h1| |karpenter.k8s.aws/instance-generation|1| @@ -5637,6 +5925,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|h| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|h1| |karpenter.k8s.aws/instance-generation|1| @@ -5652,8 +5941,8 @@ below are the resources available with some assumptions and after the instance o |--|--| |cpu|63770m| |ephemeral-storage|17Gi| - |memory|234021Mi| - |pods|737| + |memory|237794Mi| + |pods|394| ## hpc7g Family ### `hpc7g.4xlarge` #### Labels @@ -5661,6 +5950,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|hpc| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|hpc7g| |karpenter.k8s.aws/instance-generation|7| @@ -5685,6 +5975,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|hpc| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|hpc7g| |karpenter.k8s.aws/instance-generation|7| @@ -5709,6 +6000,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|hpc| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|hpc7g| |karpenter.k8s.aws/instance-generation|7| @@ -5734,6 +6026,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|i2| |karpenter.k8s.aws/instance-generation|2| @@ -5756,6 +6049,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|i2| |karpenter.k8s.aws/instance-generation|2| @@ -5778,6 +6072,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|i2| |karpenter.k8s.aws/instance-generation|2| @@ -5800,6 +6095,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|i2| |karpenter.k8s.aws/instance-generation|2| @@ -5824,6 +6120,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|i3| |karpenter.k8s.aws/instance-generation|3| @@ -5848,6 +6145,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|i3| |karpenter.k8s.aws/instance-generation|3| @@ -5872,6 +6170,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|i3| |karpenter.k8s.aws/instance-generation|3| @@ -5896,6 +6195,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|i3| |karpenter.k8s.aws/instance-generation|3| @@ -5920,6 +6220,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|i3| |karpenter.k8s.aws/instance-generation|3| @@ -5944,6 +6245,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|i3| |karpenter.k8s.aws/instance-generation|3| @@ -5968,6 +6270,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|72| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|i3| |karpenter.k8s.aws/instance-generation|3| @@ -5994,6 +6297,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i3en| |karpenter.k8s.aws/instance-generation|3| @@ -6019,6 +6323,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i3en| |karpenter.k8s.aws/instance-generation|3| @@ -6044,6 +6349,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i3en| |karpenter.k8s.aws/instance-generation|3| @@ -6069,6 +6375,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|12| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i3en| |karpenter.k8s.aws/instance-generation|3| @@ -6094,6 +6401,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|24| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i3en| |karpenter.k8s.aws/instance-generation|3| @@ -6119,6 +6427,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i3en| |karpenter.k8s.aws/instance-generation|3| @@ -6145,6 +6454,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i3en| |karpenter.k8s.aws/instance-generation|3| @@ -6171,6 +6481,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i3en| |karpenter.k8s.aws/instance-generation|3| @@ -6198,6 +6509,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i4g| |karpenter.k8s.aws/instance-generation|4| @@ -6223,6 +6535,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i4g| |karpenter.k8s.aws/instance-generation|4| @@ -6248,6 +6561,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i4g| |karpenter.k8s.aws/instance-generation|4| @@ -6273,6 +6587,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i4g| |karpenter.k8s.aws/instance-generation|4| @@ -6298,6 +6613,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i4g| |karpenter.k8s.aws/instance-generation|4| @@ -6323,6 +6639,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i4g| |karpenter.k8s.aws/instance-generation|4| @@ -6350,6 +6667,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i4i| |karpenter.k8s.aws/instance-generation|4| @@ -6374,6 +6692,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i4i| |karpenter.k8s.aws/instance-generation|4| @@ -6399,6 +6718,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i4i| |karpenter.k8s.aws/instance-generation|4| @@ -6424,6 +6744,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i4i| |karpenter.k8s.aws/instance-generation|4| @@ -6449,6 +6770,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i4i| |karpenter.k8s.aws/instance-generation|4| @@ -6474,12 +6796,14 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i4i| |karpenter.k8s.aws/instance-generation|4| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-local-nvme|11250| |karpenter.k8s.aws/instance-memory|393216| + |karpenter.k8s.aws/instance-network-bandwidth|28120| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6491,12 +6815,14 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|360795Mi| |pods|234| + |vpc.amazonaws.com/pod-eni|112| ### `i4i.16xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i4i| |karpenter.k8s.aws/instance-generation|4| @@ -6522,12 +6848,14 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i4i| |karpenter.k8s.aws/instance-generation|4| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-local-nvme|22500| |karpenter.k8s.aws/instance-memory|786432| + |karpenter.k8s.aws/instance-network-bandwidth|56250| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6539,12 +6867,14 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|722287Mi| |pods|437| + |vpc.amazonaws.com/pod-eni|105| ### `i4i.32xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i4i| |karpenter.k8s.aws/instance-generation|4| @@ -6571,6 +6901,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|i| |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|i4i| |karpenter.k8s.aws/instance-generation|4| @@ -6598,6 +6929,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|im| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|im4gn| |karpenter.k8s.aws/instance-generation|4| @@ -6623,6 +6955,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|im| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|im4gn| |karpenter.k8s.aws/instance-generation|4| @@ -6648,6 +6981,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|im| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|im4gn| |karpenter.k8s.aws/instance-generation|4| @@ -6673,6 +7007,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|im| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|im4gn| |karpenter.k8s.aws/instance-generation|4| @@ -6698,6 +7033,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|im| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|im4gn| |karpenter.k8s.aws/instance-generation|4| @@ -6723,6 +7059,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|im| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|im4gn| |karpenter.k8s.aws/instance-generation|4| @@ -6753,6 +7090,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-accelerator-name|inferentia| |karpenter.k8s.aws/instance-category|inf| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|inf1| |karpenter.k8s.aws/instance-generation|1| @@ -6781,6 +7119,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-accelerator-name|inferentia| |karpenter.k8s.aws/instance-category|inf| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|inf1| |karpenter.k8s.aws/instance-generation|1| @@ -6809,6 +7148,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-accelerator-name|inferentia| |karpenter.k8s.aws/instance-category|inf| |karpenter.k8s.aws/instance-cpu|24| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|inf1| |karpenter.k8s.aws/instance-generation|1| @@ -6837,6 +7177,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-accelerator-name|inferentia| |karpenter.k8s.aws/instance-category|inf| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|inf1| |karpenter.k8s.aws/instance-generation|1| @@ -6867,6 +7208,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-accelerator-name|inferentia| |karpenter.k8s.aws/instance-category|inf| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|inf2| |karpenter.k8s.aws/instance-generation|2| @@ -6895,6 +7237,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-accelerator-name|inferentia| |karpenter.k8s.aws/instance-category|inf| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|inf2| |karpenter.k8s.aws/instance-generation|2| @@ -6923,6 +7266,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-accelerator-name|inferentia| |karpenter.k8s.aws/instance-category|inf| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|inf2| |karpenter.k8s.aws/instance-generation|2| @@ -6951,6 +7295,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-accelerator-name|inferentia| |karpenter.k8s.aws/instance-category|inf| |karpenter.k8s.aws/instance-cpu|192| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|inf2| |karpenter.k8s.aws/instance-generation|2| @@ -6977,6 +7322,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|is| |karpenter.k8s.aws/instance-cpu|1| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|is4gen| |karpenter.k8s.aws/instance-generation|4| @@ -7002,6 +7348,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|is| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|is4gen| |karpenter.k8s.aws/instance-generation|4| @@ -7027,6 +7374,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|is| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|is4gen| |karpenter.k8s.aws/instance-generation|4| @@ -7052,6 +7400,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|is| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|is4gen| |karpenter.k8s.aws/instance-generation|4| @@ -7077,6 +7426,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|is| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|is4gen| |karpenter.k8s.aws/instance-generation|4| @@ -7102,6 +7452,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|is| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|is4gen| |karpenter.k8s.aws/instance-generation|4| @@ -7128,6 +7479,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|1| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m1| |karpenter.k8s.aws/instance-generation|1| @@ -7150,6 +7502,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|1| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m1| |karpenter.k8s.aws/instance-generation|1| @@ -7172,6 +7525,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m1| |karpenter.k8s.aws/instance-generation|1| @@ -7194,6 +7548,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m1| |karpenter.k8s.aws/instance-generation|1| @@ -7217,6 +7572,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m2| |karpenter.k8s.aws/instance-generation|2| @@ -7239,6 +7595,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m2| |karpenter.k8s.aws/instance-generation|2| @@ -7261,6 +7618,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m2| |karpenter.k8s.aws/instance-generation|2| @@ -7284,6 +7642,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|1| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m3| |karpenter.k8s.aws/instance-generation|3| @@ -7306,6 +7665,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m3| |karpenter.k8s.aws/instance-generation|3| @@ -7328,6 +7688,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m3| |karpenter.k8s.aws/instance-generation|3| @@ -7350,6 +7711,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m3| |karpenter.k8s.aws/instance-generation|3| @@ -7373,6 +7735,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m4| |karpenter.k8s.aws/instance-generation|4| @@ -7395,6 +7758,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m4| |karpenter.k8s.aws/instance-generation|4| @@ -7417,6 +7781,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m4| |karpenter.k8s.aws/instance-generation|4| @@ -7439,6 +7804,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m4| |karpenter.k8s.aws/instance-generation|4| @@ -7461,6 +7827,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|40| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m4| |karpenter.k8s.aws/instance-generation|4| @@ -7484,6 +7851,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m4| |karpenter.k8s.aws/instance-generation|4| @@ -7508,6 +7876,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5| |karpenter.k8s.aws/instance-generation|5| @@ -7532,6 +7901,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5| |karpenter.k8s.aws/instance-generation|5| @@ -7556,6 +7926,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5| |karpenter.k8s.aws/instance-generation|5| @@ -7580,6 +7951,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5| |karpenter.k8s.aws/instance-generation|5| @@ -7604,6 +7976,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5| |karpenter.k8s.aws/instance-generation|5| @@ -7628,6 +8001,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5| |karpenter.k8s.aws/instance-generation|5| @@ -7652,6 +8026,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5| |karpenter.k8s.aws/instance-generation|5| @@ -7676,6 +8051,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5| |karpenter.k8s.aws/instance-generation|5| @@ -7700,6 +8076,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5| |karpenter.k8s.aws/instance-generation|5| @@ -7725,6 +8102,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5a| |karpenter.k8s.aws/instance-generation|5| @@ -7749,6 +8127,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5a| |karpenter.k8s.aws/instance-generation|5| @@ -7773,6 +8152,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5a| |karpenter.k8s.aws/instance-generation|5| @@ -7797,6 +8177,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5a| |karpenter.k8s.aws/instance-generation|5| @@ -7821,6 +8202,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5a| |karpenter.k8s.aws/instance-generation|5| @@ -7845,6 +8227,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5a| |karpenter.k8s.aws/instance-generation|5| @@ -7869,6 +8252,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5a| |karpenter.k8s.aws/instance-generation|5| @@ -7893,6 +8277,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5a| |karpenter.k8s.aws/instance-generation|5| @@ -7918,6 +8303,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5ad| |karpenter.k8s.aws/instance-generation|5| @@ -7943,6 +8329,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5ad| |karpenter.k8s.aws/instance-generation|5| @@ -7968,6 +8355,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5ad| |karpenter.k8s.aws/instance-generation|5| @@ -7993,6 +8381,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5ad| |karpenter.k8s.aws/instance-generation|5| @@ -8018,6 +8407,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5ad| |karpenter.k8s.aws/instance-generation|5| @@ -8043,6 +8433,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5ad| |karpenter.k8s.aws/instance-generation|5| @@ -8068,6 +8459,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5ad| |karpenter.k8s.aws/instance-generation|5| @@ -8093,6 +8485,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5ad| |karpenter.k8s.aws/instance-generation|5| @@ -8119,6 +8512,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5d| |karpenter.k8s.aws/instance-generation|5| @@ -8144,6 +8538,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5d| |karpenter.k8s.aws/instance-generation|5| @@ -8169,6 +8564,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5d| |karpenter.k8s.aws/instance-generation|5| @@ -8194,6 +8590,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5d| |karpenter.k8s.aws/instance-generation|5| @@ -8219,6 +8616,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5d| |karpenter.k8s.aws/instance-generation|5| @@ -8244,6 +8642,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5d| |karpenter.k8s.aws/instance-generation|5| @@ -8269,6 +8668,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5d| |karpenter.k8s.aws/instance-generation|5| @@ -8294,6 +8694,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5d| |karpenter.k8s.aws/instance-generation|5| @@ -8319,6 +8720,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m5d| |karpenter.k8s.aws/instance-generation|5| @@ -8345,6 +8747,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5dn| |karpenter.k8s.aws/instance-generation|5| @@ -8370,6 +8773,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5dn| |karpenter.k8s.aws/instance-generation|5| @@ -8395,6 +8799,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5dn| |karpenter.k8s.aws/instance-generation|5| @@ -8420,6 +8825,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5dn| |karpenter.k8s.aws/instance-generation|5| @@ -8445,6 +8851,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5dn| |karpenter.k8s.aws/instance-generation|5| @@ -8470,6 +8877,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5dn| |karpenter.k8s.aws/instance-generation|5| @@ -8495,6 +8903,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5dn| |karpenter.k8s.aws/instance-generation|5| @@ -8520,6 +8929,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5dn| |karpenter.k8s.aws/instance-generation|5| @@ -8546,6 +8956,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5dn| |karpenter.k8s.aws/instance-generation|5| @@ -8573,6 +8984,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5n| |karpenter.k8s.aws/instance-generation|5| @@ -8597,6 +9009,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5n| |karpenter.k8s.aws/instance-generation|5| @@ -8621,6 +9034,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5n| |karpenter.k8s.aws/instance-generation|5| @@ -8645,6 +9059,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5n| |karpenter.k8s.aws/instance-generation|5| @@ -8669,6 +9084,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5n| |karpenter.k8s.aws/instance-generation|5| @@ -8693,6 +9109,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5n| |karpenter.k8s.aws/instance-generation|5| @@ -8717,6 +9134,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5n| |karpenter.k8s.aws/instance-generation|5| @@ -8741,6 +9159,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5n| |karpenter.k8s.aws/instance-generation|5| @@ -8766,6 +9185,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5n| |karpenter.k8s.aws/instance-generation|5| @@ -8792,6 +9212,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5zn| |karpenter.k8s.aws/instance-generation|5| @@ -8816,6 +9237,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5zn| |karpenter.k8s.aws/instance-generation|5| @@ -8840,6 +9262,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5zn| |karpenter.k8s.aws/instance-generation|5| @@ -8864,6 +9287,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|12| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5zn| |karpenter.k8s.aws/instance-generation|5| @@ -8888,6 +9312,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|24| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5zn| |karpenter.k8s.aws/instance-generation|5| @@ -8912,6 +9337,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5zn| |karpenter.k8s.aws/instance-generation|5| @@ -8937,6 +9363,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m5zn| |karpenter.k8s.aws/instance-generation|5| @@ -8963,6 +9390,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6a| |karpenter.k8s.aws/instance-generation|6| @@ -8987,6 +9415,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6a| |karpenter.k8s.aws/instance-generation|6| @@ -9011,6 +9440,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6a| |karpenter.k8s.aws/instance-generation|6| @@ -9035,6 +9465,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6a| |karpenter.k8s.aws/instance-generation|6| @@ -9059,6 +9490,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6a| |karpenter.k8s.aws/instance-generation|6| @@ -9083,6 +9515,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6a| |karpenter.k8s.aws/instance-generation|6| @@ -9107,6 +9540,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6a| |karpenter.k8s.aws/instance-generation|6| @@ -9131,6 +9565,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6a| |karpenter.k8s.aws/instance-generation|6| @@ -9155,6 +9590,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6a| |karpenter.k8s.aws/instance-generation|6| @@ -9179,6 +9615,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|192| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6a| |karpenter.k8s.aws/instance-generation|6| @@ -9204,6 +9641,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|192| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6a| |karpenter.k8s.aws/instance-generation|6| @@ -9230,6 +9668,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|1| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6g| |karpenter.k8s.aws/instance-generation|6| @@ -9254,6 +9693,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6g| |karpenter.k8s.aws/instance-generation|6| @@ -9278,6 +9718,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6g| |karpenter.k8s.aws/instance-generation|6| @@ -9302,6 +9743,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6g| |karpenter.k8s.aws/instance-generation|6| @@ -9326,6 +9768,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6g| |karpenter.k8s.aws/instance-generation|6| @@ -9350,6 +9793,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6g| |karpenter.k8s.aws/instance-generation|6| @@ -9374,6 +9818,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6g| |karpenter.k8s.aws/instance-generation|6| @@ -9398,6 +9843,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6g| |karpenter.k8s.aws/instance-generation|6| @@ -9422,6 +9868,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6g| |karpenter.k8s.aws/instance-generation|6| @@ -9447,6 +9894,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|1| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6gd| |karpenter.k8s.aws/instance-generation|6| @@ -9472,6 +9920,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6gd| |karpenter.k8s.aws/instance-generation|6| @@ -9497,6 +9946,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6gd| |karpenter.k8s.aws/instance-generation|6| @@ -9522,6 +9972,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6gd| |karpenter.k8s.aws/instance-generation|6| @@ -9547,6 +9998,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6gd| |karpenter.k8s.aws/instance-generation|6| @@ -9572,6 +10024,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6gd| |karpenter.k8s.aws/instance-generation|6| @@ -9597,6 +10050,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6gd| |karpenter.k8s.aws/instance-generation|6| @@ -9622,6 +10076,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6gd| |karpenter.k8s.aws/instance-generation|6| @@ -9647,6 +10102,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|m6gd| |karpenter.k8s.aws/instance-generation|6| @@ -9673,6 +10129,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6i| |karpenter.k8s.aws/instance-generation|6| @@ -9697,6 +10154,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6i| |karpenter.k8s.aws/instance-generation|6| @@ -9721,6 +10179,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6i| |karpenter.k8s.aws/instance-generation|6| @@ -9745,6 +10204,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6i| |karpenter.k8s.aws/instance-generation|6| @@ -9769,6 +10229,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6i| |karpenter.k8s.aws/instance-generation|6| @@ -9793,6 +10254,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6i| |karpenter.k8s.aws/instance-generation|6| @@ -9817,6 +10279,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6i| |karpenter.k8s.aws/instance-generation|6| @@ -9841,6 +10304,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6i| |karpenter.k8s.aws/instance-generation|6| @@ -9865,6 +10329,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6i| |karpenter.k8s.aws/instance-generation|6| @@ -9890,6 +10355,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6i| |karpenter.k8s.aws/instance-generation|6| @@ -9916,6 +10382,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6id| |karpenter.k8s.aws/instance-generation|6| @@ -9941,6 +10408,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6id| |karpenter.k8s.aws/instance-generation|6| @@ -9966,6 +10434,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6id| |karpenter.k8s.aws/instance-generation|6| @@ -9991,6 +10460,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6id| |karpenter.k8s.aws/instance-generation|6| @@ -10016,6 +10486,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6id| |karpenter.k8s.aws/instance-generation|6| @@ -10041,6 +10512,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6id| |karpenter.k8s.aws/instance-generation|6| @@ -10066,6 +10538,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6id| |karpenter.k8s.aws/instance-generation|6| @@ -10091,6 +10564,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6id| |karpenter.k8s.aws/instance-generation|6| @@ -10116,6 +10590,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6id| |karpenter.k8s.aws/instance-generation|6| @@ -10142,6 +10617,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6id| |karpenter.k8s.aws/instance-generation|6| @@ -10169,6 +10645,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6idn| |karpenter.k8s.aws/instance-generation|6| @@ -10194,6 +10671,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6idn| |karpenter.k8s.aws/instance-generation|6| @@ -10219,6 +10697,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6idn| |karpenter.k8s.aws/instance-generation|6| @@ -10244,6 +10723,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6idn| |karpenter.k8s.aws/instance-generation|6| @@ -10269,6 +10749,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6idn| |karpenter.k8s.aws/instance-generation|6| @@ -10294,6 +10775,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6idn| |karpenter.k8s.aws/instance-generation|6| @@ -10319,6 +10801,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6idn| |karpenter.k8s.aws/instance-generation|6| @@ -10344,6 +10827,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6idn| |karpenter.k8s.aws/instance-generation|6| @@ -10369,6 +10853,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6idn| |karpenter.k8s.aws/instance-generation|6| @@ -10395,6 +10880,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6idn| |karpenter.k8s.aws/instance-generation|6| @@ -10422,6 +10908,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6in| |karpenter.k8s.aws/instance-generation|6| @@ -10446,6 +10933,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6in| |karpenter.k8s.aws/instance-generation|6| @@ -10470,6 +10958,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6in| |karpenter.k8s.aws/instance-generation|6| @@ -10494,6 +10983,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6in| |karpenter.k8s.aws/instance-generation|6| @@ -10518,6 +11008,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6in| |karpenter.k8s.aws/instance-generation|6| @@ -10542,6 +11033,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6in| |karpenter.k8s.aws/instance-generation|6| @@ -10566,6 +11058,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6in| |karpenter.k8s.aws/instance-generation|6| @@ -10590,6 +11083,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6in| |karpenter.k8s.aws/instance-generation|6| @@ -10614,6 +11108,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6in| |karpenter.k8s.aws/instance-generation|6| @@ -10639,6 +11134,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m6in| |karpenter.k8s.aws/instance-generation|6| @@ -10665,6 +11161,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|1| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7a| |karpenter.k8s.aws/instance-generation|7| @@ -10689,6 +11186,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7a| |karpenter.k8s.aws/instance-generation|7| @@ -10713,6 +11211,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7a| |karpenter.k8s.aws/instance-generation|7| @@ -10737,6 +11236,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7a| |karpenter.k8s.aws/instance-generation|7| @@ -10761,6 +11261,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7a| |karpenter.k8s.aws/instance-generation|7| @@ -10785,6 +11286,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7a| |karpenter.k8s.aws/instance-generation|7| @@ -10809,6 +11311,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7a| |karpenter.k8s.aws/instance-generation|7| @@ -10833,6 +11336,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7a| |karpenter.k8s.aws/instance-generation|7| @@ -10857,6 +11361,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7a| |karpenter.k8s.aws/instance-generation|7| @@ -10881,6 +11386,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7a| |karpenter.k8s.aws/instance-generation|7| @@ -10905,6 +11411,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|192| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7a| |karpenter.k8s.aws/instance-generation|7| @@ -10930,6 +11437,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|192| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7a| |karpenter.k8s.aws/instance-generation|7| @@ -10956,6 +11464,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|1| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7g| |karpenter.k8s.aws/instance-generation|7| @@ -10980,6 +11489,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7g| |karpenter.k8s.aws/instance-generation|7| @@ -11004,6 +11514,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7g| |karpenter.k8s.aws/instance-generation|7| @@ -11028,6 +11539,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7g| |karpenter.k8s.aws/instance-generation|7| @@ -11052,6 +11564,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7g| |karpenter.k8s.aws/instance-generation|7| @@ -11076,6 +11589,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7g| |karpenter.k8s.aws/instance-generation|7| @@ -11100,6 +11614,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7g| |karpenter.k8s.aws/instance-generation|7| @@ -11124,6 +11639,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7g| |karpenter.k8s.aws/instance-generation|7| @@ -11149,6 +11665,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7g| |karpenter.k8s.aws/instance-generation|7| @@ -11175,6 +11692,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|1| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7gd| |karpenter.k8s.aws/instance-generation|7| @@ -11200,6 +11718,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7gd| |karpenter.k8s.aws/instance-generation|7| @@ -11225,6 +11744,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7gd| |karpenter.k8s.aws/instance-generation|7| @@ -11250,6 +11770,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7gd| |karpenter.k8s.aws/instance-generation|7| @@ -11275,6 +11796,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7gd| |karpenter.k8s.aws/instance-generation|7| @@ -11300,6 +11822,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7gd| |karpenter.k8s.aws/instance-generation|7| @@ -11325,6 +11848,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7gd| |karpenter.k8s.aws/instance-generation|7| @@ -11350,6 +11874,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7gd| |karpenter.k8s.aws/instance-generation|7| @@ -11370,6 +11895,32 @@ below are the resources available with some assumptions and after the instance o |pods|737| |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| +### `m7gd.metal` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|m| + |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|m7gd| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|| + |karpenter.k8s.aws/instance-local-nvme|3800| + |karpenter.k8s.aws/instance-memory|262144| + |karpenter.k8s.aws/instance-network-bandwidth|30000| + |karpenter.k8s.aws/instance-size|metal| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|m7gd.metal| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|63770m| + |ephemeral-storage|17Gi| + |memory|233962Mi| + |pods|737| + |vpc.amazonaws.com/efa|1| ## m7i Family ### `m7i.large` #### Labels @@ -11377,6 +11928,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7i| |karpenter.k8s.aws/instance-generation|7| @@ -11401,6 +11953,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7i| |karpenter.k8s.aws/instance-generation|7| @@ -11425,6 +11978,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7i| |karpenter.k8s.aws/instance-generation|7| @@ -11449,6 +12003,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7i| |karpenter.k8s.aws/instance-generation|7| @@ -11473,6 +12028,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7i| |karpenter.k8s.aws/instance-generation|7| @@ -11497,6 +12053,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7i| |karpenter.k8s.aws/instance-generation|7| @@ -11521,6 +12078,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7i| |karpenter.k8s.aws/instance-generation|7| @@ -11545,6 +12103,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7i| |karpenter.k8s.aws/instance-generation|7| @@ -11569,11 +12128,13 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7i| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|393216| + |karpenter.k8s.aws/instance-network-bandwidth|37500| |karpenter.k8s.aws/instance-size|metal-24xl| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11585,12 +12146,14 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|355262Mi| |pods|737| + |vpc.amazonaws.com/pod-eni|107| ### `m7i.48xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|192| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7i| |karpenter.k8s.aws/instance-generation|7| @@ -11616,11 +12179,13 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|192| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7i| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|786432| + |karpenter.k8s.aws/instance-network-bandwidth|50000| |karpenter.k8s.aws/instance-size|metal-48xl| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11633,6 +12198,7 @@ below are the resources available with some assumptions and after the instance o |memory|718987Mi| |pods|737| |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| ## m7i-flex Family ### `m7i-flex.large` #### Labels @@ -11640,6 +12206,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7i-flex| |karpenter.k8s.aws/instance-generation|7| @@ -11664,6 +12231,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7i-flex| |karpenter.k8s.aws/instance-generation|7| @@ -11688,6 +12256,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7i-flex| |karpenter.k8s.aws/instance-generation|7| @@ -11712,6 +12281,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7i-flex| |karpenter.k8s.aws/instance-generation|7| @@ -11736,6 +12306,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|m| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|m7i-flex| |karpenter.k8s.aws/instance-generation|7| @@ -11761,6 +12332,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|p| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|p2| |karpenter.k8s.aws/instance-generation|2| @@ -11788,6 +12360,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|p| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|p2| |karpenter.k8s.aws/instance-generation|2| @@ -11816,6 +12389,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|p| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|p2| |karpenter.k8s.aws/instance-generation|2| @@ -11845,6 +12419,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|p| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|p3| |karpenter.k8s.aws/instance-generation|3| @@ -11866,13 +12441,13 @@ below are the resources available with some assumptions and after the instance o |memory|56786Mi| |nvidia.com/gpu|1| |pods|58| - |vpc.amazonaws.com/pod-eni|38| ### `p3.8xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|p| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|p3| |karpenter.k8s.aws/instance-generation|3| @@ -11895,13 +12470,13 @@ below are the resources available with some assumptions and after the instance o |memory|228187Mi| |nvidia.com/gpu|4| |pods|234| - |vpc.amazonaws.com/pod-eni|54| ### `p3.16xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|p| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|p3| |karpenter.k8s.aws/instance-generation|3| @@ -11924,7 +12499,6 @@ below are the resources available with some assumptions and after the instance o |memory|459304Mi| |nvidia.com/gpu|8| |pods|234| - |vpc.amazonaws.com/pod-eni|114| ## p3dn Family ### `p3dn.24xlarge` #### Labels @@ -11932,6 +12506,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|p| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|p3dn| |karpenter.k8s.aws/instance-generation|3| @@ -11964,6 +12539,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|p| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|p4d| |karpenter.k8s.aws/instance-generation|4| @@ -11996,6 +12572,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|p| |karpenter.k8s.aws/instance-cpu|192| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|p5| |karpenter.k8s.aws/instance-generation|5| @@ -12028,6 +12605,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r3| |karpenter.k8s.aws/instance-generation|3| @@ -12050,6 +12628,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r3| |karpenter.k8s.aws/instance-generation|3| @@ -12072,6 +12651,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r3| |karpenter.k8s.aws/instance-generation|3| @@ -12094,6 +12674,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r3| |karpenter.k8s.aws/instance-generation|3| @@ -12116,6 +12697,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r3| |karpenter.k8s.aws/instance-generation|3| @@ -12140,6 +12722,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r4| |karpenter.k8s.aws/instance-generation|4| @@ -12163,6 +12746,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r4| |karpenter.k8s.aws/instance-generation|4| @@ -12186,6 +12770,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r4| |karpenter.k8s.aws/instance-generation|4| @@ -12209,6 +12794,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r4| |karpenter.k8s.aws/instance-generation|4| @@ -12232,6 +12818,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r4| |karpenter.k8s.aws/instance-generation|4| @@ -12255,6 +12842,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r4| |karpenter.k8s.aws/instance-generation|4| @@ -12279,6 +12867,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5| |karpenter.k8s.aws/instance-generation|5| @@ -12303,6 +12892,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5| |karpenter.k8s.aws/instance-generation|5| @@ -12327,6 +12917,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5| |karpenter.k8s.aws/instance-generation|5| @@ -12351,6 +12942,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5| |karpenter.k8s.aws/instance-generation|5| @@ -12375,6 +12967,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5| |karpenter.k8s.aws/instance-generation|5| @@ -12399,6 +12992,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5| |karpenter.k8s.aws/instance-generation|5| @@ -12423,6 +13017,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5| |karpenter.k8s.aws/instance-generation|5| @@ -12447,6 +13042,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5| |karpenter.k8s.aws/instance-generation|5| @@ -12471,6 +13067,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5| |karpenter.k8s.aws/instance-generation|5| @@ -12496,6 +13093,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5a| |karpenter.k8s.aws/instance-generation|5| @@ -12520,6 +13118,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5a| |karpenter.k8s.aws/instance-generation|5| @@ -12544,6 +13143,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5a| |karpenter.k8s.aws/instance-generation|5| @@ -12568,6 +13168,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5a| |karpenter.k8s.aws/instance-generation|5| @@ -12592,6 +13193,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5a| |karpenter.k8s.aws/instance-generation|5| @@ -12616,6 +13218,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5a| |karpenter.k8s.aws/instance-generation|5| @@ -12640,6 +13243,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5a| |karpenter.k8s.aws/instance-generation|5| @@ -12664,6 +13268,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5a| |karpenter.k8s.aws/instance-generation|5| @@ -12689,6 +13294,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5ad| |karpenter.k8s.aws/instance-generation|5| @@ -12714,6 +13320,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5ad| |karpenter.k8s.aws/instance-generation|5| @@ -12739,6 +13346,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5ad| |karpenter.k8s.aws/instance-generation|5| @@ -12764,6 +13372,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5ad| |karpenter.k8s.aws/instance-generation|5| @@ -12789,6 +13398,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5ad| |karpenter.k8s.aws/instance-generation|5| @@ -12814,6 +13424,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5ad| |karpenter.k8s.aws/instance-generation|5| @@ -12839,6 +13450,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5ad| |karpenter.k8s.aws/instance-generation|5| @@ -12864,6 +13476,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5ad| |karpenter.k8s.aws/instance-generation|5| @@ -12890,6 +13503,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5b| |karpenter.k8s.aws/instance-generation|5| @@ -12914,6 +13528,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5b| |karpenter.k8s.aws/instance-generation|5| @@ -12938,6 +13553,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5b| |karpenter.k8s.aws/instance-generation|5| @@ -12962,6 +13578,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5b| |karpenter.k8s.aws/instance-generation|5| @@ -12986,6 +13603,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5b| |karpenter.k8s.aws/instance-generation|5| @@ -13010,6 +13628,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5b| |karpenter.k8s.aws/instance-generation|5| @@ -13034,6 +13653,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5b| |karpenter.k8s.aws/instance-generation|5| @@ -13058,6 +13678,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5b| |karpenter.k8s.aws/instance-generation|5| @@ -13082,6 +13703,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5b| |karpenter.k8s.aws/instance-generation|5| @@ -13107,6 +13729,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5d| |karpenter.k8s.aws/instance-generation|5| @@ -13132,6 +13755,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5d| |karpenter.k8s.aws/instance-generation|5| @@ -13157,6 +13781,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5d| |karpenter.k8s.aws/instance-generation|5| @@ -13182,6 +13807,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5d| |karpenter.k8s.aws/instance-generation|5| @@ -13207,6 +13833,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5d| |karpenter.k8s.aws/instance-generation|5| @@ -13232,6 +13859,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5d| |karpenter.k8s.aws/instance-generation|5| @@ -13257,6 +13885,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5d| |karpenter.k8s.aws/instance-generation|5| @@ -13282,6 +13911,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5d| |karpenter.k8s.aws/instance-generation|5| @@ -13307,6 +13937,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r5d| |karpenter.k8s.aws/instance-generation|5| @@ -13333,6 +13964,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5dn| |karpenter.k8s.aws/instance-generation|5| @@ -13358,6 +13990,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5dn| |karpenter.k8s.aws/instance-generation|5| @@ -13383,6 +14016,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5dn| |karpenter.k8s.aws/instance-generation|5| @@ -13408,6 +14042,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5dn| |karpenter.k8s.aws/instance-generation|5| @@ -13433,6 +14068,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5dn| |karpenter.k8s.aws/instance-generation|5| @@ -13458,6 +14094,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5dn| |karpenter.k8s.aws/instance-generation|5| @@ -13483,6 +14120,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5dn| |karpenter.k8s.aws/instance-generation|5| @@ -13508,6 +14146,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5dn| |karpenter.k8s.aws/instance-generation|5| @@ -13534,6 +14173,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5dn| |karpenter.k8s.aws/instance-generation|5| @@ -13561,6 +14201,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5n| |karpenter.k8s.aws/instance-generation|5| @@ -13585,6 +14226,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5n| |karpenter.k8s.aws/instance-generation|5| @@ -13609,6 +14251,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5n| |karpenter.k8s.aws/instance-generation|5| @@ -13633,6 +14276,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5n| |karpenter.k8s.aws/instance-generation|5| @@ -13657,6 +14301,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5n| |karpenter.k8s.aws/instance-generation|5| @@ -13681,6 +14326,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5n| |karpenter.k8s.aws/instance-generation|5| @@ -13705,6 +14351,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5n| |karpenter.k8s.aws/instance-generation|5| @@ -13729,6 +14376,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5n| |karpenter.k8s.aws/instance-generation|5| @@ -13754,6 +14402,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r5n| |karpenter.k8s.aws/instance-generation|5| @@ -13780,6 +14429,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6a| |karpenter.k8s.aws/instance-generation|6| @@ -13804,6 +14454,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6a| |karpenter.k8s.aws/instance-generation|6| @@ -13828,6 +14479,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6a| |karpenter.k8s.aws/instance-generation|6| @@ -13852,6 +14504,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6a| |karpenter.k8s.aws/instance-generation|6| @@ -13876,6 +14529,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6a| |karpenter.k8s.aws/instance-generation|6| @@ -13900,6 +14554,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6a| |karpenter.k8s.aws/instance-generation|6| @@ -13924,6 +14579,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6a| |karpenter.k8s.aws/instance-generation|6| @@ -13948,6 +14604,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6a| |karpenter.k8s.aws/instance-generation|6| @@ -13972,6 +14629,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6a| |karpenter.k8s.aws/instance-generation|6| @@ -13996,6 +14654,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|192| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6a| |karpenter.k8s.aws/instance-generation|6| @@ -14021,6 +14680,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|192| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6a| |karpenter.k8s.aws/instance-generation|6| @@ -14047,6 +14707,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|1| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6g| |karpenter.k8s.aws/instance-generation|6| @@ -14071,6 +14732,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6g| |karpenter.k8s.aws/instance-generation|6| @@ -14095,6 +14757,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6g| |karpenter.k8s.aws/instance-generation|6| @@ -14119,6 +14782,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6g| |karpenter.k8s.aws/instance-generation|6| @@ -14143,6 +14807,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6g| |karpenter.k8s.aws/instance-generation|6| @@ -14167,6 +14832,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6g| |karpenter.k8s.aws/instance-generation|6| @@ -14191,6 +14857,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6g| |karpenter.k8s.aws/instance-generation|6| @@ -14215,6 +14882,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6g| |karpenter.k8s.aws/instance-generation|6| @@ -14239,6 +14907,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6g| |karpenter.k8s.aws/instance-generation|6| @@ -14264,6 +14933,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|1| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6gd| |karpenter.k8s.aws/instance-generation|6| @@ -14289,6 +14959,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6gd| |karpenter.k8s.aws/instance-generation|6| @@ -14314,6 +14985,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6gd| |karpenter.k8s.aws/instance-generation|6| @@ -14339,6 +15011,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6gd| |karpenter.k8s.aws/instance-generation|6| @@ -14364,6 +15037,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6gd| |karpenter.k8s.aws/instance-generation|6| @@ -14389,6 +15063,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6gd| |karpenter.k8s.aws/instance-generation|6| @@ -14414,6 +15089,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6gd| |karpenter.k8s.aws/instance-generation|6| @@ -14439,6 +15115,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6gd| |karpenter.k8s.aws/instance-generation|6| @@ -14464,6 +15141,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r6gd| |karpenter.k8s.aws/instance-generation|6| @@ -14490,6 +15168,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6i| |karpenter.k8s.aws/instance-generation|6| @@ -14514,6 +15193,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6i| |karpenter.k8s.aws/instance-generation|6| @@ -14538,6 +15218,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6i| |karpenter.k8s.aws/instance-generation|6| @@ -14562,6 +15243,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6i| |karpenter.k8s.aws/instance-generation|6| @@ -14586,6 +15268,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6i| |karpenter.k8s.aws/instance-generation|6| @@ -14610,6 +15293,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6i| |karpenter.k8s.aws/instance-generation|6| @@ -14634,6 +15318,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6i| |karpenter.k8s.aws/instance-generation|6| @@ -14658,6 +15343,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6i| |karpenter.k8s.aws/instance-generation|6| @@ -14682,6 +15368,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6i| |karpenter.k8s.aws/instance-generation|6| @@ -14707,6 +15394,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6i| |karpenter.k8s.aws/instance-generation|6| @@ -14733,6 +15421,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6id| |karpenter.k8s.aws/instance-generation|6| @@ -14758,6 +15447,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6id| |karpenter.k8s.aws/instance-generation|6| @@ -14783,6 +15473,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6id| |karpenter.k8s.aws/instance-generation|6| @@ -14808,6 +15499,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6id| |karpenter.k8s.aws/instance-generation|6| @@ -14833,6 +15525,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6id| |karpenter.k8s.aws/instance-generation|6| @@ -14858,6 +15551,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6id| |karpenter.k8s.aws/instance-generation|6| @@ -14883,6 +15577,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6id| |karpenter.k8s.aws/instance-generation|6| @@ -14908,6 +15603,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6id| |karpenter.k8s.aws/instance-generation|6| @@ -14933,6 +15629,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6id| |karpenter.k8s.aws/instance-generation|6| @@ -14959,6 +15656,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6id| |karpenter.k8s.aws/instance-generation|6| @@ -14986,6 +15684,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6idn| |karpenter.k8s.aws/instance-generation|6| @@ -15011,6 +15710,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6idn| |karpenter.k8s.aws/instance-generation|6| @@ -15036,6 +15736,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6idn| |karpenter.k8s.aws/instance-generation|6| @@ -15061,6 +15762,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6idn| |karpenter.k8s.aws/instance-generation|6| @@ -15086,6 +15788,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6idn| |karpenter.k8s.aws/instance-generation|6| @@ -15111,6 +15814,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6idn| |karpenter.k8s.aws/instance-generation|6| @@ -15136,6 +15840,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6idn| |karpenter.k8s.aws/instance-generation|6| @@ -15161,6 +15866,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6idn| |karpenter.k8s.aws/instance-generation|6| @@ -15186,6 +15892,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6idn| |karpenter.k8s.aws/instance-generation|6| @@ -15212,6 +15919,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6idn| |karpenter.k8s.aws/instance-generation|6| @@ -15239,6 +15947,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6in| |karpenter.k8s.aws/instance-generation|6| @@ -15263,6 +15972,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6in| |karpenter.k8s.aws/instance-generation|6| @@ -15287,6 +15997,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6in| |karpenter.k8s.aws/instance-generation|6| @@ -15311,6 +16022,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6in| |karpenter.k8s.aws/instance-generation|6| @@ -15335,6 +16047,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6in| |karpenter.k8s.aws/instance-generation|6| @@ -15359,6 +16072,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6in| |karpenter.k8s.aws/instance-generation|6| @@ -15383,6 +16097,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6in| |karpenter.k8s.aws/instance-generation|6| @@ -15407,6 +16122,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6in| |karpenter.k8s.aws/instance-generation|6| @@ -15431,6 +16147,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6in| |karpenter.k8s.aws/instance-generation|6| @@ -15456,6 +16173,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r6in| |karpenter.k8s.aws/instance-generation|6| @@ -15482,6 +16200,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|1| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7a| |karpenter.k8s.aws/instance-generation|7| @@ -15499,12 +16218,14 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|7134Mi| |pods|8| + |vpc.amazonaws.com/pod-eni|4| ### `r7a.large` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7a| |karpenter.k8s.aws/instance-generation|7| @@ -15522,12 +16243,14 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|14481Mi| |pods|29| + |vpc.amazonaws.com/pod-eni|9| ### `r7a.xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7a| |karpenter.k8s.aws/instance-generation|7| @@ -15545,12 +16268,14 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|29317Mi| |pods|58| + |vpc.amazonaws.com/pod-eni|18| ### `r7a.2xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7a| |karpenter.k8s.aws/instance-generation|7| @@ -15568,12 +16293,14 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|59627Mi| |pods|58| + |vpc.amazonaws.com/pod-eni|38| ### `r7a.4xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7a| |karpenter.k8s.aws/instance-generation|7| @@ -15591,12 +16318,14 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|118312Mi| |pods|234| + |vpc.amazonaws.com/pod-eni|54| ### `r7a.8xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7a| |karpenter.k8s.aws/instance-generation|7| @@ -15614,12 +16343,14 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|239554Mi| |pods|234| + |vpc.amazonaws.com/pod-eni|84| ### `r7a.12xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7a| |karpenter.k8s.aws/instance-generation|7| @@ -15637,12 +16368,14 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|360795Mi| |pods|234| + |vpc.amazonaws.com/pod-eni|114| ### `r7a.16xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7a| |karpenter.k8s.aws/instance-generation|7| @@ -15660,12 +16393,14 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|476504Mi| |pods|737| + |vpc.amazonaws.com/pod-eni|107| ### `r7a.24xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7a| |karpenter.k8s.aws/instance-generation|7| @@ -15683,12 +16418,14 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|718987Mi| |pods|737| + |vpc.amazonaws.com/pod-eni|107| ### `r7a.32xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7a| |karpenter.k8s.aws/instance-generation|7| @@ -15706,12 +16443,14 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|961470Mi| |pods|737| + |vpc.amazonaws.com/pod-eni|107| ### `r7a.48xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|192| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7a| |karpenter.k8s.aws/instance-generation|7| @@ -15730,17 +16469,20 @@ below are the resources available with some assumptions and after the instance o |memory|1446437Mi| |pods|737| |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| ### `r7a.metal-48xl` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|192| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7a| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|1572864| + |karpenter.k8s.aws/instance-network-bandwidth|50000| |karpenter.k8s.aws/instance-size|metal-48xl| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15753,6 +16495,7 @@ below are the resources available with some assumptions and after the instance o |memory|1446437Mi| |pods|737| |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| ## r7g Family ### `r7g.medium` #### Labels @@ -15760,6 +16503,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|1| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7g| |karpenter.k8s.aws/instance-generation|7| @@ -15784,6 +16528,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7g| |karpenter.k8s.aws/instance-generation|7| @@ -15808,6 +16553,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7g| |karpenter.k8s.aws/instance-generation|7| @@ -15832,6 +16578,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7g| |karpenter.k8s.aws/instance-generation|7| @@ -15856,6 +16603,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7g| |karpenter.k8s.aws/instance-generation|7| @@ -15880,6 +16628,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7g| |karpenter.k8s.aws/instance-generation|7| @@ -15904,6 +16653,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7g| |karpenter.k8s.aws/instance-generation|7| @@ -15928,6 +16678,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7g| |karpenter.k8s.aws/instance-generation|7| @@ -15953,6 +16704,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7g| |karpenter.k8s.aws/instance-generation|7| @@ -15979,6 +16731,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|1| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7gd| |karpenter.k8s.aws/instance-generation|7| @@ -16004,6 +16757,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7gd| |karpenter.k8s.aws/instance-generation|7| @@ -16029,6 +16783,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7gd| |karpenter.k8s.aws/instance-generation|7| @@ -16054,6 +16809,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7gd| |karpenter.k8s.aws/instance-generation|7| @@ -16079,6 +16835,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7gd| |karpenter.k8s.aws/instance-generation|7| @@ -16104,12 +16861,13 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7gd| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-local-nvme|1900| - |karpenter.k8s.aws/instance-memory|196608| + |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|15000| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|arm64| @@ -16120,7 +16878,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |cpu|31850m| |ephemeral-storage|17Gi| - |memory|178874Mi| + |memory|239495Mi| |pods|234| |vpc.amazonaws.com/pod-eni|54| ### `r7gd.12xlarge` @@ -16129,12 +16887,13 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7gd| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-local-nvme|2850| - |karpenter.k8s.aws/instance-memory|262144| + |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|22500| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|arm64| @@ -16145,7 +16904,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |cpu|47810m| |ephemeral-storage|17Gi| - |memory|239495Mi| + |memory|360736Mi| |pods|234| |vpc.amazonaws.com/pod-eni|54| ### `r7gd.16xlarge` @@ -16154,6 +16913,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7gd| |karpenter.k8s.aws/instance-generation|7| @@ -16174,6 +16934,32 @@ below are the resources available with some assumptions and after the instance o |pods|737| |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| +### `r7gd.metal` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7gd| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|| + |karpenter.k8s.aws/instance-local-nvme|3800| + |karpenter.k8s.aws/instance-memory|524288| + |karpenter.k8s.aws/instance-network-bandwidth|30000| + |karpenter.k8s.aws/instance-size|metal| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7gd.metal| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|63770m| + |ephemeral-storage|17Gi| + |memory|476445Mi| + |pods|737| + |vpc.amazonaws.com/efa|1| ## r7i Family ### `r7i.large` #### Labels @@ -16181,11 +16967,13 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7i| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| + |karpenter.k8s.aws/instance-network-bandwidth|781| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16197,17 +16985,20 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|14481Mi| |pods|29| + |vpc.amazonaws.com/pod-eni|9| ### `r7i.xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7i| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| + |karpenter.k8s.aws/instance-network-bandwidth|1562| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16219,17 +17010,20 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|29317Mi| |pods|58| + |vpc.amazonaws.com/pod-eni|18| ### `r7i.2xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7i| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| + |karpenter.k8s.aws/instance-network-bandwidth|3125| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16241,17 +17035,20 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|59627Mi| |pods|58| + |vpc.amazonaws.com/pod-eni|38| ### `r7i.4xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7i| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| + |karpenter.k8s.aws/instance-network-bandwidth|6250| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16263,17 +17060,20 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|118312Mi| |pods|234| + |vpc.amazonaws.com/pod-eni|54| ### `r7i.8xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7i| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| + |karpenter.k8s.aws/instance-network-bandwidth|12500| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16285,17 +17085,20 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|239554Mi| |pods|234| + |vpc.amazonaws.com/pod-eni|84| ### `r7i.12xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7i| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|393216| + |karpenter.k8s.aws/instance-network-bandwidth|18750| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16307,17 +17110,20 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|360795Mi| |pods|234| + |vpc.amazonaws.com/pod-eni|114| ### `r7i.16xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7i| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|524288| + |karpenter.k8s.aws/instance-network-bandwidth|25000| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16329,17 +17135,20 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|476504Mi| |pods|737| + |vpc.amazonaws.com/pod-eni|107| ### `r7i.24xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7i| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|786432| + |karpenter.k8s.aws/instance-network-bandwidth|37500| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16351,17 +17160,20 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|718987Mi| |pods|737| + |vpc.amazonaws.com/pod-eni|107| ### `r7i.metal-24xl` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7i| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|786432| + |karpenter.k8s.aws/instance-network-bandwidth|37500| |karpenter.k8s.aws/instance-size|metal-24xl| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16373,17 +17185,20 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|718987Mi| |pods|737| + |vpc.amazonaws.com/pod-eni|107| ### `r7i.48xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|192| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7i| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|1572864| + |karpenter.k8s.aws/instance-network-bandwidth|50000| |karpenter.k8s.aws/instance-size|48xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16396,17 +17211,20 @@ below are the resources available with some assumptions and after the instance o |memory|1446437Mi| |pods|737| |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| ### `r7i.metal-48xl` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|192| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7i| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|1572864| + |karpenter.k8s.aws/instance-network-bandwidth|50000| |karpenter.k8s.aws/instance-size|metal-48xl| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16419,6 +17237,7 @@ below are the resources available with some assumptions and after the instance o |memory|1446437Mi| |pods|737| |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| ## r7iz Family ### `r7iz.large` #### Labels @@ -16426,6 +17245,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7iz| |karpenter.k8s.aws/instance-generation|7| @@ -16443,12 +17263,14 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|14481Mi| |pods|29| + |vpc.amazonaws.com/pod-eni|9| ### `r7iz.xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7iz| |karpenter.k8s.aws/instance-generation|7| @@ -16466,12 +17288,14 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|29317Mi| |pods|58| + |vpc.amazonaws.com/pod-eni|18| ### `r7iz.2xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7iz| |karpenter.k8s.aws/instance-generation|7| @@ -16489,12 +17313,14 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|59627Mi| |pods|58| + |vpc.amazonaws.com/pod-eni|38| ### `r7iz.4xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7iz| |karpenter.k8s.aws/instance-generation|7| @@ -16512,12 +17338,14 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|118312Mi| |pods|234| + |vpc.amazonaws.com/pod-eni|54| ### `r7iz.8xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7iz| |karpenter.k8s.aws/instance-generation|7| @@ -16535,12 +17363,14 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|239554Mi| |pods|234| + |vpc.amazonaws.com/pod-eni|84| ### `r7iz.12xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7iz| |karpenter.k8s.aws/instance-generation|7| @@ -16558,12 +17388,14 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|360795Mi| |pods|234| + |vpc.amazonaws.com/pod-eni|114| ### `r7iz.16xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7iz| |karpenter.k8s.aws/instance-generation|7| @@ -16581,17 +17413,20 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|476504Mi| |pods|737| + |vpc.amazonaws.com/pod-eni|107| ### `r7iz.metal-16xl` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7iz| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|524288| + |karpenter.k8s.aws/instance-network-bandwidth|25000| |karpenter.k8s.aws/instance-size|metal-16xl| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16603,12 +17438,14 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|476504Mi| |pods|737| + |vpc.amazonaws.com/pod-eni|107| ### `r7iz.32xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7iz| |karpenter.k8s.aws/instance-generation|7| @@ -16627,17 +17464,20 @@ below are the resources available with some assumptions and after the instance o |memory|961470Mi| |pods|737| |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| ### `r7iz.metal-32xl` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|r7iz| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|1048576| + |karpenter.k8s.aws/instance-network-bandwidth|50000| |karpenter.k8s.aws/instance-size|metal-32xl| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16650,6 +17490,7 @@ below are the resources available with some assumptions and after the instance o |memory|961470Mi| |pods|737| |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| ## t1 Family ### `t1.micro` #### Labels @@ -16657,6 +17498,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|1| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t1| |karpenter.k8s.aws/instance-generation|1| @@ -16680,6 +17522,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|1| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t2| |karpenter.k8s.aws/instance-generation|2| @@ -16702,6 +17545,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|1| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t2| |karpenter.k8s.aws/instance-generation|2| @@ -16724,6 +17568,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|1| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t2| |karpenter.k8s.aws/instance-generation|2| @@ -16746,6 +17591,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t2| |karpenter.k8s.aws/instance-generation|2| @@ -16768,6 +17614,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t2| |karpenter.k8s.aws/instance-generation|2| @@ -16790,6 +17637,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t2| |karpenter.k8s.aws/instance-generation|2| @@ -16812,6 +17660,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t2| |karpenter.k8s.aws/instance-generation|2| @@ -16835,6 +17684,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t3| |karpenter.k8s.aws/instance-generation|3| @@ -16858,6 +17708,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t3| |karpenter.k8s.aws/instance-generation|3| @@ -16881,6 +17732,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t3| |karpenter.k8s.aws/instance-generation|3| @@ -16904,6 +17756,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t3| |karpenter.k8s.aws/instance-generation|3| @@ -16927,6 +17780,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t3| |karpenter.k8s.aws/instance-generation|3| @@ -16950,6 +17804,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t3| |karpenter.k8s.aws/instance-generation|3| @@ -16973,6 +17828,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t3| |karpenter.k8s.aws/instance-generation|3| @@ -16997,6 +17853,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t3a| |karpenter.k8s.aws/instance-generation|3| @@ -17020,6 +17877,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t3a| |karpenter.k8s.aws/instance-generation|3| @@ -17043,6 +17901,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t3a| |karpenter.k8s.aws/instance-generation|3| @@ -17066,6 +17925,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t3a| |karpenter.k8s.aws/instance-generation|3| @@ -17089,6 +17949,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t3a| |karpenter.k8s.aws/instance-generation|3| @@ -17112,6 +17973,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t3a| |karpenter.k8s.aws/instance-generation|3| @@ -17135,6 +17997,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|amd| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t3a| |karpenter.k8s.aws/instance-generation|3| @@ -17159,6 +18022,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t4g| |karpenter.k8s.aws/instance-generation|4| @@ -17182,6 +18046,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t4g| |karpenter.k8s.aws/instance-generation|4| @@ -17205,6 +18070,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t4g| |karpenter.k8s.aws/instance-generation|4| @@ -17228,6 +18094,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t4g| |karpenter.k8s.aws/instance-generation|4| @@ -17251,6 +18118,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t4g| |karpenter.k8s.aws/instance-generation|4| @@ -17274,6 +18142,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t4g| |karpenter.k8s.aws/instance-generation|4| @@ -17297,6 +18166,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t4g| |karpenter.k8s.aws/instance-generation|4| @@ -17324,6 +18194,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-accelerator-name|inferentia| |karpenter.k8s.aws/instance-category|trn| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|trn1| |karpenter.k8s.aws/instance-generation|1| @@ -17353,6 +18224,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-accelerator-name|inferentia| |karpenter.k8s.aws/instance-category|trn| |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|trn1| |karpenter.k8s.aws/instance-generation|1| @@ -17384,6 +18256,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-accelerator-name|inferentia| |karpenter.k8s.aws/instance-category|trn| |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|trn1n| |karpenter.k8s.aws/instance-generation|1| @@ -17412,6 +18285,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|u| |karpenter.k8s.aws/instance-cpu|448| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|u-12tb1| |karpenter.k8s.aws/instance-generation|1| @@ -17436,6 +18310,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|u| |karpenter.k8s.aws/instance-cpu|448| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|u-18tb1| |karpenter.k8s.aws/instance-generation|1| @@ -17460,6 +18335,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|u| |karpenter.k8s.aws/instance-cpu|448| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|u-24tb1| |karpenter.k8s.aws/instance-generation|1| @@ -17484,6 +18360,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|u| |karpenter.k8s.aws/instance-cpu|224| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|u-3tb1| |karpenter.k8s.aws/instance-generation|1| @@ -17509,6 +18386,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|u| |karpenter.k8s.aws/instance-cpu|224| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|u-6tb1| |karpenter.k8s.aws/instance-generation|1| @@ -17532,6 +18410,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|u| |karpenter.k8s.aws/instance-cpu|448| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|u-6tb1| |karpenter.k8s.aws/instance-generation|1| @@ -17556,6 +18435,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|u| |karpenter.k8s.aws/instance-cpu|448| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|u-9tb1| |karpenter.k8s.aws/instance-generation|1| @@ -17580,6 +18460,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|vt| |karpenter.k8s.aws/instance-cpu|12| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|vt1| |karpenter.k8s.aws/instance-generation|1| @@ -17604,6 +18485,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|vt| |karpenter.k8s.aws/instance-cpu|24| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|vt1| |karpenter.k8s.aws/instance-generation|1| @@ -17628,6 +18510,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|vt| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|vt1| |karpenter.k8s.aws/instance-generation|1| @@ -17654,6 +18537,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x1| |karpenter.k8s.aws/instance-generation|1| @@ -17677,6 +18561,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x1| |karpenter.k8s.aws/instance-generation|1| @@ -17701,6 +18586,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x1e| |karpenter.k8s.aws/instance-generation|1| @@ -17724,6 +18610,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x1e| |karpenter.k8s.aws/instance-generation|1| @@ -17747,6 +18634,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x1e| |karpenter.k8s.aws/instance-generation|1| @@ -17770,6 +18658,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x1e| |karpenter.k8s.aws/instance-generation|1| @@ -17793,6 +18682,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x1e| |karpenter.k8s.aws/instance-generation|1| @@ -17816,6 +18706,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x1e| |karpenter.k8s.aws/instance-generation|1| @@ -17840,6 +18731,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|1| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x2gd| |karpenter.k8s.aws/instance-generation|2| @@ -17865,6 +18757,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x2gd| |karpenter.k8s.aws/instance-generation|2| @@ -17890,6 +18783,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x2gd| |karpenter.k8s.aws/instance-generation|2| @@ -17915,6 +18809,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x2gd| |karpenter.k8s.aws/instance-generation|2| @@ -17940,6 +18835,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x2gd| |karpenter.k8s.aws/instance-generation|2| @@ -17965,6 +18861,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x2gd| |karpenter.k8s.aws/instance-generation|2| @@ -17990,6 +18887,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x2gd| |karpenter.k8s.aws/instance-generation|2| @@ -18015,6 +18913,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x2gd| |karpenter.k8s.aws/instance-generation|2| @@ -18040,6 +18939,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|aws| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|x2gd| |karpenter.k8s.aws/instance-generation|2| @@ -18066,6 +18966,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2idn| |karpenter.k8s.aws/instance-generation|2| @@ -18091,6 +18992,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2idn| |karpenter.k8s.aws/instance-generation|2| @@ -18116,6 +19018,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2idn| |karpenter.k8s.aws/instance-generation|2| @@ -18142,6 +19045,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2idn| |karpenter.k8s.aws/instance-generation|2| @@ -18169,6 +19073,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2iedn| |karpenter.k8s.aws/instance-generation|2| @@ -18194,6 +19099,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2iedn| |karpenter.k8s.aws/instance-generation|2| @@ -18219,6 +19125,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2iedn| |karpenter.k8s.aws/instance-generation|2| @@ -18244,6 +19151,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2iedn| |karpenter.k8s.aws/instance-generation|2| @@ -18269,6 +19177,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2iedn| |karpenter.k8s.aws/instance-generation|2| @@ -18294,6 +19203,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2iedn| |karpenter.k8s.aws/instance-generation|2| @@ -18319,6 +19229,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2iedn| |karpenter.k8s.aws/instance-generation|2| @@ -18345,6 +19256,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2iedn| |karpenter.k8s.aws/instance-generation|2| @@ -18372,6 +19284,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2iezn| |karpenter.k8s.aws/instance-generation|2| @@ -18396,6 +19309,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2iezn| |karpenter.k8s.aws/instance-generation|2| @@ -18420,6 +19334,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|24| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2iezn| |karpenter.k8s.aws/instance-generation|2| @@ -18444,6 +19359,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2iezn| |karpenter.k8s.aws/instance-generation|2| @@ -18468,6 +19384,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2iezn| |karpenter.k8s.aws/instance-generation|2| @@ -18493,6 +19410,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|x| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| |karpenter.k8s.aws/instance-family|x2iezn| |karpenter.k8s.aws/instance-generation|2| @@ -18519,6 +19437,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|z| |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|z1d| |karpenter.k8s.aws/instance-generation|1| @@ -18544,6 +19463,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|z| |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|z1d| |karpenter.k8s.aws/instance-generation|1| @@ -18569,6 +19489,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|z| |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|z1d| |karpenter.k8s.aws/instance-generation|1| @@ -18594,6 +19515,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|z| |karpenter.k8s.aws/instance-cpu|12| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|z1d| |karpenter.k8s.aws/instance-generation|1| @@ -18619,6 +19541,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|z| |karpenter.k8s.aws/instance-cpu|24| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|z1d| |karpenter.k8s.aws/instance-generation|1| @@ -18644,6 +19567,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|z| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|z1d| |karpenter.k8s.aws/instance-generation|1| @@ -18669,6 +19593,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |karpenter.k8s.aws/instance-category|z| |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-cpu-manufacturer|intel| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|z1d| |karpenter.k8s.aws/instance-generation|1| diff --git a/website/content/en/preview/reference/metrics.md b/website/content/en/preview/reference/metrics.md index 4c3e2a0fdff3..c717023a2cad 100644 --- a/website/content/en/preview/reference/metrics.md +++ b/website/content/en/preview/reference/metrics.md @@ -45,6 +45,9 @@ Node system daemon overhead are the resources reserved for system overhead, the ### `karpenter_nodes_leases_deleted` Number of deleted leaked leases. +### `karpenter_nodes_eviction_queue_depth` +The number of pods currently waiting for a successful eviction in the eviction queue. + ### `karpenter_nodes_created` Number of nodes created in total by Karpenter. Labeled by owning nodepool. @@ -64,6 +67,9 @@ The time from pod creation until the pod is running. ### `karpenter_provisioner_scheduling_simulation_duration_seconds` Duration of scheduling simulations used for deprovisioning and provisioning in seconds. +### `karpenter_provisioner_scheduling_queue_depth` +The number of pods currently waiting to be scheduled. + ### `karpenter_provisioner_scheduling_duration_seconds` Duration of scheduling process in seconds. @@ -115,27 +121,47 @@ The number of times that Karpenter failed to launch a replacement node for disru ### `karpenter_disruption_queue_depth` The number of commands currently being waited on in the disruption orchestration queue. +### `karpenter_disruption_pods_disrupted_total` +Total number of reschedulable pods disrupted on nodes. Labeled by NodePool, disruption action, method, and consolidation type. + +### `karpenter_disruption_nodes_disrupted_total` +Total number of nodes disrupted. Labeled by NodePool, disruption action, method, and consolidation type. + ### `karpenter_disruption_evaluation_duration_seconds` -Duration of the disruption evaluation process in seconds. +Duration of the disruption evaluation process in seconds. Labeled by method and consolidation type. ### `karpenter_disruption_eligible_nodes` -Number of nodes eligible for disruption by Karpenter. Labeled by disruption method. +Number of nodes eligible for disruption by Karpenter. Labeled by disruption method and consolidation type. ### `karpenter_disruption_consolidation_timeouts_total` Number of times the Consolidation algorithm has reached a timeout. Labeled by consolidation type. +### `karpenter_disruption_budgets_allowed_disruptions` +The number of nodes for a given NodePool that can be disrupted at a point in time. Labeled by NodePool. Note that allowed disruptions can change very rapidly, as new nodes may be created and others may be deleted at any point. + ### `karpenter_disruption_actions_performed_total` -Number of disruption actions performed. Labeled by disruption method. +Number of disruption actions performed. Labeled by disruption action, method, and consolidation type. ## Consistency Metrics ### `karpenter_consistency_errors` Number of consistency checks that have failed. +## Cluster State Metrics + +### `karpenter_cluster_state_synced` +Returns 1 if cluster state is synced and 0 otherwise. Synced checks that nodeclaims and nodes that are stored in the APIServer have the same representation as Karpenter's cluster state + +### `karpenter_cluster_state_node_count` +Current count of nodes in cluster state + ## Cloudprovider Metrics -### `karpenter_cloudprovider_instance_type_price_estimate` -Estimated hourly price used when making informed decisions on node cost calculation. This is updated once on startup and then every 12 hours. +### `karpenter_cloudprovider_instance_type_offering_price_estimate` +Instance type offering estimated hourly price used when making informed decisions on node cost calculation, based on instance type, capacity type, and zone. + +### `karpenter_cloudprovider_instance_type_offering_available` +Instance type offering availability, based on instance type, capacity type, and zone ### `karpenter_cloudprovider_instance_type_memory_bytes` Memory, in bytes, for a given instance type. diff --git a/website/content/en/preview/reference/settings.md b/website/content/en/preview/reference/settings.md index 4150586483ea..753e9ef7b707 100644 --- a/website/content/en/preview/reference/settings.md +++ b/website/content/en/preview/reference/settings.md @@ -24,7 +24,7 @@ Karpenter surfaces environment variables and CLI parameters to allow you to conf | FEATURE_GATES | \-\-feature-gates | Optional features can be enabled / disabled using feature gates. Current options are: Drift,SpotToSpotConsolidation (default = Drift=true,SpotToSpotConsolidation=false)| | HEALTH_PROBE_PORT | \-\-health-probe-port | The port the health probe endpoint binds to for reporting controller health (default = 8081)| | INTERRUPTION_QUEUE | \-\-interruption-queue | Interruption queue is disabled if not specified. Enabling interruption handling may require additional permissions on the controller service account. Additional permissions are outlined in the docs.| -| ISOLATED_VPC | \-\-isolated-vpc | If true, then assume we can't reach AWS services which don't have a VPC endpoint. This also has the effect of disabling look-ups to the AWS pricing endpoint.| +| ISOLATED_VPC | \-\-isolated-vpc | If true, then assume we can't reach AWS services which don't have a VPC endpoint. This also has the effect of disabling look-ups to the AWS on-demand pricing endpoint.| | KARPENTER_SERVICE | \-\-karpenter-service | The Karpenter Service name for the dynamic webhook certificate| | KUBE_CLIENT_BURST | \-\-kube-client-burst | The maximum allowed burst of queries to the kube-apiserver (default = 300)| | KUBE_CLIENT_QPS | \-\-kube-client-qps | The smoothed rate of qps to kube-apiserver (default = 200)| diff --git a/website/content/en/preview/reference/threat-model.md b/website/content/en/preview/reference/threat-model.md index 316ca5cff181..e316ac40788d 100644 --- a/website/content/en/preview/reference/threat-model.md +++ b/website/content/en/preview/reference/threat-model.md @@ -63,16 +63,16 @@ Karpenter has permissions to create and manage cloud instances. Karpenter has Ku ### Threat: Using EC2 CreateTag/DeleteTag Permissions to Orchestrate Instance Creation/Deletion -**Background**: As of v0.28.0, Karpenter creates a mapping between CloudProvider instances and CustomResources in the cluster for capacity tracking. To ensure this mapping is consistent, Karpenter utilizes the following tag keys: +**Background**: As of `0.28.0`, Karpenter creates a mapping between CloudProvider instances and CustomResources in the cluster for capacity tracking. To ensure this mapping is consistent, Karpenter utilizes the following tag keys: * `karpenter.sh/managed-by` * `karpenter.sh/nodepool` * `kubernetes.io/cluster/${CLUSTER_NAME}` -* `karpenter.sh/provisioner-name` (prior to `v0.32.0`) +* `karpenter.sh/provisioner-name` (prior to `0.32.0`) Any user that has the ability to Create/Delete tags on CloudProvider instances will have the ability to orchestrate Karpenter to Create/Delete CloudProvider instances as a side effect. -In addition, as of v0.29.0, Karpenter will Drift on Security Groups and Subnets. If a user has the Create/Delete tags permission for either of resources, they can orchestrate Karpenter to Create/Delete CloudProvider instances as a side effect. +In addition, as of `0.29.0`, Karpenter will Drift on Security Groups and Subnets. If a user has the Create/Delete tags permission for either of resources, they can orchestrate Karpenter to Create/Delete CloudProvider instances as a side effect. **Threat:** A Cluster Operator attempts to create or delete a tag on a resource discovered by Karpenter. If it has the ability to create a tag it can effectively create or delete CloudProvider instances associated with the tagged resources. diff --git a/website/content/en/preview/tasks/_index.md b/website/content/en/preview/tasks/_index.md new file mode 100644 index 000000000000..7d4ac8605f4e --- /dev/null +++ b/website/content/en/preview/tasks/_index.md @@ -0,0 +1,10 @@ +--- +title: "Tasks" +linkTitle: "Tasks" +weight: 25 +description: > + Tasks to run with Karpenter +cascade: + type: docs +--- + diff --git a/website/content/en/preview/tasks/managing-amis.md b/website/content/en/preview/tasks/managing-amis.md new file mode 100644 index 000000000000..1ef31f141e08 --- /dev/null +++ b/website/content/en/preview/tasks/managing-amis.md @@ -0,0 +1,153 @@ +--- +title: "Managing AMIs" +linkTitle: "Managing AMIs" +weight: 10 +description: > + Task for managing AMIs in Karpenter +--- + +Understanding how Karpenter assigns AMIs to nodes can help ensure that your workloads will run successfully on those nodes and continue to run if the nodes are upgraded to newer AMIs. +Below we describe how Karpenter assigns AMIs to nodes when they are first deployed and how newer AMIs are assigned later when nodes are spun up to replace old ones. +Later, it describes the options you have to assert control over how AMIs are used by Karpenter for your clusters. + +Features for managing AMIs described here should be considered as part of the larger upgrade policies that you have for your clusters. +See [How do I upgrade an EKS Cluster with Karpenter]({{< relref "../faq/#how-do-i-upgrade-an-eks-cluster-with-karpenter" >}}) for details on this process. + +## How Karpenter assigns AMIs to nodes + +Here is how Karpenter assigns AMIs nodes: + +* When you create an `EC2NodeClass`, you are required to set the family of AMIs to use. For example, for the AL2 family, you would set `amiFamily: AL2`. +* With that `amiFamily` set, any time Karpenter spins up a new node, it uses the latest [Amazon EKS optimized Amazon Linux 2 AMIs](https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami.html) release. +* Later, if an existing node needs to be replaced, Karpenter checks to see if a newer AMI in the AL2 family is available and automatically uses the new AMI instead to spin up the new node. In other words, you may automatically get an AMI that you have not tested with your workloads. + +You can manually delete a node managed by Karpenter, which will cause the default behavior just described to take effect. +However, there are situations that will cause node replacements with newer AMIs to happen automatically. +These include: Expiration (if node expiry is set, the node is marked for deletion at a certain time after the node is created), [**Consolidation**]({{< relref "../concepts/disruption/#consolidation" >}}) (if a node is empty of workloads, or deemed to be inefficiently running workloads, nodes can be deleted and more appropriately featured nodes are brought up to consolidate workloads), [Drift]({{< relref "../concepts/disruption/#drift" >}}) (nodes are set for deletion when they drift from the desired state of the `NodeClaims` and new nodes are brought up to replace them), and [Interruption]({{< relref "../concepts/disruption/#interruption" >}}) (nodes are sometimes involuntarily disrupted by things like Spot interruption, health changes, and instance events, requiring new nodes to be deployed). + +See [**Automated Methods**]({{< relref "../concepts/disruption/#automated-methods" >}}) for details on how Karpenter uses these automated actions to replace nodes. + +With these types of automated updates in place, there is some risk that the new AMI being used when replacing instances will introduce some regressions or bugs that cause your workloads to be degraded or fail altogether. +The options described below tell you how to take more control over the ways in which Karpenter selects AMIs for your nodes. + +{{% alert title="Important" color="warning" %}} +If you are new to Karpenter, you should know that the behavior described here is different than you get with Managed Node Groups (MNG). MNG will always use the assigned AMI when it creates a new node and will never automatically upgrade to a new AMI when a new node is required. See [Updating a Managed Node Group](https://docs.aws.amazon.com/eks/latest/userguide/update-managed-node-group.html) to see how you would manually update MNG to use new AMIs. +{{% /alert %}} + +## Choosing AMI options +One of Karpenter's greatest assets is its ability to provide the right node at the right time, with little intervention from the person managing the cluster. +Its default behavior of using a later AMI if one becomes available in the selected family means you automatically get the latest security fixes and features. +However, with this comes the risk that the new AMI could break or degrade your workloads. + +As the Karpenter team looks for new ways to manage AMIs, the options below offer some means of reducing these risks, based on your own security and ease-of-use requirements. +Here are the advantages and challenges of each of the options described below: + +* [Option 1]({{< relref "#option-1-manage-how-amis-are-tested-and-rolled-out" >}}) (Test AMIs): The safest way, and the one we recommend, for ensuring that a new AMI doesn't break your workloads is to test it before putting it into production. This takes the most effort on your part, but most effectively models how your workloads will run in production, allowing you to catch issues ahead of time. Note that you can sometimes get different results from your test environment when you roll a new AMI into production, since issues like scale and other factors can elevate problems you might not see in test. So combining this with other options, that do things like slow rollouts, can allow you to catch problems before they impact your whole cluster. +* [Option 2]({{< relref "#option-2-lock-down-which-amis-are-selected" >}}) (Lock down AMIs): If workloads require a particluar AMI, this option can make sure that it is the only AMI used by Karpenter. This can be used in combination with Option 1, where you lock down the AMI in production, but allow the newest AMIs in a test cluster while you test your workloads before upgrading production. Keep in mind that this makes upgrades a manual process for you. +* [Option 3]({{< relref "#option-3-control-the-pace-of-node-disruptions" >}}) ([Disruption budgets]({{< relref "../concepts/disruption/" >}})): This option can be used as a way of mitigating the scope of impact if a new AMI causes problems with your workloads. With Disruption budgets you can slow the pace of upgrades to nodes with new AMIs or make sure that upgrades only happen during selected dates and times (using `schedule`). This doesn't prevent a bad AMI from being deployed, but it allows you to control when nodes are upgraded, and gives you more time respond to rollout issues. + +## Options + +The following lays out the options you have to impact Karpenter’s behavior as it relates to how nodes are created and AMIs are consumed. + +### Option 1: Manage how AMIs are tested and rolled out + +Instead of just avoiding AMI upgrades, you can set up test clusters where you can try out new AMI releases before they are put into production. +For example, you could have: + +* **Test clusters**: On lower environment clusters, you can run the latest AMIs for your workloads in a safe environment. The `EC2NodeClass` for these clusters could be set with a chosen `amiFamily`, but no `amiSelectorTerms` set. For example, the `NodePool` and `EC2NodeClass` could begin with the following: + + ```yaml + apiVersion: karpenter.sh/v1beta1 + kind: NodePool + metadata: + name: default + spec: + template: + spec: + nodeClassRef: + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass + name: default + --- + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass + metadata: + name: default + spec: + # The latest AMI in this family will be used + amiFamily: AL2 + ``` +* **Production clusters**: After you've confirmed that the AMI works in your lower environments, you can pin the latest AMIs to be deployed in your production clusters to roll out the AMI. One way to do that is to use `amiSelectorTerms` to set the tested AMI to be used in your production cluster. Refer to Option 2 for how to choose a particular AMI by `name` or `id`. Remember that it is still best practice to gradually roll new AMIs into your cluster, even if they have been tested. So consider implementing that for your production clusters as described in Option 3. + +### Option 2: Lock down which AMIs are selected + +Instead of letting Karpenter always run the latest AMI, you can change Karpenter’s default behavior. +When you configure the [**EC2NodeClass**]({{< relref "../concepts/nodeclasses" >}}), you can set a specific AMI that you want Karpenter to always choose, using the `amiSelectorTerms` field. +This prevents a new and potentially untested AMI from replacing existing nodes when those nodes are terminated. + +With the `amiSelectorTerms` field in an `EC2NodeClass`, you can set a specific AMI for Karpenter to use, based on AMI name or id (only one is required). +These examples show two different ways to identify the same AMI: + +```yaml +amiSelectorTerms: +- tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" + environment: prod +- name: al2023-ami-2023.3.20240219.0-kernel-6.1-x86_64 +``` + +or + +```yaml +amiSelectorTerms: +- tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" + environment: prod +- id: ami-052c9ea013e6e3567 +``` + +See the [**spec.amiSelectorTerms**]({{< relref "../concepts/nodeclasses/#specamiselectorterms" >}}) section of the NodeClasses page for details. +Keep in mind, that this could prevent you from getting critical security patches when new AMIs are available, but it does give you control over exactly which AMI is running. + + +### Option 3: Control the pace of node disruptions + +To reduce the risk of entire workloads being immediately degraded when a new AMI is deployed, you can enable Karpenter [**Disruption Budgets**]({{< relref "../concepts/disruption/#disruption-budgets " >}}). +Disruption Budgets limit when and to what extent nodes can be disrupted. +You can prevent disruption based on nodes (a percentage or number of nodes that can be disrupted at a time) and schedule (excluding certain times from disrupting nodes). +You can set Disruption Budgets in a `NodePool` spec. +Here is an example: + +```yaml +disruption: + consolidationPolicy: WhenEmpty + expireAfter: 1440h + budgets: + - nodes: 15% + - nodes: "3" + - nodes: "0" + schedule: "0 7 * * sat-sun" + duration: 12h +``` + +The `disruption` settings define a few fields that indicate the state of a node that should be disrupted. +The `consolidationPolicy` field indicates that a node should be disrupted if the node is either underutilized (`WhenUnderutilized`) or not running any pods (`WhenEmpty`). +With `expireAfter` set to `1440` hours, the node expires after 60 days. +Extending those values causes longer times without disruption. + +Settings for budgets in the above example include the following: + +* **Percentage of nodes**: From the first `nodes` setting, only `15%` of the NodePool’s nodes can be disrupted at a time. +* **Number of nodes**: The second `nodes` setting limits the number of nodes that can be disrupted at a time to `3`. +* **Schedule**: The third `nodes` setting uses schedule to say that zero disruptions (`0`) are allowed starting at 7am on Saturday and Sunday and continues for 12 hours. +The format of the schedule follows the `crontab` format for identifying dates and times. +See the [crontab](https://man7.org/linux/man-pages/man5/crontab.5.html) page for information on the supported values for these fields. + +As with all disruption settings, keep in mind that avoiding updated AMIs for your nodes can result in not getting fixes for known security risks and bugs. +You need to balance that with your desire to not risk breaking the workloads on your cluster. + +## Follow-up + +The Karpenter project continues to add features to give you greater control over AMI upgrades on your clusters. +If you have opinions about features you would like to see to manage AMIs with Karpenter, feel free to enter a Karpenter [New Issue](https://github.com/aws/karpenter-provider-aws/issues/new/choose). diff --git a/website/content/en/preview/troubleshooting.md b/website/content/en/preview/troubleshooting.md index c33f04264e3b..6dc784007b75 100644 --- a/website/content/en/preview/troubleshooting.md +++ b/website/content/en/preview/troubleshooting.md @@ -29,7 +29,7 @@ Update the zap-logger-config "level" and restart the Karpenter pod(s) to enable #### Debug logging via Helm -You can enable debug logging during installation with helm by setting the option `logLevel`. +You can enable debug logging during installation with Helm by setting the option `logLevel`. ``` helm upgrade --install karpenter oci://public.ecr.aws/karpenter/karpenter \ @@ -53,6 +53,18 @@ This can be resolved by creating the [Service Linked Role](https://docs.aws.amaz aws iam create-service-linked-role --aws-service-name spot.amazonaws.com ``` +### Failed Resolving STS Credentials with I/O Timeout + +```bash +Checking EC2 API connectivity, WebIdentityErr: failed to retrieve credentials\ncaused by: RequestError: send request failed\ncaused by: Post \"https://sts.us-east-1.amazonaws.com/\": dial tcp: lookup sts.us-east-1.amazonaws.com: i/o timeout +``` + +If you see the error above when you attempt to install Karpenter, this indicates that Karpenter is unable to reach out to the STS endpoint due to failed DNS resolution. This can happen when Karpenter is running with `dnsPolicy: ClusterFirst` and your in-cluster DNS service is not yet running. + +You have two mitigations to resolve this error: +1. Let Karpenter manage your in-cluster DNS service - You can let Karpenter manage your DNS application pods' capacity by changing Karpenter's `dnsPolicy` to be `Default` (run `--set dnsPolicy=Default` with a Helm installation). This ensures that Karpenter reaches out to the [VPC DNS service](https://docs.aws.amazon.com/vpc/latest/userguide/vpc-dns.html) when running its controllers, allowing Karpenter to start-up without the DNS application pods running, enabling Karpenter to manage the capacity for these pods. +2. Let MNG/Fargate manage your in-cluster DNS service - If running a cluster with MNG, ensure that your group has enough capacity to support the DNS application pods and ensure that the application has the correct tolerations to schedule against the capacity. If running a cluster with Fargate, ensure that you have a [fargate profile](https://docs.aws.amazon.com/eks/latest/userguide/fargate-profile.html) that selects against your DNS application pods. + ### Karpenter Role names exceeding 64-character limit If you use a tool such as AWS CDK to generate your Kubernetes cluster name, when you add Karpenter to your cluster you could end up with a cluster name that is too long to incorporate into your KarpenterNodeRole name (which is limited to 64 characters). @@ -77,7 +89,7 @@ Info on whether there has been a change to the CRD between versions of Karpenter ### Unable to schedule pod due to insufficient node group instances -v0.16.0 changed the default replicas from 1 to 2. +`0.16.0` changed the default replicas from 1 to 2. Karpenter won't launch capacity to run itself (log related to the `karpenter.sh/provisioner-name DoesNotExist requirement`) so it can't provision for the second Karpenter pod. @@ -89,30 +101,30 @@ To do so on AWS increase the `minimum` and `desired` parameters on the node grou ### Helm Error When Pulling the Chart -If Helm is showing an error when trying to install Karpenter helm charts: +If Helm is showing an error when trying to install Karpenter Helm charts: -- Ensure you are using a newer Helm version, Helm started supporting OCI images since v3.8.0. -- Helm does not have an `helm repo add` concept in OCI, so to install Karpenter you no longer need this +- Ensure you are using a newer Helm version, Helm started supporting OCI images since `3.8.0`. +- Helm does not have an `helm repo add` concept in OCI, so to install Karpenter you no longer need this. +- If you get an error like `Error: public.ecr.aws/karpenter/karpenter:0.34.0: not found` make sure you're adding a `v` prefix for Karpenter versions between `0.17.0` & `0.34.x`. - Verify that the image you are trying to pull actually exists in [gallery.ecr.aws/karpenter](https://gallery.ecr.aws/karpenter/karpenter) -- Sometimes Helm generates a generic error, you can add the --debug switch to any of the helm commands in this doc for more verbose error messages -- If you are getting a 403 forbidden error, you can try `docker logout public.ecr.aws` as explained [here](https://docs.aws.amazon.com/AmazonECR/latest/public/public-troubleshooting.html) -- If you are receiving this error: `Error: failed to download "oci://public.ecr.aws/karpenter/karpenter" at version "0.17.0"`, then you need to prepend a `v` to the version number: `v0.17.0`. Before Karpenter moved to OCI helm charts (pre-v0.17.0), both `v0.16.0` and `0.16.0` would work, but OCI charts require an exact version match. +- Sometimes Helm generates a generic error, you can add the --debug switch to any of the Helm commands in this doc for more verbose error messages +- If you are getting a 403 forbidden error, you can try `docker logout public.ecr.aws` as explained [here](https://docs.aws.amazon.com/AmazonECR/latest/public/public-troubleshooting.html). ### Helm Error when installing the `karpenter-crd` chart -Karpenter v0.26.1+ introduced the `karpenter-crd` helm chart. When installing this chart on your cluster, if you have previously added the Karpenter CRDs to your cluster through the `karpenter` controller chart or through `kubectl replace`, Helm will reject the install of the chart due to `invalid ownership metadata`. +Karpenter `0.26.1` introduced the `karpenter-crd` Helm chart. When installing this chart on your cluster, if you have previously added the Karpenter CRDs to your cluster through the `karpenter` controller chart or through `kubectl replace`, Helm will reject the install of the chart due to `invalid ownership metadata`. - In the case of `invalid ownership metadata; label validation error: missing key "app.kubernetes.io/managed-by": must be set to "Helm"` run: ```shell -kubectl label crd awsnodetemplates.karpenter.k8s.aws provisioners.karpenter.sh machines.karpenter.sh app.kubernetes.io/managed-by=Helm --overwrite +kubectl label crd ec2nodeclasses.karpenter.k8s.aws nodepools.karpenter.sh nodeclaims.karpenter.sh app.kubernetes.io/managed-by=Helm --overwrite ``` - In the case of `annotation validation error: missing key "meta.helm.sh/release-namespace": must be set to "karpenter"` run: ```shell -kubectl annotate crd awsnodetemplates.karpenter.k8s.aws provisioners.karpenter.sh machines.karpenter.sh meta.helm.sh/release-name=karpenter-crd --overwrite -kubectl annotate crd awsnodetemplates.karpenter.k8s.aws provisioners.karpenter.sh machines.karpenter.sh meta.helm.sh/release-namespace=karpenter --overwrite +kubectl annotate crd ec2nodeclasses.karpenter.k8s.aws nodepools.karpenter.sh nodeclaims.karpenter.sh meta.helm.sh/release-name=karpenter-crd --overwrite +kubectl annotate crd ec2nodeclasses.karpenter.k8s.aws nodepools.karpenter.sh nodeclaims.karpenter.sh meta.helm.sh/release-namespace=karpenter --overwrite ``` ## Uninstallation @@ -137,7 +149,7 @@ kubectl get nodes -ojsonpath='{range .items[*].metadata}{@.name}:{@.finalizers}{ If you are not able to create a provisioner due to `Internal error occurred: failed calling webhook "validation.webhook.provisioners.karpenter.sh":` -Webhooks were renamed in `v0.19.0`. There's a bug in ArgoCD's upgrade workflow where webhooks are leaked. This results in Provisioner's failing to be validated, since the validation server no longer corresponds to the webhook definition. +Webhooks were renamed in `0.19.0`. There's a bug in ArgoCD's upgrade workflow where webhooks are leaked. This results in Provisioner's failing to be validated, since the validation server no longer corresponds to the webhook definition. Delete the stale webhooks. @@ -148,7 +160,7 @@ kubectl delete validatingwebhookconfiguration validation.webhook.provisioners.ka ### Failed calling webhook "defaulting.webhook.karpenter.sh" -The `defaulting.webhook.karpenter.sh` mutating webhook was removed in `v0.27.3`. If you are coming from an older version of Karpenter where this webhook existed and the webhook was not managed by Helm, you may need to delete the stale webhook. +The `defaulting.webhook.karpenter.sh` mutating webhook was removed in `0.27.3`. If you are coming from an older version of Karpenter where this webhook existed and the webhook was not managed by Helm, you may need to delete the stale webhook. ```text kubectl delete mutatingwebhookconfigurations defaulting.webhook.karpenter.sh @@ -192,11 +204,11 @@ Disabling swap will allow kubelet to join the cluster successfully, however user ### DaemonSets can result in deployment failures -For Karpenter versions 0.5.3 and earlier, DaemonSets were not properly considered when provisioning nodes. +For Karpenter versions `0.5.3` and earlier, DaemonSets were not properly considered when provisioning nodes. This sometimes caused nodes to be deployed that could not meet the needs of the requested DaemonSets and workloads. -This issue no longer occurs after Karpenter version 0.5.3 (see [PR #1155](https://github.com/aws/karpenter/pull/1155)). +This issue no longer occurs after Karpenter version `0.5.3` (see [PR #1155](https://github.com/aws/karpenter/pull/1155)). -If you are using a pre-0.5.3 version of Karpenter, one workaround is to set your provisioner to only use larger instance types that you know will be big enough for the DaemonSet and the workload. +If you are using a pre `0.5.3` version of Karpenter, one workaround is to set your provisioner to only use larger instance types that you know will be big enough for the DaemonSet and the workload. For more information, see [Issue #1084](https://github.com/aws/karpenter/issues/1084). Examples of this behavior are included in [Issue #1180](https://github.com/aws/karpenter/issues/1180). @@ -213,7 +225,7 @@ See the Karpenter [Best Practices Guide](https://aws.github.io/aws-eks-best-prac ### Missing subnetSelector and securityGroupSelector tags causes provisioning failures -Starting with Karpenter v0.5.5, if you are using Karpenter-generated launch template, provisioners require that [subnetSelector]({{}}) and [securityGroupSelector]({{}}) tags be set to match your cluster. +Starting with Karpenter `0.5.5`, if you are using Karpenter-generated launch template, provisioners require that [subnetSelector]({{}}) and [securityGroupSelector]({{}}) tags be set to match your cluster. The [Provisioner]({{}}) section in the Karpenter Getting Started Guide uses the following example: ```text @@ -284,7 +296,23 @@ Karpenter does not support [in-tree storage plugins](https://kubernetes.io/blog/ #### Pods were scheduled due to a race condition in Kubernetes -Due to [this race condition in Kubernetes](https://github.com/kubernetes/kubernetes/issues/95911), it's possible that the scheduler and the CSINode can race during node registration such that the scheduler assumes that a node can mount more volumes than the node attachments support. There is currently no solve for this problem other than enforcing `toplogySpreadConstraints` and `podAntiAffinity` on your workloads that use PVCs such that you attempt to reduce the number of PVCs that schedule to a given node. +Due to [this race condition in Kubernetes](https://github.com/kubernetes/kubernetes/issues/95911), it's possible that the scheduler and the CSINode can race during node registration such that the scheduler assumes that a node can mount more volumes than the node attachments support. There is currently no universal solve for this problem other than enforcing `toplogySpreadConstraints` and `podAntiAffinity` on your workloads that use PVCs such that you attempt to reduce the number of PVCs that schedule to a given node. + +The following is a list of known CSI drivers which support a startupTaint to eliminate this issue: +- [aws-ebs-csi-driver](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/master/docs/install.md#configure-node-startup-taint) +- [aws-efs-csi-driver](https://github.com/kubernetes-sigs/aws-efs-csi-driver/tree/master/docs#configure-node-startup-taint) + +These taints should be configured via `startupTaints` on your `NodePool`. For example, to enable this for EBS, add the following to your `NodePool`: +```yaml +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +spec: + template: + spec: + startupTaints: + - key: ebs.csi.aws.com/agent-not-ready + effect: NoExecute +``` ### CNI is unable to allocate IPs to pods @@ -642,7 +670,7 @@ Karpenter [doesn't currently support draining and terminating on spot rebalance These two components do not share information between each other, meaning if you have drain and terminate functionality enabled on NTH, NTH may remove a node for a spot rebalance recommendation. Karpenter will replace the node to fulfill the pod capacity that was being fulfilled by the old node; however, Karpenter won't be aware of the reason that that node was terminated. This means that Karpenter may launch the same instance type that was just deprovisioned, causing a spot rebalance recommendation to be sent again. This can result in very short-lived instances where NTH continually removes nodes and Karpeneter re-launches the same instance type over and over again. -Karpenter doesn't recommend reacting to spot rebalance recommendations when running Karpenter with spot nodes; however, if you absolutely require this functionality, note that the above scenario is possible. +Karpenter doesn't recommend reacting to spot rebalance recommendations when running Karpenter with spot nodes; however, if you absolutely require this functionality, note that the above scenario is possible. Spot instances are time limited and, therefore, interruptible. When a signal is sent by AWS, it triggers actions from NTH and Karpenter, where the former signals a shutdown and the later provisions, creating a recursive situation. This can be mitigated by either completely removing NTH or by setting the following values: diff --git a/website/content/en/preview/upgrading/compatibility.md b/website/content/en/preview/upgrading/compatibility.md index 27068105d7c6..0f2c8d6cab94 100644 --- a/website/content/en/preview/upgrading/compatibility.md +++ b/website/content/en/preview/upgrading/compatibility.md @@ -15,9 +15,9 @@ Before you begin upgrading Karpenter, consider Karpenter compatibility issues re [comment]: <> (the content below is generated from hack/docs/compataiblitymetrix_gen_docs.go) -| KUBERNETES | 1.23 | 1.24 | 1.25 | 1.26 | 1.27 | 1.28 | -|------------|---------|---------|---------|---------|---------|---------| -| karpenter | 0.21.x+ | 0.21.x+ | 0.25.x+ | 0.28.x+ | 0.28.x+ | 0.31.x+ | +| KUBERNETES | 1.23 | 1.24 | 1.25 | 1.26 | 1.27 | 1.28 | 1.29 | +|------------|----------|----------|----------|----------|----------|----------|------------| +| karpenter | \>= 0.21 | \>= 0.21 | \>= 0.25 | \>= 0.28 | \>= 0.28 | \>= 0.31 | \>= 0.34.0 | [comment]: <> (end docs generated content from hack/docs/compataiblitymetrix_gen_docs.go) @@ -31,7 +31,7 @@ For more information on Karpenter's support for these keys, view [this tracking {{% /alert %}} {{% alert title="Note" color="warning" %}} -Karpenter supports using [Kubernetes Common Expression Language](https://kubernetes.io/docs/reference/using-api/cel/) for validating its Custom Resource Definitions out-of-the-box; however, this feature is not supported on versions of Kubernetes < 1.25. If you are running an earlier version of Kubernetes, you will need to use the Karpenter admission webhooks for validation instead. You can enable these webhooks with `--set webhook.enabled=true` when applying the Karpenter helm chart. +Karpenter supports using [Kubernetes Common Expression Language](https://kubernetes.io/docs/reference/using-api/cel/) for validating its Custom Resource Definitions out-of-the-box; however, this feature is not supported on versions of Kubernetes < 1.25. If you are running an earlier version of Kubernetes, you will need to use the Karpenter admission webhooks for validation instead. You can enable these webhooks with `--set webhook.enabled=true` when applying the Karpenter Helm chart. {{% /alert %}} ## Compatibility issues @@ -39,7 +39,7 @@ Karpenter supports using [Kubernetes Common Expression Language](https://kuberne When we introduce a breaking change, we do so only as described in this document. Karpenter follows [Semantic Versioning 2.0.0](https://semver.org/) in its stable release versions, while in -major version zero (v0.y.z) [anything may change at any time](https://semver.org/#spec-item-4). +major version zero (`0.y.z`) [anything may change at any time](https://semver.org/#spec-item-4). However, to further protect users during this phase we will only introduce breaking changes in minor releases (releases that increment y in x.y.z). Note this does not mean every minor upgrade has a breaking change as we will also increment the minor version when we release a new feature. @@ -51,7 +51,7 @@ Users should therefore check to see if there is a breaking change every time the When there is a breaking change we will: * Increment the minor version when in major version 0 -* Add a permanent separate section named `upgrading to vx.y.z+` under [release upgrade notes](#release-upgrade-notes) +* Add a permanent separate section named `upgrading to x.y.z+` under [release upgrade notes](#release-upgrade-notes) clearly explaining the breaking change and what needs to be done on the user side to ensure a safe upgrade * Add the sentence “This is a breaking change, please refer to the above link for upgrade instructions” to the top of the release notes and in all our announcements @@ -77,7 +77,7 @@ Karpenter offers three types of releases. This section explains the purpose of e Stable releases are the most reliable releases that are released with weekly cadence. Stable releases are our only recommended versions for production environments. Sometimes we skip a stable release because we find instability or problems that need to be fixed before having a stable release. -Stable releases are tagged with Semantic Versioning. For example `v0.13.0`. +Stable releases are tagged with a semantic version prefixed by a `v`. For example `v0.13.0`. ### Release Candidates @@ -89,7 +89,7 @@ By adopting this practice we allow our users who are early adopters to test out We release a snapshot release for every commit that gets merged into [`aws/karpenter-provider-aws`](https://www.github.com/aws/karpenter-provider-aws). This enables users to immediately try a new feature or fix right after it's merged rather than waiting days or weeks for release. Snapshot releases are not made available in the same public ECR repository as other release types, they are instead published to a separate private ECR repository. -Helm charts are published to `oci://{{< param "snapshot_repo.account_id" >}}.dkr.ecr.{{< param "snapshot_repo.region" >}}.amazonaws.com/karpenter/snapshot/karpenter` and are tagged with the git commit hash prefixed by the Karpenter major version (e.g. `v0-fc17bfc89ebb30a3b102a86012b3e3992ec08adf`). +Helm charts are published to `oci://{{< param "snapshot_repo.account_id" >}}.dkr.ecr.{{< param "snapshot_repo.region" >}}.amazonaws.com/karpenter/snapshot/karpenter` and are tagged with the git commit hash prefixed by the Karpenter major version (e.g. `0-fc17bfc89ebb30a3b102a86012b3e3992ec08adf`). Anyone with an AWS account can pull from this repository, but must first authenticate: ```bash @@ -99,4 +99,3 @@ aws ecr get-login-password --region {{< param "snapshot_repo.region" >}} | docke {{% alert title="Note" color="warning" %}} Snapshot releases are suitable for testing, and troubleshooting but they should not be used in production environments. Snapshot releases are ephemeral and will be removed 90 days after they were published. {{% /alert %}} - diff --git a/website/content/en/preview/upgrading/upgrade-guide.md b/website/content/en/preview/upgrading/upgrade-guide.md index e74b001ba134..32567d981900 100644 --- a/website/content/en/preview/upgrading/upgrade-guide.md +++ b/website/content/en/preview/upgrading/upgrade-guide.md @@ -13,19 +13,19 @@ This guide contains information needed to upgrade to the latest release of Karpe ### CRD Upgrades Karpenter ships with a few Custom Resource Definitions (CRDs). These CRDs are published: -* As an independent helm chart [karpenter-crd](https://gallery.ecr.aws/karpenter/karpenter-crd) - [source](https://github.com/aws/karpenter/blob/main/charts/karpenter-crd) that can be used by Helm to manage the lifecycle of these CRDs. To upgrade or install `karpenter-crd` run: +* As an independent Helm chart [karpenter-crd](https://gallery.ecr.aws/karpenter/karpenter-crd) - [source](https://github.com/aws/karpenter/blob/main/charts/karpenter-crd) that can be used by Helm to manage the lifecycle of these CRDs. To upgrade or install `karpenter-crd` run: ```bash KARPENTER_NAMESPACE=kube-system - helm upgrade --install karpenter-crd oci://public.ecr.aws/karpenter/karpenter-crd --version vx.y.z --namespace "${KARPENTER_NAMESPACE}" --create-namespace + helm upgrade --install karpenter-crd oci://public.ecr.aws/karpenter/karpenter-crd --version x.y.z --namespace "${KARPENTER_NAMESPACE}" --create-namespace ``` {{% alert title="Note" color="warning" %}} If you get the error `invalid ownership metadata; label validation error:` while installing the `karpenter-crd` chart from an older version of Karpenter, follow the [Troubleshooting Guide]({{}}) for details on how to resolve these errors. {{% /alert %}} -* As part of the helm chart [karpenter](https://gallery.ecr.aws/karpenter/karpenter) - [source](https://github.com/aws/karpenter/blob/main/charts/karpenter/crds). Helm [does not manage the lifecycle of CRDs using this method](https://helm.sh/docs/chart_best_practices/custom_resource_definitions/), the tool will only install the CRD during the first installation of the helm chart. Subsequent chart upgrades will not add or remove CRDs, even if the CRDs have changed. When CRDs are changed, we will make a note in the version's upgrade guide. +* As part of the helm chart [karpenter](https://gallery.ecr.aws/karpenter/karpenter) - [source](https://github.com/aws/karpenter/blob/main/charts/karpenter/crds). Helm [does not manage the lifecycle of CRDs using this method](https://helm.sh/docs/chart_best_practices/custom_resource_definitions/), the tool will only install the CRD during the first installation of the Helm chart. Subsequent chart upgrades will not add or remove CRDs, even if the CRDs have changed. When CRDs are changed, we will make a note in the version's upgrade guide. -In general, you can reapply the CRDs in the `crds` directory of the Karpenter helm chart: +In general, you can reapply the CRDs in the `crds` directory of the Karpenter Helm chart: ```shell kubectl apply -f https://raw.githubusercontent.com/aws/karpenter{{< githubRelRef >}}pkg/apis/crds/karpenter.sh_nodepools.yaml @@ -33,88 +33,111 @@ kubectl apply -f https://raw.githubusercontent.com/aws/karpenter{{< githubRelRef kubectl apply -f https://raw.githubusercontent.com/aws/karpenter{{< githubRelRef >}}pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml ``` -### Upgrading to v0.34.0+ + -[comment]: <> (WHEN CREATING A NEW SECTION OF THE UPGRADE GUIDANCE FOR NEWER VERSIONS, ENSURE THAT YOU COPY THE ALERT SECTION BELOW TO PROPERLY WARN USERS OF THE RISK OF UPGRADING WITHOUT GOING TO v0.32 FIRST) +### Upgrading to `0.36.0`+ {{% alert title="Warning" color="warning" %}} -v0.34.0+ _only_ supports Karpenter v1beta1 APIs and will not work with existing Provisioner, AWSNodeTemplate or Machine alpha APIs. Do not upgrade to v0.34.0+ without first [upgrading to v0.32.x]({{}}). This version supports both the alpha and beta APIs, allowing you to migrate all of your existing APIs to beta APIs without experiencing downtime. +`0.33.0`+ _only_ supports Karpenter v1beta1 APIs and will not work with existing Provisioner, AWSNodeTemplate or Machine alpha APIs. Do not upgrade to `0.36.0`+ without first [upgrading to `0.32.x`]({{}}). This version supports both the alpha and beta APIs, allowing you to migrate all of your existing APIs to beta APIs without experiencing downtime. {{% /alert %}} -* Karpenter now supports `nodepool.spec.disruption.budgets`, which allows users to control the speed of disruption in the cluster. Since this requires an update to the Custom Resource, before upgrading, you should re-apply the new updates to the CRDs. Check out [Disruption Budgets]({{}}) for more. -* With Disruption Budgets, Karpenter will disrupt multiple batches of nodes simultaneously, which can result in overall quicker scale-down of your cluster. Before v0.34, Karpenter had a hard-coded parallelism limit for each type of disruption. In v0.34, Karpenter will now disrupt at most 10% of nodes for a given NodePool. There is no setting that will be perfectly equivalent with the behavior prior to v0.34. When considering how to configure your budgets, please refer to the following limits for versions prior to v0.34: - * `Empty Expiration / Empty Drift / Empty Consolidation`: infinite parallelism +* Karpenter changed the name of the `karpenter_cloudprovider_instance_type_price_estimate` metric to `karpenter_cloudprovider_instance_type_offering_price_estimate` to align with the new `karpenter_cloudprovider_instance_type_offering_available` metric. The `region` label was also dropped from the metric, since this can be inferred from the environment that Karpenter is running in. + +### Upgrading to `0.35.0`+ + +{{% alert title="Warning" color="warning" %}} +`0.33.0`+ _only_ supports Karpenter v1beta1 APIs and will not work with existing Provisioner, AWSNodeTemplate or Machine alpha APIs. Do not upgrade to `0.35.0`+ without first [upgrading to `0.32.x`]({{}}). This version supports both the alpha and beta APIs, allowing you to migrate all of your existing APIs to beta APIs without experiencing downtime. +{{% /alert %}} + +* Karpenter OCI tags and Helm chart version are now valid semantic versions, meaning that the `v` prefix from the git tag has been removed and they now follow the `x.y.z` pattern. + +### Upgrading to `0.34.0`+ + +{{% alert title="Warning" color="warning" %}} +`0.33.0`+ _only_ supports Karpenter v1beta1 APIs and will not work with existing Provisioner, AWSNodeTemplate or Machine alpha APIs. Do not upgrade to `0.34.0`+ without first [upgrading to `0.32.x`]({{}}). This version supports both the alpha and beta APIs, allowing you to migrate all of your existing APIs to beta APIs without experiencing downtime. +{{% /alert %}} + +{{% alert title="Warning" color="warning" %}} +The Ubuntu EKS optimized AMI has moved from 20.04 to 22.04 for Kubernetes 1.29+. This new AMI version is __not currently__ supported for users relying on AMI auto-discovery with the Ubuntu AMI family. More details can be found in this [GitHub issue](https://github.com/aws/karpenter-provider-aws/issues/5572). Please review this issue before upgrading to Kubernetes 1.29 if you are using the Ubuntu AMI family. Upgrading to 1.29 without making any changes to your EC2NodeClass will result in Karpenter being unable to create new nodes. +{{% /alert %}} + +* Karpenter now supports `nodepool.spec.disruption.budgets`, which allows users to control the speed of disruption in the cluster. Since this requires an update to the Custom Resource, before upgrading, you should re-apply the new updates to the CRDs. Check out [Disruption Budgets]({{}}) for more. +* With Disruption Budgets, Karpenter will disrupt multiple batches of nodes simultaneously, which can result in overall quicker scale-down of your cluster. Before `0.34.0`, Karpenter had a hard-coded parallelism limit for each type of disruption. In `0.34.0`+, Karpenter will now disrupt at most 10% of nodes for a given NodePool. There is no setting that will be perfectly equivalent with the behavior prior to `0.34.0`. When considering how to configure your budgets, please refer to the following limits for versions prior to `0.34.0`: + * `Empty Expiration / Empty Drift / Empty Consolidation`: infinite parallelism * `Non-Empty Expiration / Non-Empty Drift / Single-Node Consolidation`: one node at a time * `Multi-Node Consolidation`: max 100 nodes +* To support Disruption Budgets, `0.34.0`+ includes critical changes to Karpenter's core controllers, which allows Karpenter to consider multiple batches of disrupting nodes simultaneously. This increases Karpenter's performance with the potential downside of higher CPU and memory utilization from the Karpenter pod. While the magnitude of this difference varies on a case-by-case basis, when upgrading to Karpenter `0.34.0`+, please note that you may need to increase the resources allocated to the Karpenter controller pods. * Karpenter now adds a default `podSecurityContext` that configures the `fsgroup: 65536` of volumes in the pod. If you are using sidecar containers, you should review if this configuration is compatible for them. You can disable this default `podSecurityContext` through helm by performing `--set podSecurityContext=null` when installing/upgrading the chart. -* The `dnsPolicy` for the Karpenter controller pod has been changed back to the Kubernetes cluster default of `ClusterFirst`. Setting our `dnsPolicy` to `Default` (confusingly, this is not the Kubernetes cluster default) caused more confusion for any users running IPv6 clusters with dual-stack nodes or anyone running Karpenter with dependencies on cluster services (like clusters running service meshes). If you still want the old behavior here, you can change the `dnsPolicy` to point to use `Default` by setting the helm value on install/upgrade with `--set dnsPolicy=Default`. More details on this issue can be found in the following Github issues: [#2186](https://github.com/aws/karpenter-provider-aws/issues/2186) and [#4947](https://github.com/aws/karpenter-provider-aws/issues/4947). - -### Upgrading to v0.33.0+ +* The `dnsPolicy` for the Karpenter controller pod has been changed back to the Kubernetes cluster default of `ClusterFirst`. Setting our `dnsPolicy` to `Default` (confusingly, this is not the Kubernetes cluster default) caused more confusion for any users running IPv6 clusters with dual-stack nodes or anyone running Karpenter with dependencies on cluster services (like clusters running service meshes). This change may be breaking for any users on Fargate or MNG who were allowing Karpenter to manage their in-cluster DNS service (`core-dns` on most clusters). If you still want the old behavior here, you can change the `dnsPolicy` to point to use `Default` by setting the helm value on install/upgrade with `--set dnsPolicy=Default`. More details on this issue can be found in the following Github issues: [#2186](https://github.com/aws/karpenter-provider-aws/issues/2186) and [#4947](https://github.com/aws/karpenter-provider-aws/issues/4947). +* Karpenter now disallows `nodepool.spec.template.spec.resources` to be set. The webhook validation never allowed `nodepool.spec.template.spec.resources`. We are now ensuring that CEL validation also disallows `nodepool.spec.template.spec.resources` to be set. If you were previously setting the resources field on your NodePool, ensure that you remove this field before upgrading to the newest version of Karpenter or else updates to the resource may fail on the new version. -[comment]: <> (WHEN CREATING A NEW SECTION OF THE UPGRADE GUIDANCE FOR NEWER VERSIONS, ENSURE THAT YOU COPY THE ALERT SECTION BELOW TO PROPERLY WARN USERS OF THE RISK OF UPGRADING WITHOUT GOING TO v0.32 FIRST) +### Upgrading to `0.33.0`+ {{% alert title="Warning" color="warning" %}} -v0.33.0+ _only_ supports Karpenter v1beta1 APIs and will not work with existing Provisioner, AWSNodeTemplate or Machine alpha APIs. **Do not** upgrade to v0.33.0+ without first [upgrading to v0.32.x]({{}}). This version supports both the alpha and beta APIs, allowing you to migrate all of your existing APIs to beta APIs without experiencing downtime. +`0.33.0`+ _only_ supports Karpenter v1beta1 APIs and will not work with existing Provisioner, AWSNodeTemplate or Machine alpha APIs. **Do not** upgrade to `0.33.0`+ without first [upgrading to `0.32.x`]({{}}). This version supports both the alpha and beta APIs, allowing you to migrate all of your existing APIs to beta APIs without experiencing downtime. {{% /alert %}} +* Karpenter no longer supports using the `karpenter.sh/provisioner-name` label in NodePool labels and requirements or in application node selectors, affinities, or topologySpreadConstraints. If you were previously using this label to target applications to specific Provisioners, you should update your applications to use the `karpenter.sh/nodepool` label instead before upgrading. If you upgrade without changing these labels, you may begin to see pod scheduling failures for these applications. * Karpenter now tags `spot-instances-request` with the same tags that it tags instances, volumes, and primary ENIs. This means that you will now need to add `ec2:CreateTags` permission for `spot-instances-request`. You can also further scope your controller policy for the `ec2:RunInstances` action to require that it launches the `spot-instances-request` with these specific tags. You can view an example of scoping these actions in the [Getting Started Guide's default CloudFormation controller policy](https://github.com/aws/karpenter/blob/v0.33.0/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml#L61). * We now recommend that you set the installation namespace for your Karpenter controllers to `kube-system` to denote Karpenter as a critical cluster component. This ensures that requests from the Karpenter controllers are treated with higher priority by assigning them to a different [PriorityLevelConfiguration](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/#prioritylevelconfiguration) than generic requests from other namespaces. For more details on API Priority and Fairness, read the [Kubernetes API Priority and Fairness Conceptual Docs](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/). Note: Changing the namespace for your Karpenter release will cause the service account namespace to change. If you are using IRSA for authentication with AWS, you will need to change scoping set in the controller's trust policy from `karpenter:karpenter` to `kube-system:karpenter`. -* `v0.33.x` disables mutating and validating webhooks by default in favor of using [Common Expression Language for CRD validation](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#validation). The Common Expression Language Validation Feature [is enabled by default on EKS 1.25](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#validation-rules). If you are using Kubernetes version >= 1.25, no further action is required. If you are using a Kubernetes version below 1.25, you now need to set `DISABLE_WEBHOOK=false` in your container environment variables or `--set webhook.enabled=true` if using Helm. View the [Webhook Support Deprecated in Favor of CEL Section of the v1beta1 Migration Guide]({{}}). -* `v0.33.x` drops support for passing settings through the `karpenter-global-settings` ConfigMap. You should pass settings through the container environment variables in the Karpenter deployment manifest. View the [Global Settings Section of the v1beta1 Migration Guide]({{}}) for more details. -* `v0.33.x` enables `Drift=true` by default in the `FEATURE_GATES`. If you previously didn't enable the feature gate, Karpenter will now check if there is a difference between the desired state of your nodes declared in your NodePool and the actual state of your nodes. View the [Drift Section of Disruption Conceptual Docs]({{}}) for more details. -* `v0.33.x` drops looking up the `zap-logger-config` through ConfigMap discovery. Instead, Karpenter now expects the logging config to be mounted on the filesystem if you are using this to configure Zap logging. This is not enabled by default, but can be enabled through `--set logConfig.enabled=true` in the helm values. If you are setting any values in the `logConfig` from the `v0.32.x` upgrade, such as `logConfig.logEncoding`, note that you will have to explicitly set `logConfig.enabled=true` alongside it. Also, note that setting the Zap logging config is a deprecated feature in beta and is planned to be dropped at v1. View the [Logging Configuration Section of the v1beta1 Migration Guide]({{}}) for more details. -* `v0.33.x` change the default `LOG_LEVEL` from `debug` to `info` by default. If you are still enabling logging configuration through the `zap-logger-config`, no action is required. -* `v0.33.x` drops support for comma delimited lists on tags for `SubnetSelectorTerm`, `SecurityGroupsSelectorTerm`, and `AMISelectorTerm`. Karpenter now supports multiple terms for each of the selectors which means that we can specify a more explicit OR-based constraint through separate terms rather than a comma-delimited list of values. +* `0.33.0` disables mutating and validating webhooks by default in favor of using [Common Expression Language for CRD validation](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#validation). The Common Expression Language Validation Feature [is enabled by default on EKS 1.25](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#validation-rules). If you are using Kubernetes version >= 1.25, no further action is required. If you are using a Kubernetes version below 1.25, you now need to set `DISABLE_WEBHOOK=false` in your container environment variables or `--set webhook.enabled=true` if using Helm. View the [Webhook Support Deprecated in Favor of CEL Section of the v1beta1 Migration Guide]({{}}). +* `0.33.0` drops support for passing settings through the `karpenter-global-settings` ConfigMap. You should pass settings through the container environment variables in the Karpenter deployment manifest. View the [Global Settings Section of the v1beta1 Migration Guide]({{}}) for more details. +* `0.33.0` enables `Drift=true` by default in the `FEATURE_GATES`. If you previously didn't enable the feature gate, Karpenter will now check if there is a difference between the desired state of your nodes declared in your NodePool and the actual state of your nodes. View the [Drift Section of Disruption Conceptual Docs]({{}}) for more details. +* `0.33.0` drops looking up the `zap-logger-config` through ConfigMap discovery. Instead, Karpenter now expects the logging config to be mounted on the filesystem if you are using this to configure Zap logging. This is not enabled by default, but can be enabled through `--set logConfig.enabled=true` in the Helm values. If you are setting any values in the `logConfig` from the `0.32.x` upgrade, such as `logConfig.logEncoding`, note that you will have to explicitly set `logConfig.enabled=true` alongside it. Also, note that setting the Zap logging config is a deprecated feature in beta and is planned to be dropped at v1. View the [Logging Configuration Section of the v1beta1 Migration Guide]({{}}) for more details. +* `0.33.0` change the default `LOG_LEVEL` from `debug` to `info` by default. If you are still enabling logging configuration through the `zap-logger-config`, no action is required. +* `0.33.0` drops support for comma delimited lists on tags for `SubnetSelectorTerm`, `SecurityGroupsSelectorTerm`, and `AMISelectorTerm`. Karpenter now supports multiple terms for each of the selectors which means that we can specify a more explicit OR-based constraint through separate terms rather than a comma-delimited list of values. -### Upgrading to v0.32.0+ +### Upgrading to `0.32.0`+ {{% alert title="Warning" color="warning" %}} -Karpenter v0.32.0 introduces v1beta1 APIs, including _significant_ changes to the API and installation procedures for the Karpenter controllers. **Do not** upgrade to v0.32.0+ without referencing the [v1beta1 Migration Upgrade Procedure]({{}}). +Karpenter `0.32.0` introduces v1beta1 APIs, including _significant_ changes to the API and installation procedures for the Karpenter controllers. **Do not** upgrade to `0.32.0`+ without referencing the [v1beta1 Migration Upgrade Procedure]({{}}). This version includes **dual support** for both alpha and beta APIs to ensure that you can slowly migrate your existing Provisioner, AWSNodeTemplate, and Machine alpha APIs to the newer NodePool, EC2NodeClass, and NodeClaim beta APIs. -Note that if you are rolling back after upgrading to v0.32.0, note that v0.31.2 and v0.31.3 are the only versions that support handling rollback after you have deployed the v1beta1 APIs to your cluster. +Note that if you are rolling back after upgrading to `0.32.0`, note that `0.31.4` is the only version that supports handling rollback after you have deployed the v1beta1 APIs to your cluster. {{% /alert %}} * Karpenter now serves the webhook prometheus metrics server on port `8001`. If this port is already in-use on the pod or you are running in `hostNetworking` mode, you may need to change this port value. You can configure this port value through the `WEBHOOK_METRICS_PORT` environment variable or the `webhook.metrics.port` value if installing via Helm. * Karpenter now exposes the ability to disable webhooks through the `webhook.enabled=false` value. This value will disable the webhook server and will prevent any permissions, mutating or validating webhook configurations from being deployed to the cluster. * Karpenter now moves all logging configuration for the Zap logger into the `logConfig` values block. Configuring Karpenter logging with this mechanism _is_ deprecated and will be dropped at v1. Karpenter now only surfaces logLevel through the `logLevel` helm value. If you need more advanced configuration due to log parsing constraints, we recommend configuring your log parser to handle Karpenter's Zap JSON logging. -* The default log encoding changed from `console` to `json`. If you were previously not setting the type of log encoding, this default will change with the helm chart. If you were setting the value through `logEncoding`, this value will continue to work until v0.33.x but it is deprecated in favor of `logConfig.logEncoding` +* The default log encoding changed from `console` to `json`. If you were previously not setting the type of log encoding, this default will change with the Helm chart. If you were setting the value through `logEncoding`, this value will continue to work until `0.33.x` but it is deprecated in favor of `logConfig.logEncoding` * Karpenter now uses the `karpenter.sh/disruption:NoSchedule=disrupting` taint instead of the upstream `node.kubernetes.io/unschedulable` taint for nodes spawned with a NodePool to prevent pods from scheduling to nodes being disrupted. Pods that previously tolerated the `node.kubernetes.io/unschedulable` taint that previously weren't evicted during termination will now be evicted. This most notably affects DaemonSets, which have the `node.kubernetes.io/unschedulable` toleration by default, where Karpenter will now remove these pods during termination. If you want your specific pods to not be evicted when nodes are scaled down, you should add a toleration to the pods with the following: `Key=karpenter.sh/disruption, Effect=NoSchedule, Operator=Equals, Values=disrupting`. * Note: Karpenter will continue to use the old `node.kubernetes.io/unschedulable` taint for nodes spawned with a Provisioner. -### Upgrading to v0.31.0+ +### Upgrading to `0.31.0`+ * Karpenter moved its `securityContext` constraints from pod-wide to only applying to the Karpenter container exclusively. If you were previously relying on the pod-wide `securityContext` for your sidecar containers, you will now need to set these values explicitly in your sidecar container configuration. -### Upgrading to v0.30.0+ +### Upgrading to `0.30.0`+ * Karpenter will now [statically drift]({{}}) on both Provisioner and AWSNodeTemplate Fields. For Provisioner Static Drift, the `karpenter.sh/provisioner-hash` annotation must be present on both the Provisioner and Machine. For AWSNodeTemplate drift, the `karpenter.k8s.aws/nodetemplate-hash` annotation must be present on the AWSNodeTemplate and Machine. Karpenter will not add these annotations to pre-existing nodes, so each of these nodes will need to be recycled one time for the annotations to be added. * Karpenter will now fail validation on AWSNodeTemplates and Provisioner `spec.provider` that have `amiSelectors`, `subnetSelectors`, or `securityGroupSelectors` set with a combination of id selectors (`aws-ids`, `aws::ids`) and other selectors. -* Karpenter now statically sets the `securityContext` at both the pod and container-levels and doesn't allow override values to be passed through the helm chart. This change was made to adhere to [Restricted Pod Security Standard](https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted), which follows pod hardening best practices. +* Karpenter now statically sets the `securityContext` at both the pod and container-levels and doesn't allow override values to be passed through the Helm chart. This change was made to adhere to [Restricted Pod Security Standard](https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted), which follows pod hardening best practices. {{% alert title="Note" color="primary" %}} If you have sidecar containers configured to run alongside Karpenter that cannot tolerate the [pod-wide `securityContext` constraints](https://github.com/aws/karpenter/blob/v0.30.0/charts/karpenter/templates/deployment.yaml#L40), you will need to specify overrides to the sidecar `securityContext` in your deployment. {{% /alert %}} -### Upgrading to v0.29.0+ +### Upgrading to `0.29.0`+ {{% alert title="Warning" color="warning" %}} -Karpenter `v0.29.1` contains a [file descriptor and memory leak bug](https://github.com/aws/karpenter/issues/4296) that leads to Karpenter getting OOMKilled and restarting at the point that it hits its memory or file descriptor limit. Karpenter `>v0.29.2` fixes this leak. +Karpenter `0.29.1` contains a [file descriptor and memory leak bug](https://github.com/aws/karpenter/issues/4296) that leads to Karpenter getting OOMKilled and restarting at the point that it hits its memory or file descriptor limit. Karpenter `0.29.2`+ fixes this leak. {{% /alert %}} -* Karpenter has changed the default metrics service port from 8080 to 8000 and the default webhook service port from 443 to 8443. In `v0.28.0`, the Karpenter pod port was changed to 8000, but referenced the service by name, allowing users to scrape the service at port 8080 for metrics. `v0.29.0` aligns the two ports so that service and pod metrics ports are the same. These ports are set by the `controller.metrics.port` and `webhook.port` helm chart values, so if you have previously set these to non-default values, you may need to update your Prometheus scraper to match these new values. +* Karpenter has changed the default metrics service port from 8080 to 8000 and the default webhook service port from 443 to 8443. In `0.28.0`, the Karpenter pod port was changed to 8000, but referenced the service by name, allowing users to scrape the service at port 8080 for metrics. `0.29.0` aligns the two ports so that service and pod metrics ports are the same. These ports are set by the `controller.metrics.port` and `webhook.port` Helm chart values, so if you have previously set these to non-default values, you may need to update your Prometheus scraper to match these new values. * Karpenter will now reconcile nodes that are drifted due to their Security Groups or their Subnets. If your AWSNodeTemplate's Security Groups differ from the Security Groups used for an instance, Karpenter will consider it drifted. If the Subnet used by an instance is not contained in the allowed list of Subnets for an AWSNodeTemplate, Karpenter will also consider it drifted. * Since Karpenter uses tags for discovery of Subnets and SecurityGroups, check the [Threat Model]({{}}) to see how to manage this IAM Permission. -### Upgrading to v0.28.0+ +### Upgrading to `0.28.0`+ {{% alert title="Warning" color="warning" %}} -Karpenter `v0.28.0` is incompatible with Kubernetes version 1.26+, which can result in additional node scale outs when using `--cloudprovider=external`, which is the default for the EKS Optimized AMI. See: https://github.com/aws/karpenter-core/pull/375. Karpenter `>v0.28.1` fixes this issue and is compatible with Kubernetes version 1.26+. +Karpenter `0.28.0` is incompatible with Kubernetes version 1.26+, which can result in additional node scale outs when using `--cloudprovider=external`, which is the default for the EKS Optimized AMI. See: https://github.com/aws/karpenter-core/pull/375. Karpenter `0.28.1`+ fixes this issue and is compatible with Kubernetes version 1.26+. {{% /alert %}} -* The `extraObjects` value is now removed from the Helm chart. Having this value in the chart proved to not work in the majority of Karpenter installs and often led to anti-patterns, where the Karpenter resources installed to manage Karpenter's capacity were directly tied to the install of the Karpenter controller deployments. The Karpenter team recommends that, if you want to install Karpenter manifests alongside the Karpenter helm chart, to do so by creating a separate chart for the manifests, creating a dependency on the controller chart. +* The `extraObjects` value is now removed from the Helm chart. Having this value in the chart proved to not work in the majority of Karpenter installs and often led to anti-patterns, where the Karpenter resources installed to manage Karpenter's capacity were directly tied to the install of the Karpenter controller deployments. The Karpenter team recommends that, if you want to install Karpenter manifests alongside the Karpenter Helm chart, to do so by creating a separate chart for the manifests, creating a dependency on the controller chart. * The `aws.nodeNameConvention` setting is now removed from the [`karpenter-global-settings`]({{}}) ConfigMap. Because Karpenter is now driving its orchestration of capacity through Machines, it no longer needs to know the node name, making this setting obsolete. Karpenter ignores configuration that it doesn't recognize in the [`karpenter-global-settings`]({{}}) ConfigMap, so leaving the `aws.nodeNameConvention` in the ConfigMap will simply cause this setting to be ignored. * Karpenter now defines a set of "restricted tags" which can't be overridden with custom tagging in the AWSNodeTemplate or in the [`karpenter-global-settings`]({{}}) ConfigMap. If you are currently using any of these tag overrides when tagging your instances, webhook validation will now fail. These tags include: @@ -126,7 +149,7 @@ Karpenter `v0.28.0` is incompatible with Kubernetes version 1.26+, which can res * `karpenter_nodes_terminated`: Use `karpenter_machines_terminated` if you are interested in the reason why a Karpenter machine was deleted. `karpenter_nodes_terminated` now only tracks the count of terminated nodes without any additional labels. * `karpenter_nodes_created`: Use `karpenter_machines_created` if you are interested in the reason why a Karpenter machine was created. `karpenter_nodes_created` now only tracks the count of created nodes without any additional labels. * `karpenter_deprovisioning_replacement_node_initialized_seconds`: This metric has been replaced in favor of `karpenter_deprovisioning_replacement_machine_initialized_seconds`. -* `v0.28.0` introduces the Machine CustomResource into the `karpenter.sh` API Group and requires this CustomResourceDefinition to run properly. Karpenter now orchestrates its CloudProvider capacity through these in-cluster Machine CustomResources. When performing a scheduling decision, Karpenter will create a Machine, resulting in launching CloudProvider capacity. The kubelet running on the new capacity will then register the node to the cluster shortly after launch. +* `0.28.0` introduces the Machine CustomResource into the `karpenter.sh` API Group and requires this CustomResourceDefinition to run properly. Karpenter now orchestrates its CloudProvider capacity through these in-cluster Machine CustomResources. When performing a scheduling decision, Karpenter will create a Machine, resulting in launching CloudProvider capacity. The kubelet running on the new capacity will then register the node to the cluster shortly after launch. * If you are using Helm to upgrade between versions of Karpenter, note that [Helm does not automate the process of upgrading or install the new CRDs into your cluster](https://helm.sh/docs/chart_best_practices/custom_resource_definitions/#some-caveats-and-explanations). To install or upgrade the existing CRDs, follow the guidance under the [Custom Resource Definition (CRD) Upgrades]({{< relref "#custom-resource-definition-crd-upgrades" >}}) section of the upgrade guide. * Karpenter will hydrate Machines on startup for existing capacity managed by Karpenter into the cluster. Existing capacity launched by an older version of Karpenter is discovered by finding CloudProvider capacity with the `karpenter.sh/provisioner-name` tag or the `karpenter.sh/provisioner-name` label on nodes. * The metrics port for the Karpenter deployment was changed from 8080 to 8000. Users who scrape the pod directly for metrics rather than the service will need to adjust the commands they use to reference port 8000. Any users who scrape metrics from the service should be unaffected. @@ -142,19 +165,21 @@ Because Karpenter takes this dependency, any user that has the ability to Create {{% /alert %}} {{% alert title="Rolling Back" color="warning" %}} -If, after upgrading to `v0.28.0+`, a rollback to an older version of Karpenter needs to be performed, Karpenter will continue to function normally, though you will still have the Machine CustomResources on your cluster. You will need to manually delete the Machines and patch out the finalizers to fully complete the rollback. +If, after upgrading to `0.28.0`+, a rollback to an older version of Karpenter needs to be performed, Karpenter will continue to function normally, though you will still have the Machine CustomResources on your cluster. You will need to manually delete the Machines and patch out the finalizers to fully complete the rollback. -Karpenter marks CloudProvider capacity as "managed by" a Machine using the `karpenter-sh/managed-by` tag on the CloudProvider machine. It uses this tag to ensure that the Machine CustomResources in the cluster match the CloudProvider capacity managed by Karpenter. If these states don't match, Karpenter will garbage collect the capacity. Because of this, if performing an upgrade, followed by a rollback, followed by another upgrade to `v0.28.0+`, ensure you remove the `karpenter.sh/managed-by` tags from existing capacity; otherwise, Karpenter will deprovision the capacity without a Machine CR counterpart. +Karpenter marks CloudProvider capacity as "managed by" a Machine using the `karpenter-sh/managed-by` tag on the CloudProvider machine. It uses this tag to ensure that the Machine CustomResources in the cluster match the CloudProvider capacity managed by Karpenter. If these states don't match, Karpenter will garbage collect the capacity. Because of this, if performing an upgrade, followed by a rollback, followed by another upgrade to `0.28.0`+, ensure you remove the `karpenter.sh/managed-by` tags from existing capacity; otherwise, Karpenter will deprovision the capacity without a Machine CR counterpart. {{% /alert %}} -### Upgrading to v0.27.3+ -* The `defaulting.webhook.karpenter.sh` mutating webhook was removed in `v0.27.3`. If you are coming from an older version of Karpenter where this webhook existed and the webhook was not managed by Helm, you may need to delete the stale webhook. +### Upgrading to `0.27.3`+ + +* The `defaulting.webhook.karpenter.sh` mutating webhook was removed in `0.27.3`. If you are coming from an older version of Karpenter where this webhook existed and the webhook was not managed by Helm, you may need to delete the stale webhook. ```bash kubectl delete mutatingwebhookconfigurations defaulting.webhook.karpenter.sh ``` -### Upgrading to v0.27.0+ +### Upgrading to `0.27.0`+ + * The Karpenter controller pods now deploy with `kubernetes.io/hostname` self anti-affinity by default. If you are running Karpenter in HA (high-availability) mode and you do not have enough nodes to match the number of pod replicas you are deploying with, you will need to scale-out your nodes for Karpenter. * The following controller metrics changed and moved under the `controller_runtime` metrics namespace: * `karpenter_metricscraper_...` @@ -171,27 +196,33 @@ kubectl delete mutatingwebhookconfigurations defaulting.webhook.karpenter.sh * `provisioner-state` -> `provisioner_state` * The `karpenter_allocation_controller_scheduling_duration_seconds` metric name changed to `karpenter_provisioner_scheduling_duration_seconds` -### Upgrading to v0.26.0+ +### Upgrading to `0.26.0`+ + * The `karpenter.sh/do-not-evict` annotation no longer blocks node termination when running `kubectl delete node`. This annotation on pods will only block automatic deprovisioning that is considered "voluntary," that is, disruptions that can be avoided. Disruptions that Karpenter deems as "involuntary" and will ignore the `karpenter.sh/do-not-evict` annotation include spot interruption and manual deletion of the node. See [Disabling Deprovisioning]({{}}) for more details. -* Default resources `requests` and `limits` are removed from the Karpenter's controller deployment through the Helm chart. If you have not set custom resource `requests` or `limits` in your helm values and are using Karpenter's defaults, you will now need to set these values in your helm chart deployment. -* The `controller.image` value in the helm chart has been broken out to a map consisting of `controller.image.repository`, `controller.image.tag`, and `controller.image.digest`. If manually overriding the `controller.image`, you will need to update your values to the new design. +* Default resources `requests` and `limits` are removed from the Karpenter's controller deployment through the Helm chart. If you have not set custom resource `requests` or `limits` in your Helm values and are using Karpenter's defaults, you will now need to set these values in your Helm chart deployment. +* The `controller.image` value in the Helm chart has been broken out to a map consisting of `controller.image.repository`, `controller.image.tag`, and `controller.image.digest`. If manually overriding the `controller.image`, you will need to update your values to the new design. + +### Upgrading to `0.25.0`+ -### Upgrading to v0.25.0+ * Cluster Endpoint can now be automatically discovered. If you are using Amazon Elastic Kubernetes Service (EKS), you can now omit the `clusterEndpoint` field in your configuration. In order to allow the resolving, you have to add the permission `eks:DescribeCluster` to the Karpenter Controller IAM role. -### Upgrading to v0.24.0+ +### Upgrading to `0.24.0`+ + * Settings are no longer updated dynamically while Karpenter is running. If you manually make a change to the [`karpenter-global-settings`]({{}}) ConfigMap, you will need to reload the containers by restarting the deployment with `kubectl rollout restart -n karpenter deploy/karpenter` * Karpenter no longer filters out instance types internally. Previously, `g2` (not supported by the NVIDIA device plugin) and FPGA instance types were filtered. The only way to filter instance types now is to set requirements on your provisioner or pods using well-known node labels described [here]({{}}). If you are currently using overly broad requirements that allows all of the `g` instance-category, you will want to tighten the requirement, or add an instance-generation requirement. * `aws.tags` in [`karpenter-global-settings`]({{}}) ConfigMap is now a top-level field and expects the value associated with this key to be a JSON object of string to string. This is change from previous versions where keys were given implicitly by providing the key-value pair `aws.tags.: value` in the ConfigMap. -### Upgrading to v0.22.0+ +### Upgrading to `0.22.0`+ + * Do not upgrade to this version unless you are on Kubernetes >= v1.21. Karpenter no longer supports Kubernetes v1.20, but now supports Kubernetes v1.25. This change is due to the v1 PDB API, which was introduced in K8s v1.20 and subsequent removal of the v1beta1 API in K8s v1.25. -### Upgrading to v0.20.0+ -* Prior to v0.20.0, Karpenter would prioritize certain instance type categories absent of any requirements in the Provisioner. v0.20.0+ removes prioritizing these instance type categories ("m", "c", "r", "a", "t", "i") in code. Bare Metal and GPU instance types are still deprioritized and only used if no other instance types are compatible with the node requirements. Since Karpenter does not prioritize any instance types, if you do not want exotic instance types and are not using the runtime Provisioner defaults, you will need to specify this in the Provisioner. +### Upgrading to `0.20.0`+ + +* Prior to `0.20.0`, Karpenter would prioritize certain instance type categories absent of any requirements in the Provisioner. `0.20.0`+ removes prioritizing these instance type categories ("m", "c", "r", "a", "t", "i") in code. Bare Metal and GPU instance types are still deprioritized and only used if no other instance types are compatible with the node requirements. Since Karpenter does not prioritize any instance types, if you do not want exotic instance types and are not using the runtime Provisioner defaults, you will need to specify this in the Provisioner. -### Upgrading to v0.19.0+ -* The karpenter webhook and controller containers are combined into a single binary, which requires changes to the helm chart. If your Karpenter installation (helm or otherwise) currently customizes the karpenter webhook, your deployment tooling may require minor changes. +### Upgrading to `0.19.0`+ + +* The karpenter webhook and controller containers are combined into a single binary, which requires changes to the Helm chart. If your Karpenter installation (Helm or otherwise) currently customizes the karpenter webhook, your deployment tooling may require minor changes. * Karpenter now supports native interruption handling. If you were previously using Node Termination Handler for spot interruption handling and health events, you will need to remove the component from your cluster before enabling `aws.interruptionQueueName`. For more details on Karpenter's interruption handling, see the [Interruption Handling Docs]({{< ref "../concepts/disruption/#interruption" >}}). * Instance category defaults are now explicitly persisted in the Provisioner, rather than handled implicitly in memory. By default, Provisioners will limit instance category to c,m,r. If any instance type constraints are applied, it will override this default. If you have created Provisioners in the past with unconstrained instance type, family, or category, Karpenter will now more flexibly use instance types than before. If you would like to apply these constraints, they must be included in the Provisioner CRD. * Karpenter CRD raw YAML URLs have migrated from `https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.19.3/charts/karpenter/crds/...` to `https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.19.3/pkg/apis/crds/...`. If you reference static Karpenter CRDs or rely on `kubectl replace -f` to apply these CRDs from their remote location, you will need to migrate to the new location. @@ -207,38 +238,44 @@ kubectl delete mutatingwebhookconfigurations defaulting.webhook.karpenter.sh * `AWS_NODE_NAME_CONVENTION` -> `settings.aws.nodeNameConvention` * `VM_MEMORY_OVERHEAD` -> `settings.aws.vmMemoryOverheadPercent` -### Upgrading to v0.18.0+ -* v0.18.0 removes the `karpenter_consolidation_nodes_created` and `karpenter_consolidation_nodes_terminated` prometheus metrics in favor of the more generic `karpenter_nodes_created` and `karpenter_nodes_terminated` metrics. You can still see nodes created and terminated by consolidation by checking the `reason` label on the metrics. Check out all the metrics published by Karpenter [here]({{}}). +### Upgrading to `0.18.0`+ + +* `0.18.0` removes the `karpenter_consolidation_nodes_created` and `karpenter_consolidation_nodes_terminated` prometheus metrics in favor of the more generic `karpenter_nodes_created` and `karpenter_nodes_terminated` metrics. You can still see nodes created and terminated by consolidation by checking the `reason` label on the metrics. Check out all the metrics published by Karpenter [here]({{}}). + +### Upgrading to `0.17.0`+ -### Upgrading to v0.17.0+ Karpenter's Helm chart package is now stored in [Karpenter's OCI (Open Container Initiative) registry](https://gallery.ecr.aws/karpenter/karpenter). The Helm CLI supports the new format since [v3.8.0+](https://helm.sh/docs/topics/registries/). -With this change [charts.karpenter.sh](https://charts.karpenter.sh/) is no longer updated but preserved to allow using older Karpenter versions. For examples on working with the Karpenter helm charts look at [Install Karpenter Helm Chart]({{< ref "../getting-started/getting-started-with-karpenter/#install-karpenter-helm-chart" >}}). +With this change [charts.karpenter.sh](https://charts.karpenter.sh/) is no longer updated but preserved to allow using older Karpenter versions. For examples on working with the Karpenter Helm charts look at [Install Karpenter Helm Chart]({{< ref "../getting-started/getting-started-with-karpenter/#install-karpenter-helm-chart" >}}). Users who have scripted the installation or upgrading of Karpenter need to adjust their scripts with the following changes: -1. There is no longer a need to add the Karpenter helm repo to helm -2. The full URL of the Helm chart needs to be present when using the helm commands -3. If you were not prepending a `v` to the version (i.e. `0.17.0`), you will need to do so with the OCI chart, `v0.17.0`. +1. There is no longer a need to add the Karpenter Helm repo with `helm repo add` +2. The full URL of the Helm chart needs to be present when using the `helm` CLI +3. If you were not prepending a `v` to the version (i.e. `0.17.0`), you will need to do so with the OCI chart (i.e `v0.17.0`). + +### Upgrading to `0.16.2`+ -### Upgrading to v0.16.2+ -* v0.16.2 adds new kubeletConfiguration fields to the `provisioners.karpenter.sh` v1alpha5 CRD. The CRD will need to be updated to use the new parameters: +* `0.16.2` adds new kubeletConfiguration fields to the `provisioners.karpenter.sh` v1alpha5 CRD. The CRD will need to be updated to use the new parameters: ```bash kubectl replace -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.16.2/charts/karpenter/crds/karpenter.sh_provisioners.yaml ``` -### Upgrading to v0.16.0+ -* v0.16.0 adds a new weight field to the `provisioners.karpenter.sh` v1alpha5 CRD. The CRD will need to be updated to use the new parameters: +### Upgrading to `0.16.0`+ + +* `0.16.0` adds a new weight field to the `provisioners.karpenter.sh` v1alpha5 CRD. The CRD will need to be updated to use the new parameters: ```bash kubectl replace -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.16.0/charts/karpenter/crds/karpenter.sh_provisioners.yaml ``` -### Upgrading to v0.15.0+ -* v0.15.0 adds a new consolidation field to the `provisioners.karpenter.sh` v1alpha5 CRD. The CRD will need to be updated to use the new parameters: +### Upgrading to `0.15.0`+ + +* `0.15.0` adds a new consolidation field to the `provisioners.karpenter.sh` v1alpha5 CRD. The CRD will need to be updated to use the new parameters: ```bash kubectl replace -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.15.0/charts/karpenter/crds/karpenter.sh_provisioners.yaml ``` -### Upgrading to v0.14.0+ -* v0.14.0 adds new fields to the `provisioners.karpenter.sh` v1alpha5 and `awsnodetemplates.karpenter.k8s.aws` v1alpha1 CRDs. The CRDs will need to be updated to use the new parameters: +### Upgrading to `0.14.0`+ + +* `0.14.0` adds new fields to the `provisioners.karpenter.sh` v1alpha5 and `awsnodetemplates.karpenter.k8s.aws` v1alpha1 CRDs. The CRDs will need to be updated to use the new parameters: ```bash kubectl replace -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.14.0/charts/karpenter/crds/karpenter.sh_provisioners.yaml @@ -246,7 +283,7 @@ kubectl replace -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/ kubectl replace -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.14.0/charts/karpenter/crds/karpenter.k8s.aws_awsnodetemplates.yaml ``` -* v0.14.0 changes the way Karpenter discovers its dynamically generated AWS launch templates to use a tag rather than a Name scheme. The previous name scheme was `Karpenter-${CLUSTER_NAME}-*` which could collide with user created launch templates that Karpenter should not manage. The new scheme uses a tag on the launch template `karpenter.k8s.aws/cluster: ${CLUSTER_NAME}`. As a result, Karpenter will not clean-up dynamically generated launch templates using the old name scheme. You can manually clean these up with the following commands: +* `0.14.0` changes the way Karpenter discovers its dynamically generated AWS launch templates to use a tag rather than a Name scheme. The previous name scheme was `Karpenter-${CLUSTER_NAME}-*` which could collide with user created launch templates that Karpenter should not manage. The new scheme uses a tag on the launch template `karpenter.k8s.aws/cluster: ${CLUSTER_NAME}`. As a result, Karpenter will not clean-up dynamically generated launch templates using the old name scheme. You can manually clean these up with the following commands: ```bash ## Find launch templates that match the naming pattern and you do not want to keep @@ -256,52 +293,54 @@ aws ec2 describe-launch-templates --filters="Name=launch-template-name,Values=Ka aws ec2 delete-launch-template --launch-template-id ``` -* v0.14.0 introduces additional instance type filtering if there are no `node.kubernetes.io/instance-type` or `karpenter.k8s.aws/instance-family` or `karpenter.k8s.aws/instance-category` requirements that restrict instance types specified on the provisioner. This prevents Karpenter from launching bare metal and some older non-current generation instance types unless the provisioner has been explicitly configured to allow them. If you specify an instance type or family requirement that supplies a list of instance-types or families, that list will be used regardless of filtering. The filtering can also be completely eliminated by adding an `Exists` requirement for instance type or family. +* `0.14.0` introduces additional instance type filtering if there are no `node.kubernetes.io/instance-type` or `karpenter.k8s.aws/instance-family` or `karpenter.k8s.aws/instance-category` requirements that restrict instance types specified on the provisioner. This prevents Karpenter from launching bare metal and some older non-current generation instance types unless the provisioner has been explicitly configured to allow them. If you specify an instance type or family requirement that supplies a list of instance-types or families, that list will be used regardless of filtering. The filtering can also be completely eliminated by adding an `Exists` requirement for instance type or family. ```yaml - key: node.kubernetes.io/instance-type operator: Exists ``` -* v0.14.0 introduces support for custom AMIs without the need for an entire launch template. You must add the `ec2:DescribeImages` permission to the Karpenter Controller Role for this feature to work. This permission is needed for Karpenter to discover custom images specified. Read the [Custom AMI documentation here]({{}}) to get started -* v0.14.0 adds an an additional default toleration (CriticalAddonOnly=Exists) to the Karpenter helm chart. This may cause Karpenter to run on nodes with that use this Taint which previously would not have been schedulable. This can be overridden by using `--set tolerations[0]=null`. +* `0.14.0` introduces support for custom AMIs without the need for an entire launch template. You must add the `ec2:DescribeImages` permission to the Karpenter Controller Role for this feature to work. This permission is needed for Karpenter to discover custom images specified. Read the [Custom AMI documentation here]({{}}) to get started +* `0.14.0` adds an an additional default toleration (CriticalAddonOnly=Exists) to the Karpenter Helm chart. This may cause Karpenter to run on nodes with that use this Taint which previously would not have been schedulable. This can be overridden by using `--set tolerations[0]=null`. + +* `0.14.0` deprecates the `AWS_ENI_LIMITED_POD_DENSITY` environment variable in-favor of specifying `spec.kubeletConfiguration.maxPods` on the Provisioner. `AWS_ENI_LIMITED_POD_DENSITY` will continue to work when `maxPods` is not set on the Provisioner. If `maxPods` is set, it will override `AWS_ENI_LIMITED_POD_DENSITY` on that specific Provisioner. -* v0.14.0 deprecates the `AWS_ENI_LIMITED_POD_DENSITY` environment variable in-favor of specifying `spec.kubeletConfiguration.maxPods` on the Provisioner. `AWS_ENI_LIMITED_POD_DENSITY` will continue to work when `maxPods` is not set on the Provisioner. If `maxPods` is set, it will override `AWS_ENI_LIMITED_POD_DENSITY` on that specific Provisioner. +### Upgrading to `0.13.0`+ -### Upgrading to v0.13.0+ -* v0.13.0 introduces a new CRD named `AWSNodeTemplate` which can be used to specify AWS Cloud Provider parameters. Everything that was previously specified under `spec.provider` in the Provisioner resource, can now be specified in the spec of the new resource. The use of `spec.provider` is deprecated but will continue to function to maintain backwards compatibility for the current API version (v1alpha5) of the Provisioner resource. v0.13.0 also introduces support for custom user data that doesn't require the use of a custom launch template. The user data can be specified in-line in the AWSNodeTemplate resource. +* `0.13.0` introduces a new CRD named `AWSNodeTemplate` which can be used to specify AWS Cloud Provider parameters. Everything that was previously specified under `spec.provider` in the Provisioner resource, can now be specified in the spec of the new resource. The use of `spec.provider` is deprecated but will continue to function to maintain backwards compatibility for the current API version (v1alpha5) of the Provisioner resource. `0.13.0` also introduces support for custom user data that doesn't require the use of a custom launch template. The user data can be specified in-line in the AWSNodeTemplate resource. - If you are upgrading from v0.10.1 - v0.11.1, a new CRD `awsnodetemplate` was added. In v0.12.0, this crd was renamed to `awsnodetemplates`. Since helm does not manage the lifecycle of CRDs, you will need to perform a few manual steps for this CRD upgrade: + If you are upgrading from `0.10.1` - `0.11.1`, a new CRD `awsnodetemplate` was added. In `0.12.0`, this crd was renamed to `awsnodetemplates`. Since Helm does not manage the lifecycle of CRDs, you will need to perform a few manual steps for this CRD upgrade: 1. Make sure any `awsnodetemplate` manifests are saved somewhere so that they can be reapplied to the cluster. 2. `kubectl delete crd awsnodetemplate` 3. `kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.13.2/charts/karpenter/crds/karpenter.k8s.aws_awsnodetemplates.yaml` - 4. Perform the Karpenter upgrade to v0.13.x, which will install the new `awsnodetemplates` CRD. + 4. Perform the Karpenter upgrade to `0.13.0`+, which will install the new `awsnodetemplates` CRD. 5. Reapply the `awsnodetemplate` manifests you saved from step 1, if applicable. -* v0.13.0 also adds EC2/spot price fetching to Karpenter to allow making more accurate decisions regarding node deployments. Our [getting started guide]({{< ref "../getting-started/getting-started-with-karpenter" >}}) documents this, but if you are upgrading Karpenter you will need to modify your Karpenter controller policy to add the `pricing:GetProducts` and `ec2:DescribeSpotPriceHistory` permissions. +* `0.13.0` also adds EC2/spot price fetching to Karpenter to allow making more accurate decisions regarding node deployments. Our [getting started guide]({{< ref "../getting-started/getting-started-with-karpenter" >}}) documents this, but if you are upgrading Karpenter you will need to modify your Karpenter controller policy to add the `pricing:GetProducts` and `ec2:DescribeSpotPriceHistory` permissions. + +### Upgrading to `0.12.0`+ -### Upgrading to v0.12.0+ -* v0.12.0 adds an OwnerReference to each Node created by a provisioner. Previously, deleting a provisioner would orphan nodes. Now, deleting a provisioner will cause Kubernetes [cascading delete](https://kubernetes.io/docs/concepts/architecture/garbage-collection/#cascading-deletion) logic to gracefully terminate the nodes using the Karpenter node finalizer. You may still orphan nodes by removing the owner reference. -* If you are upgrading from v0.10.1 - v0.11.1, a new CRD `awsnodetemplate` was added. In v0.12.0, this crd was renamed to `awsnodetemplates`. Since helm does not manage the lifecycle of CRDs, you will need to perform a few manual steps for this CRD upgrade: +* `0.12.0` adds an OwnerReference to each Node created by a provisioner. Previously, deleting a provisioner would orphan nodes. Now, deleting a provisioner will cause Kubernetes [cascading delete](https://kubernetes.io/docs/concepts/architecture/garbage-collection/#cascading-deletion) logic to gracefully terminate the nodes using the Karpenter node finalizer. You may still orphan nodes by removing the owner reference. +* If you are upgrading from `0.10.1` - `0.11.1`, a new CRD `awsnodetemplate` was added. In `0.12.0`, this crd was renamed to `awsnodetemplates`. Since Helm does not manage the lifecycle of CRDs, you will need to perform a few manual steps for this CRD upgrade: 1. Make sure any `awsnodetemplate` manifests are saved somewhere so that they can be reapplied to the cluster. 2. `kubectl delete crd awsnodetemplate` 3. `kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.12.1/charts/karpenter/crds/karpenter.k8s.aws_awsnodetemplates.yaml` - 4. Perform the Karpenter upgrade to v0.12.x, which will install the new `awsnodetemplates` CRD. + 4. Perform the Karpenter upgrade to `0.12.0`+, which will install the new `awsnodetemplates` CRD. 5. Reapply the `awsnodetemplate` manifests you saved from step 1, if applicable. -### Upgrading to v0.11.0+ +### Upgrading to `0.11.0`+ -v0.11.0 changes the way that the `vpc.amazonaws.com/pod-eni` resource is reported. Instead of being reported for all nodes that could support the resources regardless of if the cluster is configured to support it, it is now controlled by a command line flag or environment variable. The parameter defaults to false and must be set if your cluster uses [security groups for pods](https://docs.aws.amazon.com/eks/latest/userguide/security-groups-for-pods.html). This can be enabled by setting the environment variable `AWS_ENABLE_POD_ENI` to true via the helm value `controller.env`. +`0.11.0` changes the way that the `vpc.amazonaws.com/pod-eni` resource is reported. Instead of being reported for all nodes that could support the resources regardless of if the cluster is configured to support it, it is now controlled by a command line flag or environment variable. The parameter defaults to false and must be set if your cluster uses [security groups for pods](https://docs.aws.amazon.com/eks/latest/userguide/security-groups-for-pods.html). This can be enabled by setting the environment variable `AWS_ENABLE_POD_ENI` to true via the helm value `controller.env`. Other extended resources must be registered on nodes by their respective device plugins which are typically installed as DaemonSets (e.g. the `nvidia.com/gpu` resource will be registered by the [NVIDIA device plugin](https://github.com/NVIDIA/k8s-device-plugin). Previously, Karpenter would register these resources on nodes at creation and they would be zeroed out by `kubelet` at startup. By allowing the device plugins to register the resources, pods will not bind to the nodes before any device plugin initialization has occurred. -v0.11.0 adds a `providerRef` field in the Provisioner CRD. To use this new field you will need to replace the Provisioner CRD manually: +`0.11.0` adds a `providerRef` field in the Provisioner CRD. To use this new field you will need to replace the Provisioner CRD manually: ```shell kubectl replace -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.11.0/charts/karpenter/crds/karpenter.sh_provisioners.yaml ``` -### Upgrading to v0.10.0+ +### Upgrading to `0.10.0`+ -v0.10.0 adds a new field, `startupTaints` to the provisioner spec. Standard Helm upgrades [do not upgrade CRDs](https://helm.sh/docs/chart_best_practices/custom_resource_definitions/#some-caveats-and-explanations) so the field will not be available unless the CRD is manually updated. This can be performed prior to the standard upgrade by applying the new CRD manually: +`0.10.0` adds a new field, `startupTaints` to the provisioner spec. Standard Helm upgrades [do not upgrade CRDs](https://helm.sh/docs/chart_best_practices/custom_resource_definitions/#some-caveats-and-explanations) so the field will not be available unless the CRD is manually updated. This can be performed prior to the standard upgrade by applying the new CRD manually: ```shell kubectl replace -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.10.0/charts/karpenter/crds/karpenter.sh_provisioners.yaml @@ -309,7 +348,7 @@ kubectl replace -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/ 📝 If you don't perform this manual CRD update, Karpenter will work correctly except for rejecting the creation/update of provisioners that use `startupTaints`. -### Upgrading to v0.6.2+ +### Upgrading to `0.6.2`+ If using Helm, the variable names have changed for the cluster's name and endpoint. You may need to update any configuration that sets the old variable names. diff --git a/website/content/en/v0.30/concepts/_index.md b/website/content/en/v0.30/concepts/_index.md deleted file mode 100755 index 74ec1368b573..000000000000 --- a/website/content/en/v0.30/concepts/_index.md +++ /dev/null @@ -1,177 +0,0 @@ ---- -title: "Concepts" -linkTitle: "Concepts" -weight: 10 -description: > - Understand key concepts of Karpenter ---- - -Users fall under two basic roles: Kubernetes cluster administrators and application developers. -This document describes Karpenter concepts through the lens of those two types of users. - -## Cluster administrator - -As a Kubernetes cluster administrator, you can engage with Karpenter to: - -* Install Karpenter -* Configure provisioners to set constraints and other features for managing nodes -* Deprovision nodes -* Upgrade nodes - -Concepts associated with this role are described below. - - -### Installing Karpenter - -Karpenter is designed to run on a node in your Kubernetes cluster. -As part of the installation process, you need credentials from the underlying cloud provider to allow nodes to be started up and added to the cluster as they are needed. - -[Getting Started with Karpenter on AWS](../getting-started) -describes the process of installing Karpenter on an AWS cloud provider. -Because requests to add and delete nodes and schedule pods are made through Kubernetes, AWS IAM Roles for Service Accounts (IRSA) are needed by your Kubernetes cluster to make privileged requests to AWS. -For example, Karpenter uses AWS IRSA roles to grant the permissions needed to describe EC2 instance types and create EC2 instances. - -Once privileges are in place, Karpenter is deployed with a Helm chart. - -### Configuring provisioners - -Karpenter's job is to add nodes to handle unschedulable pods, schedule pods on those nodes, and remove the nodes when they are not needed. -To configure Karpenter, you create *provisioners* that define how Karpenter manages unschedulable pods and expires nodes. -Here are some things to know about the Karpenter provisioner: - -* **Unschedulable pods**: Karpenter only attempts to schedule pods that have a status condition `Unschedulable=True`, which the kube scheduler sets when it fails to schedule the pod to existing capacity. - -* **Provisioner CR**: Karpenter defines a Custom Resource called a Provisioner to specify provisioning configuration. -Each provisioner manages a distinct set of nodes, but pods can be scheduled to any provisioner that supports its scheduling constraints. -A provisioner contains constraints that impact the nodes that can be provisioned and attributes of those nodes (such timers for removing nodes). -See [Provisioning]({{}}) docs for a description of settings and provisioner examples. - -* **Well-known labels**: The provisioner can use well-known Kubernetes labels to allow pods to request only certain instance types, architectures, operating systems, or other attributes when creating nodes. -See [Well-Known Labels, Annotations and Taints](https://kubernetes.io/docs/reference/labels-annotations-taints/) for details. -Keep in mind that only a subset of these labels are supported in Karpenter, as described later. - -* **Deprovisioning nodes**: A provisioner can also include time-to-live values to indicate when nodes should be deprovisioned after a set amount of time from when they were created or after they becomes empty of deployed pods. - -* **Multiple provisioners**: Multiple provisioners can be configured on the same cluster. -For example, you might want to configure different teams on the same cluster to run on completely separate capacity. -One team could run on nodes using BottleRocket, while another uses EKSOptimizedAMI. - -Although most use cases are addressed with a single provisioner for multiple teams, multiple provisioners are useful to isolate nodes for billing, use different node constraints (such as no GPUs for a team), or use different deprovisioning settings. - -### Deprovisioning nodes - -Karpenter deletes nodes when they are no longer needed. - -* **Finalizer**: Karpenter places a finalizer bit on each node it creates. -When a request comes in to delete one of those nodes (such as a TTL or a manual `kubectl delete node`), Karpenter will cordon the node, drain all the pods, terminate the EC2 instance, and delete the node object. -Karpenter handles all clean-up work needed to properly delete the node. -* **Node Expiry**: If a node expiry time-to-live value (`ttlSecondsUntilExpired`) is reached, that node is drained of pods and deleted (even if it is still running workloads). -* **Empty nodes**: When the last workload pod running on a Karpenter-managed node is gone, the node is annotated with an emptiness timestamp. -Once that "node empty" time-to-live (`ttlSecondsAfterEmpty`) is reached, finalization is triggered. -* **Consolidation**: If enabled, Karpenter will work to actively reduce cluster cost by identifying when nodes can be removed as their workloads will run on other nodes in the cluster and when nodes can be replaced with cheaper variants due to a change in the workloads. -* **Interruption**: If enabled, Karpenter will watch for upcoming involuntary interruption events that could affect your nodes (health events, spot interruption, etc.) and will cordon, drain, and terminate the node(s) ahead of the event to reduce workload disruption. - -For more details on how Karpenter deletes nodes, see [Deprovisioning nodes](./deprovisioning) for details. - -### Upgrading nodes - -A straight-forward way to upgrade nodes is to set `ttlSecondsUntilExpired`. -Nodes will be terminated after a set period of time and will be replaced with newer nodes using the latest discovered AMI. -See more in [AWSNodeTemplate](./node-templates). - -### Constraints - -The concept of layered constraints is key to using Karpenter. -With no constraints defined in provisioners and none requested from pods being deployed, Karpenter chooses from the entire universe of features available to your cloud provider. -Nodes can be created using any instance type and run in any zones. - -An application developer can tighten the constraints defined in a provisioner by the cluster administrator by defining additional scheduling constraints in their pod spec. -Refer to the description of Karpenter constraints in the Application Developer section below for details. - -### Scheduling - -Karpenter launches nodes in response to pods that the Kubernetes scheduler has marked unschedulable. After solving scheduling constraints and launching capacity, Karpenter launches a machine in your chosen cloud provider. - -Once Karpenter brings up a node, that node is available for the Kubernetes scheduler to schedule pods on it as well. - -### Cloud provider -Karpenter makes requests to provision new nodes to the associated cloud provider. -The first supported cloud provider is AWS, although Karpenter is designed to work with other cloud providers. -Separating Kubernetes and AWS-specific settings allows Karpenter a clean path to integrating with other cloud providers. - -While using Kubernetes well-known labels, the provisioner can set some values that are specific to the cloud provider. -So, for example, to include a certain instance type, you could use the Kubernetes label `node.kubernetes.io/instance-type`, but set its value to an AWS instance type (such as `m5.large` or `m5.2xlarge`). - -### Consolidation - -If consolidation is enabled for a provisioner, Karpenter attempts to reduce the overall cost of the nodes launched by that provisioner if workloads have changed in two ways: -- Node Deletion -- Node Replacement - -To perform these actions, Karpenter simulates all pods being evicted from a candidate node and then looks at the results of the scheduling simulation to determine if those pods can run on a combination of existing nodes in the cluster and a new cheaper node. This operation takes into consideration all scheduling constraints placed on your workloads and provisioners (e.g. taints, tolerations, node selectors, inter-pod affinity, etc). - -If as a result of the scheduling simulation all pods can run on existing nodes, the candidate node is simply deleted. If all pods can run on a combination of existing nodes and a cheaper node, we launch the cheaper node and delete the candidate node which causes the pods to be evicted and re-created by their controllers in order to be rescheduled. - -For Node Replacement to work well, your provisioner must allow selecting from a variety of instance types with varying amounts of allocatable resources. Consolidation will only consider launching nodes using instance types which are allowed by your provisioner. - -### Interruption - -If interruption-handling is enabled for the controller, Karpenter will watch for upcoming involuntary interruption events that would cause disruption to your workloads. These interruption events include: - -* Spot Interruption Warnings -* Scheduled Change Health Events (Maintenance Events) -* Instance Terminating Events -* Instance Stopping Events - -When Karpenter detects one of these events will occur to your nodes, it automatically cordons, drains, and terminates the node(s) ahead of the interruption event to give the maximum amount of time for workload cleanup prior to compute disruption. This enables scenarios where the `terminationGracePeriod` for your workloads may be long or cleanup for your workloads is critical, and you want enough time to be able to gracefully clean-up your pods. - -{{% alert title="Note" color="warning" %}} -Karpenter publishes Kubernetes events to the node for all events listed above in addition to __Spot Rebalance Recommendations__. Karpenter does not currently support cordon, drain, and terminate logic for Spot Rebalance Recommendations. -{{% /alert %}} - -### Kubernetes cluster autoscaler -Like Karpenter, [Kubernetes Cluster Autoscaler](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler) is -designed to add nodes when requests come in to run pods that cannot be met by current capacity. -Cluster autoscaler is part of the Kubernetes project, with implementations by most major Kubernetes cloud providers. -By taking a fresh look at provisioning, Karpenter offers the following improvements: - -* **Designed to handle the full flexibility of the cloud**: -Karpenter has the ability to efficiently address the full range of instance types available through AWS. -Cluster autoscaler was not originally built with the flexibility to handle hundreds of instance types, zones, and purchase options. - -* **Group-less node provisioning**: Karpenter manages each instance directly, without use of additional orchestration mechanisms like node groups. -This enables it to retry in milliseconds instead of minutes when capacity is unavailable. -It also allows Karpenter to leverage diverse instance types, availability zones, and purchase options without the creation of hundreds of node groups. - -## Application developer - -As someone deploying pods that might be evaluated by Karpenter, you should know how to request the properties that your pods need of its compute resources. -Karpenter's job is to efficiently assess and choose compute assets based on requests from pod deployments. -These can include basic Kubernetes features or features that are specific to the cloud provider (such as AWS). - -Layered *constraints* are applied when a pod makes requests for compute resources that cannot be met by current capacity. -A pod can specify `nodeAffinity` (to run in a particular zone or instance type) or a `topologySpreadConstraints` spread (to cause a set of pods to be balanced across multiple nodes). -The pod can specify a `nodeSelector` to run only on nodes with a particular label and `resource.requests` to ensure that the node has enough available memory. - -The Kubernetes scheduler tries to match those constraints with available nodes. -If the pod is unschedulable, Karpenter creates compute resources that match its needs. -When Karpenter tries to provision a node, it analyzes scheduling constraints before choosing the node to create. - -As long as the requests are not outside of the provisioner's constraints, -Karpenter will look to best match the request, comparing the same well-known labels defined by the pod's scheduling constraints. -Note that if the constraints are such that a match is not possible, the pod will remain unscheduled. - -So, what constraints can you use as an application developer deploying pods that could be managed by Karpenter? - -Kubernetes features that Karpenter supports for scheduling pods include nodeAffinity and [nodeSelector](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector). -It also supports [PodDisruptionBudget](https://kubernetes.io/docs/tasks/run-application/configure-pdb/), [topologySpreadConstraints](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/), and [inter-pod affinity and anti-affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity). - -From the Kubernetes [Well-Known Labels, Annotations and Taints](https://kubernetes.io/docs/reference/labels-annotations-taints/) page, -you can see a full list of Kubernetes labels, annotations and taints that determine scheduling. -Those that are implemented in Karpenter include: - -* **kubernetes.io/arch**: For example, kubernetes.io/arch=amd64 -* **node.kubernetes.io/instance-type**: For example, node.kubernetes.io/instance-type=m3.medium -* **topology.kubernetes.io/zone**: For example, topology.kubernetes.io/zone=us-east-1c - -For more on how, as a developer, you can add constraints to your pod deployment, see [Scheduling](./scheduling/) for details. diff --git a/website/content/en/v0.30/concepts/deprovisioning.md b/website/content/en/v0.30/concepts/deprovisioning.md deleted file mode 100644 index d2d68af7ad30..000000000000 --- a/website/content/en/v0.30/concepts/deprovisioning.md +++ /dev/null @@ -1,241 +0,0 @@ ---- -title: "Deprovisioning" -linkTitle: "Deprovisioning" -weight: 4 -description: > - Understand different ways Karpenter deprovisions nodes ---- - -## Control Flow -Karpenter sets a Kubernetes [finalizer](https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/) on each node it provisions. -The finalizer blocks deletion of the node object while the Termination Controller cordons and drains the node, before removing the underlying machine. Deprovisioning is triggered by the Deprovisioning Controller, by the user through manual deprovisioning, or through an external system that sends a delete request to the node object. - -### Deprovisioning Controller -Karpenter automatically discovers deprovisionable nodes and spins up replacements when needed. Karpenter deprovisions nodes by executing one [automatic method](#methods) at a time, in order of Expiration, Drift, Emptiness, and then Consolidation. Each method varies slightly but they all follow the standard deprovisioning process: -1. Identify a list of prioritized candidates for the deprovisioning method. - * If there are [pods that cannot be evicted](#pod-eviction) on the node, Karpenter will ignore the node and try deprovisioning it later. - * If there are no deprovisionable nodes, continue to the next deprovisioning method. -2. For each deprovisionable node, execute a scheduling simulation with the pods on the node to find if any replacement nodes are needed. -3. Cordon the node(s) to prevent pods from scheduling to it. -4. Pre-spin any replacement nodes needed as calculated in Step (2), and wait for them to become ready. - * If a replacement node fails to initialize, un-cordon the node(s), and restart from Step (1), starting at the first deprovisioning method again. -5. Delete the node(s) and wait for the Termination Controller to gracefully shutdown the node(s). -6. Once the Termination Controller terminates the node, go back to Step (1), starting at the the first deprovisioning method again. - -### Termination Controller -When a Karpenter node is deleted, the Karpenter finalizer will block deletion and the APIServer will set the `DeletionTimestamp` on the node, allowing Karpenter to gracefully shutdown the node, modeled after [K8s Graceful Node Shutdown](https://kubernetes.io/docs/concepts/architecture/nodes/#graceful-node-shutdown). Karpenter's graceful shutdown process will: -1. Cordon the node to prevent pods from scheduling to it. -2. Begin evicting the pods on the node with the [K8s Eviction API](https://kubernetes.io/docs/concepts/scheduling-eviction/api-eviction/) to respect PDBs, while ignoring all non-daemonset pods and [static pods](https://kubernetes.io/docs/tasks/configure-pod-container/static-pod/). Wait for the node to be fully drained before proceeding to Step (3). - * While waiting, if the underlying machine for the node no longer exists, remove the finalizer to allow the APIServer to delete the node, completing termination. -3. Terminate the machine in the Cloud Provider. -4. Remove the finalizer from the node to allow the APIServer to delete the node, completing termination. - -## Methods - -There are both automated and manual ways of deprovisioning nodes provisioned by Karpenter: - -### Manual Methods -* **Node Deletion**: You could use `kubectl` to manually remove a single Karpenter node: - - ```bash - # Delete a specific node - kubectl delete node $NODE_NAME - - # Delete all nodes owned any provisioner - kubectl delete nodes -l karpenter.sh/provisioner-name - - # Delete all nodes owned by a specific provisioner - kubectl delete nodes -l karpenter.sh/provisioner-name=$PROVISIONER_NAME - ``` -* **Provisioner Deletion**: Nodes are owned by the Provisioner through an [owner reference](https://kubernetes.io/docs/concepts/overview/working-with-objects/owners-dependents/#owner-references-in-object-specifications) that launched them. Karpenter will gracefully terminate nodes through cascading deletion when the owning provisioner is deleted. - -### Automated Methods -* **Emptiness**: Karpenter notes when the last workload (non-daemonset) pod stops running on a node. From that point, Karpenter waits the number of seconds set by `ttlSecondsAfterEmpty` in the provisioner, then Karpenter requests to delete the node. This feature can keep costs down by removing nodes that are no longer being used for workloads. -* **Expiration**: Karpenter will annotate nodes as expired and deprovision nodes after they have lived a set number of seconds, based on the provisioner `ttlSecondsUntilExpired` value. One use case for node expiry is to periodically recycle nodes. Old nodes (with a potentially outdated Kubernetes version or operating system) are deleted, and replaced with nodes on the current version (assuming that you requested the latest version, rather than a specific version). -* **Consolidation**: Karpenter works to actively reduce cluster cost by identifying when: - * Nodes can be removed as their workloads will run on other nodes in the cluster. - * Nodes can be replaced with cheaper variants due to a change in the workloads. -* **Drift**: Karpenter will annotate nodes as drifted and deprovision nodes that have drifted from their desired specification. See [Drift]({{}}) to see which fields are considered. -* **Interruption**: If enabled, Karpenter will watch for upcoming involuntary interruption events that could affect your nodes (health events, spot interruption, etc.) and will cordon, drain, and terminate the node(s) ahead of the event to reduce workload disruption. - -{{% alert title="Note" color="primary" %}} -- Automated deprovisioning is configured through the ProvisionerSpec `.ttlSecondsAfterEmpty`, `.ttlSecondsUntilExpired` and `.consolidation.enabled` fields. If these are not configured, Karpenter will not set default values for them and will not terminate nodes for that purpose. - -- Keep in mind that a small `ttlSecondsUntilExpired` results in a higher churn in cluster activity. For a small enough `ttlSecondsUntilExpired`, nodes may expire faster than Karpenter can safely deprovision them, resulting in constant node deprovisioning. - -- Pods without an ownerRef (also called "controllerless" or "naked" pods) will be evicted during automatic node disruption, besides [Interruption](#interruption). A pod with the annotation `karpenter.sh/do-not-evict: "true"` will cause its node to be opted out from the same deprovisioning methods. - -- Using preferred anti-affinity and topology spreads can reduce the effectiveness of consolidation. At node launch, Karpenter attempts to satisfy affinity and topology spread preferences. In order to reduce node churn, consolidation must also attempt to satisfy these constraints to avoid immediately consolidating nodes after they launch. This means that consolidation may not deprovision nodes in order to avoid violating preferences, even if kube-scheduler can fit the host pods elsewhere. Karpenter reports these pods via logging to bring awareness to the possible issues they can cause (e.g. `pod default/inflate-anti-self-55894c5d8b-522jd has a preferred Anti-Affinity which can prevent consolidation`). - -- By adding the finalizer, Karpenter improves the default Kubernetes process of node deletion. -When you run `kubectl delete node` on a node without a finalizer, the node is deleted without triggering the finalization logic. The machine will continue running in EC2, even though there is no longer a node object for it. -The kubelet isn’t watching for its own existence, so if a node is deleted, the kubelet doesn’t terminate itself. -All the pod objects get deleted by a garbage collection process later, because the pods’ node is gone. - -{{% /alert %}} - -## Consolidation - -Karpenter has two mechanisms for cluster consolidation: -- Deletion - A node is eligible for deletion if all of its pods can run on free capacity of other nodes in the cluster. -- Replace - A node can be replaced if all of its pods can run on a combination of free capacity of other nodes in the cluster and a single cheaper replacement node. - -Consolidation has three mechanisms that are performed in order to attempt to identify a consolidation action: -1) Empty Node Consolidation - Delete any entirely empty nodes in parallel -2) Multi-Node Consolidation - Try to delete two or more nodes in parallel, possibly launching a single replacement that is cheaper than the price of all nodes being removed -3) Single-Node Consolidation - Try to delete any single node, possibly launching a single replacement that is cheaper than the price of that node - -It's impractical to examine all possible consolidation options for multi-node consolidation, so Karpenter uses a heuristic to identify a likely set of nodes that can be consolidated. For single-node consolidation we consider each node in the cluster individually. - -When there are multiple nodes that could be potentially deleted or replaced, Karpenter choose to consolidate the node that overall disrupts your workloads the least by preferring to terminate: - -* nodes running fewer pods -* nodes that will expire soon -* nodes with lower priority pods - -{{% alert title="Note" color="primary" %}} -For spot nodes, Karpenter only uses the deletion consolidation mechanism. It will not replace a spot node with a cheaper spot node. Spot instance types are selected with the `price-capacity-optimized` strategy and often the cheapest spot instance type is not launched due to the likelihood of interruption. Consolidation would then replace the spot instance with a cheaper instance negating the `price-capacity-optimized` strategy entirely and increasing interruption rate. -{{% /alert %}} - -If consolidation is enabled, Karpenter periodically reports events against nodes that indicate why the node can't be consolidated. These events can be used to investigate nodes that you expect to have been consolidated, but still remain in your cluster. - -``` -Events: - Type Reason Age From Message - ---- ------ ---- ---- ------- - Normal Unconsolidatable 66s karpenter pdb default/inflate-pdb prevents pod evictions - Normal Unconsolidatable 33s (x3 over 30m) karpenter can't replace with a cheaper node - ``` - -## Interruption - -If interruption-handling is enabled, Karpenter will watch for upcoming involuntary interruption events that would cause disruption to your workloads. These interruption events include: - -* Spot Interruption Warnings -* Scheduled Change Health Events (Maintenance Events) -* Instance Terminating Events -* Instance Stopping Events - -When Karpenter detects one of these events will occur to your nodes, it automatically cordons, drains, and terminates the node(s) ahead of the interruption event to give the maximum amount of time for workload cleanup prior to compute disruption. This enables scenarios where the `terminationGracePeriod` for your workloads may be long or cleanup for your workloads is critical, and you want enough time to be able to gracefully clean-up your pods. - -For Spot interruptions, the provisioner will start a new machine as soon as it sees the Spot interruption warning. Spot interruptions have a __2 minute notice__ before Amazon EC2 reclaims the instance. Karpenter's average node startup time means that, generally, there is sufficient time for the new node to become ready and to move the pods to the new node before the machine is reclaimed. - -{{% alert title="Note" color="primary" %}} -Karpenter publishes Kubernetes events to the node for all events listed above in addition to __Spot Rebalance Recommendations__. Karpenter does not currently support cordon, drain, and terminate logic for Spot Rebalance Recommendations. -{{% /alert %}} - -Karpenter enables this feature by watching an SQS queue which receives critical events from AWS services which may affect your nodes. Karpenter requires that an SQS queue be provisioned and EventBridge rules and targets be added that forward interruption events from AWS services to the SQS queue. Karpenter provides details for provisioning this infrastructure in the [CloudFormation template in the Getting Started Guide](../../getting-started/getting-started-with-karpenter/#create-the-karpenter-infrastructure-and-iam-roles). - -To enable the interruption handling feature flag, configure the `karpenter-global-settings` ConfigMap with the following value mapped to the name of the interruption queue that handles interruption events. - -```yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: karpenter-global-settings - namespace: karpenter -data: - ... - aws.interruptionQueueName: karpenter-cluster - ... -``` - -### Drift -Drift handles changes to the NodePool/EC2NodeClass. For Drift, values in the NodePool/EC2NodeClass are reflected in the NodeClaimTemplateSpec/EC2NodeClassSpec in the same way that they’re set. A NodeClaim will be detected as drifted if the values in its owning NodePool/EC2NodeClass do not match the values in the NodeClaim. Similar to the upstream `deployment.spec.template` relationship to pods, Karpenter will annotate the owning NodePool and EC2NodeClass with a hash of the NodeClaimTemplateSpec to check for drift. Some special cases will be discovered either from Karpenter or through the CloudProvider interface, triggered by NodeClaim/Instance/NodePool/EC2NodeClass changes. - -#### Special Cases on Drift -In special cases, drift can correspond to multiple values and must be handled differently. Drift on resolved fields can create cases where drift occurs without changes to CRDs, or where CRD changes do not result in drift. For example, if a NodeClaim has `node.kubernetes.io/instance-type: m5.large`, and requirements change from `node.kubernetes.io/instance-type In [m5.large]` to `node.kubernetes.io/instance-type In [m5.large, m5.2xlarge]`, the NodeClaim will not be drifted because its value is still compatible with the new requirements. Conversely, if a NodeClaim is using a NodeClaim image `ami: ami-abc`, but a new image is published, Karpenter's `EC2NodeClass.spec.amiSelectorTerms` will discover that the new correct value is `ami: ami-xyz`, and detect the NodeClaim as drifted. - -##### NodePool -| Fields | -|----------------| -| Requirements | - -##### EC2NodeClass -| Fields | -|-------------------------------| -| Subnet Selector Terms | -| Security Group Selector Terms | -| AMI Selector Terms | - -#### Behavioral Fields -Behavioral Fields are treated as over-arching settings on the NodePool to dictate how Karpenter behaves. These fields don’t correspond to settings on the NodeClaim or instance. They’re set by the user to control Karpenter’s Provisioning and disruption logic. Since these don’t map to a desired state of NodeClaims, __behavioral fields are not considered for Drift__. - -__Behavioral Fields__ -- Weight -- Limits -- ConsolidationPolicy -- ConsolidateAfter -- ExpireAfter ---- - -Read the [Drift Design](https://github.com/aws/karpenter-core/blob/main/designs/drift.md) for more. - -To enable the drift feature flag, refer to the [Settings Feature Gates]({{}}). - -Karpenter will add `MachineDrifted` status condition on the machines if the machine is drifted, and does not have the status condition, - -Karpenter will remove the `MachineDrifted` status condition for the following these scenarios: -1. The `featureGates.driftEnabled` is not enabled but the machine is drifted, karpenter will remove the status condition. -2. The machine isn't drifted, but has the status condition, karpenter will remove it. - -If the node is marked as drifted by another controller, karpenter will do nothing. - - -## Controls - -### Pod-Level Controls - -You can block Karpenter from voluntarily choosing to disrupt certain pods by setting the `karpenter.sh/do-not-evict: "true"` annotation on the pod. This is useful for pods that you want to run from start to finish without disruption. By opting pods out of this disruption, you are telling Karpenter that it should not voluntarily remove a node containing this pod. - -Examples of pods that you might want to opt-out of disruption include an interactive game that you don't want to interrupt or a long batch job (such as you might have with machine learning) that would need to start over if it were interrupted. - -```yaml -apiVersion: apps/v1 -kind: Deployment -spec: - template: - metadata: - annotations: - karpenter.sh/do-not-evict: "true" -``` - -{{% alert title="Note" color="primary" %}} -This annotation will be ignored for [terminating pods](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase), [terminal pods](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase) (Failed/Succeeded), [DaemonSet pods](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/), or [static pods](https://kubernetes.io/docs/tasks/configure-pod-container/static-pod/). -{{% /alert %}} - -Examples of voluntary node removal that will be prevented by this annotation include: -- [Consolidation]({{}}) -- [Drift]({{}}) -- Emptiness -- Expiration - -{{% alert title="Note" color="primary" %}} -Voluntary node removal does not include [Interruption]({{}}) or manual deletion initiated through `kubectl delete node`. Both of these are considered involuntary events, since node removal cannot be delayed. -{{% /alert %}} - -### Node-Level Controls - -Nodes can be opted out of consolidation deprovisioning by setting the annotation `karpenter.sh/do-not-consolidate: "true"` on the node. - -```yaml -apiVersion: v1 -kind: Node -metadata: - annotations: - karpenter.sh/do-not-consolidate: "true" -``` - -#### Example: Disable Consolidation on Provisioner - -Provisioner `.spec.annotations` allow you to set annotations that will be applied to all nodes launched by this provisioner. By setting the annotation `karpenter.sh/do-not-consolidate: "true"` on the provisioner, you will selectively prevent all nodes launched by this Provisioner from being considered in consolidation calculations. - -```yaml -apiVersion: karpenter.sh/v1alpha5 -kind: Provisioner -metadata: - name: default -spec: - annotations: # will be applied to all nodes - karpenter.sh/do-not-consolidate: "true" -``` diff --git a/website/content/en/v0.30/concepts/node-templates.md b/website/content/en/v0.30/concepts/node-templates.md deleted file mode 100644 index 6202c881bc97..000000000000 --- a/website/content/en/v0.30/concepts/node-templates.md +++ /dev/null @@ -1,687 +0,0 @@ ---- -title: "Node Templates" -linkTitle: "Node Templates" -weight: 2 -description: > - Configure AWS specific settings ---- - -Node Templates enable configuration of AWS specific settings. -Each provisioner must reference an AWSNodeTemplate using `spec.providerRef`. -Multiple provisioners may point to the same AWSNodeTemplate. - -```yaml -apiVersion: karpenter.sh/v1alpha5 -kind: Provisioner -metadata: - name: default -spec: - providerRef: - name: default ---- -apiVersion: karpenter.k8s.aws/v1alpha1 -kind: AWSNodeTemplate -metadata: - name: default -spec: - subnetSelector: { ... } # required, discovers tagged subnets to attach to instances - securityGroupSelector: { ... } # required, discovers tagged security groups to attach to instances - instanceProfile: "..." # optional, overrides the node's identity from global settings - amiFamily: "..." # optional, resolves a default ami and userdata - amiSelector: { ... } # optional, discovers tagged amis to override the amiFamily's default - userData: "..." # optional, overrides autogenerated userdata with a merge semantic - tags: { ... } # optional, propagates tags to underlying EC2 resources - metadataOptions: { ... } # optional, configures IMDS for the instance - blockDeviceMappings: [ ... ] # optional, configures storage devices for the instance - detailedMonitoring: "..." # optional, configures detailed monitoring for the instance -status: - subnets: { ... } # resolved subnets - securityGroups: { ... } # resolved security groups - amis: { ... } # resolved AMIs -``` -Refer to the [Provisioner docs]({{}}) for settings applicable to all providers. To explore various `AWSNodeTemplate` configurations, refer to the examples provided [in the Karpenter Github repository](https://github.com/aws/karpenter/blob/main/examples/provisioner/). - -See below for other AWS provider-specific parameters. - -## spec.subnetSelector - -The `AWSNodeTemplate` discovers subnets using [AWS tags](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html). -Subnets may be specified by any AWS tag, including `Name`. Selecting tag values using wildcards (`*`) is supported. -Subnet IDs may be specified by using the key `aws-ids` and then passing the IDs as a comma-separated string value. -When launching nodes, a subnet is automatically chosen that matches the desired zone. -If multiple subnets exist for a zone, the one with the most available IP addresses will be used. - -**Examples** - -Select all with a specified tag key: -```yaml -spec: - subnetSelector: - karpenter.sh/discovery/MyClusterName: '*' -``` - -Select by name and tag (all criteria must match):: -```yaml -spec: - subnetSelector: - Name: my-subnet - MyTag: '' # matches all resources with the tag -``` - -Select using comma separated tag values: -```yaml -spec: - subnetSelector: - Name: "my-subnet-1,my-subnet-2" -``` - -Select using wildcards: -```yaml -spec: - subnetSelector: - Name: "*Public*" - -``` - -Select by ID: -```yaml -spec: - subnetSelector: - aws-ids: "subnet-09fa4a0a8f233a921,subnet-0471ca205b8a129ae" -``` - -## spec.securityGroupSelector - -The security group of an instance is comparable to a set of firewall rules. -EKS creates at least two security groups by default, [review the documentation](https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html) for more info. -Security groups may be specified by any AWS tag, including "Name". Selecting tags using wildcards (`*`) is supported. - -{{% alert title="Note" color="primary" %}} -When launching nodes, Karpenter uses all the security groups that match the selector. If you choose to use the `kubernetes.io/cluster/$CLUSTER_NAME` tag for discovery, note that this may result in failures using the AWS Load Balancer controller. The Load Balancer controller only supports a single security group having that tag key. See [this issue](https://github.com/kubernetes-sigs/aws-load-balancer-controller/issues/2367) for more details. -{{% /alert %}} - -To verify if this restriction affects you, run the following commands. -```bash -CLUSTER_VPC_ID="$(aws eks describe-cluster --name $CLUSTER_NAME --query cluster.resourcesVpcConfig.vpcId --output text)" - -aws ec2 describe-security-groups --filters Name=vpc-id,Values=$CLUSTER_VPC_ID Name=tag-key,Values=kubernetes.io/cluster/$CLUSTER_NAME --query 'SecurityGroups[].[GroupName]' --output text -``` - -If multiple securityGroups are printed, you will need a more specific securityGroupSelector. We generally recommend that you use the `karpenter.sh/discovery: $CLUSTER_NAME` tag selector instead. - -**Examples** - -Select all assigned to a cluster: -```yaml -spec: - securityGroupSelector: - karpenter.sh/discovery: "${CLUSTER_NAME}" -``` - -Select all with a specified tag key: -```yaml -spec: - securityGroupSelector: - MyTag: '*' -``` - -Select by name and tag (all criteria must match): -```yaml -spec: - securityGroupSelector: - Name: my-security-group - MyTag: '' # matches all resources with the tag -``` - -Select by comma-separated tag values: -```yaml -spec: - securityGroupSelector: - Name: "my-security-group-1,my-security-group-2" -``` - -Select by name using a wildcard: -```yaml -spec: - securityGroupSelector: - Name: "*Public*" -``` - -Select by ID: -```yaml -spec: - securityGroupSelector: - aws-ids: "sg-063d7acfb4b06c82c,sg-06e0cf9c198874591" -``` - -## spec.instanceProfile - -An `InstanceProfile` is a way to pass a single IAM role to EC2 instance launched the provisioner. -A default profile is configured in global settings, but may be overridden here. -The `AWSNodeTemplate` will not create an `InstanceProfile` automatically. -The `InstanceProfile` must refer to a `Role` that has permission to connect to the cluster. -```yaml -spec: - instanceProfile: MyInstanceProfile -``` - -## spec.amiFamily - -The AMI used when provisioning nodes can be controlled by the `amiFamily` field. Based on the value set for `amiFamily`, Karpenter will automatically query for the appropriate [EKS optimized AMI](https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-amis.html) via AWS Systems Manager (SSM). When an `amiFamily` of `Custom` is chosen, then an `amiSelector` must be specified that informs Karpenter on which custom AMIs are to be used. - -Currently, Karpenter supports `amiFamily` values `AL2`, `Bottlerocket`, `Ubuntu`, `Windows2019`, `Windows2022` and `Custom`. GPUs are only supported with `AL2` and `Bottlerocket`. The `AL2` amiFamily does not support ARM64 GPU instance types unless you specify a custom amiSelector. - -{{% alert title="Defaults" color="secondary" %}} -If no `amiFamily` is defined, Karpenter will set the default `amiFamily` to AL2 - -```yaml -spec: - amiFamily: AL2 -``` -{{% /alert %}} - -## spec.amiSelector - -AMISelector is used to configure custom AMIs for Karpenter to use, where the AMIs are discovered through `aws::` prefixed filters (`aws::ids`, `aws::owners` and `aws::name`) and [AWS tags](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html). This field is optional, and Karpenter will use the latest EKS-optimized AMIs if an amiSelector is not specified. - -To select an AMI by name, use `aws::name`. EC2 AMIs may be specified by any AWS tag, including `Name`. Selecting by tag or by name using wildcards (`*`) is supported. - -EC2 AMI IDs may be specified by using the key `aws::ids` (`aws-ids` is also supported) and then passing the IDs as a comma-separated string value. - -To ensure that AMIs are owned by the expected owner, use `aws::owners` which expects a comma-separated list of AWS account owners - you can use a combination of account aliases (e.g. `self` `amazon`, `your-aws-account-name`) and account IDs. If this is not set, *and* `aws::ids`/`aws-ids` are not set, it defaults to `self,amazon`. - -{{% alert title="Note" color="primary" %}} -If you use only `aws::owners`, Karpenter will discover all images that are owned by those specified, selecting the most recently created ones to be used. If you specify `aws::owners`, but nothing else, there is a larger chance that Karpenter could select an image that is not compatible with your instance type. To lower this chance, it is recommended to use `aws::name` or `aws::ids` if you're using `aws::owners` to select a subset of images that you have validated are compatible with your selected instance types. -{{% /alert %}} - -### AMI Selection - -If an `amiSelector` matches more than one AMI, Karpenter will automatically determine which AMI best fits the workloads on the launched worker node under the following constraints: - -* When launching nodes, Karpenter automatically determines which architecture a custom AMI is compatible with and will use images that match an instanceType's requirements. -* If multiple AMIs are found that can be used, Karpenter will choose the latest one. -* If no AMIs are found that can be used, then no nodes will be provisioned. - -If you need to express other constraints for an AMI beyond architecture, you can express these constraints as tags on the AMI. For example, if you want to limit an EC2 AMI to only be used with instanceTypes that have an `nvidia` GPU, you can specify an EC2 tag with a key of `karpenter.k8s.aws/instance-gpu-manufacturer` and value `nvidia` on that AMI. - -All labels defined [in the scheduling documentation](../scheduling#well-known-labels) can be used as requirements for an EC2 AMI. - -```bash -> aws ec2 describe-images --image-id ami-123 --query Images[0].Tags -[ - { - "Key": "karpenter.sh/discovery", - "Value": "my-cluster" - }, - { - "Key": "Name", - "Value": "amazon-eks-node-1.21-customized-v0" - }, - { - "Key": "karpenter.k8s.aws/instance-gpu-manufacturer", - "Value": "nvidia" - } -] -``` - -#### Examples - -Select all AMIs with a specified tag: -```yaml - amiSelector: - karpenter.sh/discovery/MyClusterName: '*' -``` - -Select AMIs by the AMI name: -```yaml - amiSelector: - aws::name: my-ami -``` -Select AMIs by the Name tag: -```yaml - amiSelector: - Name: my-ami -``` - -Select AMIs by name and a specific owner: -```yaml - amiSelector: - aws::name: my-ami - aws::owners: self/ownerAccountID -``` - -Select AMIs by an arbitrary AWS tag key/value pair: -```yaml - amiSelector: - MyAMITag: value -``` - -Specify AMIs explicitly by ID: -```yaml - amiSelector: - aws::ids: "ami-123,ami-456" -``` - -## spec.tags - -Karpenter adds tags to all resources it creates, including EC2 Instances, EBS volumes, and Launch Templates. The default set of AWS tags are listed below. - -``` -Name: karpenter.sh/provisioner-name/ -karpenter.sh/provisioner-name: -kubernetes.io/cluster/: owned -``` - -Additional tags can be added in the AWSNodeTemplate tags section which are merged with global tags in `aws.tags` (located in karpenter-global-settings ConfigMap). -```yaml -spec: - tags: - InternalAccountingTag: 1234 - dev.corp.net/app: Calculator - dev.corp.net/team: MyTeam -``` - -Karpenter allows overrides of the default "Name" tag but does not allow overrides to restricted domains (such as "karpenter.sh", "karpenter.k8s.aws", and "kubernetes.io/cluster"). This ensures that Karpenter is able to correctly auto-discover machines that it owns. - -## spec.metadataOptions - -Control the exposure of [Instance Metadata Service](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) on EC2 Instances launched by this provisioner using a generated launch template. - -Refer to [recommended, security best practices](https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node) for limiting exposure of Instance Metadata and User Data to pods. - -If metadataOptions are omitted from this provisioner, the following default settings will be used. - -```yaml -spec: - metadataOptions: - httpEndpoint: enabled - httpProtocolIPv6: disabled - httpPutResponseHopLimit: 2 - httpTokens: required -``` - -## spec.blockDeviceMappings - -The `blockDeviceMappings` field in an AWSNodeTemplate can be used to control the Elastic Block Storage (EBS) volumes that Karpenter attaches to provisioned nodes. Karpenter uses default block device mappings for the AMI Family specified. For example, the `Bottlerocket` AMI Family defaults with two block device mappings, one for Bottlerocket's control volume and the other for container resources such as images and logs. - -Learn more about [block device mappings](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/block-device-mapping-concepts.html). - -### Examples - -```yaml -apiVersion: karpenter.k8s.aws/v1alpha1 -kind: AWSNodeTemplate -spec: - blockDeviceMappings: - - deviceName: /dev/xvda - ebs: - volumeSize: 100Gi - volumeType: gp3 - iops: 10000 - encrypted: true - kmsKeyID: "arn:aws:kms:us-west-2:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab" - deleteOnTermination: true - throughput: 125 - snapshotID: snap-0123456789 -``` - -{{% alert title="Defaults" color="secondary" %}} -If no `blockDeviceMappings` is defined, Karpenter will set the default `blockDeviceMappings` to the following for the given AMI family. - -#### AL2 -```yaml -apiVersion: karpenter.k8s.aws/v1alpha1 -kind: AWSNodeTemplate -spec: - blockDeviceMappings: - - deviceName: /dev/xvda - ebs: - volumeSize: 20Gi - volumeType: gp3 - encrypted: true -``` - -#### Bottlerocket -```yaml -apiVersion: karpenter.k8s.aws/v1alpha1 -kind: AWSNodeTemplate -spec: - blockDeviceMappings: - # Root device - - deviceName: /dev/xvda - ebs: - volumeSize: 4Gi - volumeType: gp3 - encrypted: true - # Data device: Container resources such as images and logs - - deviceName: /dev/xvdb - ebs: - volumeSize: 20Gi - volumeType: gp3 - encrypted: true -``` - -#### Ubuntu -```yaml -apiVersion: karpenter.k8s.aws/v1alpha1 -kind: AWSNodeTemplate -spec: - blockDeviceMappings: - - deviceName: /dev/sda1 - ebs: - volumeSize: 20Gi - volumeType: gp3 - encrypted: true -``` - -#### Windows2019, Windows2022 -```yaml -apiVersion: karpenter.k8s.aws/v1alpha1 -kind: AWSNodeTemplate -spec: - blockDeviceMappings: - - deviceName: /dev/sda1 - ebs: - volumeSize: 50Gi - volumeType: gp3 - encrypted: true -``` -{{% /alert %}} - -## spec.userData - -You can control the UserData that is applied to your worker nodes via this field. - -```yaml -apiVersion: karpenter.k8s.aws/v1alpha1 -kind: AWSNodeTemplate -metadata: - name: bottlerocket-example -spec: - amiFamily: Bottlerocket - instanceProfile: MyInstanceProfile - subnetSelector: - karpenter.sh/discovery: my-cluster - securityGroupSelector: - karpenter.sh/discovery: my-cluster - userData: | - [settings.kubernetes] - kube-api-qps = 30 - [settings.kubernetes.eviction-hard] - "memory.available" = "20%" - amiSelector: - karpenter.sh/discovery: my-cluster -``` - -This example adds SSH keys to allow remote login to the node (replace *my-authorized_keys* with your key file): - -{{% alert title="Note" color="primary" %}} -Instead of using SSH as set up in this example, you can use Session Manager (SSM) or EC2 Instance Connect to gain shell access to Karpenter nodes. -See [Node NotReady]({{< ref "../troubleshooting/#node-notready" >}}) troubleshooting for an example of starting an SSM session from the command line or [EC2 Instance Connect](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-connect-set-up.html) documentation to connect to nodes using SSH. -{{% /alert %}} - -```yaml -apiVersion: karpenter.k8s.aws/v1alpha1 -kind: AWSNodeTemplate -metadata: - name: al2-example -spec: - amiFamily: AL2 - instanceProfile: MyInstanceProfile - subnetSelector: - karpenter.sh/discovery: my-cluster - securityGroupSelector: - karpenter.sh/discovery: my-cluster - userData: | - #!/bin/bash - mkdir -p ~ec2-user/.ssh/ - touch ~ec2-user/.ssh/authorized_keys - cat >> ~ec2-user/.ssh/authorized_keys < >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 -/etc/eks/bootstrap.sh 'test-cluster' --apiserver-endpoint 'https://test-cluster' --b64-cluster-ca 'ca-bundle' \ ---use-max-pods false \ ---container-runtime containerd \ ---kubelet-extra-args '--node-labels=karpenter.sh/capacity-type=on-demand,karpenter.sh/provisioner-name=test --max-pods=110' ---//-- -``` - -You can also set kubelet-config properties by modifying the kubelet-config.json file before the EKS bootstrap script starts the kubelet: - -``` -apiVersion: karpenter.k8s.aws/v1alpha1 -kind: AWSNodeTemplate -metadata: - name: kubelet-config-example -spec: - subnetSelector: - karpenter.sh/discovery: my-cluster - securityGroupSelector: - karpenter.sh/discovery: my-cluster - userData: | - #!/bin/bash - echo "$(jq '.kubeAPIQPS=50' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json -``` - -#### Windows - -* Your UserData must be specified as PowerShell commands. -* The UserData specified will be prepended to a Karpenter managed section that will bootstrap the kubelet. -* Karpenter will continue to set ClusterDNS and all other parameters defined in spec.kubeletConfiguration as before. - -Consider the following example to understand how your custom UserData settings will be merged in. - -Your UserData - -``` -Write-Host "Running custom user data script" -``` - -Final merged UserData - -``` - -Write-Host "Running custom user data script" -[string]$EKSBootstrapScriptFile = "$env:ProgramFiles\Amazon\EKS\Start-EKSBootstrap.ps1" -& $EKSBootstrapScriptFile -EKSClusterName 'test-cluster' -APIServerEndpoint 'https://test-cluster' -Base64ClusterCA 'ca-bundle' -KubeletExtraArgs '--node-labels="karpenter.sh/capacity-type=spot,karpenter.sh/provisioner-name=windows2022" --max-pods=110' -DNSClusterIP '10.0.100.10' - -``` - -{{% alert title="Windows Support Notice" color="warning" %}} -Currently, Karpenter does not specify `-ServiceCIDR` to [EKS Windows AMI Bootstrap script](https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-windows-ami.html#bootstrap-script-configuration-parameters). -Windows worker nodes will use `172.20.0.0/16` or `10.100.0.0/16` for Kubernetes service IP address ranges based on the IP address of the primary interface. -The effective ServiceCIDR can be verified at `$env:ProgramData\Amazon\EKS\cni\config\vpc-bridge.conf` on the worker node. - -Support for the Windows ServiceCIDR argument can be tracked in a [Karpenter Github Issue](https://github.com/aws/karpenter/issues/4088). Currently, if the effective ServiceCIDR is incorrect for your windows worker nodes, you can add the following userData as a workaround. - -```yaml -spec: - userData: | - $global:EKSCluster = Get-EKSCluster -Name my-cluster -``` -{{% /alert %}} - -## spec.detailedMonitoring - -Enabling detailed monitoring on the node template controls the [EC2 detailed monitoring](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-cloudwatch-new.html) feature. If you enable this option, the Amazon EC2 console displays monitoring graphs with a 1-minute period for the instances that Karpenter launches. -```yaml -spec: - detailedMonitoring: true -``` - -## status.subnets -`status.subnets` contains the `id` and `zone` of the subnets utilized during node launch. The subnets are sorted by the available IP address count in decreasing order. - -**Examples** - -```yaml -status: - subnets: - - id: subnet-0a462d98193ff9fac - zone: us-east-2b - - id: subnet-0322dfafd76a609b6 - zone: us-east-2c - - id: subnet-0727ef01daf4ac9fe - zone: us-east-2b - - id: subnet-00c99aeafe2a70304 - zone: us-east-2a - - id: subnet-023b232fd5eb0028e - zone: us-east-2c - - id: subnet-03941e7ad6afeaa72 - zone: us-east-2a -``` - -## status.securityGroups -`status.securityGroups` contains the `id` and `name` of the security groups utilized during node launch. - -**Examples** - -```yaml - status: - securityGroups: - - id: sg-041513b454818610b - name: ClusterSharedNodeSecurityGroup - - id: sg-0286715698b894bca - name: ControlPlaneSecurityGroup-1AQ073TSAAPW -``` - -## status.amis -`status.amis` contains the `id`, `name`, and `requirements` of the amis utilized during node launch. - -**Examples** - -```yaml - amis: - - id: ami-03c3a3dcda64f5b75 - name: amazon-linux-2-gpu - requirements: - - key: kubernetes.io/arch - operator: In - values: - - amd64 - - key: karpenter.k8s.aws/instance-accelerator-manufacturer - operator: In - values: - - aws - - nvidia - - id: ami-06afb2d101cc4b8bd - name: amazon-linux-2-arm64 - requirements: - - key: kubernetes.io/arch - operator: In - values: - - arm64 - - key: karpenter.k8s.aws/instance-accelerator-manufacturer - operator: NotIn - values: - - aws - - nvidia - - id: ami-0e28b76d768af234e - name: amazon-linux-2 - requirements: - - key: kubernetes.io/arch - operator: In - values: - - amd64 - - key: karpenter.k8s.aws/instance-accelerator-manufacturer - operator: NotIn - values: - - aws - - nvidia -``` \ No newline at end of file diff --git a/website/content/en/v0.30/concepts/pod-density.md b/website/content/en/v0.30/concepts/pod-density.md deleted file mode 100644 index 6a8a41b6af17..000000000000 --- a/website/content/en/v0.30/concepts/pod-density.md +++ /dev/null @@ -1,91 +0,0 @@ ---- -title: "Control Pod Density" -linkTitle: "Control Pod Density" -weight: 6 -description: > - Learn ways to specify pod density with Karpenter ---- - -Pod density is the number of pods per node. - -Kubernetes has a default limit of 110 pods per node. If you are using the EKS Optimized AMI on AWS, the [number of pods is limited by instance type](https://github.com/awslabs/amazon-eks-ami/blob/master/files/eni-max-pods.txt) in the default configuration. - -## Increase Pod Density - -### Networking Limitations - -*☁️ AWS Specific* - -By default, the number of pods on a node is limited by both the number of networking interfaces (ENIs) that may be attached to an instance type and the number of IP addresses that can be assigned to each ENI. See [IP addresses per network interface per instance type](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-eni.html#AvailableIpPerENI) for a more detailed information on these instance types' limits. - -Karpenter can be configured to disable nodes' ENI-based pod density. This is especially useful for small to medium instance types which have a lower ENI-based pod density. - -{{% alert title="Note" color="primary" %}} -When using small instance types, it may be necessary to enable [prefix assignment mode](https://aws.amazon.com/blogs/containers/amazon-vpc-cni-increases-pods-per-node-limits/) in the AWS VPC CNI plugin to more pods per node. Prefix assignment mode was introduced in AWS VPC CNI v1.9 and allows ENIs to manage a broader set of IP addresses. Much higher pod densities are supported as a result. -{{% /alert %}} - -{{% alert title="Windows Support Notice" color="warning" %}} -Presently, Windows worker nodes do not support using more than one ENI. -As a consequence, the number of IP addresses, and subsequently, the number of pods that a Windows worker node can support is limited by the number of IPv4 addresses available on the primary ENI. -At the moment, Karpenter will only consider individual secondary IP addresses when calculating the pod density limit. -{{% /alert %}} - -### Provisioner-Specific Pod Density - -#### Static Pod Density - -Static pod density can be configured at the provisioner level by specifying `maxPods` within the `.spec.kubeletConfiguration`. All nodes spawned by this provisioner will set this `maxPods` value on their kubelet and will account for this value during scheduling. - -See [Provisioner API Kubelet Configuration](../provisioners/#max-pods) for more details. - -#### Dynamic Pod Density - -Dynamic pod density (density that scales with the instance size) can be configured at the provisioner level by specifying `podsPerCore` within the `.spec.kubeletConfiguration`. Karpenter will calculate the expected pod density for each instance based on the instance's number of logical cores (vCPUs) and will account for this during scheduling. - -See [Provisioner API Kubelet Configuration](../provisioners/#pod-density) for more details. - -### Controller-Wide Pod Density - -{{% alert title="Deprecation Warning" color="warning" %}} -`AWS_ENI_LIMITED_POD_DENSITY` is deprecated in favor of the `.spec.kubeletConfiguration.maxPods` set at the Provisioner-level -{{% /alert %}} - -Set the environment variable `AWS_ENI_LIMITED_POD_DENSITY: "false"` (or the argument `--aws-eni-limited-pod-density=false`) in the Karpenter controller to allow nodes to host up to 110 pods by default. - -Environment variables for the Karpenter controller may be specified as [helm chart values](https://github.com/aws/karpenter/blob/c73f425e924bb64c3f898f30ca5035a1d8591183/charts/karpenter/values.yaml#L15). - -### VPC CNI Custom Networking - -By default, the VPC CNI allocates IPs for a node and pods from the same subnet. With [VPC CNI Custom Networking](https://aws.github.io/aws-eks-best-practices/networking/custom-networking), the pods will receive IP addresses from another subnet dedicated to pod IPs. This approach makes it easier to manage IP addresses and allows for separate Network Access Control Lists (NACLs) applied to your pods. VPC CNI Custom Networking reduces the pod density of a node since one of the ENI attachments will be used for the node and cannot share the allocated IPs on the interface to pods. Karpenter supports VPC CNI Custom Networking and similar CNI setups where the primary node interface is separated from the pods interfaces through a global [setting](./settings.md#configmap) within the karpenter-global-settings configmap: `aws.reservedENIs`. In the common case, `aws.reservedENIs` should be set to `"1"` if using Custom Networking. - -{{% alert title="Windows Support Notice" color="warning" %}} -It's currently not possible to specify custom networking with Windows nodes. -{{% /alert %}} - -## Limit Pod Density - -Generally, increasing pod density is more efficient. However, some use cases exist for limiting pod density. - -### Topology Spread - -You can use [topology spread]({{< relref "scheduling.md#topology-spread" >}}) features to reduce blast radius. For example, spreading workloads across EC2 Availability Zones. - - -### Restrict Instance Types - -Exclude large instance sizes to reduce the blast radius of an EC2 instance failure. - -Consider setting up upper or lower boundaries on target instance sizes with the node.kubernetes.io/instance-type key. - -The following example shows how to avoid provisioning large Graviton instances in order to reduce the impact of individual instance failures: - -``` --key: node.kubernetes.io/instance-type - operator: NotIn - values: - 'm6g.16xlarge' - 'm6gd.16xlarge' - 'r6g.16xlarge' - 'r6gd.16xlarge' - 'c6g.16xlarge' -``` diff --git a/website/content/en/v0.30/concepts/provisioners.md b/website/content/en/v0.30/concepts/provisioners.md deleted file mode 100644 index 186b50594962..000000000000 --- a/website/content/en/v0.30/concepts/provisioners.md +++ /dev/null @@ -1,483 +0,0 @@ ---- -title: "Provisioners" -linkTitle: "Provisioners" -weight: 1 -description: > - Learn about Karpenter Provisioners ---- - -When you first installed Karpenter, you set up a default Provisioner. -The Provisioner sets constraints on the nodes that can be created by Karpenter and the pods that can run on those nodes. -The Provisioner can be set to do things like: - -* Define taints to limit the pods that can run on nodes Karpenter creates -* Define any startup taints to inform Karpenter that it should taint the node initially, but that the taint is temporary. -* Limit node creation to certain zones, instance types, and computer architectures -* Set defaults for node expiration - -You can change your Provisioner or add other Provisioners to Karpenter. -Here are things you should know about Provisioners: - -* Karpenter won't do anything if there is not at least one Provisioner configured. -* Each Provisioner that is configured is looped through by Karpenter. -* If Karpenter encounters a taint in the Provisioner that is not tolerated by a Pod, Karpenter won't use that Provisioner to provision the pod. -* If Karpenter encounters a startup taint in the Provisioner it will be applied to nodes that are provisioned, but pods do not need to tolerate the taint. Karpenter assumes that the taint is temporary and some other system will remove the taint. -* It is recommended to create Provisioners that are mutually exclusive. So no Pod should match multiple Provisioners. If multiple Provisioners are matched, Karpenter will use the Provisioner with the highest [weight](#specweight). - -For some example `Provisioner` configurations, see the [examples in the Karpenter GitHub repository](https://github.com/aws/karpenter/blob/main/examples/provisioner/). - -```yaml -apiVersion: karpenter.sh/v1alpha5 -kind: Provisioner -metadata: - name: default -spec: - # References cloud provider-specific custom resource, see your cloud provider specific documentation - providerRef: - name: default - - # Provisioned nodes will have these taints - # Taints may prevent pods from scheduling if they are not tolerated by the pod. - taints: - - key: example.com/special-taint - effect: NoSchedule - - # Provisioned nodes will have these taints, but pods do not need to tolerate these taints to be provisioned by this - # provisioner. These taints are expected to be temporary and some other entity (e.g. a DaemonSet) is responsible for - # removing the taint after it has finished initializing the node. - startupTaints: - - key: example.com/another-taint - effect: NoSchedule - - # Labels are arbitrary key-values that are applied to all nodes - labels: - billing-team: my-team - - # Annotations are arbitrary key-values that are applied to all nodes - annotations: - example.com/owner: "my-team" - - # Requirements that constrain the parameters of provisioned nodes. - # These requirements are combined with pod.spec.topologySpreadConstraints, pod.spec.affinity.nodeAffinity, pod.spec.affinity.podAffinity, and pod.spec.nodeSelector rules. - # Operators { In, NotIn } are supported to enable including or excluding values - requirements: - - key: "karpenter.k8s.aws/instance-category" - operator: In - values: ["c", "m", "r"] - - key: "karpenter.k8s.aws/instance-cpu" - operator: In - values: ["4", "8", "16", "32"] - - key: "karpenter.k8s.aws/instance-hypervisor" - operator: In - values: ["nitro"] - - key: "karpenter.k8s.aws/instance-generation" - operator: Gt - values: ["2"] - - key: "topology.kubernetes.io/zone" - operator: In - values: ["us-west-2a", "us-west-2b"] - - key: "kubernetes.io/arch" - operator: In - values: ["arm64", "amd64"] - - key: "karpenter.sh/capacity-type" # If not included, the webhook for the AWS cloud provider will default to on-demand - operator: In - values: ["spot", "on-demand"] - - # Karpenter provides the ability to specify a few additional Kubelet args. - # These are all optional and provide support for additional customization and use cases. - kubeletConfiguration: - clusterDNS: ["10.0.1.100"] - containerRuntime: containerd - systemReserved: - cpu: 100m - memory: 100Mi - ephemeral-storage: 1Gi - kubeReserved: - cpu: 200m - memory: 100Mi - ephemeral-storage: 3Gi - evictionHard: - memory.available: 5% - nodefs.available: 10% - nodefs.inodesFree: 10% - evictionSoft: - memory.available: 500Mi - nodefs.available: 15% - nodefs.inodesFree: 15% - evictionSoftGracePeriod: - memory.available: 1m - nodefs.available: 1m30s - nodefs.inodesFree: 2m - evictionMaxPodGracePeriod: 60 - imageGCHighThresholdPercent: 85 - imageGCLowThresholdPercent: 80 - cpuCFSQuota: true - podsPerCore: 2 - maxPods: 20 - - - # Resource limits constrain the total size of the cluster. - # Limits prevent Karpenter from creating new instances once the limit is exceeded. - limits: - resources: - cpu: "1000" - memory: 1000Gi - - # Enables consolidation which attempts to reduce cluster cost by both removing un-needed nodes and down-sizing those - # that can't be removed. Mutually exclusive with the ttlSecondsAfterEmpty parameter. - consolidation: - enabled: true - - # If omitted, the feature is disabled and nodes will never expire. If set to less time than it requires for a node - # to become ready, the node may expire before any pods successfully start. - ttlSecondsUntilExpired: 2592000 # 30 Days = 60 * 60 * 24 * 30 Seconds; - - # If omitted, the feature is disabled, nodes will never scale down due to low utilization - ttlSecondsAfterEmpty: 30 - - # Priority given to the provisioner when the scheduler considers which provisioner - # to select. Higher weights indicate higher priority when comparing provisioners. - # Specifying no weight is equivalent to specifying a weight of 0. - weight: 10 -``` - -## spec.requirements - -Kubernetes defines the following [Well-Known Labels](https://kubernetes.io/docs/reference/labels-annotations-taints/), and cloud providers (e.g., AWS) implement them. They are defined at the "spec.requirements" section of the Provisioner API. - -In addition to the well-known labels from Kubernetes, Karpenter supports AWS-specific labels for more advanced scheduling. See the full list [here](../scheduling/#well-known-labels). - -These well-known labels may be specified at the provisioner level, or in a workload definition (e.g., nodeSelector on a pod.spec). Nodes are chosen using both the provisioner's and pod's requirements. If there is no overlap, nodes will not be launched. In other words, a pod's requirements must be within the provisioner's requirements. If a requirement is not defined for a well known label, any value available to the cloud provider may be chosen. - -For example, an instance type may be specified using a nodeSelector in a pod spec. If the instance type requested is not included in the provisioner list and the provisioner has instance type requirements, Karpenter will not create a node or schedule the pod. - -📝 None of these values are required. - -### Instance Types - -- key: `node.kubernetes.io/instance-type` -- key: `karpenter.k8s.aws/instance-family` -- key: `karpenter.k8s.aws/instance-category` -- key: `karpenter.k8s.aws/instance-generation` - -Generally, instance types should be a list and not a single value. Leaving these requirements undefined is recommended, as it maximizes choices for efficiently placing pods. - -Review [AWS instance types](../instance-types). Most instance types are supported with the exclusion of [non-HVM](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/virtualization_types.html). - -{{% alert title="Defaults" color="secondary" %}} -If no instance type constraints are defined, Karpenter will set default instance type constraints on your Provisioner that supports most common user workloads: - -```yaml -requirements: - - key: karpenter.k8s.aws/instance-category - operator: In - values: ["c", "m", "r"] - - key: karpenter.k8s.aws/instance-generation - operator: Gt - values: ["2"] -``` -{{% /alert %}} - -### Availability Zones - -- key: `topology.kubernetes.io/zone` -- value example: `us-east-1c` -- value list: `aws ec2 describe-availability-zones --region ` - -Karpenter can be configured to create nodes in a particular zone. Note that the Availability Zone `us-east-1a` for your AWS account might not have the same location as `us-east-1a` for another AWS account. - -[Learn more about Availability Zone -IDs.](https://docs.aws.amazon.com/ram/latest/userguide/working-with-az-ids.html) - -### Architecture - -- key: `kubernetes.io/arch` -- values - - `amd64` - - `arm64` - -Karpenter supports `amd64` nodes, and `arm64` nodes. - -{{% alert title="Defaults" color="secondary" %}} -If no architecture constraint is defined, Karpenter will set the default architecture constraint on your Provisioner that supports most common user workloads: - -```yaml -requirements: - - key: kubernetes.io/arch - operator: In - values: ["amd64"] -``` -{{% /alert %}} - -### Operating System - - key: `kubernetes.io/os` - - values - - `linux` - - `windows` - -Karpenter supports `linux` and `windows` operating systems. - -{{% alert title="Defaults" color="secondary" %}} -If no operating system constraint is defined, Karpenter will set the default operating system constraint on your Provisioner that supports most common user workloads: - -```yaml -requirements: - - key: kubernetes.io/os - operator: In - values: ["linux"] -``` -{{% /alert %}} - -### Capacity Type - -- key: `karpenter.sh/capacity-type` -- values - - `spot` - - `on-demand` - -Karpenter supports specifying capacity type, which is analogous to [EC2 purchase options](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-purchasing-options.html). - -Karpenter prioritizes Spot offerings if the provisioner allows Spot and on-demand instances. If the provider API (e.g. EC2 Fleet's API) indicates Spot capacity is unavailable, Karpenter caches that result across all attempts to provision EC2 capacity for that instance type and zone for the next 45 seconds. If there are no other possible offerings available for Spot, Karpenter will attempt to provision on-demand instances, generally within milliseconds. - -Karpenter also allows `karpenter.sh/capacity-type` to be used as a topology key for enforcing topology-spread. - -{{% alert title="Defaults" color="secondary" %}} -If no capacity type constraint is defined, Karpenter will set the default capacity type constraint on your Provisioner that supports most common user workloads: - -```yaml -requirements: - - key: karpenter.sh/capacity-type - operator: In - values: ["on-demand"] -``` -{{% /alert %}} - -## spec.weight - -Karpenter allows you to describe provisioner preferences through a `weight` mechanism similar to how weight is described with [pod and node affinities](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity). - -For more information on weighting Provisioners, see the [Weighting Provisioners section](../scheduling#weighting-provisioners) in the scheduling details. - -## spec.kubeletConfiguration - -Karpenter provides the ability to specify a few additional Kubelet args. These are all optional and provide support for -additional customization and use cases. Adjust these only if you know you need to do so. For more details on kubelet configuration arguments, [see the KubeletConfiguration API specification docs](https://kubernetes.io/docs/reference/config-api/kubelet-config.v1beta1/). The implemented fields are a subset of the full list of upstream kubelet configuration arguments. Please cut an issue if you'd like to see another field implemented. - -```yaml -spec: - ... - kubeletConfiguration: - clusterDNS: ["10.0.1.100"] - containerRuntime: containerd - systemReserved: - cpu: 100m - memory: 100Mi - ephemeral-storage: 1Gi - kubeReserved: - cpu: 200m - memory: 100Mi - ephemeral-storage: 3Gi - evictionHard: - memory.available: 5% - nodefs.available: 10% - nodefs.inodesFree: 10% - evictionSoft: - memory.available: 500Mi - nodefs.available: 15% - nodefs.inodesFree: 15% - evictionSoftGracePeriod: - memory.available: 1m - nodefs.available: 1m30s - nodefs.inodesFree: 2m - evictionMaxPodGracePeriod: 60 - imageGCHighThresholdPercent: 85 - imageGCLowThresholdPercent: 80 - cpuCFSQuota: true - podsPerCore: 2 - maxPods: 20 -``` - -You can specify the container runtime to be either `dockerd` or `containerd`. By default, `containerd` is used. - -* `containerd` is the only valid container runtime when using the `Bottlerocket` AMIFamily or when using Kubernetes version 1.24+ and the `AL2`, `Windows2019`, or `Windows2022` AMIFamilies. - -### Reserved Resources - -Karpenter will automatically configure the system and kube reserved resource requests on the fly on your behalf. These requests are used to configure your node and to make scheduling decisions for your pods. If you have specific requirements or know that you will have additional capacity requirements, you can optionally override the `--system-reserved` configuration defaults with the `.spec.kubeletConfiguration.systemReserved` values and the `--kube-reserved` configuration defaults with the `.spec.kubeletConfiguration.kubeReserved` values. - -For more information on the default `--system-reserved` and `--kube-reserved` configuration refer to the [Kubelet Docs](https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/#kube-reserved) - -### Eviction Thresholds - -The kubelet supports eviction thresholds by default. When enough memory or file system pressure is exerted on the node, the kubelet will begin to evict pods to ensure that system daemons and other system processes can continue to run in a healthy manner. - -Kubelet has the notion of [hard evictions](https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#hard-eviction-thresholds) and [soft evictions](https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#soft-eviction-thresholds). In hard evictions, pods are evicted as soon as a threshold is met, with no grace period to terminate. Soft evictions, on the other hand, provide an opportunity for pods to be terminated gracefully. They do so by sending a termination signal to pods that are planning to be evicted and allowing those pods to terminate up to their grace period. - -Karpenter supports [hard evictions](https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#hard-eviction-thresholds) through the `.spec.kubeletConfiguration.evictionHard` field and [soft evictions](https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#soft-eviction-thresholds) through the `.spec.kubeletConfiguration.evictionSoft` field. `evictionHard` and `evictionSoft` are configured by listing [signal names](https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#eviction-signals) with either percentage values or resource values. - -```yaml -spec: - ... - kubeletConfiguration: - evictionHard: - memory.available: 500Mi - nodefs.available: 10% - nodefs.inodesFree: 10% - imagefs.available: 5% - imagefs.inodesFree: 5% - pid.available: 7% - evictionSoft: - memory.available: 1Gi - nodefs.available: 15% - nodefs.inodesFree: 15% - imagefs.available: 10% - imagefs.inodesFree: 10% - pid.available: 10% -``` - -#### Supported Eviction Signals - -| Eviction Signal | Description | -| --------------- | ----------- | -| memory.available | memory.available := node.status.capacity[memory] - node.stats.memory.workingSet | -| nodefs.available | nodefs.available := node.stats.fs.available | -| nodefs.inodesFree | nodefs.inodesFree := node.stats.fs.inodesFree | -| imagefs.available | imagefs.available := node.stats.runtime.imagefs.available | -| imagefs.inodesFree | imagefs.inodesFree := node.stats.runtime.imagefs.inodesFree | -| pid.available | pid.available := node.stats.rlimit.maxpid - node.stats.rlimit.curproc | - -For more information on eviction thresholds, view the [Node-pressure Eviction](https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction) section of the official Kubernetes docs. - -#### Soft Eviction Grace Periods - -Soft eviction pairs an eviction threshold with a specified grace period. With soft eviction thresholds, the kubelet will only begin evicting pods when the node exceeds its soft eviction threshold over the entire duration of its grace period. For example, if you specify `evictionSoft[memory.available]` of `500Mi` and a `evictionSoftGracePeriod[memory.available]` of `1m30`, the node must have less than `500Mi` of available memory over a minute and a half in order for the kubelet to begin evicting pods. - -Optionally, you can specify an `evictionMaxPodGracePeriod` which defines the administrator-specified maximum pod termination grace period to use during soft eviction. If a namespace-owner had specified a pod `terminationGracePeriodInSeconds` on pods in their namespace, the minimum of `evictionPodGracePeriod` and `terminationGracePeriodInSeconds` would be used. - -```yaml -spec: - ... - kubeletConfiguration: - evictionSoftGracePeriod: - memory.available: 1m - nodefs.available: 1m30s - nodefs.inodesFree: 2m - imagefs.available: 1m30s - imagefs.inodesFree: 2m - pid.available: 2m - evictionMaxPodGracePeriod: 60 -``` - -### Pod Density - -#### Max Pods - -By default, AWS will configure the maximum density of pods on a node [based on the node instance type](https://github.com/awslabs/amazon-eks-ami/blob/master/files/eni-max-pods.txt). For small instances that require an increased pod density or large instances that require a reduced pod density, you can override this default value with `.spec.kubeletConfiguration.maxPods`. This value will be used during Karpenter pod scheduling and passed through to `--max-pods` on kubelet startup. - -{{% alert title="Note" color="primary" %}} -When using small instance types, it may be necessary to enable [prefix assignment mode](https://aws.amazon.com/blogs/containers/amazon-vpc-cni-increases-pods-per-node-limits/) in the AWS VPC CNI plugin to support a higher pod density per node. Prefix assignment mode was introduced in AWS VPC CNI v1.9 and allows ENIs to manage a broader set of IP addresses. Much higher pod densities are supported as a result. -{{% /alert %}} - -#### Pods Per Core - -An alternative way to dynamically set the maximum density of pods on a node is to use the `.spec.kubeletConfiguration.podsPerCore` value. Karpenter will calculate the pod density during scheduling by multiplying this value by the number of logical cores (vCPUs) on an instance type. This value will also be passed through to the `--pods-per-core` value on kubelet startup to configure the number of allocatable pods the kubelet can assign to the node instance. - -The value generated from `podsPerCore` cannot exceed `maxPods`, meaning, if both are set, the minimum of the `podsPerCore` dynamic pod density and the static `maxPods` value will be used for scheduling. - -{{% alert title="Note" color="primary" %}} -`maxPods` may not be set in the `kubeletConfiguration` of a Provisioner, but may still be restricted by the `ENI_LIMITED_POD_DENSITY` value. You may want to ensure that the `podsPerCore` value that will be used for instance families associated with the Provisioner will not cause unexpected behavior by exceeding the `maxPods` value. -{{% /alert %}} - -{{% alert title="Pods Per Core on Bottlerocket" color="warning" %}} -Bottlerocket AMIFamily currently does not support `podsPerCore` configuration. If a Provisioner contains a `provider` or `providerRef` to a node template that will launch a Bottlerocket instance, the `podsPerCore` value will be ignored for scheduling and for configuring the kubelet. -{{% /alert %}} - -## spec.limits.resources - -The provisioner spec includes a limits section (`spec.limits.resources`), which constrains the maximum amount of resources that the provisioner will manage. - -Karpenter supports limits of any resource type reported by your cloudprovider. It limits instance types when scheduling to those that will not exceed the specified limits. If a limit has been exceeded, nodes provisioning is prevented until some nodes have been terminated. - -```yaml -apiVersion: karpenter.sh/v1alpha5 -kind: Provisioner -metadata: - name: default -spec: - requirements: - - key: karpenter.sh/capacity-type - operator: In - values: ["spot"] - limits: - resources: - cpu: 1000 - memory: 1000Gi - nvidia.com/gpu: 2 -``` - -{{% alert title="Note" color="primary" %}} -Karpenter provisioning is highly parallel. Because of this, limit checking is eventually consistent, which can result in overrun during rapid scale outs. -{{% /alert %}} - -CPU limits are described with a `DecimalSI` value. Note that the Kubernetes API will coerce this into a string, so we recommend against using integers to avoid GitOps skew. - -Memory limits are described with a [`BinarySI` value, such as 1000Gi.](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#meaning-of-memory) - -You can view the current consumption of cpu and memory on your cluster by running: -``` -kubectl get provisioner -o=jsonpath='{.items[0].status}' -``` - -Review the [Kubernetes core API](https://github.com/kubernetes/api/blob/37748cca582229600a3599b40e9a82a951d8bbbf/core/v1/resource.go#L23) (`k8s.io/api/core/v1`) for more information on `resources`. - -## spec.providerRef - -This field points to the cloud provider-specific custom resource. Learn more about [AWSNodeTemplates](../node-templates/). - -## spec.consolidation - -You can configure Karpenter to deprovision instances through your Provisioner in multiple ways. You can use `spec.ttlSecondsAfterEmpty`, `spec.ttlSecondsUntilExpired` or `spec.consolidation.enabled`. Read [Deprovisioning](../deprovisioning/) for more. - -## Example Use-Cases - -### Isolating Expensive Hardware - -A provisioner can be set up to only provision nodes on particular processor types. -The following example sets a taint that only allows pods with tolerations for Nvidia GPUs to be scheduled: - -```yaml -apiVersion: karpenter.sh/v1alpha5 -kind: Provisioner -metadata: - name: gpu -spec: - consolidation: - enabled: true - requirements: - - key: node.kubernetes.io/instance-type - operator: In - values: ["p3.8xlarge", "p3.16xlarge"] - taints: - - key: nvidia.com/gpu - value: "true" - effect: NoSchedule -``` -In order for a pod to run on a node defined in this provisioner, it must tolerate `nvidia.com/gpu` in its pod spec. - -### Cilium Startup Taint - -Per the Cilium [docs](https://docs.cilium.io/en/stable/installation/taints/#taint-effects), it's recommended to place a taint of `node.cilium.io/agent-not-ready=true:NoExecute` on nodes to allow Cilium to configure networking prior to other pods starting. This can be accomplished via the use of Karpenter `startupTaints`. These taints are placed on the node, but pods aren't required to tolerate these taints to be considered for provisioning. - -```yaml -apiVersion: karpenter.sh/v1alpha5 -kind: Provisioner -metadata: - name: cilium-startup -spec: - consolidation: - enabled: true - startupTaints: - - key: node.cilium.io/agent-not-ready - value: "true" - effect: NoExecute -``` diff --git a/website/content/en/v0.30/concepts/settings.md b/website/content/en/v0.30/concepts/settings.md deleted file mode 100644 index cfe4416f652d..000000000000 --- a/website/content/en/v0.30/concepts/settings.md +++ /dev/null @@ -1,124 +0,0 @@ ---- -title: "Settings" -linkTitle: "Settings" -weight: 5 -description: > - Configure Karpenter ---- - -There are two main configuration mechanisms that can be used to configure Karpenter: Environment Variables / CLI parameters to the controller and webhook binaries and the `karpenter-global-settings` config-map. - -## Environment Variables / CLI Flags - -[comment]: <> (the content below is generated from hack/docs/configuration_gen_docs.go) - -| Environment Variable | CLI Flag | Description | -|--|--|--| -| DISABLE_WEBHOOK | \-\-disable-webhook | Disable the admission and validation webhooks (default = false)| -| ENABLE_PROFILING | \-\-enable-profiling | Enable the profiling on the metric endpoint (default = false)| -| HEALTH_PROBE_PORT | \-\-health-probe-port | The port the health probe endpoint binds to for reporting controller health (default = 8081)| -| KARPENTER_SERVICE | \-\-karpenter-service | The Karpenter Service name for the dynamic webhook certificate| -| KUBE_CLIENT_BURST | \-\-kube-client-burst | The maximum allowed burst of queries to the kube-apiserver (default = 300)| -| KUBE_CLIENT_QPS | \-\-kube-client-qps | The smoothed rate of qps to kube-apiserver (default = 200)| -| LEADER_ELECT | \-\-leader-elect | Start leader election client and gain leadership before executing the main loop. Enable this when running replicated components for high availability. (default = true)| -| MEMORY_LIMIT | \-\-memory-limit | Memory limit on the container running the controller. The GC soft memory limit is set to 90% of this value. (default = -1)| -| METRICS_PORT | \-\-metrics-port | The port the metric endpoint binds to for operating metrics about the controller itself (default = 8000)| -| WEBHOOK_PORT | \-\-webhook-port | The port the webhook endpoint binds to for validation and mutation of resources (default = 8443)| - -[comment]: <> (end docs generated content from hack/docs/configuration_gen_docs.go) - -## ConfigMap - -Karpenter installs a default configuration via its Helm chart that should work for most. Additional configuration can be performed by editing the `karpenter-global-settings` configmap within the namespace that Karpenter was installed in. - -```yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: karpenter-global-settings - namespace: karpenter -data: - # The maximum length of a batch window. The longer this is, the more pods we can consider for provisioning at one - # time which usually results in fewer but larger nodes. - batchMaxDuration: 10s - # The maximum amount of time with no new pending pods that if exceeded ends the current batching window. If pods arrive - # faster than this time, the batching window will be extended up to the maxDuration. If they arrive slower, the pods - # will be batched separately. - batchIdleDuration: 1s - # Role to assume for calling AWS services. - aws.assumeRoleARN: arn:aws:iam::111222333444:role/examplerole - # Duration of assumed credentials in minutes. Default value is 15 minutes. Not used unless aws.assumeRole set. - aws.assumeRoleDuration: 15m - # Cluster CA bundle for nodes to use for TLS connections with the API server. If not set, this is taken from the controller's TLS configuration. - aws.clusterCABundle: "LS0tLS1..." - # [REQUIRED] The kubernetes cluster name for resource discovery - aws.clusterName: karpenter-cluster - # The external kubernetes cluster endpoint for new nodes to connect with. If not specified, will discover the cluster endpoint using DescribeCluster API - aws.clusterEndpoint: https://00000000000000000000000000000000.gr7.us-west-2.eks.amazonaws.com - # The default instance profile to use when provisioning nodes - aws.defaultInstanceProfile: karpenter-instance-profile - # If true, then instances that support pod ENI will report a vpc.amazonaws.com/pod-eni resource - aws.enablePodENI: "false" - # Indicates whether new nodes should use ENI-based pod density. DEPRECATED: Use `.spec.kubeletConfiguration.maxPods` to set pod density on a per-provisioner basis - aws.enableENILimitedPodDensity: "true" - # If true, then assume we can't reach AWS services which don't have a VPC endpoint - # This also has the effect of disabling look-ups to the AWS pricing endpoint - aws.isolatedVPC: "false" - # The VM memory overhead as a percent that will be subtracted - # from the total memory for all instance types - aws.vmMemoryOverheadPercent: "0.075" - # aws.interruptionQueueName is disabled if not specified. Enabling interruption handling may - # require additional permissions on the controller service account. Additional permissions are outlined in the docs - aws.interruptionQueueName: karpenter-cluster - # Global tags are specified by including a JSON object of string to string from tag key to tag value - aws.tags: '{"custom-tag1-key": "custom-tag-value", "custom-tag2-key": "custom-tag-value"}' - # Reserved ENIs are not included in the calculations for max-pods or kube-reserved - # This is most often used in the VPC CNI custom networking setup https://docs.aws.amazon.com/eks/latest/userguide/cni-custom-network.html - aws.reservedENIs: "1" -``` - -### Feature Gates -Karpenter uses [feature gates](https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/#feature-gates-for-alpha-or-beta-features). You can add a feature gate's ConfigKey to the `karpenter-global-settings` ConfigMap above with the desired value. - -| Feature | Default | Config Key | Stage | Since | Until | -|---------|---------|---------------------------|-------|---------|-------| -| Drift | false | featureGates.driftEnabled | Alpha | v0.21.0 | | - - -### Batching Parameters - -The batching parameters control how Karpenter batches an incoming stream of pending pods. Reducing these values may trade off a slightly faster time from pending pod to node launch, in exchange for launching smaller nodes. Increasing the values can do the inverse. Karpenter provides reasonable defaults for these values, but if you have specific knowledge about your workloads you can tweak these parameters to match the expected rate of incoming pods. - -For a standard deployment scale-up, the pods arrive at the QPS setting of the `kube-controller-manager`, and the default values are typically fine. These settings are intended for use cases where other systems may create large numbers of pods over a period of many seconds or minutes and there is a desire to batch them together. - -#### `batchIdleDuration` - -The `batchIdleDuration` is the period of time that a new pending pod extends the current batching window. This can be increased to handle scenarios where pods arrive slower than one second part, but it would be preferable if they were batched together onto a single larger node. - -This value is expressed as a string value like `10s`, `1m` or `2h45m`. The valid time units are `ns`, `us` (or `µs`), `ms`, `s`, `m`, `h`. - -#### `batchMaxDuration` - -The `batchMaxDuration` is the maximum period of time a batching window can be extended to. Increasing this value will allow the maximum batch window size to increase to collect more pending pods into a single batch at the expense of a longer delay from when the first pending pod was created. - -This value is expressed as a string value like `10s`, `1m` or `2h45m`. The valid time units are `ns`, `us` (or `µs`), `ms`, `s`, `m`, `h`. - -### AWS Parameters - -#### `aws.tags` - -Global tags are applied to __all__ AWS infrastructure resources deployed by Karpenter. These resources include: - -- Launch Templates -- Volumes -- Instances - -Tags are specified by including a JSON object of string to string from tag key to tag value. - -```yaml - aws.tags: '{"custom-tag1-key": "custom-tag-value", "custom-tag2-key": "custom-tag-value"}' -``` - -{{% alert title="Note" color="primary" %}} -Since you can specify tags at the global level and in the `AWSNodeTemplate` resource, if a key is specified in both locations, the `AWSNodeTemplate` tag value will override the global tag. -{{% /alert %}} diff --git a/website/content/en/v0.30/concepts/threat-model.md b/website/content/en/v0.30/concepts/threat-model.md deleted file mode 100644 index 200a83848221..000000000000 --- a/website/content/en/v0.30/concepts/threat-model.md +++ /dev/null @@ -1,102 +0,0 @@ ---- -title: "Threat Model" -linkTitle: "Threat Model" -weight: 999 ---- - -Karpenter observes Kubernetes pods and launches nodes in response to those pods’ scheduling constraints. Karpenter does not perform the actual scheduling and instead waits for [kube-scheduler](https://kubernetes.io/docs/concepts/scheduling-eviction/kube-scheduler/) to schedule the pods. - -When running in AWS, Karpenter is typically installed onto EC2 instances that run in EKS Clusters. Karpenter relies on public facing AWS APIs and standard IAM Permissions. Karpenter uses AWS-SDK-Go v1, and AWS advises that credentials are provided using [IAM Roles for Service Accounts](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html). - - -## Architecture & Actors - -1. **Cluster Operator**: An identity that installs and configures Karpenter in a Kubernetes cluster, and configures Karpenter's cloud identity and permissions. -2. **Cluster Developer**: An identity that can create pods, typically through Deployments, DaemonSets, or other pod-controller types. -3. **Karpenter Controller:** The Karpenter application pod that operates inside a cluster. - -![threat-model](/threat-model.png) - -## Capabilities - -### Cluster Operator - -The Cluster Operator has full control over Kubernetes resources to install and configure Karpenter, its CRDs, and Provisioners and NodeTemplates. The Cluster Operator has privileges to manage the cloud identities and permissions for Nodes, and the cloud identity and permissions for Karpenter. - -### Cluster Developer - -A Cluster Developer has the ability to create pods via Deployments, ReplicaSets, StatefulSets, Jobs, etc. This assumes that the Cluster Developer cannot modify the Karpenter pod or launch pods using Karpenter’s service account and gain access to Karpenter’s IAM role. - -### Karpenter Controller - -Karpenter has permissions to create and manage cloud instances. Karpenter has Kubernetes API permissions to create, update, and remove nodes, as well as evict pods. For a full list of the permissions, see the RBAC rules in the helm chart template. Karpenter also has AWS IAM permissions to create instances with IAM roles. - -* [aggregate-clusterrole.yaml](https://github.com/aws/karpenter/blob/v0.30.0/charts/karpenter/templates/aggregate-clusterrole.yaml) -* [clusterrole-core.yaml](https://github.com/aws/karpenter/blob/v0.30.0/charts/karpenter/templates/clusterrole-core.yaml) -* [clusterrole.yaml](https://github.com/aws/karpenter/blob/v0.30.0/charts/karpenter/templates/clusterrole.yaml) -* [rolebinding.yaml](https://github.com/aws/karpenter/blob/v0.30.0/charts/karpenter/templates/rolebinding.yaml) -* [role.yaml](https://github.com/aws/karpenter/blob/v0.30.0/charts/karpenter/templates/role.yaml) - -## Assumptions - -| Category | Assumption | Comment | -| --- | --- | --- | -| Generic | The Karpenter pod is operated on a node in the cluster, and uses a Service Account for authentication to the Kubernetes API | Cluster Operators may want to isolate the node running the Karpenter pod to a system-pool of nodes to mitigate the possibility of container breakout with Karpenter’s permissions. | -| Generic | Cluster Developer does not have any Kubernetes permissions to manage Karpenter running in the cluster (The deployment, pods, clusterrole, etc) | | -| Generic | Restrictions on the fields of pods a Cluster Developer can create are out of scope. | Cluster Operators can use policy frameworks to enforce restrictions on Pod capabilities | -| Generic | No sensitive data is included in non-Secret resources in the Kubernetes API. The Karpenter controller has the ability to list all pods, nodes, deployments, and many other pod-controller and storage resource types. | Karpenter does not have permission to list/watch cluster-wide ConfigMaps or Secrets | -| Generic | Karpenter has permissions to create, modify, and delete nodes from the cluster, and evict any pod. | Cluster Operators running applications with varying security profiles in the same cluster may want to configure dedicated nodes and scheduling rules for Karpenter to mitigate potential container escapes from other containers | -| AWS-Specific | The Karpenter IAM policy is encoded in the GitHub repo. Any additional permissions possibly granted to that role by the administrator are out of scope | | -| AWS-Specific | The Karpenter pod uses IRSA for AWS credentials | Setup of IRSA is out of scope for this document | - -## Generic Threats and Mitigations - -### Threat: Cluster Developer can influence creation of an arbitrary number of nodes - -**Background**: Karpenter creates new instances based on the count of pending pods. - -**Threat**: A Cluster Developer attempts to have Karpenter create more instances than intended by creating a large number of pods or by using anti-affinity to schedule one pod per node. - -**Mitigation**: In addition to [Kubernetes resource limits](https://kubernetes.io/docs/concepts/policy/resource-quotas/#object-count-quota), Cluster Operators can [configure limits on a Provisioner](https://karpenter.sh/preview/concepts/provisioners/#speclimitsresources) to limit the total amount of memory, CPU, or other resources provisioned across all nodes. - -## AWS-Specific Threats - -### Threat: Using EC2 CreateTag/DeleteTag Permissions to Orchestrate Machine Creation/Deletion - -**Background**: As of v0.28.0, Karpenter creates a mapping between CloudProvider machines and CustomResources in the cluster for capacity tracking. To ensure this mapping is consistent, Karpenter utilizes the following tag keys: - -* `karpenter.sh/managed-by` -* `karpenter.sh/provisioner-name` -* `kubernetes.io/cluster/${CLUSTER_NAME}` - -Any user that has the ability to Create/Delete tags on CloudProvider machines will have the ability to orchestrate Karpenter to Create/Delete CloudProvider machines as a side effect. - -In addition, as of v0.29.0, Karpenter will Drift on Security Groups and Subnets. If a user has the Create/Delete tags permission for either of resources, they can orchestrate Karpenter to Create/Delete CloudProvider machines as a side effect. - -**Threat:** A Cluster Operator attempts to create or delete a tag on a resource discovered by Karpenter. If it has the ability to create a tag it can effectively create or delete CloudProvider machines associated with the tagged resources. - -**Mitigation** Cluster Operators should [enforce tag-based IAM policies](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_tags.html) on these tags against any EC2 instance resource (`i-*`) for any users that might have [CreateTags](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateTags.html)/[DeleteTags](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DeleteTags.html) permissions but should not have [RunInstances](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_RunInstances.html)/[TerminateInstances](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_TerminateInstances.html) permissions. - -### Threat: Launching EC2 instances with IAM roles not intended for Karpenter nodes - -**Background**: Many IAM roles in an AWS account may trust the EC2 service principal. IAM administrators must grant the `iam:PassRole` permission to IAM principals to allow those principals in the account to launch instances with specific roles. - -**Threat:** A Cluster Operator attempts to create a Node Template with an IAM role not intended for Karpenter - -**Mitigation**: Cluster Operators must enumerate the roles in the resource section of the IAM policy granted to the Karpenter role for the `iam:PassRole` action. - -### Threat: Karpenter can be used to create or terminate EC2 instances outside of the cluster - -**Background**: EC2 instances can exist in an AWS account outside of the Kubernetes cluster. - -**Threat**: An actor who obtains control of the Karpenter pod’s IAM role may be able to create or terminate EC2 instances not part of the Kubernetes cluster managed by Karpenter. - -**Mitigation**: Karpenter creates instances with tags, several of which can be enforced in the IAM policy granted to the Karpenter IAM role that restrict the instances Karpenter can terminate. One tag can require that the instance was provisioned by a Karpenter controller, another tag can include a cluster name to mitigate any termination between two clusters with Karpenter in the same account. Cluster Operators also can restrict the region to prevent two clusters in the same account with the same name in different regions. - -### Threat: Karpenter launches an EC2 instance using an unintended AMI - -**Background**: Cluster Developers can create Node Templates that refer to an AMI by metadata, such as a name rather than an AMI resource ID. - -**Threat:** A threat actor creates a public AMI with the same name as a customer’s AMI in an attempt to get Karpenter to select the threat actor’s AMI instead of the intended AMI. - -**Mitigation**: When selecting AMIs by name or tags, Karpenter defaults to adding an ownership filter of `self,amazon` so AMI images external to the account are not used. diff --git a/website/content/en/v0.30/getting-started/getting-started-with-karpenter/_index.md b/website/content/en/v0.30/getting-started/getting-started-with-karpenter/_index.md deleted file mode 100644 index 9c02da0d7131..000000000000 --- a/website/content/en/v0.30/getting-started/getting-started-with-karpenter/_index.md +++ /dev/null @@ -1,171 +0,0 @@ - ---- -title: "Getting Started with Karpenter" -linkTitle: "Getting Started with Karpenter" -weight: 10 -description: > - Set up a cluster and add Karpenter ---- - -Karpenter automatically provisions new nodes in response to unschedulable pods. Karpenter does this by observing events within the Kubernetes cluster, and then sending commands to the underlying cloud provider. - -This guide shows how to get started with Karpenter by creating a Kubernetes cluster and installing Karpenter. -To use Karpenter, you must be running a supported Kubernetes cluster on a supported cloud provider. -Currently, only EKS on AWS is supported. - -## Create a cluster and add Karpenter - -This guide uses `eksctl` to create the cluster. -It should take less than 1 hour to complete, and cost less than $0.25. -Follow the clean-up instructions to reduce any charges. - -### 1. Install utilities - -Karpenter is installed in clusters with a Helm chart. - -Karpenter requires cloud provider permissions to provision nodes, for AWS IAM -Roles for Service Accounts (IRSA) should be used. IRSA permits Karpenter -(within the cluster) to make privileged requests to AWS (as the cloud provider) -via a ServiceAccount. - -Install these tools before proceeding: - -1. [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2-linux.html) -2. `kubectl` - [the Kubernetes CLI](https://kubernetes.io/docs/tasks/tools/install-kubectl-linux/) -3. `eksctl` - [the CLI for AWS EKS](https://docs.aws.amazon.com/eks/latest/userguide/eksctl.html) -4. `helm` - [the package manager for Kubernetes](https://helm.sh/docs/intro/install/) - -[Configure the AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-quickstart.html) -with a user that has sufficient privileges to create an EKS cluster. Verify that the CLI can -authenticate properly by running `aws sts get-caller-identity`. - -### 2. Set environment variables - -After setting up the tools, set the Karpenter version number: - -```bash -export KARPENTER_VERSION=v0.30.0 -``` - -Then set the following environment variable: - -{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step01-config.sh" language="bash"%}} - -{{% alert title="Warning" color="warning" %}} -If you open a new shell to run steps in this procedure, you need to set some or all of the environment variables again. -To remind yourself of these values, type: - -```bash -echo $KARPENTER_VERSION $CLUSTER_NAME $AWS_DEFAULT_REGION $AWS_ACCOUNT_ID $TEMPOUT -``` - -{{% /alert %}} - - -### 3. Create a Cluster - -Create a basic cluster with `eksctl`. -The following cluster configuration will: - -* Use CloudFormation to set up the infrastructure needed by the EKS cluster. -* Create a Kubernetes service account and AWS IAM Role, and associate them using IRSA to let Karpenter launch instances. -* Add the Karpenter node role to the aws-auth configmap to allow nodes to connect. -* Use [AWS EKS managed node groups](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html) for the kube-system and karpenter namespaces. Uncomment fargateProfiles settings (and comment out managedNodeGroups settings) to use Fargate for both namespaces instead. -* Set KARPENTER_IAM_ROLE_ARN variables. -* Create a role to allow spot instances. -* Run helm to install karpenter - -{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step02-create-cluster.sh" language="bash"%}} - -{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step06-add-spot-role.sh" language="bash"%}} - -{{% alert title="Windows Support Notice" color="warning" %}} -In order to run Windows workloads, Windows support should be enabled in your EKS Cluster. -See [Enabling Windows support](https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html#enable-windows-support) to learn more. -{{% /alert %}} - -### 4. Install Karpenter - -{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step08-apply-helm-chart.sh" language="bash"%}} - -{{% alert title="Warning" color="warning" %}} -Karpenter creates a mapping between CloudProvider machines and CustomResources in the cluster for capacity tracking. To ensure this mapping is consistent, Karpenter utilizes the following tag keys: - -* `karpenter.sh/managed-by` -* `karpenter.sh/provisioner-name` -* `kubernetes.io/cluster/${CLUSTER_NAME}` - -Because Karpenter takes this dependency, any user that has the ability to Create/Delete these tags on CloudProvider machines will have the ability to orchestrate Karpenter to Create/Delete CloudProvider machines as a side effect. We recommend that you [enforce tag-based IAM policies](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_tags.html) on these tags against any EC2 instance resource (`i-*`) for any users that might have [CreateTags](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateTags.html)/[DeleteTags](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DeleteTags.html) permissions but should not have [RunInstances](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_RunInstances.html)/[TerminateInstances](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_TerminateInstances.html) permissions. -{{% /alert %}} - -### 5. Create Provisioner - -A single Karpenter provisioner is capable of handling many different pod -shapes. Karpenter makes scheduling and provisioning decisions based on pod -attributes such as labels and affinity. In other words, Karpenter eliminates -the need to manage many different node groups. - -Create a default provisioner using the command below. -This provisioner uses `securityGroupSelector` and `subnetSelector` to discover resources used to launch nodes. -We applied the tag `karpenter.sh/discovery` in the `eksctl` command above. -Depending how these resources are shared between clusters, you may need to use different tagging schemes. - -The `consolidation` value configures Karpenter to reduce cluster cost by removing and replacing nodes. As a result, consolidation will terminate any empty nodes on the cluster. This behavior can be disabled by leaving the value undefined or setting `consolidation.enabled` to `false`. Review the [provisioner CRD]({{}}) for more information. - -Review the [provisioner CRD]({{}}) for more information. For example, -`ttlSecondsUntilExpired` configures Karpenter to terminate nodes when a maximum age is reached. - -Note: This provisioner will create capacity as long as the sum of all created capacity is less than the specified limit. - -{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step12-add-provisioner.sh" language="bash"%}} - -Karpenter is now active and ready to begin provisioning nodes. - -## First Use - -Create some pods using a deployment and watch Karpenter provision nodes in response. - -### Scale up deployment - -This deployment uses the [pause image](https://www.ianlewis.org/en/almighty-pause-container) and starts with zero replicas. - -{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step13-automatic-node-provisioning.sh" language="bash"%}} - -### Scale down deployment - -Now, delete the deployment. After a short amount of time, Karpenter should terminate the empty nodes due to consolidation. - -{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step14-deprovisioning.sh" language="bash"%}} - -## Add optional monitoring with Grafana - -This section describes optional ways to configure Karpenter to enhance its capabilities. -In particular, the following commands deploy a Prometheus and Grafana stack that is suitable for this guide but does not include persistent storage or other configurations that would be necessary for monitoring a production deployment of Karpenter. -This deployment includes two Karpenter dashboards that are automatically onboarded to Grafana. They provide a variety of visualization examples on Karpenter metrics. - -{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step09-add-prometheus-grafana.sh" language="bash"%}} - -The Grafana instance may be accessed using port forwarding. - -{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step10-add-grafana-port-forward.sh" language="bash"%}} - -The new stack has only one user, `admin`, and the password is stored in a secret. The following command will retrieve the password. - -{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step11-grafana-get-password.sh" language="bash"%}} - -## Cleanup - -### Delete Karpenter nodes manually - -If you delete a node with kubectl, Karpenter will gracefully cordon, drain, -and shutdown the corresponding instance. Under the hood, Karpenter adds a -finalizer to the node object, which blocks deletion until all pods are -drained and the instance is terminated. Keep in mind, this only works for -nodes provisioned by Karpenter. - -{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step15-delete-node.sh" language="bash"%}} - -### Delete the cluster -To avoid additional charges, remove the demo infrastructure from your AWS account. - -{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step16-cleanup.sh" language="bash"%}} diff --git a/website/content/en/v0.30/getting-started/getting-started-with-karpenter/scripts/add-monitoring.sh b/website/content/en/v0.30/getting-started/getting-started-with-karpenter/scripts/add-monitoring.sh deleted file mode 100755 index 6d2332b06be2..000000000000 --- a/website/content/en/v0.30/getting-started/getting-started-with-karpenter/scripts/add-monitoring.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -set -euo pipefail #fail if one step fails - -if [ "$#" -ne 1 ] -then - echo "Missing required Karpenter version. Usage: add-monitoring.sh v0.0.1" - exit 1 -fi - -export KARPENTER_VERSION=$1 - -declare -a steps=( - step01-config.sh - step09-add-prometheus-grafana.sh - step10-add-grafana-port-forward.sh - step11-grafana-get-password.sh -) - -for step in "${steps[@]}"; do - echo "$step" - source $step -done diff --git a/website/content/en/v0.30/getting-started/getting-started-with-karpenter/scripts/add-provisioner.sh b/website/content/en/v0.30/getting-started/getting-started-with-karpenter/scripts/add-provisioner.sh deleted file mode 100755 index 51f286de0c3c..000000000000 --- a/website/content/en/v0.30/getting-started/getting-started-with-karpenter/scripts/add-provisioner.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -set -euo pipefail #fail if one step fails - -declare -a steps=( - step01-config.sh - step12-add-provisioner.sh - step13-automatic-node-provisioning.sh - step14-automatic-node-termination.sh -) - -for step in "${steps[@]}"; do - echo "$step" - source $step -done diff --git a/website/content/en/v0.30/getting-started/getting-started-with-karpenter/scripts/add-roles.sh b/website/content/en/v0.30/getting-started/getting-started-with-karpenter/scripts/add-roles.sh deleted file mode 100755 index cf0656f3db3d..000000000000 --- a/website/content/en/v0.30/getting-started/getting-started-with-karpenter/scripts/add-roles.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -set -euo pipefail #fail if one step fails - -if [ "$#" -ne 1 ] -then - echo "Missing required Karpenter version. Usage: setup-roles.sh v0.0.1" - exit 1 -fi - -export KARPENTER_VERSION=$1 -SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) - -declare -a steps=( - step03-iam-cloud-formation.sh - step04-grant-access.sh - step05-controller-iam.sh - step06-add-spot-role.sh -) - -for step in "${steps[@]}"; do - echo "$step" - source "$SCRIPT_DIR/$step" -done diff --git a/website/content/en/v0.30/getting-started/getting-started-with-karpenter/scripts/cleanup.sh b/website/content/en/v0.30/getting-started/getting-started-with-karpenter/scripts/cleanup.sh deleted file mode 100755 index 52851e7f975a..000000000000 --- a/website/content/en/v0.30/getting-started/getting-started-with-karpenter/scripts/cleanup.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -declare -a steps=( - step01-config.sh - step16-cleanup.sh -) - -for step in "${steps[@]}"; do - echo "$step" - source $step -done diff --git a/website/content/en/v0.30/getting-started/getting-started-with-karpenter/scripts/install.sh b/website/content/en/v0.30/getting-started/getting-started-with-karpenter/scripts/install.sh deleted file mode 100755 index e0ebfb32e511..000000000000 --- a/website/content/en/v0.30/getting-started/getting-started-with-karpenter/scripts/install.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -set -euo pipefail #fail if one step fails - -if [ "$#" -ne 1 ] -then - echo "Missing required Karpenter version. Usage: install.sh v0.0.1" - exit 1 -fi - -export KARPENTER_VERSION=$1 - -declare -a steps=( - step01-config.sh - step02-create-cluster.sh - step03-iam-cloud-formation.sh - step04-grant-access.sh - step05-controller-iam.sh - step06-add-spot-role.sh - step08-apply-helm-chart.sh -) - -for step in "${steps[@]}"; do - echo "$step" - source $step -done diff --git a/website/content/en/v0.30/getting-started/getting-started-with-karpenter/scripts/step01-config.sh b/website/content/en/v0.30/getting-started/getting-started-with-karpenter/scripts/step01-config.sh deleted file mode 100755 index a3af512d02ac..000000000000 --- a/website/content/en/v0.30/getting-started/getting-started-with-karpenter/scripts/step01-config.sh +++ /dev/null @@ -1,5 +0,0 @@ -export AWS_PARTITION="aws" # if you are not using standard partitions, you may need to configure to aws-cn / aws-us-gov -export CLUSTER_NAME="${USER}-karpenter-demo" -export AWS_DEFAULT_REGION="us-west-2" -export AWS_ACCOUNT_ID="$(aws sts get-caller-identity --query Account --output text)" -export TEMPOUT=$(mktemp) diff --git a/website/content/en/v0.30/getting-started/getting-started-with-karpenter/scripts/step12-add-provisioner.sh b/website/content/en/v0.30/getting-started/getting-started-with-karpenter/scripts/step12-add-provisioner.sh deleted file mode 100755 index 2d487c0449dc..000000000000 --- a/website/content/en/v0.30/getting-started/getting-started-with-karpenter/scripts/step12-add-provisioner.sh +++ /dev/null @@ -1,28 +0,0 @@ -cat < karpenter.yaml diff --git a/website/content/en/v0.30/getting-started/migrating-from-cas/scripts/step11-create-provisioner.sh b/website/content/en/v0.30/getting-started/migrating-from-cas/scripts/step11-create-provisioner.sh deleted file mode 100644 index 76642ce51122..000000000000 --- a/website/content/en/v0.30/getting-started/migrating-from-cas/scripts/step11-create-provisioner.sh +++ /dev/null @@ -1,26 +0,0 @@ -cat < - Learn about upgrading Karpenter ---- - -Karpenter is a controller that runs in your cluster, but it is not tied to a specific Kubernetes version, as the Cluster Autoscaler is. -Use your existing upgrade mechanisms to upgrade your core add-ons in Kubernetes and keep Karpenter up to date on bug fixes and new features. - -To make upgrading easier we aim to minimize introduction of breaking changes with the following: - -## Compatibility Matrix - -[comment]: <> (the content below is generated from hack/docs/compataiblitymetrix_gen_docs.go) - -| KUBERNETES | 1.23 | 1.24 | 1.25 | 1.26 | 1.27 | -|------------|---------|---------|---------|---------|---------| -| karpenter | 0.21.x+ | 0.21.x+ | 0.25.x+ | 0.28.x+ | 0.28.x+ | - -[comment]: <> (end docs generated content from hack/docs/compataiblitymetrix_gen_docs.go) - -{{% alert title="Note" color="warning" %}} -Karpenter currently does not support the following [new `topologySpreadConstraints` keys](https://kubernetes.io/blog/2023/04/17/fine-grained-pod-topology-spread-features-beta/), promoted to beta in Kubernetes 1.27: -- `matchLabelKeys` -- `nodeAffinityPolicy` -- `nodeTaintsPolicy` - -For more information on Karpenter's support for these keys, view [this tracking issue](https://github.com/aws/karpenter-core/issues/430). -{{% /alert %}} - -## Compatibility issues - -To make upgrading easier, we aim to minimize the introduction of breaking changes with the following components: - -* Provisioner API -* Helm Chart - -We try to maintain compatibility with: - -* The application itself -* The documentation of the application - -When we introduce a breaking change, we do so only as described in this document. - -Karpenter follows [Semantic Versioning 2.0.0](https://semver.org/) in its stable release versions, while in -major version zero (v0.y.z) [anything may change at any time](https://semver.org/#spec-item-4). -However, to further protect users during this phase we will only introduce breaking changes in minor releases (releases that increment y in x.y.z). -Note this does not mean every minor upgrade has a breaking change as we will also increment the -minor version when we release a new feature. - -Users should therefore check to see if there is a breaking change every time they are upgrading to a new minor version. - -### Custom Resource Definition (CRD) Upgrades - -Karpenter ships with a few Custom Resource Definitions (CRDs). These CRDs are published: -* As an independent helm chart [karpenter-crd](https://gallery.ecr.aws/karpenter/karpenter-crd) - [source](https://github.com/aws/karpenter/blob/main/charts/karpenter-crd) that can be used by Helm to manage the lifecycle of these CRDs. - * To upgrade or install `karpenter-crd` run: - ``` - helm upgrade --install karpenter-crd oci://public.ecr.aws/karpenter/karpenter-crd --version vx.y.z --namespace karpenter --create-namespace - ``` - -{{% alert title="Note" color="warning" %}} -If you get the error `invalid ownership metadata; label validation error:` while installing the `karpenter-crd` chart from an older version of Karpenter, follow the [Troubleshooting Guide]({{}}) for details on how to resolve these errors. -{{% /alert %}} - -* As part of the helm chart [karpenter](https://gallery.ecr.aws/karpenter/karpenter) - [source](https://github.com/aws/karpenter/blob/main/charts/karpenter/crds). Helm [does not manage the lifecycle of CRDs using this method](https://helm.sh/docs/chart_best_practices/custom_resource_definitions/), the tool will only install the CRD during the first installation of the helm chart. Subsequent chart upgrades will not add or remove CRDs, even if the CRDs have changed. When CRDs are changed, we will make a note in the version's upgrade guide. - -In general, you can reapply the CRDs in the `crds` directory of the Karpenter helm chart: - -```shell -kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.30.0/pkg/apis/crds/karpenter.sh_provisioners.yaml -kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.30.0/pkg/apis/crds/karpenter.sh_machines.yaml -kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.30.0/pkg/apis/crds/karpenter.k8s.aws_awsnodetemplates.yaml -``` - -### How Do We Break Incompatibility? - -When there is a breaking change we will: - -* Increment the minor version when in major version 0 -* Add a permanent separate section named `upgrading to vx.y.z+` under [released upgrade notes](#released-upgrade-notes) - clearly explaining the breaking change and what needs to be done on the user side to ensure a safe upgrade -* Add the sentence “This is a breaking change, please refer to the above link for upgrade instructions” to the top of the release notes and in all our announcements - -### How Do We Find Incompatibilities? - -Besides the peer review process for all changes to the code base we also do the followings in order to find -incompatibilities: -* (To be implemented) To check the compatibility of the application, we will automate tests for installing, uninstalling, upgrading from an older version, and downgrading to an older version -* (To be implemented) To check the compatibility of the documentation with the application, we will turn the commands in our documentation into scripts that we can automatically run - -### Security Patches - -While we are in major version 0 we will not release security patches to older versions. -Rather we provide the patches in the latest versions. -When at major version 1 we will have an EOL (end of life) policy where we provide security patches -for a subset of older versions and deprecate the others. - -## Release Types - -Karpenter offers three types of releases. This section explains the purpose of each release type and how the images for each release type are tagged in our [public image repository](https://gallery.ecr.aws/karpenter). - -### Stable Releases - -Stable releases are the most reliable releases that are released with weekly cadence. Stable releases are our only recommended versions for production environments. -Sometimes we skip a stable release because we find instability or problems that need to be fixed before having a stable release. -Stable releases are tagged with Semantic Versioning. For example `v0.13.0`. - -### Release Candidates - -We consider having release candidates for major and important minor versions. Our release candidates are tagged like `vx.y.z-rc.0`, `vx.y.z-rc.1`. The release candidate will then graduate to `vx.y.z` as a normal stable release. -By adopting this practice we allow our users who are early adopters to test out new releases before they are available to the wider community, thereby providing us with early feedback resulting in more stable releases. - -### Snapshot Releases - -We release a snapshot release for every commit that gets merged into [`aws/karpenter-provider-aws`](https://www.github.com/aws/karpenter-provider-aws). This enables users to immediately try a new feature or fix right after it's merged rather than waiting days or weeks for release. - -Snapshot releases are not made available in the same public ECR repository as other release types, they are instead published to a separate private ECR repository. -Helm charts are published to `oci://{{< param "snapshot_repo.account_id" >}}.dkr.ecr.{{< param "snapshot_repo.region" >}}.amazonaws.com/karpenter/snapshot/karpenter` and are tagged with the git commit hash prefixed by the Karpenter major version (e.g. `v0-fc17bfc89ebb30a3b102a86012b3e3992ec08adf`). -Anyone with an AWS account can pull from this repository, but must first authenticate: - -```bash -aws ecr get-login-password --region {{< param "snapshot_repo.region" >}} | docker login --username AWS --password-stdin {{< param "snapshot_repo.account_id" >}}.dkr.ecr.{{< param "snapshot_repo.region" >}}.amazonaws.com -``` - -{{% alert title="Note" color="warning" %}} -Snapshot releases are suitable for testing, and troubleshooting but they should not be used in production environments. Snapshot releases are ephemeral and will be removed 90 days after they were published. -{{% /alert %}} - -## Released Upgrade Notes - -### Upgrading to v0.30.0+ - -* Karpenter will now [statically drift]({{}}) on both Provisioner and AWSNodeTemplate Fields. For Provisioner Static Drift, the `karpenter.sh/provisioner-hash` annotation must be present on both the Provisioner and Machine. For AWSNodeTemplate drift, the `karpenter.k8s.aws/nodetemplate-hash` annotation must be present on the AWSNodeTemplate and Machine. Karpenter will not add these annotations to pre-existing nodes, so each of these nodes will need to be recycled one time for the annotations to be added. -* Karpenter will now fail validation on AWSNodeTemplates and Provisioner `spec.provider` that have `amiSelectors`, `subnetSelectors`, or `securityGroupSelectors` set with a combination of id selectors (`aws-ids`, `aws::ids`) and other selectors. -* Karpenter now statically sets the `securityContext` at both the pod and container-levels and doesn't allow override values to be passed through the helm chart. This change was made to adhere to [Restricted Pod Security Standard](https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted), which follows pod hardening best practices. - -{{% alert title="Note" color="primary" %}} -If you have sidecar containers configured to run alongside Karpenter that cannot tolerate the [pod-wide `securityContext` constraints](https://github.com/aws/karpenter/blob/v0.30.0/charts/karpenter/templates/deployment.yaml#L40), you will need to specify overrides to the sidecar `securityContext` in your deployment. -{{% /alert %}} - -### Upgrading to v0.29.0+ - -{{% alert title="Warning" color="warning" %}} -Karpenter `v0.29.1` contains a [file descriptor and memory leak bug](https://github.com/aws/karpenter/issues/4296) that leads to Karpenter getting OOMKilled and restarting at the point that it hits its memory or file descriptor limit. Karpenter `>v0.29.2` fixes this leak. -{{% /alert %}} - -* Karpenter has changed the default metrics service port from 8080 to 8000 and the default webhook service port from 443 to 8443. In `v0.28.0`, the Karpenter pod port was changed to 8000, but referenced the service by name, allowing users to scrape the service at port 8080 for metrics. `v0.29.0` aligns the two ports so that service and pod metrics ports are the same. These ports are set by the `controller.metrics.port` and `webhook.port` helm chart values, so if you have previously set these to non-default values, you may need to update your Prometheus scraper to match these new values. - -* Karpenter will now reconcile nodes that are drifted due to their Security Groups or their Subnets. If your AWSNodeTemplate's Security Groups differ from the Security Groups used for an instance, Karpenter will consider it drifted. If the Subnet used by an instance is not contained in the allowed list of Subnets for an AWSNodeTemplate, Karpenter will also consider it drifted. - * Since Karpenter uses tags for discovery of Subnets and SecurityGroups, check the [Threat Model]({{}}) to see how to manage this IAM Permission. - -### Upgrading to v0.28.0+ - -{{% alert title="Warning" color="warning" %}} -Karpenter `v0.28.0` is incompatible with Kubernetes version 1.26+, which can result in additional node scale outs when using `--cloudprovider=external`, which is the default for the EKS Optimized AMI. See: https://github.com/aws/karpenter-core/pull/375. Karpenter `>v0.28.1` fixes this issue and is compatible with Kubernetes version 1.26+. -{{% /alert %}} - -* The `extraObjects` value is now removed from the Helm chart. Having this value in the chart proved to not work in the majority of Karpenter installs and often led to anti-patterns, where the Karpenter resources installed to manage Karpenter's capacity were directly tied to the install of the Karpenter controller deployments. The Karpenter team recommends that, if you want to install Karpenter manifests alongside the Karpenter helm chart, to do so by creating a separate chart for the manifests, creating a dependency on the controller chart. -* The `aws.nodeNameConvention` setting is now removed from the [`karpenter-global-settings`]({{}}) ConfigMap. Because Karpenter is now driving its orchestration of capacity through Machines, it no longer needs to know the node name, making this setting obsolete. Karpenter ignores configuration that it doesn't recognize in the [`karpenter-global-settings`]({{}}) ConfigMap, so leaving the `aws.nodeNameConvention` in the ConfigMap will simply cause this setting to be ignored. -* Karpenter now defines a set of "restricted tags" which can't be overridden with custom tagging in the AWSNodeTemplate or in the [`karpenter-global-settings`]({{}}) ConfigMap. If you are currently using any of these tag overrides when tagging your instances, webhook validation will now fail. These tags include: - - * `karpenter.sh/managed-by` - * `karpenter.sh/provisioner-name` - * `kubernetes.io/cluster/${CLUSTER_NAME}` - -* The following metrics changed their meaning, based on the introduction of the Machine resource: - * `karpenter_nodes_terminated`: Use `karpenter_machines_terminated` if you are interested in the reason why a Karpenter machine was deleted. `karpenter_nodes_terminated` now only tracks the count of terminated nodes without any additional labels. - * `karpenter_nodes_created`: Use `karpenter_machines_created` if you are interested in the reason why a Karpenter machine was created. `karpenter_nodes_created` now only tracks the count of created nodes without any additional labels. - * `karpenter_deprovisioning_replacement_node_initialized_seconds`: This metric has been replaced in favor of `karpenter_deprovisioning_replacement_machine_initialized_seconds`. -* `v0.28.0` introduces the Machine CustomResource into the `karpenter.sh` API Group and requires this CustomResourceDefinition to run properly. Karpenter now orchestrates its CloudProvider capacity through these in-cluster Machine CustomResources. When performing a scheduling decision, Karpenter will create a Machine, resulting in launching CloudProvider capacity. The kubelet running on the new capacity will then register the node to the cluster shortly after launch. - * If you are using Helm to upgrade between versions of Karpenter, note that [Helm does not automate the process of upgrading or install the new CRDs into your cluster](https://helm.sh/docs/chart_best_practices/custom_resource_definitions/#some-caveats-and-explanations). To install or upgrade the existing CRDs, follow the guidance under the [Custom Resource Definition (CRD) Upgrades]({{< relref "#custom-resource-definition-crd-upgrades" >}}) section of the upgrade guide. - * Karpenter will hydrate Machines on startup for existing capacity managed by Karpenter into the cluster. Existing capacity launched by an older version of Karpenter is discovered by finding CloudProvider capacity with the `karpenter.sh/provisioner-name` tag or the `karpenter.sh/provisioner-name` label on nodes. -* The metrics port for the Karpenter deployment was changed from 8080 to 8000. Users who scrape the pod directly for metrics rather than the service will need to adjust the commands they use to reference port 8000. Any users who scrape metrics from the service should be unaffected. - -{{% alert title="Warning" color="warning" %}} -Karpenter creates a mapping between CloudProvider machines and CustomResources in the cluster for capacity tracking. To ensure this mapping is consistent, Karpenter utilizes the following tag keys: - -* `karpenter.sh/managed-by` -* `karpenter.sh/provisioner-name` -* `kubernetes.io/cluster/${CLUSTER_NAME}` - -Because Karpenter takes this dependency, any user that has the ability to Create/Delete these tags on CloudProvider machines will have the ability to orchestrate Karpenter to Create/Delete CloudProvider machines as a side effect. Check the [Threat Model]({{}}) to see how this might affect you, and ways to mitigate this. -{{% /alert %}} - -{{% alert title="Rolling Back" color="warning" %}} -If, after upgrading to `v0.28.0+`, a rollback to an older version of Karpenter needs to be performed, Karpenter will continue to function normally, though you will still have the Machine CustomResources on your cluster. You will need to manually delete the Machines and patch out the finalizers to fully complete the rollback. - -Karpenter marks CloudProvider capacity as "managed by" a Machine using the `karpenter-sh/managed-by` tag on the CloudProvider machine. It uses this tag to ensure that the Machine CustomResources in the cluster match the CloudProvider capacity managed by Karpenter. If these states don't match, Karpenter will garbage collect the capacity. Because of this, if performing an upgrade, followed by a rollback, followed by another upgrade to `v0.28.0+`, ensure you remove the `karpenter.sh/managed-by` tags from existing capacity; otherwise, Karpenter will deprovision the capacity without a Machine CR counterpart. -{{% /alert %}} - - -### Upgrading to v0.27.3+ -* The `defaulting.webhook.karpenter.sh` mutating webhook was removed in `v0.27.3`. If you are coming from an older version of Karpenter where this webhook existed and the webhook was not managed by Helm, you may need to delete the stale webhook. - -```console -kubectl delete mutatingwebhookconfigurations defaulting.webhook.karpenter.sh -``` - -### Upgrading to v0.27.0+ -* The Karpenter controller pods now deploy with `kubernetes.io/hostname` self anti-affinity by default. If you are running Karpenter in HA (high-availability) mode and you do not have enough nodes to match the number of pod replicas you are deploying with, you will need to scale-out your nodes for Karpenter. -* The following controller metrics changed and moved under the `controller_runtime` metrics namespace: - * `karpenter_metricscraper_...` - * `karpenter_deprovisioning_...` - * `karpenter_provisioner_...` - * `karpenter_interruption_...` -* The following controller metric names changed, affecting the `controller` label value under `controller_runtime_...` metrics. These metrics include: - * `podmetrics` -> `pod_metrics` - * `provisionermetrics` -> `provisioner_metrics` - * `metricscraper` -> `metric_scraper` - * `provisioning` -> `provisioner_trigger` - * `node-state` -> `node_state` - * `pod-state` -> `pod_state` - * `provisioner-state` -> `provisioner_state` -* The `karpenter_allocation_controller_scheduling_duration_seconds` metric name changed to `karpenter_provisioner_scheduling_duration_seconds` - -### Upgrading to v0.26.0+ -* The `karpenter.sh/do-not-evict` annotation no longer blocks node termination when running `kubectl delete node`. This annotation on pods will only block automatic deprovisioning that is considered "voluntary," that is, disruptions that can be avoided. Disruptions that Karpenter deems as "involuntary" and will ignore the `karpenter.sh/do-not-evict` annotation include spot interruption and manual deletion of the node. See [Disabling Deprovisioning]({{}}) for more details. -* Default resources `requests` and `limits` are removed from the Karpenter's controller deployment through the Helm chart. If you have not set custom resource `requests` or `limits` in your helm values and are using Karpenter's defaults, you will now need to set these values in your helm chart deployment. -* The `controller.image` value in the helm chart has been broken out to a map consisting of `controller.image.repository`, `controller.image.tag`, and `controller.image.digest`. If manually overriding the `controller.image`, you will need to update your values to the new design. - -### Upgrading to v0.25.0+ -* Cluster Endpoint can now be automatically discovered. If you are using Amazon Elastic Kubernetes Service (EKS), you can now omit the `clusterEndpoint` field in your configuration. In order to allow the resolving, you have to add the permission `eks:DescribeCluster` to the Karpenter Controller IAM role. - -### Upgrading to v0.24.0+ -* Settings are no longer updated dynamically while Karpenter is running. If you manually make a change to the [`karpenter-global-settings`]({{}}) ConfigMap, you will need to reload the containers by restarting the deployment with `kubectl rollout restart -n karpenter deploy/karpenter` -* Karpenter no longer filters out instance types internally. Previously, `g2` (not supported by the NVIDIA device plugin) and FPGA instance types were filtered. The only way to filter instance types now is to set requirements on your provisioner or pods using well-known node labels described [here]({{}}). If you are currently using overly broad requirements that allows all of the `g` instance-category, you will want to tighten the requirement, or add an instance-generation requirement. -* `aws.tags` in [`karpenter-global-settings`]({{}}) ConfigMap is now a top-level field and expects the value associated with this key to be a JSON object of string to string. This is change from previous versions where keys were given implicitly by providing the key-value pair `aws.tags.: value` in the ConfigMap. - -### Upgrading to v0.22.0+ -* Do not upgrade to this version unless you are on Kubernetes >= v1.21. Karpenter no longer supports Kubernetes v1.20, but now supports Kubernetes v1.25. This change is due to the v1 PDB API, which was introduced in K8s v1.20 and subsequent removal of the v1beta1 API in K8s v1.25. - -### Upgrading to v0.20.0+ -* Prior to v0.20.0, Karpenter would prioritize certain instance type categories absent of any requirements in the Provisioner. v0.20.0+ removes prioritizing these instance type categories ("m", "c", "r", "a", "t", "i") in code. Bare Metal and GPU instance types are still deprioritized and only used if no other instance types are compatible with the node requirements. Since Karpenter does not prioritize any instance types, if you do not want exotic instance types and are not using the runtime Provisioner defaults, you will need to specify this in the Provisioner. - -### Upgrading to v0.19.0+ -* The karpenter webhook and controller containers are combined into a single binary, which requires changes to the helm chart. If your Karpenter installation (helm or otherwise) currently customizes the karpenter webhook, your deployment tooling may require minor changes. -* Karpenter now supports native interruption handling. If you were previously using Node Termination Handler for spot interruption handling and health events, you will need to remove the component from your cluster before enabling `aws.interruptionQueueName`. For more details on Karpenter's interruption handling, see the [Interruption Handling Docs]({{< ref "./concepts/deprovisioning/#interruption" >}}). For common questions on the migration process, see the [FAQ]({{< ref "./faq/#interruption-handling" >}}) -* Instance category defaults are now explicitly persisted in the Provisioner, rather than handled implicitly in memory. By default, Provisioners will limit instance category to c,m,r. If any instance type constraints are applied, it will override this default. If you have created Provisioners in the past with unconstrained instance type, family, or category, Karpenter will now more flexibly use instance types than before. If you would like to apply these constraints, they must be included in the Provisioner CRD. -* Karpenter CRD raw YAML URLs have migrated from `https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.19.3/charts/karpenter/crds/...` to `https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.19.3/pkg/apis/crds/...`. If you reference static Karpenter CRDs or rely on `kubectl replace -f` to apply these CRDs from their remote location, you will need to migrate to the new location. -* Pods without an ownerRef (also called "controllerless" or "naked" pods) will now be evicted by default during node termination and consolidation. Users can prevent controllerless pods from being voluntarily disrupted by applying the `karpenter.sh/do-not-evict: "true"` annotation to the pods in question. -* The following CLI options/environment variables are now removed and replaced in favor of pulling settings dynamically from the [`karpenter-global-settings`]({{}}) ConfigMap. See the [Settings docs]({{}}) for more details on configuring the new values in the ConfigMap. - - * `CLUSTER_NAME` -> `settings.aws.clusterName` - * `CLUSTER_ENDPOINT` -> `settings.aws.clusterEndpoint` - * `AWS_DEFAULT_INSTANCE_PROFILE` -> `settings.aws.defaultInstanceProfile` - * `AWS_ENABLE_POD_ENI` -> `settings.aws.enablePodENI` - * `AWS_ENI_LIMITED_POD_DENSITY` -> `settings.aws.enableENILimitedPodDensity` - * `AWS_ISOLATED_VPC` -> `settings.aws.isolatedVPC` - * `AWS_NODE_NAME_CONVENTION` -> `settings.aws.nodeNameConvention` - * `VM_MEMORY_OVERHEAD` -> `settings.aws.vmMemoryOverheadPercent` - -### Upgrading to v0.18.0+ -* v0.18.0 removes the `karpenter_consolidation_nodes_created` and `karpenter_consolidation_nodes_terminated` prometheus metrics in favor of the more generic `karpenter_nodes_created` and `karpenter_nodes_terminated` metrics. You can still see nodes created and terminated by consolidation by checking the `reason` label on the metrics. Check out all the metrics published by Karpenter [here]({{}}). - -### Upgrading to v0.17.0+ -Karpenter's Helm chart package is now stored in [Karpenter's OCI (Open Container Initiative) registry](https://gallery.ecr.aws/karpenter/karpenter). The Helm CLI supports the new format since [v3.8.0+](https://helm.sh/docs/topics/registries/). -With this change [charts.karpenter.sh](https://charts.karpenter.sh/) is no longer updated but preserved to allow using older Karpenter versions. For examples on working with the Karpenter helm charts look at [Install Karpenter Helm Chart]({{< ref "./getting-started/getting-started-with-karpenter/#install-karpenter-helm-chart" >}}). - -Users who have scripted the installation or upgrading of Karpenter need to adjust their scripts with the following changes: -1. There is no longer a need to add the Karpenter helm repo to helm -2. The full URL of the Helm chart needs to be present when using the helm commands -3. If you were not prepending a `v` to the version (i.e. `0.17.0`), you will need to do so with the OCI chart, `v0.17.0`. - -### Upgrading to v0.16.2+ -* v0.16.2 adds new kubeletConfiguration fields to the `provisioners.karpenter.sh` v1alpha5 CRD. The CRD will need to be updated to use the new parameters: -```bash -kubectl replace -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.16.2/charts/karpenter/crds/karpenter.sh_provisioners.yaml -``` - -### Upgrading to v0.16.0+ -* v0.16.0 adds a new weight field to the `provisioners.karpenter.sh` v1alpha5 CRD. The CRD will need to be updated to use the new parameters: -```bash -kubectl replace -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.16.0/charts/karpenter/crds/karpenter.sh_provisioners.yaml -``` - -### Upgrading to v0.15.0+ -* v0.15.0 adds a new consolidation field to the `provisioners.karpenter.sh` v1alpha5 CRD. The CRD will need to be updated to use the new parameters: -```bash -kubectl replace -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.15.0/charts/karpenter/crds/karpenter.sh_provisioners.yaml -``` - -### Upgrading to v0.14.0+ -* v0.14.0 adds new fields to the `provisioners.karpenter.sh` v1alpha5 and `awsnodetemplates.karpenter.k8s.aws` v1alpha1 CRDs. The CRDs will need to be updated to use the new parameters: - -```bash -kubectl replace -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.14.0/charts/karpenter/crds/karpenter.sh_provisioners.yaml - -kubectl replace -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.14.0/charts/karpenter/crds/karpenter.k8s.aws_awsnodetemplates.yaml -``` - -* v0.14.0 changes the way Karpenter discovers its dynamically generated AWS launch templates to use a tag rather than a Name scheme. The previous name scheme was `Karpenter-${CLUSTER_NAME}-*` which could collide with user created launch templates that Karpenter should not manage. The new scheme uses a tag on the launch template `karpenter.k8s.aws/cluster: ${CLUSTER_NAME}`. As a result, Karpenter will not clean-up dynamically generated launch templates using the old name scheme. You can manually clean these up with the following commands: - -```bash -## Find launch templates that match the naming pattern and you do not want to keep -aws ec2 describe-launch-templates --filters="Name=launch-template-name,Values=Karpenter-${CLUSTER_NAME}-*" - -## Delete launch template(s) that match the name but do not have the "karpenter.k8s.aws/cluster" tag -aws ec2 delete-launch-template --launch-template-id -``` - -* v0.14.0 introduces additional instance type filtering if there are no `node.kubernetes.io/instance-type` or `karpenter.k8s.aws/instance-family` or `karpenter.k8s.aws/instance-category` requirements that restrict instance types specified on the provisioner. This prevents Karpenter from launching bare metal and some older non-current generation instance types unless the provisioner has been explicitly configured to allow them. If you specify an instance type or family requirement that supplies a list of instance-types or families, that list will be used regardless of filtering. The filtering can also be completely eliminated by adding an `Exists` requirement for instance type or family. -```yaml - - key: node.kubernetes.io/instance-type - operator: Exists -``` - -* v0.14.0 introduces support for custom AMIs without the need for an entire launch template. You must add the `ec2:DescribeImages` permission to the Karpenter Controller Role for this feature to work. This permission is needed for Karpenter to discover custom images specified. Read the [Custom AMI documentation here]({{}}) to get started -* v0.14.0 adds an an additional default toleration (CriticalAddonOnly=Exists) to the Karpenter helm chart. This may cause Karpenter to run on nodes with that use this Taint which previously would not have been schedulable. This can be overridden by using `--set tolerations[0]=null`. - -* v0.14.0 deprecates the `AWS_ENI_LIMITED_POD_DENSITY` environment variable in-favor of specifying `spec.kubeletConfiguration.maxPods` on the Provisioner. `AWS_ENI_LIMITED_POD_DENSITY` will continue to work when `maxPods` is not set on the Provisioner. If `maxPods` is set, it will override `AWS_ENI_LIMITED_POD_DENSITY` on that specific Provisioner. - -### Upgrading to v0.13.0+ -* v0.13.0 introduces a new CRD named `AWSNodeTemplate` which can be used to specify AWS Cloud Provider parameters. Everything that was previously specified under `spec.provider` in the Provisioner resource, can now be specified in the spec of the new resource. The use of `spec.provider` is deprecated but will continue to function to maintain backwards compatibility for the current API version (v1alpha5) of the Provisioner resource. v0.13.0 also introduces support for custom user data that doesn't require the use of a custom launch template. The user data can be specified in-line in the AWSNodeTemplate resource. Read the [UserData documentation here](../aws/operating-systems) to get started. - - If you are upgrading from v0.10.1 - v0.11.1, a new CRD `awsnodetemplate` was added. In v0.12.0, this crd was renamed to `awsnodetemplates`. Since helm does not manage the lifecycle of CRDs, you will need to perform a few manual steps for this CRD upgrade: - 1. Make sure any `awsnodetemplate` manifests are saved somewhere so that they can be reapplied to the cluster. - 2. `kubectl delete crd awsnodetemplate` - 3. `kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.13.2/charts/karpenter/crds/karpenter.k8s.aws_awsnodetemplates.yaml` - 4. Perform the Karpenter upgrade to v0.13.x, which will install the new `awsnodetemplates` CRD. - 5. Reapply the `awsnodetemplate` manifests you saved from step 1, if applicable. -* v0.13.0 also adds EC2/spot price fetching to Karpenter to allow making more accurate decisions regarding node deployments. Our getting started guide documents this, but if you are upgrading Karpenter you will need to modify your Karpenter controller policy to add the `pricing:GetProducts` and `ec2:DescribeSpotPriceHistory` permissions. - - -### Upgrading to v0.12.0+ -* v0.12.0 adds an OwnerReference to each Node created by a provisioner. Previously, deleting a provisioner would orphan nodes. Now, deleting a provisioner will cause Kubernetes [cascading delete](https://kubernetes.io/docs/concepts/architecture/garbage-collection/#cascading-deletion) logic to gracefully terminate the nodes using the Karpenter node finalizer. You may still orphan nodes by removing the owner reference. -* If you are upgrading from v0.10.1 - v0.11.1, a new CRD `awsnodetemplate` was added. In v0.12.0, this crd was renamed to `awsnodetemplates`. Since helm does not manage the lifecycle of CRDs, you will need to perform a few manual steps for this CRD upgrade: - 1. Make sure any `awsnodetemplate` manifests are saved somewhere so that they can be reapplied to the cluster. - 2. `kubectl delete crd awsnodetemplate` - 3. `kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.12.1/charts/karpenter/crds/karpenter.k8s.aws_awsnodetemplates.yaml` - 4. Perform the Karpenter upgrade to v0.12.x, which will install the new `awsnodetemplates` CRD. - 5. Reapply the `awsnodetemplate` manifests you saved from step 1, if applicable. - -### Upgrading to v0.11.0+ - -v0.11.0 changes the way that the `vpc.amazonaws.com/pod-eni` resource is reported. Instead of being reported for all nodes that could support the resources regardless of if the cluster is configured to support it, it is now controlled by a command line flag or environment variable. The parameter defaults to false and must be set if your cluster uses [security groups for pods](https://docs.aws.amazon.com/eks/latest/userguide/security-groups-for-pods.html). This can be enabled by setting the environment variable `AWS_ENABLE_POD_ENI` to true via the helm value `controller.env`. - -Other extended resources must be registered on nodes by their respective device plugins which are typically installed as DaemonSets (e.g. the `nvidia.com/gpu` resource will be registered by the [NVIDIA device plugin](https://github.com/NVIDIA/k8s-device-plugin). Previously, Karpenter would register these resources on nodes at creation and they would be zeroed out by `kubelet` at startup. By allowing the device plugins to register the resources, pods will not bind to the nodes before any device plugin initialization has occurred. - -v0.11.0 adds a `providerRef` field in the Provisioner CRD. To use this new field you will need to replace the Provisioner CRD manually: - -```shell -kubectl replace -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.11.0/charts/karpenter/crds/karpenter.sh_provisioners.yaml -``` - -### Upgrading to v0.10.0+ - -v0.10.0 adds a new field, `startupTaints` to the provisioner spec. Standard Helm upgrades [do not upgrade CRDs](https://helm.sh/docs/chart_best_practices/custom_resource_definitions/#some-caveats-and-explanations) so the field will not be available unless the CRD is manually updated. This can be performed prior to the standard upgrade by applying the new CRD manually: - -```shell -kubectl replace -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.10.0/charts/karpenter/crds/karpenter.sh_provisioners.yaml -``` - -📝 If you don't perform this manual CRD update, Karpenter will work correctly except for rejecting the creation/update of provisioners that use `startupTaints`. - -### Upgrading to v0.6.2+ - -If using Helm, the variable names have changed for the cluster's name and endpoint. You may need to update any configuration -that sets the old variable names. - -- `controller.clusterName` is now `clusterName` -- `controller.clusterEndpoint` is now `clusterEndpoint` diff --git a/website/content/en/v0.31/concepts/_index.md b/website/content/en/v0.31/concepts/_index.md deleted file mode 100755 index fec50b86b6d7..000000000000 --- a/website/content/en/v0.31/concepts/_index.md +++ /dev/null @@ -1,179 +0,0 @@ ---- -title: "Concepts" -linkTitle: "Concepts" -weight: 10 -description: > - Understand key concepts of Karpenter ---- - -Users fall under two basic roles: Kubernetes cluster administrators and application developers. -This document describes Karpenter concepts through the lens of those two types of users. - -## Cluster administrator - -As a Kubernetes cluster administrator, you can engage with Karpenter to: - -* Install Karpenter -* Configure provisioners to set constraints and other features for managing nodes -* Deprovision nodes -* Upgrade nodes - -Concepts associated with this role are described below. - - -### Installing Karpenter - -Karpenter is designed to run on a node in your Kubernetes cluster. -As part of the installation process, you need credentials from the underlying cloud provider to allow nodes to be started up and added to the cluster as they are needed. - -[Getting Started with Karpenter on AWS](../getting-started) -describes the process of installing Karpenter on an AWS cloud provider. -Because requests to add and delete nodes and schedule pods are made through Kubernetes, AWS IAM Roles for Service Accounts (IRSA) are needed by your Kubernetes cluster to make privileged requests to AWS. -For example, Karpenter uses AWS IRSA roles to grant the permissions needed to describe EC2 instance types and create EC2 instances. - -Once privileges are in place, Karpenter is deployed with a Helm chart. - -### Configuring provisioners - -Karpenter's job is to add nodes to handle unschedulable pods, schedule pods on those nodes, and remove the nodes when they are not needed. -To configure Karpenter, you create *provisioners* that define how Karpenter manages unschedulable pods and expires nodes. -Here are some things to know about the Karpenter provisioner: - -* **Unschedulable pods**: Karpenter only attempts to schedule pods that have a status condition `Unschedulable=True`, which the kube scheduler sets when it fails to schedule the pod to existing capacity. - -* **Provisioner CR**: Karpenter defines a Custom Resource called a Provisioner to specify provisioning configuration. -Each provisioner manages a distinct set of nodes, but pods can be scheduled to any provisioner that supports its scheduling constraints. -A provisioner contains constraints that impact the nodes that can be provisioned and attributes of those nodes (such timers for removing nodes). -See [Provisioning]({{}}) docs for a description of settings and provisioner examples. - -* **Well-known labels**: The provisioner can use well-known Kubernetes labels to allow pods to request only certain instance types, architectures, operating systems, or other attributes when creating nodes. -See [Well-Known Labels, Annotations and Taints](https://kubernetes.io/docs/reference/labels-annotations-taints/) for details. -Keep in mind that only a subset of these labels are supported in Karpenter, as described later. - -* **Deprovisioning nodes**: A provisioner can also include time-to-live values to indicate when nodes should be deprovisioned after a set amount of time from when they were created or after they becomes empty of deployed pods. - -* **Multiple provisioners**: Multiple provisioners can be configured on the same cluster. -For example, you might want to configure different teams on the same cluster to run on completely separate capacity. -One team could run on nodes using BottleRocket, while another uses EKSOptimizedAMI. - -Although most use cases are addressed with a single provisioner for multiple teams, multiple provisioners are useful to isolate nodes for billing, use different node constraints (such as no GPUs for a team), or use different deprovisioning settings. - -### Deprovisioning nodes - -Karpenter deletes nodes when they are no longer needed. - -* **Finalizer**: Karpenter places a finalizer bit on each node it creates. -When a request comes in to delete one of those nodes (such as a TTL or a manual `kubectl delete node`), Karpenter will cordon the node, drain all the pods, terminate the EC2 instance, and delete the node object. -Karpenter handles all clean-up work needed to properly delete the node. -* **Node Expiry**: If a node expiry time-to-live value (`ttlSecondsUntilExpired`) is reached, that node is drained of pods and deleted (even if it is still running workloads). -* **Empty nodes**: When the last workload pod running on a Karpenter-managed node is gone, the node is annotated with an emptiness timestamp. -Once that "node empty" time-to-live (`ttlSecondsAfterEmpty`) is reached, finalization is triggered. -* **Consolidation**: If enabled, Karpenter will work to actively reduce cluster cost by identifying when nodes can be removed as their workloads will run on other nodes in the cluster and when nodes can be replaced with cheaper variants due to a change in the workloads. -* **Interruption**: If enabled, Karpenter will watch for upcoming involuntary interruption events that could affect your nodes (health events, spot interruption, etc.) and will cordon, drain, and terminate the node(s) ahead of the event to reduce workload disruption. - -For more details on how Karpenter deletes nodes, see [Deprovisioning nodes](./deprovisioning) for details. - -### Upgrading nodes - -A straight-forward way to upgrade nodes is to set `ttlSecondsUntilExpired`. -Nodes will be terminated after a set period of time and will be replaced with newer nodes using the latest discovered AMI. -See more in [AWSNodeTemplate](./node-templates). - -### Constraints - -The concept of layered constraints is key to using Karpenter. -With no constraints defined in provisioners and none requested from pods being deployed, Karpenter chooses from the entire universe of features available to your cloud provider. -Nodes can be created using any instance type and run in any zones. - -An application developer can tighten the constraints defined in a provisioner by the cluster administrator by defining additional scheduling constraints in their pod spec. -Refer to the description of Karpenter constraints in the Application Developer section below for details. - -### Scheduling - -Karpenter launches nodes in response to pods that the Kubernetes scheduler has marked unschedulable. After solving scheduling constraints and launching capacity, Karpenter launches a machine in your chosen cloud provider. - -Once Karpenter brings up a node, that node is available for the Kubernetes scheduler to schedule pods on it as well. - -### Cloud provider -Karpenter makes requests to provision new nodes to the associated cloud provider. -The first supported cloud provider is AWS, although Karpenter is designed to work with other cloud providers. -Separating Kubernetes and AWS-specific settings allows Karpenter a clean path to integrating with other cloud providers. - -While using Kubernetes well-known labels, the provisioner can set some values that are specific to the cloud provider. -So, for example, to include a certain instance type, you could use the Kubernetes label `node.kubernetes.io/instance-type`, but set its value to an AWS instance type (such as `m5.large` or `m5.2xlarge`). - -### Consolidation - -If consolidation is enabled for a provisioner, Karpenter attempts to reduce the overall cost of the nodes launched by that provisioner if workloads have changed in two ways: -- Node Deletion -- Node Replacement - -To perform these actions, Karpenter simulates all pods being evicted from a candidate node and then looks at the results of the scheduling simulation to determine if those pods can run on a combination of existing nodes in the cluster and a new cheaper node. This operation takes into consideration all scheduling constraints placed on your workloads and provisioners (e.g. taints, tolerations, node selectors, inter-pod affinity, etc). - -If as a result of the scheduling simulation all pods can run on existing nodes, the candidate node is simply deleted. If all pods can run on a combination of existing nodes and a cheaper node, we launch the cheaper node and delete the candidate node which causes the pods to be evicted and re-created by their controllers in order to be rescheduled. - -For Node Replacement to work well, your provisioner must allow selecting from a variety of instance types with varying amounts of allocatable resources. Consolidation will only consider launching nodes using instance types which are allowed by your provisioner. - -### Interruption - -If interruption-handling is enabled for the controller, Karpenter will watch for upcoming involuntary interruption events that would cause disruption to your workloads. These interruption events include: - -* Spot Interruption Warnings -* Scheduled Change Health Events (Maintenance Events) -* Instance Terminating Events -* Instance Stopping Events - -When Karpenter detects one of these events will occur to your nodes, it automatically cordons, drains, and terminates the node(s) ahead of the interruption event to give the maximum amount of time for workload cleanup prior to compute disruption. This enables scenarios where the `terminationGracePeriod` for your workloads may be long or cleanup for your workloads is critical, and you want enough time to be able to gracefully clean-up your pods. - -{{% alert title="Note" color="warning" %}} -Karpenter publishes Kubernetes events to the node for all events listed above in addition to [__Spot Rebalance Recommendations__](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/rebalance-recommendations.html). Karpenter does not currently support taint, drain, and terminate logic for Spot Rebalance Recommendations. - -If you require handling for Spot Rebalance Recommendations, you can use the [AWS Node Termination Handler (NTH)](https://github.com/aws/aws-node-termination-handler) alongside Karpenter; however, note that the AWS Node Termination Handler cordons and drains nodes on rebalance recommendations, potentially causing more node churn in the cluster than with interruptions alone. Further information can be found in the [Troubleshooting Guide]({{< ref "../troubleshooting#aws-node-termination-handler-nth-interactions" >}}). -{{% /alert %}} - -### Kubernetes cluster autoscaler -Like Karpenter, [Kubernetes Cluster Autoscaler](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler) is -designed to add nodes when requests come in to run pods that cannot be met by current capacity. -Cluster autoscaler is part of the Kubernetes project, with implementations by most major Kubernetes cloud providers. -By taking a fresh look at provisioning, Karpenter offers the following improvements: - -* **Designed to handle the full flexibility of the cloud**: -Karpenter has the ability to efficiently address the full range of instance types available through AWS. -Cluster autoscaler was not originally built with the flexibility to handle hundreds of instance types, zones, and purchase options. - -* **Group-less node provisioning**: Karpenter manages each instance directly, without use of additional orchestration mechanisms like node groups. -This enables it to retry in milliseconds instead of minutes when capacity is unavailable. -It also allows Karpenter to leverage diverse instance types, availability zones, and purchase options without the creation of hundreds of node groups. - -## Application developer - -As someone deploying pods that might be evaluated by Karpenter, you should know how to request the properties that your pods need of its compute resources. -Karpenter's job is to efficiently assess and choose compute assets based on requests from pod deployments. -These can include basic Kubernetes features or features that are specific to the cloud provider (such as AWS). - -Layered *constraints* are applied when a pod makes requests for compute resources that cannot be met by current capacity. -A pod can specify `nodeAffinity` (to run in a particular zone or instance type) or a `topologySpreadConstraints` spread (to cause a set of pods to be balanced across multiple nodes). -The pod can specify a `nodeSelector` to run only on nodes with a particular label and `resource.requests` to ensure that the node has enough available memory. - -The Kubernetes scheduler tries to match those constraints with available nodes. -If the pod is unschedulable, Karpenter creates compute resources that match its needs. -When Karpenter tries to provision a node, it analyzes scheduling constraints before choosing the node to create. - -As long as the requests are not outside of the provisioner's constraints, -Karpenter will look to best match the request, comparing the same well-known labels defined by the pod's scheduling constraints. -Note that if the constraints are such that a match is not possible, the pod will remain unscheduled. - -So, what constraints can you use as an application developer deploying pods that could be managed by Karpenter? - -Kubernetes features that Karpenter supports for scheduling pods include nodeAffinity and [nodeSelector](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector). -It also supports [PodDisruptionBudget](https://kubernetes.io/docs/tasks/run-application/configure-pdb/), [topologySpreadConstraints](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/), and [inter-pod affinity and anti-affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity). - -From the Kubernetes [Well-Known Labels, Annotations and Taints](https://kubernetes.io/docs/reference/labels-annotations-taints/) page, -you can see a full list of Kubernetes labels, annotations and taints that determine scheduling. -Those that are implemented in Karpenter include: - -* **kubernetes.io/arch**: For example, kubernetes.io/arch=amd64 -* **node.kubernetes.io/instance-type**: For example, node.kubernetes.io/instance-type=m3.medium -* **topology.kubernetes.io/zone**: For example, topology.kubernetes.io/zone=us-east-1c - -For more on how, as a developer, you can add constraints to your pod deployment, see [Scheduling](./scheduling/) for details. diff --git a/website/content/en/v0.31/concepts/deprovisioning.md b/website/content/en/v0.31/concepts/deprovisioning.md deleted file mode 100644 index 466e96f5bae6..000000000000 --- a/website/content/en/v0.31/concepts/deprovisioning.md +++ /dev/null @@ -1,240 +0,0 @@ ---- -title: "Deprovisioning" -linkTitle: "Deprovisioning" -weight: 4 -description: > - Understand different ways Karpenter deprovisions nodes ---- - -## Control Flow -Karpenter sets a Kubernetes [finalizer](https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/) on each node it provisions. -The finalizer blocks deletion of the node object while the Termination Controller cordons and drains the node, before removing the underlying machine. Deprovisioning is triggered by the Deprovisioning Controller, by the user through manual deprovisioning, or through an external system that sends a delete request to the node object. - -### Deprovisioning Controller -Karpenter automatically discovers deprovisionable nodes and spins up replacements when needed. Karpenter deprovisions nodes by executing one [automatic method](#methods) at a time, in order of Expiration, Drift, Emptiness, and then Consolidation. Each method varies slightly but they all follow the standard deprovisioning process: -1. Identify a list of prioritized candidates for the deprovisioning method. - * If there are [pods that cannot be evicted](#pod-eviction) on the node, Karpenter will ignore the node and try deprovisioning it later. - * If there are no deprovisionable nodes, continue to the next deprovisioning method. -2. For each deprovisionable node, execute a scheduling simulation with the pods on the node to find if any replacement nodes are needed. -3. Cordon the node(s) to prevent pods from scheduling to it. -4. Pre-spin any replacement nodes needed as calculated in Step (2), and wait for them to become ready. - * If a replacement node fails to initialize, un-cordon the node(s), and restart from Step (1), starting at the first deprovisioning method again. -5. Delete the node(s) and wait for the Termination Controller to gracefully shutdown the node(s). -6. Once the Termination Controller terminates the node, go back to Step (1), starting at the the first deprovisioning method again. - -### Termination Controller -When a Karpenter node is deleted, the Karpenter finalizer will block deletion and the APIServer will set the `DeletionTimestamp` on the node, allowing Karpenter to gracefully shutdown the node, modeled after [K8s Graceful Node Shutdown](https://kubernetes.io/docs/concepts/architecture/nodes/#graceful-node-shutdown). Karpenter's graceful shutdown process will: -1. Cordon the node to prevent pods from scheduling to it. -2. Begin evicting the pods on the node with the [K8s Eviction API](https://kubernetes.io/docs/concepts/scheduling-eviction/api-eviction/) to respect PDBs, while ignoring all non-daemonset pods and [static pods](https://kubernetes.io/docs/tasks/configure-pod-container/static-pod/). Wait for the node to be fully drained before proceeding to Step (3). - * While waiting, if the underlying machine for the node no longer exists, remove the finalizer to allow the APIServer to delete the node, completing termination. -3. Terminate the machine in the Cloud Provider. -4. Remove the finalizer from the node to allow the APIServer to delete the node, completing termination. - -## Methods - -There are both automated and manual ways of deprovisioning nodes provisioned by Karpenter: - -### Manual Methods -* **Node Deletion**: You could use `kubectl` to manually remove a single Karpenter node: - - ```bash - # Delete a specific node - kubectl delete node $NODE_NAME - - # Delete all nodes owned any provisioner - kubectl delete nodes -l karpenter.sh/provisioner-name - - # Delete all nodes owned by a specific provisioner - kubectl delete nodes -l karpenter.sh/provisioner-name=$PROVISIONER_NAME - ``` -* **Provisioner Deletion**: Nodes are owned by the Provisioner through an [owner reference](https://kubernetes.io/docs/concepts/overview/working-with-objects/owners-dependents/#owner-references-in-object-specifications) that launched them. Karpenter will gracefully terminate nodes through cascading deletion when the owning provisioner is deleted. - -### Automated Methods -* **Emptiness**: Karpenter notes when the last workload (non-daemonset) pod stops running on a node. From that point, Karpenter waits the number of seconds set by `ttlSecondsAfterEmpty` in the provisioner, then Karpenter requests to delete the node. This feature can keep costs down by removing nodes that are no longer being used for workloads. -* **Expiration**: Karpenter will annotate nodes as expired and deprovision nodes after they have lived a set number of seconds, based on the provisioner `ttlSecondsUntilExpired` value. One use case for node expiry is to periodically recycle nodes. Old nodes (with a potentially outdated Kubernetes version or operating system) are deleted, and replaced with nodes on the current version (assuming that you requested the latest version, rather than a specific version). -* **Consolidation**: Karpenter works to actively reduce cluster cost by identifying when: - * Nodes can be removed as their workloads will run on other nodes in the cluster. - * Nodes can be replaced with cheaper variants due to a change in the workloads. -* **Drift**: Karpenter will annotate nodes as drifted and deprovision nodes that have drifted from their desired specification. See [Drift]({{}}) to see which fields are considered. -* **Interruption**: If enabled, Karpenter will watch for upcoming involuntary interruption events that could affect your nodes (health events, spot interruption, etc.) and will cordon, drain, and terminate the node(s) ahead of the event to reduce workload disruption. - -{{% alert title="Note" color="primary" %}} -- Automated deprovisioning is configured through the ProvisionerSpec `.ttlSecondsAfterEmpty`, `.ttlSecondsUntilExpired` and `.consolidation.enabled` fields. If these are not configured, Karpenter will not set default values for them and will not terminate nodes for that purpose. - -- Keep in mind that a small `ttlSecondsUntilExpired` results in a higher churn in cluster activity. For a small enough `ttlSecondsUntilExpired`, nodes may expire faster than Karpenter can safely deprovision them, resulting in constant node deprovisioning. - -- Pods without an ownerRef (also called "controllerless" or "naked" pods) will be evicted during automatic node disruption, besides [Interruption](#interruption). A pod with the annotation `karpenter.sh/do-not-evict: "true"` will cause its node to be opted out from the same deprovisioning methods. - -- Using preferred anti-affinity and topology spreads can reduce the effectiveness of consolidation. At node launch, Karpenter attempts to satisfy affinity and topology spread preferences. In order to reduce node churn, consolidation must also attempt to satisfy these constraints to avoid immediately consolidating nodes after they launch. This means that consolidation may not deprovision nodes in order to avoid violating preferences, even if kube-scheduler can fit the host pods elsewhere. Karpenter reports these pods via logging to bring awareness to the possible issues they can cause (e.g. `pod default/inflate-anti-self-55894c5d8b-522jd has a preferred Anti-Affinity which can prevent consolidation`). - -- By adding the finalizer, Karpenter improves the default Kubernetes process of node deletion. -When you run `kubectl delete node` on a node without a finalizer, the node is deleted without triggering the finalization logic. The machine will continue running in EC2, even though there is no longer a node object for it. -The kubelet isn’t watching for its own existence, so if a node is deleted, the kubelet doesn’t terminate itself. -All the pod objects get deleted by a garbage collection process later, because the pods’ node is gone. - -{{% /alert %}} - -## Consolidation - -Karpenter has two mechanisms for cluster consolidation: -- Deletion - A node is eligible for deletion if all of its pods can run on free capacity of other nodes in the cluster. -- Replace - A node can be replaced if all of its pods can run on a combination of free capacity of other nodes in the cluster and a single cheaper replacement node. - -Consolidation has three mechanisms that are performed in order to attempt to identify a consolidation action: -1) Empty Node Consolidation - Delete any entirely empty nodes in parallel -2) Multi-Node Consolidation - Try to delete two or more nodes in parallel, possibly launching a single replacement that is cheaper than the price of all nodes being removed -3) Single-Node Consolidation - Try to delete any single node, possibly launching a single replacement that is cheaper than the price of that node - -It's impractical to examine all possible consolidation options for multi-node consolidation, so Karpenter uses a heuristic to identify a likely set of nodes that can be consolidated. For single-node consolidation we consider each node in the cluster individually. - -When there are multiple nodes that could be potentially deleted or replaced, Karpenter choose to consolidate the node that overall disrupts your workloads the least by preferring to terminate: - -* nodes running fewer pods -* nodes that will expire soon -* nodes with lower priority pods - -{{% alert title="Note" color="primary" %}} -For spot nodes, Karpenter only uses the deletion consolidation mechanism. It will not replace a spot node with a cheaper spot node. Spot instance types are selected with the `price-capacity-optimized` strategy and often the cheapest spot instance type is not launched due to the likelihood of interruption. Consolidation would then replace the spot instance with a cheaper instance negating the `price-capacity-optimized` strategy entirely and increasing interruption rate. -{{% /alert %}} - -If consolidation is enabled, Karpenter periodically reports events against nodes that indicate why the node can't be consolidated. These events can be used to investigate nodes that you expect to have been consolidated, but still remain in your cluster. - -``` -Events: - Type Reason Age From Message - ---- ------ ---- ---- ------- - Normal Unconsolidatable 66s karpenter pdb default/inflate-pdb prevents pod evictions - Normal Unconsolidatable 33s (x3 over 30m) karpenter can't replace with a cheaper node - ``` - -## Interruption - -If interruption-handling is enabled, Karpenter will watch for upcoming involuntary interruption events that would cause disruption to your workloads. These interruption events include: - -* Spot Interruption Warnings -* Scheduled Change Health Events (Maintenance Events) -* Instance Terminating Events -* Instance Stopping Events - -When Karpenter detects one of these events will occur to your nodes, it automatically cordons, drains, and terminates the node(s) ahead of the interruption event to give the maximum amount of time for workload cleanup prior to compute disruption. This enables scenarios where the `terminationGracePeriod` for your workloads may be long or cleanup for your workloads is critical, and you want enough time to be able to gracefully clean-up your pods. - -For Spot interruptions, the provisioner will start a new machine as soon as it sees the Spot interruption warning. Spot interruptions have a __2 minute notice__ before Amazon EC2 reclaims the instance. Karpenter's average node startup time means that, generally, there is sufficient time for the new node to become ready and to move the pods to the new node before the machine is reclaimed. - -{{% alert title="Note" color="primary" %}} -Karpenter publishes Kubernetes events to the node for all events listed above in addition to __Spot Rebalance Recommendations__. Karpenter does not currently support cordon, drain, and terminate logic for Spot Rebalance Recommendations. -{{% /alert %}} - -Karpenter enables this feature by watching an SQS queue which receives critical events from AWS services which may affect your nodes. Karpenter requires that an SQS queue be provisioned and EventBridge rules and targets be added that forward interruption events from AWS services to the SQS queue. Karpenter provides details for provisioning this infrastructure in the [CloudFormation template in the Getting Started Guide](../../getting-started/getting-started-with-karpenter/#create-the-karpenter-infrastructure-and-iam-roles). - -To enable the interruption handling feature flag, configure the `karpenter-global-settings` ConfigMap with the following value mapped to the name of the interruption queue that handles interruption events. - -```yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: karpenter-global-settings - namespace: karpenter -data: - ... - aws.interruptionQueueName: karpenter-cluster - ... -``` - -### Drift -Drift handles changes to the NodePool/EC2NodeClass. For Drift, values in the NodePool/EC2NodeClass are reflected in the NodeClaimTemplateSpec/EC2NodeClassSpec in the same way that they’re set. A NodeClaim will be detected as drifted if the values in its owning NodePool/EC2NodeClass do not match the values in the NodeClaim. Similar to the upstream `deployment.spec.template` relationship to pods, Karpenter will annotate the owning NodePool and EC2NodeClass with a hash of the NodeClaimTemplateSpec to check for drift. Some special cases will be discovered either from Karpenter or through the CloudProvider interface, triggered by NodeClaim/Instance/NodePool/EC2NodeClass changes. - -#### Special Cases on Drift -In special cases, drift can correspond to multiple values and must be handled differently. Drift on resolved fields can create cases where drift occurs without changes to CRDs, or where CRD changes do not result in drift. For example, if a NodeClaim has `node.kubernetes.io/instance-type: m5.large`, and requirements change from `node.kubernetes.io/instance-type In [m5.large]` to `node.kubernetes.io/instance-type In [m5.large, m5.2xlarge]`, the NodeClaim will not be drifted because its value is still compatible with the new requirements. Conversely, if a NodeClaim is using a NodeClaim image `ami: ami-abc`, but a new image is published, Karpenter's `EC2NodeClass.spec.amiSelectorTerms` will discover that the new correct value is `ami: ami-xyz`, and detect the NodeClaim as drifted. - -##### NodePool -| Fields | -|----------------| -| Requirements | - -##### EC2NodeClass -| Fields | -|-------------------------------| -| Subnet Selector Terms | -| Security Group Selector Terms | -| AMI Selector Terms | - -#### Behavioral Fields -Behavioral Fields are treated as over-arching settings on the NodePool to dictate how Karpenter behaves. These fields don’t correspond to settings on the NodeClaim or instance. They’re set by the user to control Karpenter’s Provisioning and disruption logic. Since these don’t map to a desired state of NodeClaims, __behavioral fields are not considered for Drift__. - -__Behavioral Fields__ -- Weight -- Limits -- ConsolidationPolicy -- ConsolidateAfter -- ExpireAfter ---- - -Read the [Drift Design](https://github.com/aws/karpenter-core/blob/main/designs/drift.md) for more. - -To enable the drift feature flag, refer to the [Settings Feature Gates]({{}}). - -Karpenter will add `MachineDrifted` status condition on the machines if the machine is drifted, and does not have the status condition, - -Karpenter will remove the `MachineDrifted` status condition for the following these scenarios: -1. The `featureGates.driftEnabled` is not enabled but the machine is drifted, karpenter will remove the status condition. -2. The machine isn't drifted, but has the status condition, karpenter will remove it. - -If the node is marked as drifted by another controller, karpenter will do nothing. - -## Controls - -### Pod-Level Controls - -You can block Karpenter from voluntarily choosing to disrupt certain pods by setting the `karpenter.sh/do-not-evict: "true"` annotation on the pod. This is useful for pods that you want to run from start to finish without disruption. By opting pods out of this disruption, you are telling Karpenter that it should not voluntarily remove a node containing this pod. - -Examples of pods that you might want to opt-out of disruption include an interactive game that you don't want to interrupt or a long batch job (such as you might have with machine learning) that would need to start over if it were interrupted. - -```yaml -apiVersion: apps/v1 -kind: Deployment -spec: - template: - metadata: - annotations: - karpenter.sh/do-not-evict: "true" -``` - -{{% alert title="Note" color="primary" %}} -This annotation will be ignored for [terminating pods](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase), [terminal pods](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase) (Failed/Succeeded), [DaemonSet pods](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/), or [static pods](https://kubernetes.io/docs/tasks/configure-pod-container/static-pod/). -{{% /alert %}} - -Examples of voluntary node removal that will be prevented by this annotation include: -- [Consolidation]({{}}) -- [Drift]({{}}) -- Emptiness -- Expiration - -{{% alert title="Note" color="primary" %}} -Voluntary node removal does not include [Interruption]({{}}) or manual deletion initiated through `kubectl delete node`. Both of these are considered involuntary events, since node removal cannot be delayed. -{{% /alert %}} - -### Node-Level Controls - -Nodes can be opted out of consolidation deprovisioning by setting the annotation `karpenter.sh/do-not-consolidate: "true"` on the node. - -```yaml -apiVersion: v1 -kind: Node -metadata: - annotations: - karpenter.sh/do-not-consolidate: "true" -``` - -#### Example: Disable Consolidation on Provisioner - -Provisioner `.spec.annotations` allow you to set annotations that will be applied to all nodes launched by this provisioner. By setting the annotation `karpenter.sh/do-not-consolidate: "true"` on the provisioner, you will selectively prevent all nodes launched by this Provisioner from being considered in consolidation calculations. - -```yaml -apiVersion: karpenter.sh/v1alpha5 -kind: Provisioner -metadata: - name: default -spec: - annotations: # will be applied to all nodes - karpenter.sh/do-not-consolidate: "true" -``` diff --git a/website/content/en/v0.31/concepts/node-templates.md b/website/content/en/v0.31/concepts/node-templates.md deleted file mode 100644 index ecab793e0984..000000000000 --- a/website/content/en/v0.31/concepts/node-templates.md +++ /dev/null @@ -1,689 +0,0 @@ ---- -title: "Node Templates" -linkTitle: "Node Templates" -weight: 2 -description: > - Configure AWS specific settings ---- - -Node Templates enable configuration of AWS specific settings. -Each provisioner must reference an AWSNodeTemplate using `spec.providerRef`. -Multiple provisioners may point to the same AWSNodeTemplate. - -```yaml -apiVersion: karpenter.sh/v1alpha5 -kind: Provisioner -metadata: - name: default -spec: - providerRef: - name: default ---- -apiVersion: karpenter.k8s.aws/v1alpha1 -kind: AWSNodeTemplate -metadata: - name: default -spec: - subnetSelector: { ... } # required, discovers tagged subnets to attach to instances - securityGroupSelector: { ... } # required, discovers tagged security groups to attach to instances - instanceProfile: "..." # optional, overrides the node's identity from global settings - amiFamily: "..." # optional, resolves a default ami and userdata - amiSelector: { ... } # optional, discovers tagged amis to override the amiFamily's default - userData: "..." # optional, overrides autogenerated userdata with a merge semantic - tags: { ... } # optional, propagates tags to underlying EC2 resources - metadataOptions: { ... } # optional, configures IMDS for the instance - blockDeviceMappings: [ ... ] # optional, configures storage devices for the instance - detailedMonitoring: "..." # optional, configures detailed monitoring for the instance -status: - subnets: { ... } # resolved subnets - securityGroups: { ... } # resolved security groups - amis: { ... } # resolved AMIs -``` -Refer to the [Provisioner docs]({{}}) for settings applicable to all providers. To explore various `AWSNodeTemplate` configurations, refer to the examples provided [in the Karpenter Github repository](https://github.com/aws/karpenter/blob/main/examples/provisioner/). - -See below for other AWS provider-specific parameters. - -## spec.subnetSelector - -The `AWSNodeTemplate` discovers subnets using [AWS tags](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html). -Subnets may be specified by any AWS tag, including `Name`. Selecting tag values using wildcards (`*`) is supported. -Subnet IDs may be specified by using the key `aws-ids` and then passing the IDs as a comma-separated string value. -When launching nodes, a subnet is automatically chosen that matches the desired zone. -If multiple subnets exist for a zone, the one with the most available IP addresses will be used. - -**Examples** - -Select all with a specified tag key: -```yaml -spec: - subnetSelector: - karpenter.sh/discovery/MyClusterName: '*' -``` - -Select by name and tag (all criteria must match):: -```yaml -spec: - subnetSelector: - Name: my-subnet - MyTag: '' # matches all resources with the tag -``` - -Select using comma separated tag values: -```yaml -spec: - subnetSelector: - Name: "my-subnet-1,my-subnet-2" -``` - -Select using wildcards: -```yaml -spec: - subnetSelector: - Name: "*Public*" - -``` - -Select by ID: -```yaml -spec: - subnetSelector: - aws-ids: "subnet-09fa4a0a8f233a921,subnet-0471ca205b8a129ae" -``` - -## spec.securityGroupSelector - -The security group of an instance is comparable to a set of firewall rules. -EKS creates at least two security groups by default, [review the documentation](https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html) for more info. -Security groups may be specified by any AWS tag, including "Name". Selecting tags using wildcards (`*`) is supported. - -{{% alert title="Note" color="primary" %}} -When launching nodes, Karpenter uses all the security groups that match the selector. If you choose to use the `kubernetes.io/cluster/$CLUSTER_NAME` tag for discovery, note that this may result in failures using the AWS Load Balancer controller. The Load Balancer controller only supports a single security group having that tag key. See [this issue](https://github.com/kubernetes-sigs/aws-load-balancer-controller/issues/2367) for more details. -{{% /alert %}} - -To verify if this restriction affects you, run the following commands. -```bash -CLUSTER_VPC_ID="$(aws eks describe-cluster --name $CLUSTER_NAME --query cluster.resourcesVpcConfig.vpcId --output text)" - -aws ec2 describe-security-groups --filters Name=vpc-id,Values=$CLUSTER_VPC_ID Name=tag-key,Values=kubernetes.io/cluster/$CLUSTER_NAME --query 'SecurityGroups[].[GroupName]' --output text -``` - -If multiple securityGroups are printed, you will need a more specific securityGroupSelector. We generally recommend that you use the `karpenter.sh/discovery: $CLUSTER_NAME` tag selector instead. - -**Examples** - -Select all assigned to a cluster: -```yaml -spec: - securityGroupSelector: - karpenter.sh/discovery: "${CLUSTER_NAME}" -``` - -Select all with a specified tag key: -```yaml -spec: - securityGroupSelector: - MyTag: '*' -``` - -Select by name and tag (all criteria must match): -```yaml -spec: - securityGroupSelector: - Name: my-security-group - MyTag: '' # matches all resources with the tag -``` - -Select by comma-separated tag values: -```yaml -spec: - securityGroupSelector: - Name: "my-security-group-1,my-security-group-2" -``` - -Select by name using a wildcard: -```yaml -spec: - securityGroupSelector: - Name: "*Public*" -``` - -Select by ID: -```yaml -spec: - securityGroupSelector: - aws-ids: "sg-063d7acfb4b06c82c,sg-06e0cf9c198874591" -``` - -## spec.instanceProfile - -An `InstanceProfile` is a way to pass a single IAM role to EC2 instance launched the provisioner. -A default profile is configured in global settings, but may be overridden here. -The `AWSNodeTemplate` will not create an `InstanceProfile` automatically. -The `InstanceProfile` must refer to a `Role` that has permission to connect to the cluster. -```yaml -spec: - instanceProfile: MyInstanceProfile -``` - -## spec.amiFamily - -The AMI used when provisioning nodes can be controlled by the `amiFamily` field. Based on the value set for `amiFamily`, Karpenter will automatically query for the appropriate [EKS optimized AMI](https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-amis.html) via AWS Systems Manager (SSM). When an `amiFamily` of `Custom` is chosen, then an `amiSelector` must be specified that informs Karpenter on which custom AMIs are to be used. - -Currently, Karpenter supports `amiFamily` values `AL2`, `Bottlerocket`, `Ubuntu`, `Windows2019`, `Windows2022` and `Custom`. GPUs are only supported with `AL2` and `Bottlerocket`. The `AL2` amiFamily does not support ARM64 GPU instance types unless you specify a custom amiSelector. - -{{% alert title="Defaults" color="secondary" %}} -If no `amiFamily` is defined, Karpenter will set the default `amiFamily` to AL2 - -```yaml -spec: - amiFamily: AL2 -``` -{{% /alert %}} - -## spec.amiSelector - -AMISelector is used to configure custom AMIs for Karpenter to use, where the AMIs are discovered through `aws::` prefixed filters (`aws::ids`, `aws::owners` and `aws::name`) and [AWS tags](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html). This field is optional, and Karpenter will use the latest EKS-optimized AMIs if an amiSelector is not specified. - -To select an AMI by name, use `aws::name`. EC2 AMIs may be specified by any AWS tag, including `Name`. Selecting by tag or by name using wildcards (`*`) is supported. - -EC2 AMI IDs may be specified by using the key `aws::ids` (`aws-ids` is also supported) and then passing the IDs as a comma-separated string value. - -To ensure that AMIs are owned by the expected owner, use `aws::owners` which expects a comma-separated list of AWS account owners - you can use a combination of account aliases (e.g. `self` `amazon`, `your-aws-account-name`) and account IDs. If this is not set, *and* `aws::ids`/`aws-ids` are not set, it defaults to `self,amazon`. - -{{% alert title="Note" color="primary" %}} -If you use only `aws::owners`, Karpenter will discover all images that are owned by those specified, selecting the most recently created ones to be used. If you specify `aws::owners`, but nothing else, there is a larger chance that Karpenter could select an image that is not compatible with your instance type. To lower this chance, it is recommended to use `aws::name` or `aws::ids` if you're using `aws::owners` to select a subset of images that you have validated are compatible with your selected instance types. -{{% /alert %}} - -### AMI Selection - -If an `amiSelector` matches more than one AMI, Karpenter will automatically determine which AMI best fits the workloads on the launched worker node under the following constraints: - -* When launching nodes, Karpenter automatically determines which architecture a custom AMI is compatible with and will use images that match an instanceType's requirements. -* If multiple AMIs are found that can be used, Karpenter will choose the latest one. -* If no AMIs are found that can be used, then no nodes will be provisioned. - -If you need to express other constraints for an AMI beyond architecture, you can express these constraints as tags on the AMI. For example, if you want to limit an EC2 AMI to only be used with instanceTypes that have an `nvidia` GPU, you can specify an EC2 tag with a key of `karpenter.k8s.aws/instance-gpu-manufacturer` and value `nvidia` on that AMI. - -All labels defined [in the scheduling documentation](../scheduling#well-known-labels) can be used as requirements for an EC2 AMI. - -```bash -> aws ec2 describe-images --image-id ami-123 --query Images[0].Tags -[ - { - "Key": "karpenter.sh/discovery", - "Value": "my-cluster" - }, - { - "Key": "Name", - "Value": "amazon-eks-node-1.21-customized-v0" - }, - { - "Key": "karpenter.k8s.aws/instance-gpu-manufacturer", - "Value": "nvidia" - } -] -``` - -#### Examples - -Select all AMIs with a specified tag: -```yaml - amiSelector: - karpenter.sh/discovery/MyClusterName: '*' -``` - -Select AMIs by the AMI name: -```yaml - amiSelector: - aws::name: my-ami -``` -Select AMIs by the Name tag: -```yaml - amiSelector: - Name: my-ami -``` - -Select AMIs by name and a specific owner: -```yaml - amiSelector: - aws::name: my-ami - aws::owners: self/ownerAccountID -``` - -Select AMIs by an arbitrary AWS tag key/value pair: -```yaml - amiSelector: - MyAMITag: value -``` - -Specify AMIs explicitly by ID: -```yaml - amiSelector: - aws::ids: "ami-123,ami-456" -``` - -## spec.tags - -Karpenter adds tags to all resources it creates, including EC2 Instances, EBS volumes, and Launch Templates. The default set of AWS tags are listed below. - -``` -Name: karpenter.sh/provisioner-name/ -karpenter.sh/provisioner-name: -kubernetes.io/cluster/: owned -``` - -Additional tags can be added in the AWSNodeTemplate tags section which are merged with global tags in `aws.tags` (located in karpenter-global-settings ConfigMap). -```yaml -spec: - tags: - InternalAccountingTag: 1234 - dev.corp.net/app: Calculator - dev.corp.net/team: MyTeam -``` - -Karpenter allows overrides of the default "Name" tag but does not allow overrides to restricted domains (such as "karpenter.sh", "karpenter.k8s.aws", and "kubernetes.io/cluster"). This ensures that Karpenter is able to correctly auto-discover machines that it owns. - -## spec.metadataOptions - -Control the exposure of [Instance Metadata Service](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) on EC2 Instances launched by this provisioner using a generated launch template. - -Refer to [recommended, security best practices](https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node) for limiting exposure of Instance Metadata and User Data to pods. - -If metadataOptions are omitted from this provisioner, the following default settings will be used. - -```yaml -spec: - metadataOptions: - httpEndpoint: enabled - httpProtocolIPv6: disabled - httpPutResponseHopLimit: 2 - httpTokens: required -``` - -## spec.blockDeviceMappings - -The `blockDeviceMappings` field in an AWSNodeTemplate can be used to control the Elastic Block Storage (EBS) volumes that Karpenter attaches to provisioned nodes. Karpenter uses default block device mappings for the AMI Family specified. For example, the `Bottlerocket` AMI Family defaults with two block device mappings, one for Bottlerocket's control volume and the other for container resources such as images and logs. - -Learn more about [block device mappings](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/block-device-mapping-concepts.html). - -### Examples - -```yaml -apiVersion: karpenter.k8s.aws/v1alpha1 -kind: AWSNodeTemplate -spec: - blockDeviceMappings: - - deviceName: /dev/xvda - ebs: - volumeSize: 100Gi - volumeType: gp3 - iops: 10000 - encrypted: true - kmsKeyID: "arn:aws:kms:us-west-2:111122223333:key/1234abcd-12ab-34cd-56ef-1234567890ab" - deleteOnTermination: true - throughput: 125 - snapshotID: snap-0123456789 -``` - -{{% alert title="Defaults" color="secondary" %}} -If no `blockDeviceMappings` is defined, Karpenter will set the default `blockDeviceMappings` to the following for the given AMI family. - -#### AL2 -```yaml -apiVersion: karpenter.k8s.aws/v1alpha1 -kind: AWSNodeTemplate -spec: - blockDeviceMappings: - - deviceName: /dev/xvda - ebs: - volumeSize: 20Gi - volumeType: gp3 - encrypted: true -``` - -#### Bottlerocket -```yaml -apiVersion: karpenter.k8s.aws/v1alpha1 -kind: AWSNodeTemplate -spec: - blockDeviceMappings: - # Root device - - deviceName: /dev/xvda - ebs: - volumeSize: 4Gi - volumeType: gp3 - encrypted: true - # Data device: Container resources such as images and logs - - deviceName: /dev/xvdb - ebs: - volumeSize: 20Gi - volumeType: gp3 - encrypted: true -``` - -#### Ubuntu -```yaml -apiVersion: karpenter.k8s.aws/v1alpha1 -kind: AWSNodeTemplate -spec: - blockDeviceMappings: - - deviceName: /dev/sda1 - ebs: - volumeSize: 20Gi - volumeType: gp3 - encrypted: true -``` - -#### Windows2019, Windows2022 -```yaml -apiVersion: karpenter.k8s.aws/v1alpha1 -kind: AWSNodeTemplate -spec: - blockDeviceMappings: - - deviceName: /dev/sda1 - ebs: - volumeSize: 50Gi - volumeType: gp3 - encrypted: true -``` -{{% /alert %}} - -## spec.userData - -You can control the UserData that is applied to your worker nodes via this field. - -```yaml -apiVersion: karpenter.k8s.aws/v1alpha1 -kind: AWSNodeTemplate -metadata: - name: bottlerocket-example -spec: - amiFamily: Bottlerocket - instanceProfile: MyInstanceProfile - subnetSelector: - karpenter.sh/discovery: my-cluster - securityGroupSelector: - karpenter.sh/discovery: my-cluster - userData: | - [settings.kubernetes] - "kube-api-qps" = 30 - "shutdown-grace-period" = "30s" - "shutdown-grace-period-for-critical-pods" = "30s" - [settings.kubernetes.eviction-hard] - "memory.available" = "20%" - amiSelector: - karpenter.sh/discovery: my-cluster -``` - -This example adds SSH keys to allow remote login to the node (replace *my-authorized_keys* with your key file): - -{{% alert title="Note" color="primary" %}} -Instead of using SSH as set up in this example, you can use Session Manager (SSM) or EC2 Instance Connect to gain shell access to Karpenter nodes. -See [Node NotReady]({{< ref "../troubleshooting/#node-notready" >}}) troubleshooting for an example of starting an SSM session from the command line or [EC2 Instance Connect](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-connect-set-up.html) documentation to connect to nodes using SSH. -{{% /alert %}} - -```yaml -apiVersion: karpenter.k8s.aws/v1alpha1 -kind: AWSNodeTemplate -metadata: - name: al2-example -spec: - amiFamily: AL2 - instanceProfile: MyInstanceProfile - subnetSelector: - karpenter.sh/discovery: my-cluster - securityGroupSelector: - karpenter.sh/discovery: my-cluster - userData: | - #!/bin/bash - mkdir -p ~ec2-user/.ssh/ - touch ~ec2-user/.ssh/authorized_keys - cat >> ~ec2-user/.ssh/authorized_keys < >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 -/etc/eks/bootstrap.sh 'test-cluster' --apiserver-endpoint 'https://test-cluster' --b64-cluster-ca 'ca-bundle' \ ---use-max-pods false \ ---container-runtime containerd \ ---kubelet-extra-args '--node-labels=karpenter.sh/capacity-type=on-demand,karpenter.sh/provisioner-name=test --max-pods=110' ---//-- -``` - -You can also set kubelet-config properties by modifying the kubelet-config.json file before the EKS bootstrap script starts the kubelet: - -``` -apiVersion: karpenter.k8s.aws/v1alpha1 -kind: AWSNodeTemplate -metadata: - name: kubelet-config-example -spec: - subnetSelector: - karpenter.sh/discovery: my-cluster - securityGroupSelector: - karpenter.sh/discovery: my-cluster - userData: | - #!/bin/bash - echo "$(jq '.kubeAPIQPS=50' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json -``` - -#### Windows - -* Your UserData must be specified as PowerShell commands. -* The UserData specified will be prepended to a Karpenter managed section that will bootstrap the kubelet. -* Karpenter will continue to set ClusterDNS and all other parameters defined in spec.kubeletConfiguration as before. - -Consider the following example to understand how your custom UserData settings will be merged in. - -Your UserData - -``` -Write-Host "Running custom user data script" -``` - -Final merged UserData - -``` - -Write-Host "Running custom user data script" -[string]$EKSBootstrapScriptFile = "$env:ProgramFiles\Amazon\EKS\Start-EKSBootstrap.ps1" -& $EKSBootstrapScriptFile -EKSClusterName 'test-cluster' -APIServerEndpoint 'https://test-cluster' -Base64ClusterCA 'ca-bundle' -KubeletExtraArgs '--node-labels="karpenter.sh/capacity-type=spot,karpenter.sh/provisioner-name=windows2022" --max-pods=110' -DNSClusterIP '10.0.100.10' - -``` - -{{% alert title="Windows Support Notice" color="warning" %}} -Currently, Karpenter does not specify `-ServiceCIDR` to [EKS Windows AMI Bootstrap script](https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-windows-ami.html#bootstrap-script-configuration-parameters). -Windows worker nodes will use `172.20.0.0/16` or `10.100.0.0/16` for Kubernetes service IP address ranges based on the IP address of the primary interface. -The effective ServiceCIDR can be verified at `$env:ProgramData\Amazon\EKS\cni\config\vpc-bridge.conf` on the worker node. - -Support for the Windows ServiceCIDR argument can be tracked in a [Karpenter Github Issue](https://github.com/aws/karpenter/issues/4088). Currently, if the effective ServiceCIDR is incorrect for your windows worker nodes, you can add the following userData as a workaround. - -```yaml -spec: - userData: | - $global:EKSCluster = Get-EKSCluster -Name my-cluster -``` -{{% /alert %}} - -## spec.detailedMonitoring - -Enabling detailed monitoring on the node template controls the [EC2 detailed monitoring](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-cloudwatch-new.html) feature. If you enable this option, the Amazon EC2 console displays monitoring graphs with a 1-minute period for the instances that Karpenter launches. -```yaml -spec: - detailedMonitoring: true -``` - -## status.subnets -`status.subnets` contains the `id` and `zone` of the subnets utilized during node launch. The subnets are sorted by the available IP address count in decreasing order. - -**Examples** - -```yaml -status: - subnets: - - id: subnet-0a462d98193ff9fac - zone: us-east-2b - - id: subnet-0322dfafd76a609b6 - zone: us-east-2c - - id: subnet-0727ef01daf4ac9fe - zone: us-east-2b - - id: subnet-00c99aeafe2a70304 - zone: us-east-2a - - id: subnet-023b232fd5eb0028e - zone: us-east-2c - - id: subnet-03941e7ad6afeaa72 - zone: us-east-2a -``` - -## status.securityGroups -`status.securityGroups` contains the `id` and `name` of the security groups utilized during node launch. - -**Examples** - -```yaml - status: - securityGroups: - - id: sg-041513b454818610b - name: ClusterSharedNodeSecurityGroup - - id: sg-0286715698b894bca - name: ControlPlaneSecurityGroup-1AQ073TSAAPW -``` - -## status.amis -`status.amis` contains the `id`, `name`, and `requirements` of the amis utilized during node launch. - -**Examples** - -```yaml - amis: - - id: ami-03c3a3dcda64f5b75 - name: amazon-linux-2-gpu - requirements: - - key: kubernetes.io/arch - operator: In - values: - - amd64 - - key: karpenter.k8s.aws/instance-accelerator-manufacturer - operator: In - values: - - aws - - nvidia - - id: ami-06afb2d101cc4b8bd - name: amazon-linux-2-arm64 - requirements: - - key: kubernetes.io/arch - operator: In - values: - - arm64 - - key: karpenter.k8s.aws/instance-accelerator-manufacturer - operator: NotIn - values: - - aws - - nvidia - - id: ami-0e28b76d768af234e - name: amazon-linux-2 - requirements: - - key: kubernetes.io/arch - operator: In - values: - - amd64 - - key: karpenter.k8s.aws/instance-accelerator-manufacturer - operator: NotIn - values: - - aws - - nvidia -``` \ No newline at end of file diff --git a/website/content/en/v0.31/concepts/pod-density.md b/website/content/en/v0.31/concepts/pod-density.md deleted file mode 100644 index 6a8a41b6af17..000000000000 --- a/website/content/en/v0.31/concepts/pod-density.md +++ /dev/null @@ -1,91 +0,0 @@ ---- -title: "Control Pod Density" -linkTitle: "Control Pod Density" -weight: 6 -description: > - Learn ways to specify pod density with Karpenter ---- - -Pod density is the number of pods per node. - -Kubernetes has a default limit of 110 pods per node. If you are using the EKS Optimized AMI on AWS, the [number of pods is limited by instance type](https://github.com/awslabs/amazon-eks-ami/blob/master/files/eni-max-pods.txt) in the default configuration. - -## Increase Pod Density - -### Networking Limitations - -*☁️ AWS Specific* - -By default, the number of pods on a node is limited by both the number of networking interfaces (ENIs) that may be attached to an instance type and the number of IP addresses that can be assigned to each ENI. See [IP addresses per network interface per instance type](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-eni.html#AvailableIpPerENI) for a more detailed information on these instance types' limits. - -Karpenter can be configured to disable nodes' ENI-based pod density. This is especially useful for small to medium instance types which have a lower ENI-based pod density. - -{{% alert title="Note" color="primary" %}} -When using small instance types, it may be necessary to enable [prefix assignment mode](https://aws.amazon.com/blogs/containers/amazon-vpc-cni-increases-pods-per-node-limits/) in the AWS VPC CNI plugin to more pods per node. Prefix assignment mode was introduced in AWS VPC CNI v1.9 and allows ENIs to manage a broader set of IP addresses. Much higher pod densities are supported as a result. -{{% /alert %}} - -{{% alert title="Windows Support Notice" color="warning" %}} -Presently, Windows worker nodes do not support using more than one ENI. -As a consequence, the number of IP addresses, and subsequently, the number of pods that a Windows worker node can support is limited by the number of IPv4 addresses available on the primary ENI. -At the moment, Karpenter will only consider individual secondary IP addresses when calculating the pod density limit. -{{% /alert %}} - -### Provisioner-Specific Pod Density - -#### Static Pod Density - -Static pod density can be configured at the provisioner level by specifying `maxPods` within the `.spec.kubeletConfiguration`. All nodes spawned by this provisioner will set this `maxPods` value on their kubelet and will account for this value during scheduling. - -See [Provisioner API Kubelet Configuration](../provisioners/#max-pods) for more details. - -#### Dynamic Pod Density - -Dynamic pod density (density that scales with the instance size) can be configured at the provisioner level by specifying `podsPerCore` within the `.spec.kubeletConfiguration`. Karpenter will calculate the expected pod density for each instance based on the instance's number of logical cores (vCPUs) and will account for this during scheduling. - -See [Provisioner API Kubelet Configuration](../provisioners/#pod-density) for more details. - -### Controller-Wide Pod Density - -{{% alert title="Deprecation Warning" color="warning" %}} -`AWS_ENI_LIMITED_POD_DENSITY` is deprecated in favor of the `.spec.kubeletConfiguration.maxPods` set at the Provisioner-level -{{% /alert %}} - -Set the environment variable `AWS_ENI_LIMITED_POD_DENSITY: "false"` (or the argument `--aws-eni-limited-pod-density=false`) in the Karpenter controller to allow nodes to host up to 110 pods by default. - -Environment variables for the Karpenter controller may be specified as [helm chart values](https://github.com/aws/karpenter/blob/c73f425e924bb64c3f898f30ca5035a1d8591183/charts/karpenter/values.yaml#L15). - -### VPC CNI Custom Networking - -By default, the VPC CNI allocates IPs for a node and pods from the same subnet. With [VPC CNI Custom Networking](https://aws.github.io/aws-eks-best-practices/networking/custom-networking), the pods will receive IP addresses from another subnet dedicated to pod IPs. This approach makes it easier to manage IP addresses and allows for separate Network Access Control Lists (NACLs) applied to your pods. VPC CNI Custom Networking reduces the pod density of a node since one of the ENI attachments will be used for the node and cannot share the allocated IPs on the interface to pods. Karpenter supports VPC CNI Custom Networking and similar CNI setups where the primary node interface is separated from the pods interfaces through a global [setting](./settings.md#configmap) within the karpenter-global-settings configmap: `aws.reservedENIs`. In the common case, `aws.reservedENIs` should be set to `"1"` if using Custom Networking. - -{{% alert title="Windows Support Notice" color="warning" %}} -It's currently not possible to specify custom networking with Windows nodes. -{{% /alert %}} - -## Limit Pod Density - -Generally, increasing pod density is more efficient. However, some use cases exist for limiting pod density. - -### Topology Spread - -You can use [topology spread]({{< relref "scheduling.md#topology-spread" >}}) features to reduce blast radius. For example, spreading workloads across EC2 Availability Zones. - - -### Restrict Instance Types - -Exclude large instance sizes to reduce the blast radius of an EC2 instance failure. - -Consider setting up upper or lower boundaries on target instance sizes with the node.kubernetes.io/instance-type key. - -The following example shows how to avoid provisioning large Graviton instances in order to reduce the impact of individual instance failures: - -``` --key: node.kubernetes.io/instance-type - operator: NotIn - values: - 'm6g.16xlarge' - 'm6gd.16xlarge' - 'r6g.16xlarge' - 'r6gd.16xlarge' - 'c6g.16xlarge' -``` diff --git a/website/content/en/v0.31/concepts/provisioners.md b/website/content/en/v0.31/concepts/provisioners.md deleted file mode 100644 index 5881eb6046f8..000000000000 --- a/website/content/en/v0.31/concepts/provisioners.md +++ /dev/null @@ -1,484 +0,0 @@ ---- -title: "Provisioners" -linkTitle: "Provisioners" -weight: 1 -description: > - Learn about Karpenter Provisioners ---- - -When you first installed Karpenter, you set up a default Provisioner. -The Provisioner sets constraints on the nodes that can be created by Karpenter and the pods that can run on those nodes. -The Provisioner can be set to do things like: - -* Define taints to limit the pods that can run on nodes Karpenter creates -* Define any startup taints to inform Karpenter that it should taint the node initially, but that the taint is temporary. -* Limit node creation to certain zones, instance types, and computer architectures -* Set defaults for node expiration - -You can change your Provisioner or add other Provisioners to Karpenter. -Here are things you should know about Provisioners: - -* Karpenter won't do anything if there is not at least one Provisioner configured. -* Each Provisioner that is configured is looped through by Karpenter. -* If Karpenter encounters a taint in the Provisioner that is not tolerated by a Pod, Karpenter won't use that Provisioner to provision the pod. -* If Karpenter encounters a startup taint in the Provisioner it will be applied to nodes that are provisioned, but pods do not need to tolerate the taint. Karpenter assumes that the taint is temporary and some other system will remove the taint. -* It is recommended to create Provisioners that are mutually exclusive. So no Pod should match multiple Provisioners. If multiple Provisioners are matched, Karpenter will use the Provisioner with the highest [weight](#specweight). - -For some example `Provisioner` configurations, see the [examples in the Karpenter GitHub repository](https://github.com/aws/karpenter/blob/main/examples/provisioner/). - -```yaml -apiVersion: karpenter.sh/v1alpha5 -kind: Provisioner -metadata: - name: default -spec: - # References cloud provider-specific custom resource, see your cloud provider specific documentation - providerRef: - name: default - - # Provisioned nodes will have these taints - # Taints may prevent pods from scheduling if they are not tolerated by the pod. - taints: - - key: example.com/special-taint - effect: NoSchedule - - # Provisioned nodes will have these taints, but pods do not need to tolerate these taints to be provisioned by this - # provisioner. These taints are expected to be temporary and some other entity (e.g. a DaemonSet) is responsible for - # removing the taint after it has finished initializing the node. - startupTaints: - - key: example.com/another-taint - effect: NoSchedule - - # Labels are arbitrary key-values that are applied to all nodes - labels: - billing-team: my-team - - # Annotations are arbitrary key-values that are applied to all nodes - annotations: - example.com/owner: "my-team" - - # Requirements that constrain the parameters of provisioned nodes. - # These requirements are combined with pod.spec.topologySpreadConstraints, pod.spec.affinity.nodeAffinity, pod.spec.affinity.podAffinity, and pod.spec.nodeSelector rules. - # Operators { In, NotIn, Exists, DoesNotExist, Gt, and Lt } are supported. - # https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#operators - requirements: - - key: "karpenter.k8s.aws/instance-category" - operator: In - values: ["c", "m", "r"] - - key: "karpenter.k8s.aws/instance-cpu" - operator: In - values: ["4", "8", "16", "32"] - - key: "karpenter.k8s.aws/instance-hypervisor" - operator: In - values: ["nitro"] - - key: "karpenter.k8s.aws/instance-generation" - operator: Gt - values: ["2"] - - key: "topology.kubernetes.io/zone" - operator: In - values: ["us-west-2a", "us-west-2b"] - - key: "kubernetes.io/arch" - operator: In - values: ["arm64", "amd64"] - - key: "karpenter.sh/capacity-type" # If not included, the webhook for the AWS cloud provider will default to on-demand - operator: In - values: ["spot", "on-demand"] - - # Karpenter provides the ability to specify a few additional Kubelet args. - # These are all optional and provide support for additional customization and use cases. - kubeletConfiguration: - clusterDNS: ["10.0.1.100"] - containerRuntime: containerd - systemReserved: - cpu: 100m - memory: 100Mi - ephemeral-storage: 1Gi - kubeReserved: - cpu: 200m - memory: 100Mi - ephemeral-storage: 3Gi - evictionHard: - memory.available: 5% - nodefs.available: 10% - nodefs.inodesFree: 10% - evictionSoft: - memory.available: 500Mi - nodefs.available: 15% - nodefs.inodesFree: 15% - evictionSoftGracePeriod: - memory.available: 1m - nodefs.available: 1m30s - nodefs.inodesFree: 2m - evictionMaxPodGracePeriod: 60 - imageGCHighThresholdPercent: 85 - imageGCLowThresholdPercent: 80 - cpuCFSQuota: true - podsPerCore: 2 - maxPods: 20 - - - # Resource limits constrain the total size of the cluster. - # Limits prevent Karpenter from creating new instances once the limit is exceeded. - limits: - resources: - cpu: "1000" - memory: 1000Gi - - # Enables consolidation which attempts to reduce cluster cost by both removing un-needed nodes and down-sizing those - # that can't be removed. Mutually exclusive with the ttlSecondsAfterEmpty parameter. - consolidation: - enabled: true - - # If omitted, the feature is disabled and nodes will never expire. If set to less time than it requires for a node - # to become ready, the node may expire before any pods successfully start. - ttlSecondsUntilExpired: 2592000 # 30 Days = 60 * 60 * 24 * 30 Seconds; - - # If omitted, the feature is disabled, nodes will never scale down due to low utilization - ttlSecondsAfterEmpty: 30 - - # Priority given to the provisioner when the scheduler considers which provisioner - # to select. Higher weights indicate higher priority when comparing provisioners. - # Specifying no weight is equivalent to specifying a weight of 0. - weight: 10 -``` - -## spec.requirements - -Kubernetes defines the following [Well-Known Labels](https://kubernetes.io/docs/reference/labels-annotations-taints/), and cloud providers (e.g., AWS) implement them. They are defined at the "spec.requirements" section of the Provisioner API. - -In addition to the well-known labels from Kubernetes, Karpenter supports AWS-specific labels for more advanced scheduling. See the full list [here](../scheduling/#well-known-labels). - -These well-known labels may be specified at the provisioner level, or in a workload definition (e.g., nodeSelector on a pod.spec). Nodes are chosen using both the provisioner's and pod's requirements. If there is no overlap, nodes will not be launched. In other words, a pod's requirements must be within the provisioner's requirements. If a requirement is not defined for a well known label, any value available to the cloud provider may be chosen. - -For example, an instance type may be specified using a nodeSelector in a pod spec. If the instance type requested is not included in the provisioner list and the provisioner has instance type requirements, Karpenter will not create a node or schedule the pod. - -📝 None of these values are required. - -### Instance Types - -- key: `node.kubernetes.io/instance-type` -- key: `karpenter.k8s.aws/instance-family` -- key: `karpenter.k8s.aws/instance-category` -- key: `karpenter.k8s.aws/instance-generation` - -Generally, instance types should be a list and not a single value. Leaving these requirements undefined is recommended, as it maximizes choices for efficiently placing pods. - -Review [AWS instance types](../instance-types). Most instance types are supported with the exclusion of [non-HVM](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/virtualization_types.html). - -{{% alert title="Defaults" color="secondary" %}} -If no instance type constraints are defined, Karpenter will set default instance type constraints on your Provisioner that supports most common user workloads: - -```yaml -requirements: - - key: karpenter.k8s.aws/instance-category - operator: In - values: ["c", "m", "r"] - - key: karpenter.k8s.aws/instance-generation - operator: Gt - values: ["2"] -``` -{{% /alert %}} - -### Availability Zones - -- key: `topology.kubernetes.io/zone` -- value example: `us-east-1c` -- value list: `aws ec2 describe-availability-zones --region ` - -Karpenter can be configured to create nodes in a particular zone. Note that the Availability Zone `us-east-1a` for your AWS account might not have the same location as `us-east-1a` for another AWS account. - -[Learn more about Availability Zone -IDs.](https://docs.aws.amazon.com/ram/latest/userguide/working-with-az-ids.html) - -### Architecture - -- key: `kubernetes.io/arch` -- values - - `amd64` - - `arm64` - -Karpenter supports `amd64` nodes, and `arm64` nodes. - -{{% alert title="Defaults" color="secondary" %}} -If no architecture constraint is defined, Karpenter will set the default architecture constraint on your Provisioner that supports most common user workloads: - -```yaml -requirements: - - key: kubernetes.io/arch - operator: In - values: ["amd64"] -``` -{{% /alert %}} - -### Operating System - - key: `kubernetes.io/os` - - values - - `linux` - - `windows` - -Karpenter supports `linux` and `windows` operating systems. - -{{% alert title="Defaults" color="secondary" %}} -If no operating system constraint is defined, Karpenter will set the default operating system constraint on your Provisioner that supports most common user workloads: - -```yaml -requirements: - - key: kubernetes.io/os - operator: In - values: ["linux"] -``` -{{% /alert %}} - -### Capacity Type - -- key: `karpenter.sh/capacity-type` -- values - - `spot` - - `on-demand` - -Karpenter supports specifying capacity type, which is analogous to [EC2 purchase options](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-purchasing-options.html). - -Karpenter prioritizes Spot offerings if the provisioner allows Spot and on-demand instances. If the provider API (e.g. EC2 Fleet's API) indicates Spot capacity is unavailable, Karpenter caches that result across all attempts to provision EC2 capacity for that instance type and zone for the next 45 seconds. If there are no other possible offerings available for Spot, Karpenter will attempt to provision on-demand instances, generally within milliseconds. - -Karpenter also allows `karpenter.sh/capacity-type` to be used as a topology key for enforcing topology-spread. - -{{% alert title="Defaults" color="secondary" %}} -If no capacity type constraint is defined, Karpenter will set the default capacity type constraint on your Provisioner that supports most common user workloads: - -```yaml -requirements: - - key: karpenter.sh/capacity-type - operator: In - values: ["on-demand"] -``` -{{% /alert %}} - -## spec.weight - -Karpenter allows you to describe provisioner preferences through a `weight` mechanism similar to how weight is described with [pod and node affinities](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity). - -For more information on weighting Provisioners, see the [Weighting Provisioners section](../scheduling#weighting-provisioners) in the scheduling details. - -## spec.kubeletConfiguration - -Karpenter provides the ability to specify a few additional Kubelet args. These are all optional and provide support for -additional customization and use cases. Adjust these only if you know you need to do so. For more details on kubelet configuration arguments, [see the KubeletConfiguration API specification docs](https://kubernetes.io/docs/reference/config-api/kubelet-config.v1beta1/). The implemented fields are a subset of the full list of upstream kubelet configuration arguments. Please cut an issue if you'd like to see another field implemented. - -```yaml -spec: - ... - kubeletConfiguration: - clusterDNS: ["10.0.1.100"] - containerRuntime: containerd - systemReserved: - cpu: 100m - memory: 100Mi - ephemeral-storage: 1Gi - kubeReserved: - cpu: 200m - memory: 100Mi - ephemeral-storage: 3Gi - evictionHard: - memory.available: 5% - nodefs.available: 10% - nodefs.inodesFree: 10% - evictionSoft: - memory.available: 500Mi - nodefs.available: 15% - nodefs.inodesFree: 15% - evictionSoftGracePeriod: - memory.available: 1m - nodefs.available: 1m30s - nodefs.inodesFree: 2m - evictionMaxPodGracePeriod: 60 - imageGCHighThresholdPercent: 85 - imageGCLowThresholdPercent: 80 - cpuCFSQuota: true - podsPerCore: 2 - maxPods: 20 -``` - -You can specify the container runtime to be either `dockerd` or `containerd`. By default, `containerd` is used. - -* `containerd` is the only valid container runtime when using the `Bottlerocket` AMIFamily or when using Kubernetes version 1.24+ and the `AL2`, `Windows2019`, or `Windows2022` AMIFamilies. - -### Reserved Resources - -Karpenter will automatically configure the system and kube reserved resource requests on the fly on your behalf. These requests are used to configure your node and to make scheduling decisions for your pods. If you have specific requirements or know that you will have additional capacity requirements, you can optionally override the `--system-reserved` configuration defaults with the `.spec.kubeletConfiguration.systemReserved` values and the `--kube-reserved` configuration defaults with the `.spec.kubeletConfiguration.kubeReserved` values. - -For more information on the default `--system-reserved` and `--kube-reserved` configuration refer to the [Kubelet Docs](https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/#kube-reserved) - -### Eviction Thresholds - -The kubelet supports eviction thresholds by default. When enough memory or file system pressure is exerted on the node, the kubelet will begin to evict pods to ensure that system daemons and other system processes can continue to run in a healthy manner. - -Kubelet has the notion of [hard evictions](https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#hard-eviction-thresholds) and [soft evictions](https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#soft-eviction-thresholds). In hard evictions, pods are evicted as soon as a threshold is met, with no grace period to terminate. Soft evictions, on the other hand, provide an opportunity for pods to be terminated gracefully. They do so by sending a termination signal to pods that are planning to be evicted and allowing those pods to terminate up to their grace period. - -Karpenter supports [hard evictions](https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#hard-eviction-thresholds) through the `.spec.kubeletConfiguration.evictionHard` field and [soft evictions](https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#soft-eviction-thresholds) through the `.spec.kubeletConfiguration.evictionSoft` field. `evictionHard` and `evictionSoft` are configured by listing [signal names](https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#eviction-signals) with either percentage values or resource values. - -```yaml -spec: - ... - kubeletConfiguration: - evictionHard: - memory.available: 500Mi - nodefs.available: 10% - nodefs.inodesFree: 10% - imagefs.available: 5% - imagefs.inodesFree: 5% - pid.available: 7% - evictionSoft: - memory.available: 1Gi - nodefs.available: 15% - nodefs.inodesFree: 15% - imagefs.available: 10% - imagefs.inodesFree: 10% - pid.available: 10% -``` - -#### Supported Eviction Signals - -| Eviction Signal | Description | -| --------------- | ----------- | -| memory.available | memory.available := node.status.capacity[memory] - node.stats.memory.workingSet | -| nodefs.available | nodefs.available := node.stats.fs.available | -| nodefs.inodesFree | nodefs.inodesFree := node.stats.fs.inodesFree | -| imagefs.available | imagefs.available := node.stats.runtime.imagefs.available | -| imagefs.inodesFree | imagefs.inodesFree := node.stats.runtime.imagefs.inodesFree | -| pid.available | pid.available := node.stats.rlimit.maxpid - node.stats.rlimit.curproc | - -For more information on eviction thresholds, view the [Node-pressure Eviction](https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction) section of the official Kubernetes docs. - -#### Soft Eviction Grace Periods - -Soft eviction pairs an eviction threshold with a specified grace period. With soft eviction thresholds, the kubelet will only begin evicting pods when the node exceeds its soft eviction threshold over the entire duration of its grace period. For example, if you specify `evictionSoft[memory.available]` of `500Mi` and a `evictionSoftGracePeriod[memory.available]` of `1m30`, the node must have less than `500Mi` of available memory over a minute and a half in order for the kubelet to begin evicting pods. - -Optionally, you can specify an `evictionMaxPodGracePeriod` which defines the administrator-specified maximum pod termination grace period to use during soft eviction. If a namespace-owner had specified a pod `terminationGracePeriodInSeconds` on pods in their namespace, the minimum of `evictionPodGracePeriod` and `terminationGracePeriodInSeconds` would be used. - -```yaml -spec: - ... - kubeletConfiguration: - evictionSoftGracePeriod: - memory.available: 1m - nodefs.available: 1m30s - nodefs.inodesFree: 2m - imagefs.available: 1m30s - imagefs.inodesFree: 2m - pid.available: 2m - evictionMaxPodGracePeriod: 60 -``` - -### Pod Density - -#### Max Pods - -By default, AWS will configure the maximum density of pods on a node [based on the node instance type](https://github.com/awslabs/amazon-eks-ami/blob/master/files/eni-max-pods.txt). For small instances that require an increased pod density or large instances that require a reduced pod density, you can override this default value with `.spec.kubeletConfiguration.maxPods`. This value will be used during Karpenter pod scheduling and passed through to `--max-pods` on kubelet startup. - -{{% alert title="Note" color="primary" %}} -When using small instance types, it may be necessary to enable [prefix assignment mode](https://aws.amazon.com/blogs/containers/amazon-vpc-cni-increases-pods-per-node-limits/) in the AWS VPC CNI plugin to support a higher pod density per node. Prefix assignment mode was introduced in AWS VPC CNI v1.9 and allows ENIs to manage a broader set of IP addresses. Much higher pod densities are supported as a result. -{{% /alert %}} - -#### Pods Per Core - -An alternative way to dynamically set the maximum density of pods on a node is to use the `.spec.kubeletConfiguration.podsPerCore` value. Karpenter will calculate the pod density during scheduling by multiplying this value by the number of logical cores (vCPUs) on an instance type. This value will also be passed through to the `--pods-per-core` value on kubelet startup to configure the number of allocatable pods the kubelet can assign to the node instance. - -The value generated from `podsPerCore` cannot exceed `maxPods`, meaning, if both are set, the minimum of the `podsPerCore` dynamic pod density and the static `maxPods` value will be used for scheduling. - -{{% alert title="Note" color="primary" %}} -`maxPods` may not be set in the `kubeletConfiguration` of a Provisioner, but may still be restricted by the `ENI_LIMITED_POD_DENSITY` value. You may want to ensure that the `podsPerCore` value that will be used for instance families associated with the Provisioner will not cause unexpected behavior by exceeding the `maxPods` value. -{{% /alert %}} - -{{% alert title="Pods Per Core on Bottlerocket" color="warning" %}} -Bottlerocket AMIFamily currently does not support `podsPerCore` configuration. If a Provisioner contains a `provider` or `providerRef` to a node template that will launch a Bottlerocket instance, the `podsPerCore` value will be ignored for scheduling and for configuring the kubelet. -{{% /alert %}} - -## spec.limits.resources - -The provisioner spec includes a limits section (`spec.limits.resources`), which constrains the maximum amount of resources that the provisioner will manage. - -Karpenter supports limits of any resource type reported by your cloudprovider. It limits instance types when scheduling to those that will not exceed the specified limits. If a limit has been exceeded, nodes provisioning is prevented until some nodes have been terminated. - -```yaml -apiVersion: karpenter.sh/v1alpha5 -kind: Provisioner -metadata: - name: default -spec: - requirements: - - key: karpenter.sh/capacity-type - operator: In - values: ["spot"] - limits: - resources: - cpu: 1000 - memory: 1000Gi - nvidia.com/gpu: 2 -``` - -{{% alert title="Note" color="primary" %}} -Karpenter provisioning is highly parallel. Because of this, limit checking is eventually consistent, which can result in overrun during rapid scale outs. -{{% /alert %}} - -CPU limits are described with a `DecimalSI` value. Note that the Kubernetes API will coerce this into a string, so we recommend against using integers to avoid GitOps skew. - -Memory limits are described with a [`BinarySI` value, such as 1000Gi.](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#meaning-of-memory) - -You can view the current consumption of cpu and memory on your cluster by running: -``` -kubectl get provisioner -o=jsonpath='{.items[0].status}' -``` - -Review the [Kubernetes core API](https://github.com/kubernetes/api/blob/37748cca582229600a3599b40e9a82a951d8bbbf/core/v1/resource.go#L23) (`k8s.io/api/core/v1`) for more information on `resources`. - -## spec.providerRef - -This field points to the cloud provider-specific custom resource. Learn more about [AWSNodeTemplates](../node-templates/). - -## spec.consolidation - -You can configure Karpenter to deprovision instances through your Provisioner in multiple ways. You can use `spec.ttlSecondsAfterEmpty`, `spec.ttlSecondsUntilExpired` or `spec.consolidation.enabled`. Read [Deprovisioning](../deprovisioning/) for more. - -## Example Use-Cases - -### Isolating Expensive Hardware - -A provisioner can be set up to only provision nodes on particular processor types. -The following example sets a taint that only allows pods with tolerations for Nvidia GPUs to be scheduled: - -```yaml -apiVersion: karpenter.sh/v1alpha5 -kind: Provisioner -metadata: - name: gpu -spec: - consolidation: - enabled: true - requirements: - - key: node.kubernetes.io/instance-type - operator: In - values: ["p3.8xlarge", "p3.16xlarge"] - taints: - - key: nvidia.com/gpu - value: "true" - effect: NoSchedule -``` -In order for a pod to run on a node defined in this provisioner, it must tolerate `nvidia.com/gpu` in its pod spec. - -### Cilium Startup Taint - -Per the Cilium [docs](https://docs.cilium.io/en/stable/installation/taints/#taint-effects), it's recommended to place a taint of `node.cilium.io/agent-not-ready=true:NoExecute` on nodes to allow Cilium to configure networking prior to other pods starting. This can be accomplished via the use of Karpenter `startupTaints`. These taints are placed on the node, but pods aren't required to tolerate these taints to be considered for provisioning. - -```yaml -apiVersion: karpenter.sh/v1alpha5 -kind: Provisioner -metadata: - name: cilium-startup -spec: - consolidation: - enabled: true - startupTaints: - - key: node.cilium.io/agent-not-ready - value: "true" - effect: NoExecute -``` diff --git a/website/content/en/v0.31/concepts/settings.md b/website/content/en/v0.31/concepts/settings.md deleted file mode 100644 index cfe4416f652d..000000000000 --- a/website/content/en/v0.31/concepts/settings.md +++ /dev/null @@ -1,124 +0,0 @@ ---- -title: "Settings" -linkTitle: "Settings" -weight: 5 -description: > - Configure Karpenter ---- - -There are two main configuration mechanisms that can be used to configure Karpenter: Environment Variables / CLI parameters to the controller and webhook binaries and the `karpenter-global-settings` config-map. - -## Environment Variables / CLI Flags - -[comment]: <> (the content below is generated from hack/docs/configuration_gen_docs.go) - -| Environment Variable | CLI Flag | Description | -|--|--|--| -| DISABLE_WEBHOOK | \-\-disable-webhook | Disable the admission and validation webhooks (default = false)| -| ENABLE_PROFILING | \-\-enable-profiling | Enable the profiling on the metric endpoint (default = false)| -| HEALTH_PROBE_PORT | \-\-health-probe-port | The port the health probe endpoint binds to for reporting controller health (default = 8081)| -| KARPENTER_SERVICE | \-\-karpenter-service | The Karpenter Service name for the dynamic webhook certificate| -| KUBE_CLIENT_BURST | \-\-kube-client-burst | The maximum allowed burst of queries to the kube-apiserver (default = 300)| -| KUBE_CLIENT_QPS | \-\-kube-client-qps | The smoothed rate of qps to kube-apiserver (default = 200)| -| LEADER_ELECT | \-\-leader-elect | Start leader election client and gain leadership before executing the main loop. Enable this when running replicated components for high availability. (default = true)| -| MEMORY_LIMIT | \-\-memory-limit | Memory limit on the container running the controller. The GC soft memory limit is set to 90% of this value. (default = -1)| -| METRICS_PORT | \-\-metrics-port | The port the metric endpoint binds to for operating metrics about the controller itself (default = 8000)| -| WEBHOOK_PORT | \-\-webhook-port | The port the webhook endpoint binds to for validation and mutation of resources (default = 8443)| - -[comment]: <> (end docs generated content from hack/docs/configuration_gen_docs.go) - -## ConfigMap - -Karpenter installs a default configuration via its Helm chart that should work for most. Additional configuration can be performed by editing the `karpenter-global-settings` configmap within the namespace that Karpenter was installed in. - -```yaml -apiVersion: v1 -kind: ConfigMap -metadata: - name: karpenter-global-settings - namespace: karpenter -data: - # The maximum length of a batch window. The longer this is, the more pods we can consider for provisioning at one - # time which usually results in fewer but larger nodes. - batchMaxDuration: 10s - # The maximum amount of time with no new pending pods that if exceeded ends the current batching window. If pods arrive - # faster than this time, the batching window will be extended up to the maxDuration. If they arrive slower, the pods - # will be batched separately. - batchIdleDuration: 1s - # Role to assume for calling AWS services. - aws.assumeRoleARN: arn:aws:iam::111222333444:role/examplerole - # Duration of assumed credentials in minutes. Default value is 15 minutes. Not used unless aws.assumeRole set. - aws.assumeRoleDuration: 15m - # Cluster CA bundle for nodes to use for TLS connections with the API server. If not set, this is taken from the controller's TLS configuration. - aws.clusterCABundle: "LS0tLS1..." - # [REQUIRED] The kubernetes cluster name for resource discovery - aws.clusterName: karpenter-cluster - # The external kubernetes cluster endpoint for new nodes to connect with. If not specified, will discover the cluster endpoint using DescribeCluster API - aws.clusterEndpoint: https://00000000000000000000000000000000.gr7.us-west-2.eks.amazonaws.com - # The default instance profile to use when provisioning nodes - aws.defaultInstanceProfile: karpenter-instance-profile - # If true, then instances that support pod ENI will report a vpc.amazonaws.com/pod-eni resource - aws.enablePodENI: "false" - # Indicates whether new nodes should use ENI-based pod density. DEPRECATED: Use `.spec.kubeletConfiguration.maxPods` to set pod density on a per-provisioner basis - aws.enableENILimitedPodDensity: "true" - # If true, then assume we can't reach AWS services which don't have a VPC endpoint - # This also has the effect of disabling look-ups to the AWS pricing endpoint - aws.isolatedVPC: "false" - # The VM memory overhead as a percent that will be subtracted - # from the total memory for all instance types - aws.vmMemoryOverheadPercent: "0.075" - # aws.interruptionQueueName is disabled if not specified. Enabling interruption handling may - # require additional permissions on the controller service account. Additional permissions are outlined in the docs - aws.interruptionQueueName: karpenter-cluster - # Global tags are specified by including a JSON object of string to string from tag key to tag value - aws.tags: '{"custom-tag1-key": "custom-tag-value", "custom-tag2-key": "custom-tag-value"}' - # Reserved ENIs are not included in the calculations for max-pods or kube-reserved - # This is most often used in the VPC CNI custom networking setup https://docs.aws.amazon.com/eks/latest/userguide/cni-custom-network.html - aws.reservedENIs: "1" -``` - -### Feature Gates -Karpenter uses [feature gates](https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/#feature-gates-for-alpha-or-beta-features). You can add a feature gate's ConfigKey to the `karpenter-global-settings` ConfigMap above with the desired value. - -| Feature | Default | Config Key | Stage | Since | Until | -|---------|---------|---------------------------|-------|---------|-------| -| Drift | false | featureGates.driftEnabled | Alpha | v0.21.0 | | - - -### Batching Parameters - -The batching parameters control how Karpenter batches an incoming stream of pending pods. Reducing these values may trade off a slightly faster time from pending pod to node launch, in exchange for launching smaller nodes. Increasing the values can do the inverse. Karpenter provides reasonable defaults for these values, but if you have specific knowledge about your workloads you can tweak these parameters to match the expected rate of incoming pods. - -For a standard deployment scale-up, the pods arrive at the QPS setting of the `kube-controller-manager`, and the default values are typically fine. These settings are intended for use cases where other systems may create large numbers of pods over a period of many seconds or minutes and there is a desire to batch them together. - -#### `batchIdleDuration` - -The `batchIdleDuration` is the period of time that a new pending pod extends the current batching window. This can be increased to handle scenarios where pods arrive slower than one second part, but it would be preferable if they were batched together onto a single larger node. - -This value is expressed as a string value like `10s`, `1m` or `2h45m`. The valid time units are `ns`, `us` (or `µs`), `ms`, `s`, `m`, `h`. - -#### `batchMaxDuration` - -The `batchMaxDuration` is the maximum period of time a batching window can be extended to. Increasing this value will allow the maximum batch window size to increase to collect more pending pods into a single batch at the expense of a longer delay from when the first pending pod was created. - -This value is expressed as a string value like `10s`, `1m` or `2h45m`. The valid time units are `ns`, `us` (or `µs`), `ms`, `s`, `m`, `h`. - -### AWS Parameters - -#### `aws.tags` - -Global tags are applied to __all__ AWS infrastructure resources deployed by Karpenter. These resources include: - -- Launch Templates -- Volumes -- Instances - -Tags are specified by including a JSON object of string to string from tag key to tag value. - -```yaml - aws.tags: '{"custom-tag1-key": "custom-tag-value", "custom-tag2-key": "custom-tag-value"}' -``` - -{{% alert title="Note" color="primary" %}} -Since you can specify tags at the global level and in the `AWSNodeTemplate` resource, if a key is specified in both locations, the `AWSNodeTemplate` tag value will override the global tag. -{{% /alert %}} diff --git a/website/content/en/v0.31/concepts/threat-model.md b/website/content/en/v0.31/concepts/threat-model.md deleted file mode 100644 index 3ae71fb51861..000000000000 --- a/website/content/en/v0.31/concepts/threat-model.md +++ /dev/null @@ -1,102 +0,0 @@ ---- -title: "Threat Model" -linkTitle: "Threat Model" -weight: 999 ---- - -Karpenter observes Kubernetes pods and launches nodes in response to those pods’ scheduling constraints. Karpenter does not perform the actual scheduling and instead waits for [kube-scheduler](https://kubernetes.io/docs/concepts/scheduling-eviction/kube-scheduler/) to schedule the pods. - -When running in AWS, Karpenter is typically installed onto EC2 instances that run in EKS Clusters. Karpenter relies on public facing AWS APIs and standard IAM Permissions. Karpenter uses AWS-SDK-Go v1, and AWS advises that credentials are provided using [IAM Roles for Service Accounts](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html). - - -## Architecture & Actors - -1. **Cluster Operator**: An identity that installs and configures Karpenter in a Kubernetes cluster, and configures Karpenter's cloud identity and permissions. -2. **Cluster Developer**: An identity that can create pods, typically through Deployments, DaemonSets, or other pod-controller types. -3. **Karpenter Controller:** The Karpenter application pod that operates inside a cluster. - -![threat-model](/threat-model.png) - -## Capabilities - -### Cluster Operator - -The Cluster Operator has full control over Kubernetes resources to install and configure Karpenter, its CRDs, and Provisioners and NodeTemplates. The Cluster Operator has privileges to manage the cloud identities and permissions for Nodes, and the cloud identity and permissions for Karpenter. - -### Cluster Developer - -A Cluster Developer has the ability to create pods via Deployments, ReplicaSets, StatefulSets, Jobs, etc. This assumes that the Cluster Developer cannot modify the Karpenter pod or launch pods using Karpenter’s service account and gain access to Karpenter’s IAM role. - -### Karpenter Controller - -Karpenter has permissions to create and manage cloud instances. Karpenter has Kubernetes API permissions to create, update, and remove nodes, as well as evict pods. For a full list of the permissions, see the RBAC rules in the helm chart template. Karpenter also has AWS IAM permissions to create instances with IAM roles. - -* [aggregate-clusterrole.yaml](https://github.com/aws/karpenter/blob/v0.31.0/charts/karpenter/templates/aggregate-clusterrole.yaml) -* [clusterrole-core.yaml](https://github.com/aws/karpenter/blob/v0.31.0/charts/karpenter/templates/clusterrole-core.yaml) -* [clusterrole.yaml](https://github.com/aws/karpenter/blob/v0.31.0/charts/karpenter/templates/clusterrole.yaml) -* [rolebinding.yaml](https://github.com/aws/karpenter/blob/v0.31.0/charts/karpenter/templates/rolebinding.yaml) -* [role.yaml](https://github.com/aws/karpenter/blob/v0.31.0/charts/karpenter/templates/role.yaml) - -## Assumptions - -| Category | Assumption | Comment | -| --- | --- | --- | -| Generic | The Karpenter pod is operated on a node in the cluster, and uses a Service Account for authentication to the Kubernetes API | Cluster Operators may want to isolate the node running the Karpenter pod to a system-pool of nodes to mitigate the possibility of container breakout with Karpenter’s permissions. | -| Generic | Cluster Developer does not have any Kubernetes permissions to manage Karpenter running in the cluster (The deployment, pods, clusterrole, etc) | | -| Generic | Restrictions on the fields of pods a Cluster Developer can create are out of scope. | Cluster Operators can use policy frameworks to enforce restrictions on Pod capabilities | -| Generic | No sensitive data is included in non-Secret resources in the Kubernetes API. The Karpenter controller has the ability to list all pods, nodes, deployments, and many other pod-controller and storage resource types. | Karpenter does not have permission to list/watch cluster-wide ConfigMaps or Secrets | -| Generic | Karpenter has permissions to create, modify, and delete nodes from the cluster, and evict any pod. | Cluster Operators running applications with varying security profiles in the same cluster may want to configure dedicated nodes and scheduling rules for Karpenter to mitigate potential container escapes from other containers | -| AWS-Specific | The Karpenter IAM policy is encoded in the GitHub repo. Any additional permissions possibly granted to that role by the administrator are out of scope | | -| AWS-Specific | The Karpenter pod uses IRSA for AWS credentials | Setup of IRSA is out of scope for this document | - -## Generic Threats and Mitigations - -### Threat: Cluster Developer can influence creation of an arbitrary number of nodes - -**Background**: Karpenter creates new instances based on the count of pending pods. - -**Threat**: A Cluster Developer attempts to have Karpenter create more instances than intended by creating a large number of pods or by using anti-affinity to schedule one pod per node. - -**Mitigation**: In addition to [Kubernetes resource limits](https://kubernetes.io/docs/concepts/policy/resource-quotas/#object-count-quota), Cluster Operators can [configure limits on a Provisioner](https://karpenter.sh/preview/concepts/provisioners/#speclimitsresources) to limit the total amount of memory, CPU, or other resources provisioned across all nodes. - -## AWS-Specific Threats - -### Threat: Using EC2 CreateTag/DeleteTag Permissions to Orchestrate Machine Creation/Deletion - -**Background**: As of v0.28.0, Karpenter creates a mapping between CloudProvider machines and CustomResources in the cluster for capacity tracking. To ensure this mapping is consistent, Karpenter utilizes the following tag keys: - -* `karpenter.sh/managed-by` -* `karpenter.sh/provisioner-name` -* `kubernetes.io/cluster/${CLUSTER_NAME}` - -Any user that has the ability to Create/Delete tags on CloudProvider machines will have the ability to orchestrate Karpenter to Create/Delete CloudProvider machines as a side effect. - -In addition, as of v0.29.0, Karpenter will Drift on Security Groups and Subnets. If a user has the Create/Delete tags permission for either of resources, they can orchestrate Karpenter to Create/Delete CloudProvider machines as a side effect. - -**Threat:** A Cluster Operator attempts to create or delete a tag on a resource discovered by Karpenter. If it has the ability to create a tag it can effectively create or delete CloudProvider machines associated with the tagged resources. - -**Mitigation** Cluster Operators should [enforce tag-based IAM policies](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_tags.html) on these tags against any EC2 instance resource (`i-*`) for any users that might have [CreateTags](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateTags.html)/[DeleteTags](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DeleteTags.html) permissions but should not have [RunInstances](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_RunInstances.html)/[TerminateInstances](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_TerminateInstances.html) permissions. - -### Threat: Launching EC2 instances with IAM roles not intended for Karpenter nodes - -**Background**: Many IAM roles in an AWS account may trust the EC2 service principal. IAM administrators must grant the `iam:PassRole` permission to IAM principals to allow those principals in the account to launch instances with specific roles. - -**Threat:** A Cluster Operator attempts to create a Node Template with an IAM role not intended for Karpenter - -**Mitigation**: Cluster Operators must enumerate the roles in the resource section of the IAM policy granted to the Karpenter role for the `iam:PassRole` action. - -### Threat: Karpenter can be used to create or terminate EC2 instances outside of the cluster - -**Background**: EC2 instances can exist in an AWS account outside of the Kubernetes cluster. - -**Threat**: An actor who obtains control of the Karpenter pod’s IAM role may be able to create or terminate EC2 instances not part of the Kubernetes cluster managed by Karpenter. - -**Mitigation**: Karpenter creates instances with tags, several of which can be enforced in the IAM policy granted to the Karpenter IAM role that restrict the instances Karpenter can terminate. One tag can require that the instance was provisioned by a Karpenter controller, another tag can include a cluster name to mitigate any termination between two clusters with Karpenter in the same account. Cluster Operators also can restrict the region to prevent two clusters in the same account with the same name in different regions. - -### Threat: Karpenter launches an EC2 instance using an unintended AMI - -**Background**: Cluster Developers can create Node Templates that refer to an AMI by metadata, such as a name rather than an AMI resource ID. - -**Threat:** A threat actor creates a public AMI with the same name as a customer’s AMI in an attempt to get Karpenter to select the threat actor’s AMI instead of the intended AMI. - -**Mitigation**: When selecting AMIs by name or tags, Karpenter defaults to adding an ownership filter of `self,amazon` so AMI images external to the account are not used. diff --git a/website/content/en/v0.31/getting-started/getting-started-with-karpenter/_index.md b/website/content/en/v0.31/getting-started/getting-started-with-karpenter/_index.md deleted file mode 100644 index 963381e369a8..000000000000 --- a/website/content/en/v0.31/getting-started/getting-started-with-karpenter/_index.md +++ /dev/null @@ -1,171 +0,0 @@ - ---- -title: "Getting Started with Karpenter" -linkTitle: "Getting Started with Karpenter" -weight: 10 -description: > - Set up a cluster and add Karpenter ---- - -Karpenter automatically provisions new nodes in response to unschedulable pods. Karpenter does this by observing events within the Kubernetes cluster, and then sending commands to the underlying cloud provider. - -This guide shows how to get started with Karpenter by creating a Kubernetes cluster and installing Karpenter. -To use Karpenter, you must be running a supported Kubernetes cluster on a supported cloud provider. -Currently, only EKS on AWS is supported. - -## Create a cluster and add Karpenter - -This guide uses `eksctl` to create the cluster. -It should take less than 1 hour to complete, and cost less than $0.25. -Follow the clean-up instructions to reduce any charges. - -### 1. Install utilities - -Karpenter is installed in clusters with a Helm chart. - -Karpenter requires cloud provider permissions to provision nodes, for AWS IAM -Roles for Service Accounts (IRSA) should be used. IRSA permits Karpenter -(within the cluster) to make privileged requests to AWS (as the cloud provider) -via a ServiceAccount. - -Install these tools before proceeding: - -1. [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2-linux.html) -2. `kubectl` - [the Kubernetes CLI](https://kubernetes.io/docs/tasks/tools/install-kubectl-linux/) -3. `eksctl` - [the CLI for AWS EKS](https://docs.aws.amazon.com/eks/latest/userguide/eksctl.html) -4. `helm` - [the package manager for Kubernetes](https://helm.sh/docs/intro/install/) - -[Configure the AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-quickstart.html) -with a user that has sufficient privileges to create an EKS cluster. Verify that the CLI can -authenticate properly by running `aws sts get-caller-identity`. - -### 2. Set environment variables - -After setting up the tools, set the Karpenter version number: - -```bash -export KARPENTER_VERSION=v0.31.3 -``` - -Then set the following environment variable: - -{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step01-config.sh" language="bash"%}} - -{{% alert title="Warning" color="warning" %}} -If you open a new shell to run steps in this procedure, you need to set some or all of the environment variables again. -To remind yourself of these values, type: - -```bash -echo $KARPENTER_VERSION $CLUSTER_NAME $AWS_DEFAULT_REGION $AWS_ACCOUNT_ID $TEMPOUT -``` - -{{% /alert %}} - - -### 3. Create a Cluster - -Create a basic cluster with `eksctl`. -The following cluster configuration will: - -* Use CloudFormation to set up the infrastructure needed by the EKS cluster. -* Create a Kubernetes service account and AWS IAM Role, and associate them using IRSA to let Karpenter launch instances. -* Add the Karpenter node role to the aws-auth configmap to allow nodes to connect. -* Use [AWS EKS managed node groups](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html) for the kube-system and karpenter namespaces. Uncomment fargateProfiles settings (and comment out managedNodeGroups settings) to use Fargate for both namespaces instead. -* Set KARPENTER_IAM_ROLE_ARN variables. -* Create a role to allow spot instances. -* Run helm to install karpenter - -{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step02-create-cluster.sh" language="bash"%}} - -{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step06-add-spot-role.sh" language="bash"%}} - -{{% alert title="Windows Support Notice" color="warning" %}} -In order to run Windows workloads, Windows support should be enabled in your EKS Cluster. -See [Enabling Windows support](https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html#enable-windows-support) to learn more. -{{% /alert %}} - -### 4. Install Karpenter - -{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step08-apply-helm-chart.sh" language="bash"%}} - -{{% alert title="Warning" color="warning" %}} -Karpenter creates a mapping between CloudProvider machines and CustomResources in the cluster for capacity tracking. To ensure this mapping is consistent, Karpenter utilizes the following tag keys: - -* `karpenter.sh/managed-by` -* `karpenter.sh/provisioner-name` -* `kubernetes.io/cluster/${CLUSTER_NAME}` - -Because Karpenter takes this dependency, any user that has the ability to Create/Delete these tags on CloudProvider machines will have the ability to orchestrate Karpenter to Create/Delete CloudProvider machines as a side effect. We recommend that you [enforce tag-based IAM policies](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_tags.html) on these tags against any EC2 instance resource (`i-*`) for any users that might have [CreateTags](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateTags.html)/[DeleteTags](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DeleteTags.html) permissions but should not have [RunInstances](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_RunInstances.html)/[TerminateInstances](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_TerminateInstances.html) permissions. -{{% /alert %}} - -### 5. Create Provisioner - -A single Karpenter provisioner is capable of handling many different pod -shapes. Karpenter makes scheduling and provisioning decisions based on pod -attributes such as labels and affinity. In other words, Karpenter eliminates -the need to manage many different node groups. - -Create a default provisioner using the command below. -This provisioner uses `securityGroupSelector` and `subnetSelector` to discover resources used to launch nodes. -We applied the tag `karpenter.sh/discovery` in the `eksctl` command above. -Depending how these resources are shared between clusters, you may need to use different tagging schemes. - -The `consolidation` value configures Karpenter to reduce cluster cost by removing and replacing nodes. As a result, consolidation will terminate any empty nodes on the cluster. This behavior can be disabled by leaving the value undefined or setting `consolidation.enabled` to `false`. Review the [provisioner CRD]({{}}) for more information. - -Review the [provisioner CRD]({{}}) for more information. For example, -`ttlSecondsUntilExpired` configures Karpenter to terminate nodes when a maximum age is reached. - -Note: This provisioner will create capacity as long as the sum of all created capacity is less than the specified limit. - -{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step12-add-provisioner.sh" language="bash"%}} - -Karpenter is now active and ready to begin provisioning nodes. - -## First Use - -Create some pods using a deployment and watch Karpenter provision nodes in response. - -### Scale up deployment - -This deployment uses the [pause image](https://www.ianlewis.org/en/almighty-pause-container) and starts with zero replicas. - -{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step13-automatic-node-provisioning.sh" language="bash"%}} - -### Scale down deployment - -Now, delete the deployment. After a short amount of time, Karpenter should terminate the empty nodes due to consolidation. - -{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step14-deprovisioning.sh" language="bash"%}} - -## Add optional monitoring with Grafana - -This section describes optional ways to configure Karpenter to enhance its capabilities. -In particular, the following commands deploy a Prometheus and Grafana stack that is suitable for this guide but does not include persistent storage or other configurations that would be necessary for monitoring a production deployment of Karpenter. -This deployment includes two Karpenter dashboards that are automatically onboarded to Grafana. They provide a variety of visualization examples on Karpenter metrics. - -{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step09-add-prometheus-grafana.sh" language="bash"%}} - -The Grafana instance may be accessed using port forwarding. - -{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step10-add-grafana-port-forward.sh" language="bash"%}} - -The new stack has only one user, `admin`, and the password is stored in a secret. The following command will retrieve the password. - -{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step11-grafana-get-password.sh" language="bash"%}} - -## Cleanup - -### Delete Karpenter nodes manually - -If you delete a node with kubectl, Karpenter will gracefully cordon, drain, -and shutdown the corresponding instance. Under the hood, Karpenter adds a -finalizer to the node object, which blocks deletion until all pods are -drained and the instance is terminated. Keep in mind, this only works for -nodes provisioned by Karpenter. - -{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step15-delete-node.sh" language="bash"%}} - -### Delete the cluster -To avoid additional charges, remove the demo infrastructure from your AWS account. - -{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step16-cleanup.sh" language="bash"%}} diff --git a/website/content/en/v0.31/getting-started/getting-started-with-karpenter/scripts/add-monitoring.sh b/website/content/en/v0.31/getting-started/getting-started-with-karpenter/scripts/add-monitoring.sh deleted file mode 100755 index 6d2332b06be2..000000000000 --- a/website/content/en/v0.31/getting-started/getting-started-with-karpenter/scripts/add-monitoring.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -set -euo pipefail #fail if one step fails - -if [ "$#" -ne 1 ] -then - echo "Missing required Karpenter version. Usage: add-monitoring.sh v0.0.1" - exit 1 -fi - -export KARPENTER_VERSION=$1 - -declare -a steps=( - step01-config.sh - step09-add-prometheus-grafana.sh - step10-add-grafana-port-forward.sh - step11-grafana-get-password.sh -) - -for step in "${steps[@]}"; do - echo "$step" - source $step -done diff --git a/website/content/en/v0.31/getting-started/getting-started-with-karpenter/scripts/add-provisioner.sh b/website/content/en/v0.31/getting-started/getting-started-with-karpenter/scripts/add-provisioner.sh deleted file mode 100755 index 51f286de0c3c..000000000000 --- a/website/content/en/v0.31/getting-started/getting-started-with-karpenter/scripts/add-provisioner.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -set -euo pipefail #fail if one step fails - -declare -a steps=( - step01-config.sh - step12-add-provisioner.sh - step13-automatic-node-provisioning.sh - step14-automatic-node-termination.sh -) - -for step in "${steps[@]}"; do - echo "$step" - source $step -done diff --git a/website/content/en/v0.31/getting-started/getting-started-with-karpenter/scripts/add-roles.sh b/website/content/en/v0.31/getting-started/getting-started-with-karpenter/scripts/add-roles.sh deleted file mode 100755 index cf0656f3db3d..000000000000 --- a/website/content/en/v0.31/getting-started/getting-started-with-karpenter/scripts/add-roles.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -set -euo pipefail #fail if one step fails - -if [ "$#" -ne 1 ] -then - echo "Missing required Karpenter version. Usage: setup-roles.sh v0.0.1" - exit 1 -fi - -export KARPENTER_VERSION=$1 -SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) - -declare -a steps=( - step03-iam-cloud-formation.sh - step04-grant-access.sh - step05-controller-iam.sh - step06-add-spot-role.sh -) - -for step in "${steps[@]}"; do - echo "$step" - source "$SCRIPT_DIR/$step" -done diff --git a/website/content/en/v0.31/getting-started/getting-started-with-karpenter/scripts/cleanup.sh b/website/content/en/v0.31/getting-started/getting-started-with-karpenter/scripts/cleanup.sh deleted file mode 100755 index 52851e7f975a..000000000000 --- a/website/content/en/v0.31/getting-started/getting-started-with-karpenter/scripts/cleanup.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -declare -a steps=( - step01-config.sh - step16-cleanup.sh -) - -for step in "${steps[@]}"; do - echo "$step" - source $step -done diff --git a/website/content/en/v0.31/getting-started/getting-started-with-karpenter/scripts/install.sh b/website/content/en/v0.31/getting-started/getting-started-with-karpenter/scripts/install.sh deleted file mode 100755 index e0ebfb32e511..000000000000 --- a/website/content/en/v0.31/getting-started/getting-started-with-karpenter/scripts/install.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -set -euo pipefail #fail if one step fails - -if [ "$#" -ne 1 ] -then - echo "Missing required Karpenter version. Usage: install.sh v0.0.1" - exit 1 -fi - -export KARPENTER_VERSION=$1 - -declare -a steps=( - step01-config.sh - step02-create-cluster.sh - step03-iam-cloud-formation.sh - step04-grant-access.sh - step05-controller-iam.sh - step06-add-spot-role.sh - step08-apply-helm-chart.sh -) - -for step in "${steps[@]}"; do - echo "$step" - source $step -done diff --git a/website/content/en/v0.31/getting-started/getting-started-with-karpenter/scripts/step01-config.sh b/website/content/en/v0.31/getting-started/getting-started-with-karpenter/scripts/step01-config.sh deleted file mode 100755 index a3af512d02ac..000000000000 --- a/website/content/en/v0.31/getting-started/getting-started-with-karpenter/scripts/step01-config.sh +++ /dev/null @@ -1,5 +0,0 @@ -export AWS_PARTITION="aws" # if you are not using standard partitions, you may need to configure to aws-cn / aws-us-gov -export CLUSTER_NAME="${USER}-karpenter-demo" -export AWS_DEFAULT_REGION="us-west-2" -export AWS_ACCOUNT_ID="$(aws sts get-caller-identity --query Account --output text)" -export TEMPOUT=$(mktemp) diff --git a/website/content/en/v0.31/getting-started/getting-started-with-karpenter/scripts/step12-add-provisioner.sh b/website/content/en/v0.31/getting-started/getting-started-with-karpenter/scripts/step12-add-provisioner.sh deleted file mode 100755 index 2d487c0449dc..000000000000 --- a/website/content/en/v0.31/getting-started/getting-started-with-karpenter/scripts/step12-add-provisioner.sh +++ /dev/null @@ -1,28 +0,0 @@ -cat < controller-trust-policy.json -{ - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Principal": { - "Federated": "arn:${AWS_PARTITION}:iam::${AWS_ACCOUNT_ID}:oidc-provider/${OIDC_ENDPOINT#*//}" - }, - "Action": "sts:AssumeRoleWithWebIdentity", - "Condition": { - "StringEquals": { - "${OIDC_ENDPOINT#*//}:aud": "sts.amazonaws.com", - "${OIDC_ENDPOINT#*//}:sub": "system:serviceaccount:karpenter:karpenter" - } - } - } - ] -} -EOF - -aws iam create-role --role-name KarpenterControllerRole-${CLUSTER_NAME} \ - --assume-role-policy-document file://controller-trust-policy.json - -cat << EOF > controller-policy.json -{ - "Statement": [ - { - "Action": [ - "ssm:GetParameter", - "ec2:DescribeImages", - "ec2:RunInstances", - "ec2:DescribeSubnets", - "ec2:DescribeSecurityGroups", - "ec2:DescribeLaunchTemplates", - "ec2:DescribeInstances", - "ec2:DescribeInstanceTypes", - "ec2:DescribeInstanceTypeOfferings", - "ec2:DescribeAvailabilityZones", - "ec2:DeleteLaunchTemplate", - "ec2:CreateTags", - "ec2:CreateLaunchTemplate", - "ec2:CreateFleet", - "ec2:DescribeSpotPriceHistory", - "pricing:GetProducts" - ], - "Effect": "Allow", - "Resource": "*", - "Sid": "Karpenter" - }, - { - "Action": "ec2:TerminateInstances", - "Condition": { - "StringLike": { - "ec2:ResourceTag/karpenter.sh/provisioner-name": "*" - } - }, - "Effect": "Allow", - "Resource": "*", - "Sid": "ConditionalEC2Termination" - }, - { - "Effect": "Allow", - "Action": "iam:PassRole", - "Resource": "arn:${AWS_PARTITION}:iam::${AWS_ACCOUNT_ID}:role/KarpenterNodeRole-${CLUSTER_NAME}", - "Sid": "PassNodeIAMRole" - }, - { - "Effect": "Allow", - "Action": "eks:DescribeCluster", - "Resource": "arn:${AWS_PARTITION}:eks:${AWS_REGION}:${AWS_ACCOUNT_ID}:cluster/${CLUSTER_NAME}", - "Sid": "EKSClusterEndpointLookup" - } - ], - "Version": "2012-10-17" -} -EOF - -aws iam put-role-policy --role-name KarpenterControllerRole-${CLUSTER_NAME} \ - --policy-name KarpenterControllerPolicy-${CLUSTER_NAME} \ - --policy-document file://controller-policy.json diff --git a/website/content/en/v0.31/getting-started/migrating-from-cas/scripts/step06-tag-subnets.sh b/website/content/en/v0.31/getting-started/migrating-from-cas/scripts/step06-tag-subnets.sh deleted file mode 100644 index de972ea2bddd..000000000000 --- a/website/content/en/v0.31/getting-started/migrating-from-cas/scripts/step06-tag-subnets.sh +++ /dev/null @@ -1,6 +0,0 @@ -for NODEGROUP in $(aws eks list-nodegroups --cluster-name ${CLUSTER_NAME} \ - --query 'nodegroups' --output text); do aws ec2 create-tags \ - --tags "Key=karpenter.sh/discovery,Value=${CLUSTER_NAME}" \ - --resources $(aws eks describe-nodegroup --cluster-name ${CLUSTER_NAME} \ - --nodegroup-name $NODEGROUP --query 'nodegroup.subnets' --output text ) -done diff --git a/website/content/en/v0.31/getting-started/migrating-from-cas/scripts/step07-tag-security-groups.sh b/website/content/en/v0.31/getting-started/migrating-from-cas/scripts/step07-tag-security-groups.sh deleted file mode 100644 index 397e40904cee..000000000000 --- a/website/content/en/v0.31/getting-started/migrating-from-cas/scripts/step07-tag-security-groups.sh +++ /dev/null @@ -1,22 +0,0 @@ -NODEGROUP=$(aws eks list-nodegroups --cluster-name ${CLUSTER_NAME} \ - --query 'nodegroups[0]' --output text) - -LAUNCH_TEMPLATE=$(aws eks describe-nodegroup --cluster-name ${CLUSTER_NAME} \ - --nodegroup-name ${NODEGROUP} --query 'nodegroup.launchTemplate.{id:id,version:version}' \ - --output text | tr -s "\t" ",") - -# If your EKS setup is configured to use only Cluster security group, then please execute - - -SECURITY_GROUPS=$(aws eks describe-cluster \ - --name ${CLUSTER_NAME} --query "cluster.resourcesVpcConfig.clusterSecurityGroupId" --output text) - -# If your setup uses the security groups in the Launch template of a managed node group, then : - -SECURITY_GROUPS=$(aws ec2 describe-launch-template-versions \ - --launch-template-id ${LAUNCH_TEMPLATE%,*} --versions ${LAUNCH_TEMPLATE#*,} \ - --query 'LaunchTemplateVersions[0].LaunchTemplateData.[NetworkInterfaces[0].Groups||SecurityGroupIds]' \ - --output text) - -aws ec2 create-tags \ - --tags "Key=karpenter.sh/discovery,Value=${CLUSTER_NAME}" \ - --resources ${SECURITY_GROUPS} diff --git a/website/content/en/v0.31/getting-started/migrating-from-cas/scripts/step09-generate-chart.sh b/website/content/en/v0.31/getting-started/migrating-from-cas/scripts/step09-generate-chart.sh deleted file mode 100644 index f2bc603e0eeb..000000000000 --- a/website/content/en/v0.31/getting-started/migrating-from-cas/scripts/step09-generate-chart.sh +++ /dev/null @@ -1,8 +0,0 @@ -helm template karpenter oci://public.ecr.aws/karpenter/karpenter --version ${KARPENTER_VERSION} --namespace karpenter \ - --set settings.aws.defaultInstanceProfile=KarpenterNodeInstanceProfile-${CLUSTER_NAME} \ - --set settings.aws.clusterName=${CLUSTER_NAME} \ - --set serviceAccount.annotations."eks\.amazonaws\.com/role-arn"="arn:${AWS_PARTITION}:iam::${AWS_ACCOUNT_ID}:role/KarpenterControllerRole-${CLUSTER_NAME}" \ - --set controller.resources.requests.cpu=1 \ - --set controller.resources.requests.memory=1Gi \ - --set controller.resources.limits.cpu=1 \ - --set controller.resources.limits.memory=1Gi > karpenter.yaml diff --git a/website/content/en/v0.31/getting-started/migrating-from-cas/scripts/step10-deploy.sh b/website/content/en/v0.31/getting-started/migrating-from-cas/scripts/step10-deploy.sh deleted file mode 100644 index d66941f52ed0..000000000000 --- a/website/content/en/v0.31/getting-started/migrating-from-cas/scripts/step10-deploy.sh +++ /dev/null @@ -1,8 +0,0 @@ -kubectl create namespace karpenter -kubectl create -f \ - https://raw.githubusercontent.com/aws/karpenter-provider-aws/${KARPENTER_VERSION}/pkg/apis/crds/karpenter.sh_provisioners.yaml -kubectl create -f \ - https://raw.githubusercontent.com/aws/karpenter-provider-aws/${KARPENTER_VERSION}/pkg/apis/crds/karpenter.k8s.aws_awsnodetemplates.yaml -kubectl create -f \ - https://raw.githubusercontent.com/aws/karpenter-provider-aws/${KARPENTER_VERSION}/pkg/apis/crds/karpenter.sh_machines.yaml -kubectl apply -f karpenter.yaml diff --git a/website/content/en/v0.31/getting-started/migrating-from-cas/scripts/step11-create-provisioner.sh b/website/content/en/v0.31/getting-started/migrating-from-cas/scripts/step11-create-provisioner.sh deleted file mode 100644 index 76642ce51122..000000000000 --- a/website/content/en/v0.31/getting-started/migrating-from-cas/scripts/step11-create-provisioner.sh +++ /dev/null @@ -1,26 +0,0 @@ -cat <} ### Node-Level Controls -Nodes can be opted out of consolidation disruption by setting the annotation `karpenter.sh/do-not-disrupt: "true"` on the node. +You can block Karpenter from voluntarily choosing to disrupt certain nodes by setting the `karpenter.sh/do-not-disrupt: "true"` annotation on the node. This will prevent disruption actions on the node. ```yaml apiVersion: v1 @@ -221,7 +221,7 @@ metadata: #### Example: Disable Disruption on a NodePool -NodePool `.spec.annotations` allow you to set annotations that will be applied to all nodes launched by this NodePool. By setting the annotation `karpenter.sh/do-not-disrupt: "true"` on the NodePool, you will selectively prevent all nodes launched by this NodePool from being considered in consolidation calculations. +NodePool `.spec.annotations` allow you to set annotations that will be applied to all nodes launched by this NodePool. By setting the annotation `karpenter.sh/do-not-disrupt: "true"` on the NodePool, you will selectively prevent all nodes launched by this NodePool from being considered in disruption actions. ```yaml apiVersion: karpenter.sh/v1beta1 diff --git a/website/content/en/v0.32/concepts/nodeclasses.md b/website/content/en/v0.32/concepts/nodeclasses.md index 12c4c60a373f..a5fbbd0636cb 100644 --- a/website/content/en/v0.32/concepts/nodeclasses.md +++ b/website/content/en/v0.32/concepts/nodeclasses.md @@ -35,7 +35,7 @@ spec: # Each term in the array of subnetSelectorTerms is ORed together # Within a single term, all conditions are ANDed subnetSelectorTerms: - # Select on any subnet that has the "karpenter.sh/discovery: ${CLUSTER_NAME}" + # Select on any subnet that has the "karpenter.sh/discovery: ${CLUSTER_NAME}" # AND the "environment: test" tag OR any subnet with ID "subnet-09fa4a0a8f233a921" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" @@ -46,8 +46,8 @@ spec: # Each term in the array of securityGroupSelectorTerms is ORed together # Within a single term, all conditions are ANDed securityGroupSelectorTerms: - # Select on any security group that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag - # AND the "environment: test" tag OR any security group with the "my-security-group" name + # Select on any security group that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any security group with the "my-security-group" name # OR any security group with ID "sg-063d7acfb4b06c82c" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" @@ -70,8 +70,8 @@ spec: # Each term in the array of amiSelectorTerms is ORed together # Within a single term, all conditions are ANDed amiSelectorTerms: - # Select on any AMI that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag - # AND the "environment: test" tag OR any AMI with the "my-ami" name + # Select on any AMI that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any AMI with the "my-ami" name # OR any AMI with ID "ami-123" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" @@ -246,7 +246,7 @@ This selection logic is modeled as terms, where each term contains multiple cond ```yaml subnetSelectorTerms: - # Select on any subnet that has the "karpenter.sh/discovery: ${CLUSTER_NAME}" + # Select on any subnet that has the "karpenter.sh/discovery: ${CLUSTER_NAME}" # AND the "environment: test" tag OR any subnet with ID "subnet-09fa4a0a8f233a921" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" @@ -313,8 +313,8 @@ This selection logic is modeled as terms, where each term contains multiple cond ```yaml securityGroupSelectorTerms: - # Select on any security group that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag - # AND the "environment: test" tag OR any security group with the "my-security-group" name + # Select on any security group that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any security group with the "my-security-group" name # OR any security group with ID "sg-063d7acfb4b06c82c" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" @@ -400,8 +400,8 @@ This selection logic is modeled as terms, where each term contains multiple cond ```yaml amiSelectorTerms: - # Select on any AMI that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag - # AND the "environment: test" tag OR any AMI with the "my-ami" name + # Select on any AMI that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any AMI with the "my-ami" name # OR any AMI with ID "ami-123" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" @@ -422,6 +422,7 @@ AMIs may be specified by any AWS tag, including `Name`. Selecting by tag or by n If `amiSelectorTerms` match more than one AMI, Karpenter will automatically determine which AMI best fits the workloads on the launched worker node under the following constraints: * When launching nodes, Karpenter automatically determines which architecture a custom AMI is compatible with and will use images that match an instanceType's requirements. + * Note that Karpenter **cannot** detect any requirement other than architecture. If you need to specify different AMIs for different kind of nodes (e.g. accelerated GPU AMIs), you should use a separate `EC2NodeClass`. * If multiple AMIs are found that can be used, Karpenter will choose the latest one. * If no AMIs are found that can be used, then no nodes will be provisioned. {{% /alert %}} @@ -481,7 +482,7 @@ Specify using ids: ## spec.role -`Role` is an optional field and is necessary to tell Karpenter which identity nodes from this `EC2NodeClass` should assume. You must specify one of `role` or `instanceProfile` when creating a Karpenter `EC2NodeClass`. If using the [Karpenter Getting Started Guide]({{}}) to deploy Karpenter, you can use the `KarpenterNodeRole-$CLUSTER_NAME` role provisioned by that process. +`Role` is an optional field and tells Karpenter which IAM identity nodes should assume. You must specify one of `role` or `instanceProfile` when creating a Karpenter `EC2NodeClass`. If using the [Karpenter Getting Started Guide]({{}}) to deploy Karpenter, you can use the `KarpenterNodeRole-$CLUSTER_NAME` role provisioned by that process. ```yaml spec: @@ -490,7 +491,9 @@ spec: ## spec.instanceProfile -`InstanceProfile` is an optional field and is necessary to tell Karpenter which identity nodes from this `EC2NodeClass` should assume. You must specify one of `role` or `instanceProfile` when creating a Karpenter `EC2NodeClasss`. If you use the `instanceProfile` field instead of `role`, Karpenter will not manage the InstanceProfile on your behalf. +`InstanceProfile` is an optional field and tells Karpenter which IAM identity nodes should assume. You must specify one of `role` or `instanceProfile` when creating a Karpenter `EC2NodeClass`. If you use the `instanceProfile` field instead of `role`, Karpenter will not manage the InstanceProfile on your behalf; instead, it expects that you have pre-provisioned an IAM instance profile and assigned it a role. + +You can provision and assign a role to an IAM instance profile using [CloudFormation](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iam-instanceprofile.html) or by using the [`aws iam create-instance-profile`](https://docs.aws.amazon.com/cli/latest/reference/iam/create-instance-profile.html) and [`aws iam add-role-to-instance-profile`](https://docs.aws.amazon.com/cli/latest/reference/iam/add-role-to-instance-profile.html) commands in the CLI. {{% alert title="Note" color="primary" %}} @@ -506,6 +509,7 @@ Karpenter adds tags to all resources it creates, including EC2 Instances, EBS vo Name: karpenter.sh/nodeclaim: karpenter.sh/nodepool: +karpenter.k8s.aws/ec2nodeclass: kubernetes.io/cluster/: owned ``` diff --git a/website/content/en/v0.32/concepts/nodepools.md b/website/content/en/v0.32/concepts/nodepools.md index 7ff7f1f64402..607311c5ef6e 100644 --- a/website/content/en/v0.32/concepts/nodepools.md +++ b/website/content/en/v0.32/concepts/nodepools.md @@ -46,6 +46,8 @@ spec: spec: # References the Cloud Provider's NodeClass resource, see your cloud provider specific documentation nodeClassRef: + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass name: default # Provisioned nodes will have these taints @@ -127,12 +129,12 @@ spec: # If using 'WhenUnderutilized', Karpenter will consider all nodes for consolidation and attempt to remove or replace Nodes when it discovers that the Node is underutilized and could be changed to reduce cost # If using `WhenEmpty`, Karpenter will only consider nodes for consolidation that contain no workload pods consolidationPolicy: WhenUnderutilized | WhenEmpty - + # The amount of time Karpenter should wait after discovering a consolidation decision # This value can currently only be set when the consolidationPolicy is 'WhenEmpty' # You can choose to disable consolidation entirely by setting the string value 'Never' here consolidateAfter: 30s - + # The amount of time a Node can live on the cluster before being removed # Avoiding long-running Nodes helps to reduce security vulnerabilities as well as to reduce the chance of issues that can plague Nodes with long uptimes such as file fragmentation or memory leaks from system processes # You can choose to disable expiration entirely by setting the string value 'Never' here @@ -285,7 +287,12 @@ kubelet: Karpenter will automatically configure the system and kube reserved resource requests on the fly on your behalf. These requests are used to configure your node and to make scheduling decisions for your pods. If you have specific requirements or know that you will have additional capacity requirements, you can optionally override the `--system-reserved` configuration defaults with the `.spec.template.spec.kubelet.systemReserved` values and the `--kube-reserved` configuration defaults with the `.spec.template.spec.kubelet.kubeReserved` values. -For more information on the default `--system-reserved` and `--kube-reserved` configuration refer to the [Kubelet Docs](https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/#kube-reserved) +{{% alert title="Note" color="primary" %}} +Karpenter considers these reserved resources when computing the allocatable ephemeral storage on a given instance type. +If `kubeReserved` is not specified, Karpenter will compute the default reserved [CPU](https://github.com/awslabs/amazon-eks-ami/blob/db28da15d2b696bc08ac3aacc9675694f4a69933/files/bootstrap.sh#L251) and [memory](https://github.com/awslabs/amazon-eks-ami/blob/db28da15d2b696bc08ac3aacc9675694f4a69933/files/bootstrap.sh#L235) resources for the purpose of ephemeral storage computation. +These defaults are based on the defaults on Karpenter's supported AMI families, which are not the same as the kubelet defaults. +You should be aware of the CPU and memory default calculation when using Custom AMI Families. If they don't align, there may be a difference in Karpenter's computed allocatable ephemeral storage and the actually ephemeral storage available on the node. +{{% /alert %}} ### Eviction Thresholds @@ -431,7 +438,7 @@ Review the [Kubernetes core API](https://github.com/kubernetes/api/blob/37748cca Karpenter allows you to describe NodePool preferences through a `weight` mechanism similar to how weight is described with [pod and node affinities](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity). -For more information on weighting NodePools, see the [Weighting NodePools section]({{}}) in the scheduling details. +For more information on weighting NodePools, see the [Weighted NodePools section]({{}}) in the scheduling docs. ## Examples diff --git a/website/content/en/v0.32/concepts/scheduling.md b/website/content/en/v0.32/concepts/scheduling.md index cecc70c0c4a0..48816feebe6c 100755 --- a/website/content/en/v0.32/concepts/scheduling.md +++ b/website/content/en/v0.32/concepts/scheduling.md @@ -213,7 +213,7 @@ All examples below assume that the NodePool doesn't have constraints to prevent - matchExpressions: - key: "topology.kubernetes.io/zone" operator: "In" - values: ["us-west-2a, us-west-2b"] + values: ["us-west-2a", "us-west-2b"] - key: "topology.kubernetes.io/zone" operator: "In" values: ["us-west-2b"] @@ -224,7 +224,7 @@ Changing the second operator to `NotIn` would allow the pod to run in `us-west-2 ```yaml - key: "topology.kubernetes.io/zone" operator: "In" - values: ["us-west-2a, us-west-2b"] + values: ["us-west-2a", "us-west-2b"] - key: "topology.kubernetes.io/zone" operator: "NotIn" values: ["us-west-2b"] @@ -242,7 +242,7 @@ Here, if `us-west-2a` is not available, the second term will cause the pod to ru - matchExpressions: # OR - key: "topology.kubernetes.io/zone" # AND operator: "In" - values: ["us-west-2a, us-west-2b"] + values: ["us-west-2a", "us-west-2b"] - key: "topology.kubernetes.io/zone" # AND operator: "NotIn" values: ["us-west-2b"] @@ -351,6 +351,10 @@ The three supported `topologyKey` values that Karpenter supports are: See [Pod Topology Spread Constraints](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/) for details. +{{% alert title="Note" color="primary" %}} +NodePools do not attempt to balance or rebalance the availability zones for their nodes. Availability zone balancing may be achieved by defining zonal Topology Spread Constraints for Pods that require multi-zone durability, and NodePools will respect these constraints while optimizing for compute costs. +{{% /alert %}} + ## Pod affinity/anti-affinity By using the `podAffinity` and `podAntiAffinity` configuration on a pod spec, you can inform the Karpenter scheduler of your desire for pods to schedule together or apart with respect to different topology domains. For example: diff --git a/website/content/en/v0.32/faq.md b/website/content/en/v0.32/faq.md index dc30ec64d18b..eb3a46e16351 100644 --- a/website/content/en/v0.32/faq.md +++ b/website/content/en/v0.32/faq.md @@ -14,7 +14,7 @@ See [Configuring NodePools]({{< ref "./concepts/#configuring-nodepools" >}}) for AWS is the first cloud provider supported by Karpenter, although it is designed to be used with other cloud providers as well. ### Can I write my own cloud provider for Karpenter? -Yes, but there is no documentation yet for it. Start with Karpenter's GitHub [cloudprovider](https://github.com/aws/karpenter-core/tree/v0.32.5/pkg/cloudprovider) documentation to see how the AWS provider is built, but there are other sections of the code that will require changes too. +Yes, but there is no documentation yet for it. Start with Karpenter's GitHub [cloudprovider](https://github.com/aws/karpenter-core/tree/v0.32.9/pkg/cloudprovider) documentation to see how the AWS provider is built, but there are other sections of the code that will require changes too. ### What operating system nodes does Karpenter deploy? When using v1beta1 APIs, Karpenter uses the OS defined by the [AMI Family in your EC2NodeClass]({{< ref "./concepts/nodeclasses#specamifamily" >}}). @@ -27,7 +27,7 @@ Karpenter has multiple mechanisms for configuring the [operating system]({{< ref Karpenter is flexible to multi-architecture configurations using [well known labels]({{< ref "./concepts/scheduling/#supported-labels">}}). ### What RBAC access is required? -All the required RBAC rules can be found in the helm chart template. See [clusterrole-core.yaml](https://github.com/aws/karpenter/blob/v0.32.5/charts/karpenter/templates/clusterrole-core.yaml), [clusterrole.yaml](https://github.com/aws/karpenter/blob/v0.32.5/charts/karpenter/templates/clusterrole.yaml), [rolebinding.yaml](https://github.com/aws/karpenter/blob/v0.32.5/charts/karpenter/templates/rolebinding.yaml), and [role.yaml](https://github.com/aws/karpenter/blob/v0.32.5/charts/karpenter/templates/role.yaml) files for details. +All the required RBAC rules can be found in the helm chart template. See [clusterrole-core.yaml](https://github.com/aws/karpenter/blob/v0.32.9/charts/karpenter/templates/clusterrole-core.yaml), [clusterrole.yaml](https://github.com/aws/karpenter/blob/v0.32.9/charts/karpenter/templates/clusterrole.yaml), [rolebinding.yaml](https://github.com/aws/karpenter/blob/v0.32.9/charts/karpenter/templates/rolebinding.yaml), and [role.yaml](https://github.com/aws/karpenter/blob/v0.32.9/charts/karpenter/templates/role.yaml) files for details. ### Can I run Karpenter outside of a Kubernetes cluster? Yes, as long as the controller has network and IAM/RBAC access to the Kubernetes API and your provider API. @@ -180,10 +180,10 @@ Yes, see the [KubeletConfiguration Section in the NodePool docs]({{}} by configuring an `amiSelector` that references the AMI name. +You can specify the [Amazon EKS optimized AMI](https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-windows-ami.html) with Windows Server 2022 Full for Kubernetes 1.28 by configuring an `amiSelector` that references the AMI name. ```yaml amiSelectorTerms: - - name: Windows_Server-2022-English-Full-EKS_Optimized-{{< param "latest_k8s_version" >}}* + - name: Windows_Server-2022-English-Full-EKS_Optimized-1.28* ``` ## Deprovisioning diff --git a/website/content/en/v0.32/getting-started/_index.md b/website/content/en/v0.32/getting-started/_index.md index a70f6bbc422b..cebdf66a99fd 100644 --- a/website/content/en/v0.32/getting-started/_index.md +++ b/website/content/en/v0.32/getting-started/_index.md @@ -21,6 +21,6 @@ Learn more about Karpenter and how to get started below. * [Karpenter EKS Best Practices](https://aws.github.io/aws-eks-best-practices/karpenter/) guide * [EC2 Spot Workshop for Karpenter](https://ec2spotworkshops.com/karpenter.html) * [EKS Karpenter Workshop](https://www.eksworkshop.com/docs/autoscaling/compute/karpenter/) -* [Advanced EKS Immersion Karpenter Workshop](https://catalog.workshops.aws/eks-advanced/karpenter/) +* [Advanced EKS Immersion Karpenter Workshop](https://catalog.us-east-1.prod.workshops.aws/workshops/76a5dd80-3249-4101-8726-9be3eeee09b2/en-US/autoscaling/karpenter) * [Karpenter Blueprints](https://github.com/aws-samples/karpenter-blueprints) * [Tutorial: Run Kubernetes Clusters for Less with Amazon EC2 Spot and Karpenter](https://community.aws/tutorials/run-kubernetes-clusters-for-less-with-amazon-ec2-spot-and-karpenter#step-6-optional-simulate-spot-interruption) diff --git a/website/content/en/v0.32/getting-started/getting-started-with-karpenter/_index.md b/website/content/en/v0.32/getting-started/getting-started-with-karpenter/_index.md index 2d99cb3acca4..d03eb9c5a546 100644 --- a/website/content/en/v0.32/getting-started/getting-started-with-karpenter/_index.md +++ b/website/content/en/v0.32/getting-started/getting-started-with-karpenter/_index.md @@ -45,8 +45,8 @@ After setting up the tools, set the Karpenter and Kubernetes version: ```bash export KARPENTER_NAMESPACE=karpenter -export KARPENTER_VERSION=v0.32.5 -export K8S_VERSION={{< param "latest_k8s_version" >}} +export KARPENTER_VERSION=v0.32.9 +export K8S_VERSION=1.28 ``` Then set the following environment variable: @@ -58,7 +58,7 @@ If you open a new shell to run steps in this procedure, you need to set some or To remind yourself of these values, type: ```bash -echo $KARPENTER_NAMESPACE $KARPENTER_VERSION $K8S_VERSION $CLUSTER_NAME $AWS_DEFAULT_REGION $AWS_ACCOUNT_ID $TEMPOUT +echo "${KARPENTER_NAMESPACE}" "${KARPENTER_VERSION}" "${K8S_VERSION}" "${CLUSTER_NAME}" "${AWS_DEFAULT_REGION}" "${AWS_ACCOUNT_ID}" "${TEMPOUT}" "${ARM_AMI_ID}" "${AMD_AMI_ID}" "${GPU_AMI_ID}" ``` {{% /alert %}} @@ -180,6 +180,7 @@ com.amazonaws..s3 – For pulling container images com.amazonaws..sts – For IAM roles for service accounts com.amazonaws..ssm - For resolving default AMIs com.amazonaws..sqs - For accessing SQS if using interruption handling +com.amazonaws..eks - For Karpenter to discover the cluster endpoint ``` If you do not currently have these endpoints surfaced in your VPC, you can add the endpoints by running diff --git a/website/content/en/v0.32/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json b/website/content/en/v0.32/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json index 9d9cb6ad14a8..d474d01f4e16 100644 --- a/website/content/en/v0.32/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json +++ b/website/content/en/v0.32/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json @@ -484,7 +484,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by ($distribution_filter)(\n karpenter_pods_state{arch=~\"$arch\", capacity_type=~\"$capacity_type\", instance_type=~\"$instance_type\", provisioner=~\"$provisioner\"}\n)", + "expr": "sum by ($distribution_filter)(\n karpenter_pods_state{arch=~\"$arch\", capacity_type=~\"$capacity_type\", instance_type=~\"$instance_type\", nodepool=~\"$nodepool\"}\n)", "legendFormat": "{{label_name}}", "range": true, "refId": "A" @@ -588,7 +588,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "karpenter_provisioner_usage{resource_type=\"cpu\"} / karpenter_provisioner_limit{resource_type=\"cpu\"}", + "expr": "karpenter_nodepool_usage{resource_type=\"cpu\"} / karpenter_nodepool_limit{resource_type=\"cpu\"}", "format": "table", "instant": true, "legendFormat": "CPU Limit Utilization", @@ -602,7 +602,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "count by (provisioner)(karpenter_nodes_allocatable{provisioner!=\"N/A\",resource_type=\"cpu\"}) # Selects a single resource type to get node count", + "expr": "count by (nodepool)(karpenter_nodes_allocatable{nodepool!=\"N/A\",resource_type=\"cpu\"}) # Selects a single resource type to get node count", "format": "table", "hide": false, "instant": true, @@ -616,7 +616,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "karpenter_provisioner_usage{resource_type=\"memory\"} / karpenter_provisioner_limit{resource_type=\"memory\"}", + "expr": "karpenter_nodepool_usage{resource_type=\"memory\"} / karpenter_nodepool_limit{resource_type=\"memory\"}", "format": "table", "hide": false, "instant": true, @@ -631,7 +631,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum by (provisioner)(karpenter_nodes_allocatable{provisioner!=\"N/A\",resource_type=\"cpu\"})", + "expr": "sum by (nodepool)(karpenter_nodes_allocatable{nodepool!=\"N/A\",resource_type=\"cpu\"})", "format": "table", "hide": false, "instant": true, @@ -645,7 +645,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum by (provisioner)(karpenter_nodes_allocatable{provisioner!=\"N/A\",resource_type=\"memory\"})", + "expr": "sum by (nodepool)(karpenter_nodes_allocatable{nodepool!=\"N/A\",resource_type=\"memory\"})", "format": "table", "hide": false, "instant": true, @@ -653,12 +653,12 @@ "refId": "Memory Capacity" } ], - "title": "Provisioner Summary", + "title": "Nodepool Summary", "transformations": [ { "id": "seriesToColumns", "options": { - "byField": "provisioner" + "byField": "nodepool" } }, { @@ -697,7 +697,7 @@ "instance 2": 12, "job 1": 9, "job 2": 13, - "provisioner": 0, + "nodepool": 0, "resource_type 1": 10, "resource_type 2": 14 }, @@ -714,7 +714,7 @@ "instance": "", "instance 1": "", "job": "", - "provisioner": "Provisioner" + "nodepool": "Nodepool" } } } @@ -804,7 +804,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "(count(karpenter_nodes_allocatable{arch=~\"$arch\",capacity_type=\"spot\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"}) or vector(0)) / count(karpenter_nodes_allocatable{arch=~\"$arch\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"})", + "expr": "(count(karpenter_nodes_allocatable{arch=~\"$arch\",capacity_type=\"spot\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}) or vector(0)) / count(karpenter_nodes_allocatable{arch=~\"$arch\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"})", "legendFormat": "Percentage", "range": true, "refId": "A" @@ -946,7 +946,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "((karpenter_nodes_total_daemon_requests{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0) + \n(karpenter_nodes_total_pod_requests{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0)) / \nkarpenter_nodes_allocatable{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"}", + "expr": "((karpenter_nodes_total_daemon_requests{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0) + \n(karpenter_nodes_total_pod_requests{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0)) / \nkarpenter_nodes_allocatable{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}", "format": "table", "hide": false, "instant": true, @@ -961,7 +961,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "((karpenter_nodes_total_daemon_requests{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0) + \n(karpenter_nodes_total_pod_requests{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0)) / \nkarpenter_nodes_allocatable{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"}", + "expr": "((karpenter_nodes_total_daemon_requests{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0) + \n(karpenter_nodes_total_pod_requests{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0)) / \nkarpenter_nodes_allocatable{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}", "format": "table", "hide": false, "instant": true, @@ -976,7 +976,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "karpenter_nodes_total_daemon_requests{resource_type=\"pods\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"} + \nkarpenter_nodes_total_pod_requests{resource_type=\"pods\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"}", + "expr": "karpenter_nodes_total_daemon_requests{resource_type=\"pods\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} + \nkarpenter_nodes_total_pod_requests{resource_type=\"pods\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}", "format": "table", "hide": false, "instant": true, @@ -1091,9 +1091,9 @@ "os 1": true, "os 2": true, "os 3": true, - "provisioner 1": false, - "provisioner 2": true, - "provisioner 3": true, + "nodepool 1": false, + "nodepool 2": true, + "nodepool 3": true, "resource_type": true, "resource_type 1": true, "resource_type 2": true, @@ -1161,9 +1161,9 @@ "os 1": 23, "os 2": 41, "os 3": 61, - "provisioner 1": 2, - "provisioner 2": 42, - "provisioner 3": 62, + "nodepool 1": 2, + "nodepool 2": 42, + "nodepool 3": 62, "resource_type 1": 24, "resource_type 2": 43, "resource_type 3": 63, @@ -1190,7 +1190,7 @@ "instance_type": "Instance Type", "instance_type 1": "Instance Type", "node_name": "Node Name", - "provisioner 1": "Provisioner", + "nodepool 1": "Nodepool", "zone 1": "Zone" } } @@ -1237,14 +1237,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "definition": "label_values(karpenter_nodes_allocatable, provisioner)", + "definition": "label_values(karpenter_nodes_allocatable, nodepool)", "hide": 0, "includeAll": true, "multi": true, - "name": "provisioner", + "name": "nodepool", "options": [], "query": { - "query": "label_values(karpenter_nodes_allocatable, provisioner)", + "query": "label_values(karpenter_nodes_allocatable, nodepool)", "refId": "StandardVariableQuery" }, "refresh": 2, @@ -1376,8 +1376,8 @@ { "current": { "selected": true, - "text": "provisioner", - "value": "provisioner" + "text": "nodepool", + "value": "nodepool" }, "hide": 0, "includeAll": false, @@ -1411,8 +1411,8 @@ }, { "selected": true, - "text": "provisioner", - "value": "provisioner" + "text": "nodepool", + "value": "nodepool" }, { "selected": false, @@ -1420,7 +1420,7 @@ "value": "zone" } ], - "query": "arch,capacity_type,instance_type,namespace,node,provisioner,zone", + "query": "arch,capacity_type,instance_type,namespace,node,nodepool,zone", "queryValue": "", "skipUrlSync": false, "type": "custom" diff --git a/website/content/en/v0.32/getting-started/getting-started-with-karpenter/prometheus-values.yaml b/website/content/en/v0.32/getting-started/getting-started-with-karpenter/prometheus-values.yaml index 4cd78495351b..f939dd2b4888 100644 --- a/website/content/en/v0.32/getting-started/getting-started-with-karpenter/prometheus-values.yaml +++ b/website/content/en/v0.32/getting-started/getting-started-with-karpenter/prometheus-values.yaml @@ -13,7 +13,7 @@ extraScrapeConfigs: | - role: endpoints namespaces: names: - - karpenter + - $KARPENTER_NAMESPACE relabel_configs: - source_labels: [__meta_kubernetes_endpoint_port_name] regex: http-metrics diff --git a/website/content/en/v0.32/getting-started/getting-started-with-karpenter/scripts/add-monitoring.sh b/website/content/en/v0.32/getting-started/getting-started-with-karpenter/scripts/add-monitoring.sh deleted file mode 100755 index 6d2332b06be2..000000000000 --- a/website/content/en/v0.32/getting-started/getting-started-with-karpenter/scripts/add-monitoring.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -set -euo pipefail #fail if one step fails - -if [ "$#" -ne 1 ] -then - echo "Missing required Karpenter version. Usage: add-monitoring.sh v0.0.1" - exit 1 -fi - -export KARPENTER_VERSION=$1 - -declare -a steps=( - step01-config.sh - step09-add-prometheus-grafana.sh - step10-add-grafana-port-forward.sh - step11-grafana-get-password.sh -) - -for step in "${steps[@]}"; do - echo "$step" - source $step -done diff --git a/website/content/en/v0.32/getting-started/getting-started-with-karpenter/scripts/add-provisioner.sh b/website/content/en/v0.32/getting-started/getting-started-with-karpenter/scripts/add-provisioner.sh deleted file mode 100755 index 51f286de0c3c..000000000000 --- a/website/content/en/v0.32/getting-started/getting-started-with-karpenter/scripts/add-provisioner.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -set -euo pipefail #fail if one step fails - -declare -a steps=( - step01-config.sh - step12-add-provisioner.sh - step13-automatic-node-provisioning.sh - step14-automatic-node-termination.sh -) - -for step in "${steps[@]}"; do - echo "$step" - source $step -done diff --git a/website/content/en/v0.32/getting-started/getting-started-with-karpenter/scripts/add-roles.sh b/website/content/en/v0.32/getting-started/getting-started-with-karpenter/scripts/add-roles.sh deleted file mode 100755 index cf0656f3db3d..000000000000 --- a/website/content/en/v0.32/getting-started/getting-started-with-karpenter/scripts/add-roles.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -set -euo pipefail #fail if one step fails - -if [ "$#" -ne 1 ] -then - echo "Missing required Karpenter version. Usage: setup-roles.sh v0.0.1" - exit 1 -fi - -export KARPENTER_VERSION=$1 -SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) - -declare -a steps=( - step03-iam-cloud-formation.sh - step04-grant-access.sh - step05-controller-iam.sh - step06-add-spot-role.sh -) - -for step in "${steps[@]}"; do - echo "$step" - source "$SCRIPT_DIR/$step" -done diff --git a/website/content/en/v0.32/getting-started/getting-started-with-karpenter/scripts/cleanup.sh b/website/content/en/v0.32/getting-started/getting-started-with-karpenter/scripts/cleanup.sh deleted file mode 100755 index 52851e7f975a..000000000000 --- a/website/content/en/v0.32/getting-started/getting-started-with-karpenter/scripts/cleanup.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -declare -a steps=( - step01-config.sh - step16-cleanup.sh -) - -for step in "${steps[@]}"; do - echo "$step" - source $step -done diff --git a/website/content/en/v0.32/getting-started/getting-started-with-karpenter/scripts/install.sh b/website/content/en/v0.32/getting-started/getting-started-with-karpenter/scripts/install.sh deleted file mode 100755 index e0ebfb32e511..000000000000 --- a/website/content/en/v0.32/getting-started/getting-started-with-karpenter/scripts/install.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -set -euo pipefail #fail if one step fails - -if [ "$#" -ne 1 ] -then - echo "Missing required Karpenter version. Usage: install.sh v0.0.1" - exit 1 -fi - -export KARPENTER_VERSION=$1 - -declare -a steps=( - step01-config.sh - step02-create-cluster.sh - step03-iam-cloud-formation.sh - step04-grant-access.sh - step05-controller-iam.sh - step06-add-spot-role.sh - step08-apply-helm-chart.sh -) - -for step in "${steps[@]}"; do - echo "$step" - source $step -done diff --git a/website/content/en/v0.32/getting-started/getting-started-with-karpenter/scripts/step01-config.sh b/website/content/en/v0.32/getting-started/getting-started-with-karpenter/scripts/step01-config.sh index a3af512d02ac..6b948de3c793 100755 --- a/website/content/en/v0.32/getting-started/getting-started-with-karpenter/scripts/step01-config.sh +++ b/website/content/en/v0.32/getting-started/getting-started-with-karpenter/scripts/step01-config.sh @@ -3,3 +3,6 @@ export CLUSTER_NAME="${USER}-karpenter-demo" export AWS_DEFAULT_REGION="us-west-2" export AWS_ACCOUNT_ID="$(aws sts get-caller-identity --query Account --output text)" export TEMPOUT=$(mktemp) +export ARM_AMI_ID="$(aws ssm get-parameter --name /aws/service/eks/optimized-ami/${K8S_VERSION}/amazon-linux-2-arm64/recommended/image_id --query Parameter.Value --output text)" +export AMD_AMI_ID="$(aws ssm get-parameter --name /aws/service/eks/optimized-ami/${K8S_VERSION}/amazon-linux-2/recommended/image_id --query Parameter.Value --output text)" +export GPU_AMI_ID="$(aws ssm get-parameter --name /aws/service/eks/optimized-ami/${K8S_VERSION}/amazon-linux-2-gpu/recommended/image_id --query Parameter.Value --output text)" diff --git a/website/content/en/v0.32/getting-started/getting-started-with-karpenter/scripts/step09-add-prometheus-grafana.sh b/website/content/en/v0.32/getting-started/getting-started-with-karpenter/scripts/step09-add-prometheus-grafana.sh index a539f7491e10..951813a865cc 100755 --- a/website/content/en/v0.32/getting-started/getting-started-with-karpenter/scripts/step09-add-prometheus-grafana.sh +++ b/website/content/en/v0.32/getting-started/getting-started-with-karpenter/scripts/step09-add-prometheus-grafana.sh @@ -4,7 +4,7 @@ helm repo update kubectl create namespace monitoring -curl -fsSL https://raw.githubusercontent.com/aws/karpenter-provider-aws/"${KARPENTER_VERSION}"/website/content/en/preview/getting-started/getting-started-with-karpenter/prometheus-values.yaml | tee prometheus-values.yaml +curl -fsSL https://raw.githubusercontent.com/aws/karpenter-provider-aws/"${KARPENTER_VERSION}"/website/content/en/preview/getting-started/getting-started-with-karpenter/prometheus-values.yaml | envsubst | tee prometheus-values.yaml helm install --namespace monitoring prometheus prometheus-community/prometheus --values prometheus-values.yaml curl -fsSL https://raw.githubusercontent.com/aws/karpenter-provider-aws/"${KARPENTER_VERSION}"/website/content/en/preview/getting-started/getting-started-with-karpenter/grafana-values.yaml | tee grafana-values.yaml diff --git a/website/content/en/v0.32/getting-started/getting-started-with-karpenter/scripts/step12-add-nodepool.sh b/website/content/en/v0.32/getting-started/getting-started-with-karpenter/scripts/step12-add-nodepool.sh index 8c518e9e74b8..33f1cb553b1b 100755 --- a/website/content/en/v0.32/getting-started/getting-started-with-karpenter/scripts/step12-add-nodepool.sh +++ b/website/content/en/v0.32/getting-started/getting-started-with-karpenter/scripts/step12-add-nodepool.sh @@ -23,6 +23,8 @@ spec: operator: Gt values: ["2"] nodeClassRef: + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass name: default limits: cpu: 1000 @@ -43,4 +45,9 @@ spec: securityGroupSelectorTerms: - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + amiSelectorTerms: + - id: "${ARM_AMI_ID}" + - id: "${AMD_AMI_ID}" +# - id: "${GPU_AMI_ID}" # <- GPU Optimized AMD AMI +# - name: "amazon-eks-node-${K8S_VERSION}-*" # <- automatically upgrade when a new AL2 EKS Optimized AMI is released. This is unsafe for production workloads. Validate AMIs in lower environments before deploying them to production. EOF diff --git a/website/content/en/v0.32/getting-started/migrating-from-cas/_index.md b/website/content/en/v0.32/getting-started/migrating-from-cas/_index.md index b1a794b954f0..9c7e128cbff4 100644 --- a/website/content/en/v0.32/getting-started/migrating-from-cas/_index.md +++ b/website/content/en/v0.32/getting-started/migrating-from-cas/_index.md @@ -92,7 +92,7 @@ One for your Karpenter node role and one for your existing node group. First set the Karpenter release you want to deploy. ```bash -export KARPENTER_VERSION=v0.32.5 +export KARPENTER_VERSION=v0.32.9 ``` We can now generate a full Karpenter deployment yaml from the helm chart. @@ -133,7 +133,7 @@ Now that our deployment is ready we can create the karpenter namespace, create t ## Create default NodePool -We need to create a default NodePool so Karpenter knows what types of nodes we want for unscheduled workloads. You can refer to some of the [example NodePool](https://github.com/aws/karpenter/tree/v0.32.5/examples/v1beta1) for specific needs. +We need to create a default NodePool so Karpenter knows what types of nodes we want for unscheduled workloads. You can refer to some of the [example NodePool](https://github.com/aws/karpenter/tree/v0.32.9/examples/v1beta1) for specific needs. {{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step10-create-nodepool.sh" language="bash" %}} diff --git a/website/content/en/v0.32/getting-started/migrating-from-cas/scripts/step01-env.sh b/website/content/en/v0.32/getting-started/migrating-from-cas/scripts/step01-env.sh index 20645685137b..6276d7aa1439 100644 --- a/website/content/en/v0.32/getting-started/migrating-from-cas/scripts/step01-env.sh +++ b/website/content/en/v0.32/getting-started/migrating-from-cas/scripts/step01-env.sh @@ -4,3 +4,7 @@ OIDC_ENDPOINT="$(aws eks describe-cluster --name ${CLUSTER_NAME} \ --query "cluster.identity.oidc.issuer" --output text)" AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query 'Account' \ --output text) +K8S_VERSION=1.28 +ARM_AMI_ID="$(aws ssm get-parameter --name /aws/service/eks/optimized-ami/${K8S_VERSION}/amazon-linux-2-arm64/recommended/image_id --query Parameter.Value --output text)" +AMD_AMI_ID="$(aws ssm get-parameter --name /aws/service/eks/optimized-ami/${K8S_VERSION}/amazon-linux-2/recommended/image_id --query Parameter.Value --output text)" +GPU_AMI_ID="$(aws ssm get-parameter --name /aws/service/eks/optimized-ami/${K8S_VERSION}/amazon-linux-2-gpu/recommended/image_id --query Parameter.Value --output text)" diff --git a/website/content/en/v0.32/getting-started/migrating-from-cas/scripts/step10-create-nodepool.sh b/website/content/en/v0.32/getting-started/migrating-from-cas/scripts/step10-create-nodepool.sh index 8c518e9e74b8..34bf95bdce4c 100644 --- a/website/content/en/v0.32/getting-started/migrating-from-cas/scripts/step10-create-nodepool.sh +++ b/website/content/en/v0.32/getting-started/migrating-from-cas/scripts/step10-create-nodepool.sh @@ -23,6 +23,8 @@ spec: operator: Gt values: ["2"] nodeClassRef: + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass name: default limits: cpu: 1000 @@ -43,4 +45,10 @@ spec: securityGroupSelectorTerms: - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + amiSelectorTerms: + - id: "${ARM_AMI_ID}" + - id: "${AMD_AMI_ID}" +# - id: "${GPU_AMI_ID}" # <- GPU Optimized AMD AMI +# - name: "amazon-eks-node-${K8S_VERSION}-*" # <- automatically upgrade when a new AL2 EKS Optimized AMI is released. This is unsafe for production workloads. Validate AMIs in lower environments before deploying them to production. EOF + diff --git a/website/content/en/v0.32/reference/cloudformation.md b/website/content/en/v0.32/reference/cloudformation.md index bb9ca50fe213..a14970e6e383 100644 --- a/website/content/en/v0.32/reference/cloudformation.md +++ b/website/content/en/v0.32/reference/cloudformation.md @@ -17,7 +17,7 @@ These descriptions should allow you to understand: To download a particular version of `cloudformation.yaml`, set the version and use `curl` to pull the file to your local system: ```bash -export KARPENTER_VERSION=v0.32.5 +export KARPENTER_VERSION=v0.32.9 curl https://raw.githubusercontent.com/aws/karpenter-provider-aws/"${KARPENTER_VERSION}"/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml > cloudformation.yaml ``` diff --git a/website/content/en/v0.32/reference/threat-model.md b/website/content/en/v0.32/reference/threat-model.md index fff13f8917f5..0e40d8a4669d 100644 --- a/website/content/en/v0.32/reference/threat-model.md +++ b/website/content/en/v0.32/reference/threat-model.md @@ -31,11 +31,11 @@ A Cluster Developer has the ability to create pods via `Deployments`, `ReplicaSe Karpenter has permissions to create and manage cloud instances. Karpenter has Kubernetes API permissions to create, update, and remove nodes, as well as evict pods. For a full list of the permissions, see the RBAC rules in the helm chart template. Karpenter also has AWS IAM permissions to create instances with IAM roles. -* [aggregate-clusterrole.yaml](https://github.com/aws/karpenter/blob/v0.32.5/charts/karpenter/templates/aggregate-clusterrole.yaml) -* [clusterrole-core.yaml](https://github.com/aws/karpenter/blob/v0.32.5/charts/karpenter/templates/clusterrole-core.yaml) -* [clusterrole.yaml](https://github.com/aws/karpenter/blob/v0.32.5/charts/karpenter/templates/clusterrole.yaml) -* [rolebinding.yaml](https://github.com/aws/karpenter/blob/v0.32.5/charts/karpenter/templates/rolebinding.yaml) -* [role.yaml](https://github.com/aws/karpenter/blob/v0.32.5/charts/karpenter/templates/role.yaml) +* [aggregate-clusterrole.yaml](https://github.com/aws/karpenter/blob/v0.32.9/charts/karpenter/templates/aggregate-clusterrole.yaml) +* [clusterrole-core.yaml](https://github.com/aws/karpenter/blob/v0.32.9/charts/karpenter/templates/clusterrole-core.yaml) +* [clusterrole.yaml](https://github.com/aws/karpenter/blob/v0.32.9/charts/karpenter/templates/clusterrole.yaml) +* [rolebinding.yaml](https://github.com/aws/karpenter/blob/v0.32.9/charts/karpenter/templates/rolebinding.yaml) +* [role.yaml](https://github.com/aws/karpenter/blob/v0.32.9/charts/karpenter/templates/role.yaml) ## Assumptions diff --git a/website/content/en/v0.32/tasks/_index.md b/website/content/en/v0.32/tasks/_index.md new file mode 100644 index 000000000000..7d4ac8605f4e --- /dev/null +++ b/website/content/en/v0.32/tasks/_index.md @@ -0,0 +1,10 @@ +--- +title: "Tasks" +linkTitle: "Tasks" +weight: 25 +description: > + Tasks to run with Karpenter +cascade: + type: docs +--- + diff --git a/website/content/en/v0.32/tasks/managing-amis.md b/website/content/en/v0.32/tasks/managing-amis.md new file mode 100644 index 000000000000..1ef31f141e08 --- /dev/null +++ b/website/content/en/v0.32/tasks/managing-amis.md @@ -0,0 +1,153 @@ +--- +title: "Managing AMIs" +linkTitle: "Managing AMIs" +weight: 10 +description: > + Task for managing AMIs in Karpenter +--- + +Understanding how Karpenter assigns AMIs to nodes can help ensure that your workloads will run successfully on those nodes and continue to run if the nodes are upgraded to newer AMIs. +Below we describe how Karpenter assigns AMIs to nodes when they are first deployed and how newer AMIs are assigned later when nodes are spun up to replace old ones. +Later, it describes the options you have to assert control over how AMIs are used by Karpenter for your clusters. + +Features for managing AMIs described here should be considered as part of the larger upgrade policies that you have for your clusters. +See [How do I upgrade an EKS Cluster with Karpenter]({{< relref "../faq/#how-do-i-upgrade-an-eks-cluster-with-karpenter" >}}) for details on this process. + +## How Karpenter assigns AMIs to nodes + +Here is how Karpenter assigns AMIs nodes: + +* When you create an `EC2NodeClass`, you are required to set the family of AMIs to use. For example, for the AL2 family, you would set `amiFamily: AL2`. +* With that `amiFamily` set, any time Karpenter spins up a new node, it uses the latest [Amazon EKS optimized Amazon Linux 2 AMIs](https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami.html) release. +* Later, if an existing node needs to be replaced, Karpenter checks to see if a newer AMI in the AL2 family is available and automatically uses the new AMI instead to spin up the new node. In other words, you may automatically get an AMI that you have not tested with your workloads. + +You can manually delete a node managed by Karpenter, which will cause the default behavior just described to take effect. +However, there are situations that will cause node replacements with newer AMIs to happen automatically. +These include: Expiration (if node expiry is set, the node is marked for deletion at a certain time after the node is created), [**Consolidation**]({{< relref "../concepts/disruption/#consolidation" >}}) (if a node is empty of workloads, or deemed to be inefficiently running workloads, nodes can be deleted and more appropriately featured nodes are brought up to consolidate workloads), [Drift]({{< relref "../concepts/disruption/#drift" >}}) (nodes are set for deletion when they drift from the desired state of the `NodeClaims` and new nodes are brought up to replace them), and [Interruption]({{< relref "../concepts/disruption/#interruption" >}}) (nodes are sometimes involuntarily disrupted by things like Spot interruption, health changes, and instance events, requiring new nodes to be deployed). + +See [**Automated Methods**]({{< relref "../concepts/disruption/#automated-methods" >}}) for details on how Karpenter uses these automated actions to replace nodes. + +With these types of automated updates in place, there is some risk that the new AMI being used when replacing instances will introduce some regressions or bugs that cause your workloads to be degraded or fail altogether. +The options described below tell you how to take more control over the ways in which Karpenter selects AMIs for your nodes. + +{{% alert title="Important" color="warning" %}} +If you are new to Karpenter, you should know that the behavior described here is different than you get with Managed Node Groups (MNG). MNG will always use the assigned AMI when it creates a new node and will never automatically upgrade to a new AMI when a new node is required. See [Updating a Managed Node Group](https://docs.aws.amazon.com/eks/latest/userguide/update-managed-node-group.html) to see how you would manually update MNG to use new AMIs. +{{% /alert %}} + +## Choosing AMI options +One of Karpenter's greatest assets is its ability to provide the right node at the right time, with little intervention from the person managing the cluster. +Its default behavior of using a later AMI if one becomes available in the selected family means you automatically get the latest security fixes and features. +However, with this comes the risk that the new AMI could break or degrade your workloads. + +As the Karpenter team looks for new ways to manage AMIs, the options below offer some means of reducing these risks, based on your own security and ease-of-use requirements. +Here are the advantages and challenges of each of the options described below: + +* [Option 1]({{< relref "#option-1-manage-how-amis-are-tested-and-rolled-out" >}}) (Test AMIs): The safest way, and the one we recommend, for ensuring that a new AMI doesn't break your workloads is to test it before putting it into production. This takes the most effort on your part, but most effectively models how your workloads will run in production, allowing you to catch issues ahead of time. Note that you can sometimes get different results from your test environment when you roll a new AMI into production, since issues like scale and other factors can elevate problems you might not see in test. So combining this with other options, that do things like slow rollouts, can allow you to catch problems before they impact your whole cluster. +* [Option 2]({{< relref "#option-2-lock-down-which-amis-are-selected" >}}) (Lock down AMIs): If workloads require a particluar AMI, this option can make sure that it is the only AMI used by Karpenter. This can be used in combination with Option 1, where you lock down the AMI in production, but allow the newest AMIs in a test cluster while you test your workloads before upgrading production. Keep in mind that this makes upgrades a manual process for you. +* [Option 3]({{< relref "#option-3-control-the-pace-of-node-disruptions" >}}) ([Disruption budgets]({{< relref "../concepts/disruption/" >}})): This option can be used as a way of mitigating the scope of impact if a new AMI causes problems with your workloads. With Disruption budgets you can slow the pace of upgrades to nodes with new AMIs or make sure that upgrades only happen during selected dates and times (using `schedule`). This doesn't prevent a bad AMI from being deployed, but it allows you to control when nodes are upgraded, and gives you more time respond to rollout issues. + +## Options + +The following lays out the options you have to impact Karpenter’s behavior as it relates to how nodes are created and AMIs are consumed. + +### Option 1: Manage how AMIs are tested and rolled out + +Instead of just avoiding AMI upgrades, you can set up test clusters where you can try out new AMI releases before they are put into production. +For example, you could have: + +* **Test clusters**: On lower environment clusters, you can run the latest AMIs for your workloads in a safe environment. The `EC2NodeClass` for these clusters could be set with a chosen `amiFamily`, but no `amiSelectorTerms` set. For example, the `NodePool` and `EC2NodeClass` could begin with the following: + + ```yaml + apiVersion: karpenter.sh/v1beta1 + kind: NodePool + metadata: + name: default + spec: + template: + spec: + nodeClassRef: + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass + name: default + --- + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass + metadata: + name: default + spec: + # The latest AMI in this family will be used + amiFamily: AL2 + ``` +* **Production clusters**: After you've confirmed that the AMI works in your lower environments, you can pin the latest AMIs to be deployed in your production clusters to roll out the AMI. One way to do that is to use `amiSelectorTerms` to set the tested AMI to be used in your production cluster. Refer to Option 2 for how to choose a particular AMI by `name` or `id`. Remember that it is still best practice to gradually roll new AMIs into your cluster, even if they have been tested. So consider implementing that for your production clusters as described in Option 3. + +### Option 2: Lock down which AMIs are selected + +Instead of letting Karpenter always run the latest AMI, you can change Karpenter’s default behavior. +When you configure the [**EC2NodeClass**]({{< relref "../concepts/nodeclasses" >}}), you can set a specific AMI that you want Karpenter to always choose, using the `amiSelectorTerms` field. +This prevents a new and potentially untested AMI from replacing existing nodes when those nodes are terminated. + +With the `amiSelectorTerms` field in an `EC2NodeClass`, you can set a specific AMI for Karpenter to use, based on AMI name or id (only one is required). +These examples show two different ways to identify the same AMI: + +```yaml +amiSelectorTerms: +- tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" + environment: prod +- name: al2023-ami-2023.3.20240219.0-kernel-6.1-x86_64 +``` + +or + +```yaml +amiSelectorTerms: +- tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" + environment: prod +- id: ami-052c9ea013e6e3567 +``` + +See the [**spec.amiSelectorTerms**]({{< relref "../concepts/nodeclasses/#specamiselectorterms" >}}) section of the NodeClasses page for details. +Keep in mind, that this could prevent you from getting critical security patches when new AMIs are available, but it does give you control over exactly which AMI is running. + + +### Option 3: Control the pace of node disruptions + +To reduce the risk of entire workloads being immediately degraded when a new AMI is deployed, you can enable Karpenter [**Disruption Budgets**]({{< relref "../concepts/disruption/#disruption-budgets " >}}). +Disruption Budgets limit when and to what extent nodes can be disrupted. +You can prevent disruption based on nodes (a percentage or number of nodes that can be disrupted at a time) and schedule (excluding certain times from disrupting nodes). +You can set Disruption Budgets in a `NodePool` spec. +Here is an example: + +```yaml +disruption: + consolidationPolicy: WhenEmpty + expireAfter: 1440h + budgets: + - nodes: 15% + - nodes: "3" + - nodes: "0" + schedule: "0 7 * * sat-sun" + duration: 12h +``` + +The `disruption` settings define a few fields that indicate the state of a node that should be disrupted. +The `consolidationPolicy` field indicates that a node should be disrupted if the node is either underutilized (`WhenUnderutilized`) or not running any pods (`WhenEmpty`). +With `expireAfter` set to `1440` hours, the node expires after 60 days. +Extending those values causes longer times without disruption. + +Settings for budgets in the above example include the following: + +* **Percentage of nodes**: From the first `nodes` setting, only `15%` of the NodePool’s nodes can be disrupted at a time. +* **Number of nodes**: The second `nodes` setting limits the number of nodes that can be disrupted at a time to `3`. +* **Schedule**: The third `nodes` setting uses schedule to say that zero disruptions (`0`) are allowed starting at 7am on Saturday and Sunday and continues for 12 hours. +The format of the schedule follows the `crontab` format for identifying dates and times. +See the [crontab](https://man7.org/linux/man-pages/man5/crontab.5.html) page for information on the supported values for these fields. + +As with all disruption settings, keep in mind that avoiding updated AMIs for your nodes can result in not getting fixes for known security risks and bugs. +You need to balance that with your desire to not risk breaking the workloads on your cluster. + +## Follow-up + +The Karpenter project continues to add features to give you greater control over AMI upgrades on your clusters. +If you have opinions about features you would like to see to manage AMIs with Karpenter, feel free to enter a Karpenter [New Issue](https://github.com/aws/karpenter-provider-aws/issues/new/choose). diff --git a/website/content/en/v0.32/troubleshooting.md b/website/content/en/v0.32/troubleshooting.md index db6a5c867350..cb51859789d9 100644 --- a/website/content/en/v0.32/troubleshooting.md +++ b/website/content/en/v0.32/troubleshooting.md @@ -105,14 +105,14 @@ Karpenter v0.26.1+ introduced the `karpenter-crd` helm chart. When installing th - In the case of `invalid ownership metadata; label validation error: missing key "app.kubernetes.io/managed-by": must be set to "Helm"` run: ```shell -kubectl label crd awsnodetemplates.karpenter.k8s.aws provisioners.karpenter.sh machines.karpenter.sh app.kubernetes.io/managed-by=Helm --overwrite +kubectl label crd ec2nodeclasses.karpenter.k8s.aws nodepools.karpenter.sh nodeclaims.karpenter.sh app.kubernetes.io/managed-by=Helm --overwrite ``` - In the case of `annotation validation error: missing key "meta.helm.sh/release-namespace": must be set to "karpenter"` run: ```shell -kubectl annotate crd awsnodetemplates.karpenter.k8s.aws provisioners.karpenter.sh machines.karpenter.sh meta.helm.sh/release-name=karpenter-crd --overwrite -kubectl annotate crd awsnodetemplates.karpenter.k8s.aws provisioners.karpenter.sh machines.karpenter.sh meta.helm.sh/release-namespace=karpenter --overwrite +kubectl annotate crd ec2nodeclasses.karpenter.k8s.aws nodepools.karpenter.sh nodeclaims.karpenter.sh meta.helm.sh/release-name=karpenter-crd --overwrite +kubectl annotate crd ec2nodeclasses.karpenter.k8s.aws nodepools.karpenter.sh nodeclaims.karpenter.sh meta.helm.sh/release-namespace=karpenter --overwrite ``` ## Uninstallation @@ -284,7 +284,23 @@ Karpenter does not support [in-tree storage plugins](https://kubernetes.io/blog/ #### Pods were scheduled due to a race condition in Kubernetes -Due to [this race condition in Kubernetes](https://github.com/kubernetes/kubernetes/issues/95911), it's possible that the scheduler and the CSINode can race during node registration such that the scheduler assumes that a node can mount more volumes than the node attachments support. There is currently no solve for this problem other than enforcing `toplogySpreadConstraints` and `podAntiAffinity` on your workloads that use PVCs such that you attempt to reduce the number of PVCs that schedule to a given node. +Due to [this race condition in Kubernetes](https://github.com/kubernetes/kubernetes/issues/95911), it's possible that the scheduler and the CSINode can race during node registration such that the scheduler assumes that a node can mount more volumes than the node attachments support. There is currently no universal solve for this problem other than enforcing `toplogySpreadConstraints` and `podAntiAffinity` on your workloads that use PVCs such that you attempt to reduce the number of PVCs that schedule to a given node. + +The following is a list of known CSI drivers which support a startupTaint to eliminate this issue: +- [aws-ebs-csi-driver](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/master/docs/install.md#configure-node-startup-taint) +- [aws-efs-csi-driver](https://github.com/kubernetes-sigs/aws-efs-csi-driver/tree/master/docs#configure-node-startup-taint) + +These taints should be configured via `startupTaints` on your `NodePool`. For example, to enable this for EBS, add the following to your `NodePool`: +```yaml +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +spec: + template: + spec: + startupTaints: + - key: ebs.csi.aws.com/agent-not-ready + effect: NoExecute +``` ### CNI is unable to allocate IPs to pods @@ -358,7 +374,7 @@ then the following solution(s) may resolve your issue. #### Solution(s) 1. Verify that the instance role of the Windows node includes the RBAC permission group `eks:kube-proxy-windows` as shown below. This group is required for Windows nodes because in Windows, `kube-proxy` runs as a process on the node, and as such, the node requires the necessary RBAC cluster permissions to allow access to the resources required by `kube-proxy`. - For more information, see https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html. + For more information, see https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html. ```yaml ... username: system:node:{{EC2PrivateDNSName}} @@ -642,7 +658,7 @@ Karpenter [doesn't currently support draining and terminating on spot rebalance These two components do not share information between each other, meaning if you have drain and terminate functionality enabled on NTH, NTH may remove a node for a spot rebalance recommendation. Karpenter will replace the node to fulfill the pod capacity that was being fulfilled by the old node; however, Karpenter won't be aware of the reason that that node was terminated. This means that Karpenter may launch the same instance type that was just deprovisioned, causing a spot rebalance recommendation to be sent again. This can result in very short-lived instances where NTH continually removes nodes and Karpeneter re-launches the same instance type over and over again. -Karpenter doesn't recommend reacting to spot rebalance recommendations when running Karpenter with spot nodes; however, if you absolutely require this functionality, note that the above scenario is possible. +Karpenter doesn't recommend reacting to spot rebalance recommendations when running Karpenter with spot nodes; however, if you absolutely require this functionality, note that the above scenario is possible. Spot instances are time limited and, therefore, interruptible. When a signal is sent by AWS, it triggers actions from NTH and Karpenter, where the former signals a shutdown and the later provisions, creating a recursive situation. This can be mitigated by either completely removing NTH or by setting the following values: diff --git a/website/content/en/v0.32/upgrading/compatibility.md b/website/content/en/v0.32/upgrading/compatibility.md index 3b0ea36e3f2f..836df2552a15 100644 --- a/website/content/en/v0.32/upgrading/compatibility.md +++ b/website/content/en/v0.32/upgrading/compatibility.md @@ -15,9 +15,9 @@ Before you begin upgrading Karpenter, consider Karpenter compatibility issues re [comment]: <> (the content below is generated from hack/docs/compataiblitymetrix_gen_docs.go) -| KUBERNETES | 1.23 | 1.24 | 1.25 | 1.26 | 1.27 | 1.28 | -|------------|---------|---------|---------|---------|---------|---------| -| karpenter | 0.21.x+ | 0.21.x+ | 0.25.x+ | 0.28.x+ | 0.28.x+ | 0.31.x+ | +| KUBERNETES | 1.23 | 1.24 | 1.25 | 1.26 | 1.27 | 1.28 | 1.29 | +|------------|----------|----------|----------|----------|----------|----------|------------| +| karpenter | \>= 0.21 | \>= 0.21 | \>= 0.25 | \>= 0.28 | \>= 0.28 | \>= 0.31 | \>= 0.34.0 | [comment]: <> (end docs generated content from hack/docs/compataiblitymetrix_gen_docs.go) diff --git a/website/content/en/v0.32/upgrading/upgrade-guide.md b/website/content/en/v0.32/upgrading/upgrade-guide.md index a36b8e0a36c4..5deb0741f0b2 100644 --- a/website/content/en/v0.32/upgrading/upgrade-guide.md +++ b/website/content/en/v0.32/upgrading/upgrade-guide.md @@ -28,12 +28,12 @@ If you get the error `invalid ownership metadata; label validation error:` while In general, you can reapply the CRDs in the `crds` directory of the Karpenter helm chart: ```shell -kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.5/pkg/apis/crds/karpenter.sh_provisioners.yaml -kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.5/pkg/apis/crds/karpenter.sh_machines.yaml -kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.5/pkg/apis/crds/karpenter.k8s.aws_awsnodetemplates.yaml -kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.5/pkg/apis/crds/karpenter.sh_nodepools.yaml -kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.5/pkg/apis/crds/karpenter.sh_nodeclaims.yaml -kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.5/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml +kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.9/pkg/apis/crds/karpenter.sh_provisioners.yaml +kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.9/pkg/apis/crds/karpenter.sh_machines.yaml +kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.9/pkg/apis/crds/karpenter.k8s.aws_awsnodetemplates.yaml +kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.9/pkg/apis/crds/karpenter.sh_nodepools.yaml +kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.9/pkg/apis/crds/karpenter.sh_nodeclaims.yaml +kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.9/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml ``` ### Upgrading to v0.32.0+ @@ -43,7 +43,7 @@ Karpenter v0.32.0 introduces v1beta1 APIs, including _significant_ changes to th This version includes **dual support** for both alpha and beta APIs to ensure that you can slowly migrate your existing Provisioner, AWSNodeTemplate, and Machine alpha APIs to the newer NodePool, EC2NodeClass, and NodeClaim beta APIs. -Note that if you are rolling back after upgrading to v0.32.0, note that v0.31.2 and v0.31.3 are the only versions that support handling rollback after you have deployed the v1beta1 APIs to your cluster. +Note that if you are rolling back after upgrading to v0.32.0, note that v0.31.4 is the only version that supports handling rollback after you have deployed the v1beta1 APIs to your cluster. {{% /alert %}} * Karpenter now serves the webhook prometheus metrics server on port `8001`. If this port is already in-use on the pod or you are running in `hostNetworking` mode, you may need to change this port value. You can configure this port value through the `WEBHOOK_METRICS_PORT` environment variable or the `webhook.metrics.port` value if installing via Helm. diff --git a/website/content/en/v0.32/upgrading/v1beta1-controller-policy.json b/website/content/en/v0.32/upgrading/v1beta1-controller-policy.json index c65c402f9c21..fb3c335631d5 100644 --- a/website/content/en/v0.32/upgrading/v1beta1-controller-policy.json +++ b/website/content/en/v0.32/upgrading/v1beta1-controller-policy.json @@ -142,7 +142,7 @@ { "Sid": "AllowInterruptionQueueActions", "Effect": "Allow", - "Resource": "arn:aws:sqs:${AWS_REGION}:${AWS_ACCOUNT_ID}:${CLUSTER_NAME}", + "Resource": "arn:${AWS_PARTITION}:sqs:${AWS_REGION}:${AWS_ACCOUNT_ID}:${CLUSTER_NAME}", "Action": [ "sqs:DeleteMessage", "sqs:GetQueueUrl", diff --git a/website/content/en/v0.32/upgrading/v1beta1-migration.md b/website/content/en/v0.32/upgrading/v1beta1-migration.md index 65b2e1d3eb4a..84e5fb5042ad 100644 --- a/website/content/en/v0.32/upgrading/v1beta1-migration.md +++ b/website/content/en/v0.32/upgrading/v1beta1-migration.md @@ -38,7 +38,7 @@ This procedure assumes you are running the Karpenter controller on cluster and w ``` {{% alert title="Warning" color="warning" %}} - v0.31.2 introduces minor changes to Karpenter so that rollback from v0.32.0 is supported. If you are coming from some other patch version of minor version v0.31.x, note that v0.31.2 and v0.31.3 are the _only_ patch versions that support rollback for v1beta1. + v0.31.2 introduces minor changes to Karpenter so that rollback from v0.32.0 is supported. If you are coming from some other patch version of minor version v0.31.x, note that v0.31.4 is the _only_ patch version that supports rollback for v1beta1. {{% /alert %}} 2. Review for breaking changes: If you are already running Karpenter v0.31.x, you can skip this step. If you are running an earlier Karpenter version, you need to review the upgrade notes for each minor release. @@ -47,7 +47,7 @@ This procedure assumes you are running the Karpenter controller on cluster and w ```bash export KARPENTER_NAMESPACE=karpenter - export KARPENTER_VERSION=v0.32.5 + export KARPENTER_VERSION=v0.32.9 export AWS_PARTITION="aws" # if you are not using standard partitions, you may need to configure to aws-cn / aws-us-gov export CLUSTER_NAME="${USER}-karpenter-demo" export AWS_REGION="us-west-2" @@ -60,7 +60,7 @@ This procedure assumes you are running the Karpenter controller on cluster and w ```bash TEMPOUT=$(mktemp) - curl -fsSL https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.5/website/content/en/preview/upgrading/v1beta1-controller-policy.json > ${TEMPOUT} + curl -fsSL https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.9/website/content/en/preview/upgrading/v1beta1-controller-policy.json > ${TEMPOUT} AWS_REGION=${AWS_REGION:=$AWS_DEFAULT_REGION} # use the default region if AWS_REGION isn't defined POLICY_DOCUMENT=$(envsubst < ${TEMPOUT}) @@ -71,15 +71,15 @@ This procedure assumes you are running the Karpenter controller on cluster and w aws iam attach-role-policy --role-name "${ROLE_NAME}" --policy-arn "${POLICY_ARN}" ``` -5. Apply the v0.32.5 Custom Resource Definitions (CRDs): +5. Apply the v0.32.9 Custom Resource Definitions (CRDs): ```bash - kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.5/pkg/apis/crds/karpenter.sh_provisioners.yaml - kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.5/pkg/apis/crds/karpenter.sh_machines.yaml - kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.5/pkg/apis/crds/karpenter.k8s.aws_awsnodetemplates.yaml - kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.5/pkg/apis/crds/karpenter.sh_nodepools.yaml - kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.5/pkg/apis/crds/karpenter.sh_nodeclaims.yaml - kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.5/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml + kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.9/pkg/apis/crds/karpenter.sh_provisioners.yaml + kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.9/pkg/apis/crds/karpenter.sh_machines.yaml + kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.9/pkg/apis/crds/karpenter.k8s.aws_awsnodetemplates.yaml + kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.9/pkg/apis/crds/karpenter.sh_nodepools.yaml + kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.9/pkg/apis/crds/karpenter.sh_nodeclaims.yaml + kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.9/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml ``` 6. Upgrade Karpenter to the new version: @@ -321,6 +321,8 @@ apiVersion: karpenter.sh/v1beta1 kind: NodePool ... nodeClassRef: + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass name: default ``` @@ -756,7 +758,7 @@ Karpenter v1beta1 introduces changes to some common labels, annotations, and sta v1beta1 introduces changes to the IAM permissions assigned to the Karpenter controller policy used when deploying Karpenter to your cluster with [IRSA](https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/setting-up-enable-IAM.html) or [EKS Pod Identity](https://docs.aws.amazon.com/eks/latest/userguide/pod-identities.html). -You can see a full example of the v1beta1 required controller permissions by viewing the [v1beta1 Controller Policy](https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.5/website/content/en/preview/upgrading/v1beta1-controller-policy.json). +You can see a full example of the v1beta1 required controller permissions by viewing the [v1beta1 Controller Policy](https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.32.9/website/content/en/preview/upgrading/v1beta1-controller-policy.json). Additionally, read more detail about the full set of permissions assigned to the Karpenter controller policy in the [CloudFormation Reference Guide]({{< ref "../reference/cloudformation" >}}). diff --git a/website/content/en/v0.33/concepts/disruption.md b/website/content/en/v0.33/concepts/disruption.md index c4fcf66ef923..408e591147a0 100644 --- a/website/content/en/v0.33/concepts/disruption.md +++ b/website/content/en/v0.33/concepts/disruption.md @@ -13,7 +13,7 @@ The finalizer blocks deletion of the node object while the Termination Controlle ### Disruption Controller -Karpenter automatically discovers disruptable nodes and spins up replacements when needed. Karpenter disrupts nodes by executing one [automatic method](#automatic-methods) at a time, in order of Expiration, Drift, and then Consolidation. Each method varies slightly, but they all follow the standard disruption process: +Karpenter automatically discovers disruptable nodes and spins up replacements when needed. Karpenter disrupts nodes by executing one [automated method](#automated-methods) at a time, in order of Expiration, Drift, and then Consolidation. Each method varies slightly, but they all follow the standard disruption process: 1. Identify a list of prioritized candidates for the disruption method. * If there are [pods that cannot be evicted](#pod-eviction) on the node, Karpenter will ignore the node and try disrupting it later. * If there are no disruptable nodes, continue to the next disruption method. @@ -28,7 +28,7 @@ Karpenter automatically discovers disruptable nodes and spins up replacements wh When a Karpenter node is deleted, the Karpenter finalizer will block deletion and the APIServer will set the `DeletionTimestamp` on the node, allowing Karpenter to gracefully shutdown the node, modeled after [Kubernetes Graceful Node Shutdown](https://kubernetes.io/docs/concepts/architecture/nodes/#graceful-node-shutdown). Karpenter's graceful shutdown process will: 1. Add the `karpenter.sh/disruption:NoSchedule` taint to the node to prevent pods from scheduling to it. -2. Begin evicting the pods on the node with the [Kubernetes Eviction API](https://kubernetes.io/docs/concepts/scheduling-eviction/api-eviction/) to respect PDBs, while ignoring all non-daemonset pods and [static pods](https://kubernetes.io/docs/tasks/configure-pod-container/static-pod/). Wait for the node to be fully drained before proceeding to Step (3). +2. Begin evicting the pods on the node with the [Kubernetes Eviction API](https://kubernetes.io/docs/concepts/scheduling-eviction/api-eviction/) to respect PDBs, while ignoring all daemonset pods and [static pods](https://kubernetes.io/docs/tasks/configure-pod-container/static-pod/). Wait for the node to be fully drained before proceeding to Step (3). * While waiting, if the underlying NodeClaim for the node no longer exists, remove the finalizer to allow the APIServer to delete the node, completing termination. 3. Terminate the NodeClaim in the Cloud Provider. 4. Remove the finalizer from the node to allow the APIServer to delete the node, completing termination. @@ -209,7 +209,7 @@ Voluntary node removal does not include [Interruption]({{} ### Node-Level Controls -Nodes can be opted out of consolidation disruption by setting the annotation `karpenter.sh/do-not-disrupt: "true"` on the node. +You can block Karpenter from voluntarily choosing to disrupt certain nodes by setting the `karpenter.sh/do-not-disrupt: "true"` annotation on the node. This will prevent disruption actions on the node. ```yaml apiVersion: v1 @@ -221,7 +221,7 @@ metadata: #### Example: Disable Disruption on a NodePool -NodePool `.spec.annotations` allow you to set annotations that will be applied to all nodes launched by this NodePool. By setting the annotation `karpenter.sh/do-not-disrupt: "true"` on the NodePool, you will selectively prevent all nodes launched by this NodePool from being considered in consolidation calculations. +NodePool `.spec.annotations` allow you to set annotations that will be applied to all nodes launched by this NodePool. By setting the annotation `karpenter.sh/do-not-disrupt: "true"` on the NodePool, you will selectively prevent all nodes launched by this NodePool from being considered in disruption actions. ```yaml apiVersion: karpenter.sh/v1beta1 diff --git a/website/content/en/v0.33/concepts/nodeclasses.md b/website/content/en/v0.33/concepts/nodeclasses.md index 12c4c60a373f..a5fbbd0636cb 100644 --- a/website/content/en/v0.33/concepts/nodeclasses.md +++ b/website/content/en/v0.33/concepts/nodeclasses.md @@ -35,7 +35,7 @@ spec: # Each term in the array of subnetSelectorTerms is ORed together # Within a single term, all conditions are ANDed subnetSelectorTerms: - # Select on any subnet that has the "karpenter.sh/discovery: ${CLUSTER_NAME}" + # Select on any subnet that has the "karpenter.sh/discovery: ${CLUSTER_NAME}" # AND the "environment: test" tag OR any subnet with ID "subnet-09fa4a0a8f233a921" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" @@ -46,8 +46,8 @@ spec: # Each term in the array of securityGroupSelectorTerms is ORed together # Within a single term, all conditions are ANDed securityGroupSelectorTerms: - # Select on any security group that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag - # AND the "environment: test" tag OR any security group with the "my-security-group" name + # Select on any security group that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any security group with the "my-security-group" name # OR any security group with ID "sg-063d7acfb4b06c82c" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" @@ -70,8 +70,8 @@ spec: # Each term in the array of amiSelectorTerms is ORed together # Within a single term, all conditions are ANDed amiSelectorTerms: - # Select on any AMI that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag - # AND the "environment: test" tag OR any AMI with the "my-ami" name + # Select on any AMI that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any AMI with the "my-ami" name # OR any AMI with ID "ami-123" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" @@ -246,7 +246,7 @@ This selection logic is modeled as terms, where each term contains multiple cond ```yaml subnetSelectorTerms: - # Select on any subnet that has the "karpenter.sh/discovery: ${CLUSTER_NAME}" + # Select on any subnet that has the "karpenter.sh/discovery: ${CLUSTER_NAME}" # AND the "environment: test" tag OR any subnet with ID "subnet-09fa4a0a8f233a921" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" @@ -313,8 +313,8 @@ This selection logic is modeled as terms, where each term contains multiple cond ```yaml securityGroupSelectorTerms: - # Select on any security group that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag - # AND the "environment: test" tag OR any security group with the "my-security-group" name + # Select on any security group that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any security group with the "my-security-group" name # OR any security group with ID "sg-063d7acfb4b06c82c" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" @@ -400,8 +400,8 @@ This selection logic is modeled as terms, where each term contains multiple cond ```yaml amiSelectorTerms: - # Select on any AMI that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag - # AND the "environment: test" tag OR any AMI with the "my-ami" name + # Select on any AMI that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any AMI with the "my-ami" name # OR any AMI with ID "ami-123" - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" @@ -422,6 +422,7 @@ AMIs may be specified by any AWS tag, including `Name`. Selecting by tag or by n If `amiSelectorTerms` match more than one AMI, Karpenter will automatically determine which AMI best fits the workloads on the launched worker node under the following constraints: * When launching nodes, Karpenter automatically determines which architecture a custom AMI is compatible with and will use images that match an instanceType's requirements. + * Note that Karpenter **cannot** detect any requirement other than architecture. If you need to specify different AMIs for different kind of nodes (e.g. accelerated GPU AMIs), you should use a separate `EC2NodeClass`. * If multiple AMIs are found that can be used, Karpenter will choose the latest one. * If no AMIs are found that can be used, then no nodes will be provisioned. {{% /alert %}} @@ -481,7 +482,7 @@ Specify using ids: ## spec.role -`Role` is an optional field and is necessary to tell Karpenter which identity nodes from this `EC2NodeClass` should assume. You must specify one of `role` or `instanceProfile` when creating a Karpenter `EC2NodeClass`. If using the [Karpenter Getting Started Guide]({{}}) to deploy Karpenter, you can use the `KarpenterNodeRole-$CLUSTER_NAME` role provisioned by that process. +`Role` is an optional field and tells Karpenter which IAM identity nodes should assume. You must specify one of `role` or `instanceProfile` when creating a Karpenter `EC2NodeClass`. If using the [Karpenter Getting Started Guide]({{}}) to deploy Karpenter, you can use the `KarpenterNodeRole-$CLUSTER_NAME` role provisioned by that process. ```yaml spec: @@ -490,7 +491,9 @@ spec: ## spec.instanceProfile -`InstanceProfile` is an optional field and is necessary to tell Karpenter which identity nodes from this `EC2NodeClass` should assume. You must specify one of `role` or `instanceProfile` when creating a Karpenter `EC2NodeClasss`. If you use the `instanceProfile` field instead of `role`, Karpenter will not manage the InstanceProfile on your behalf. +`InstanceProfile` is an optional field and tells Karpenter which IAM identity nodes should assume. You must specify one of `role` or `instanceProfile` when creating a Karpenter `EC2NodeClass`. If you use the `instanceProfile` field instead of `role`, Karpenter will not manage the InstanceProfile on your behalf; instead, it expects that you have pre-provisioned an IAM instance profile and assigned it a role. + +You can provision and assign a role to an IAM instance profile using [CloudFormation](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iam-instanceprofile.html) or by using the [`aws iam create-instance-profile`](https://docs.aws.amazon.com/cli/latest/reference/iam/create-instance-profile.html) and [`aws iam add-role-to-instance-profile`](https://docs.aws.amazon.com/cli/latest/reference/iam/add-role-to-instance-profile.html) commands in the CLI. {{% alert title="Note" color="primary" %}} @@ -506,6 +509,7 @@ Karpenter adds tags to all resources it creates, including EC2 Instances, EBS vo Name: karpenter.sh/nodeclaim: karpenter.sh/nodepool: +karpenter.k8s.aws/ec2nodeclass: kubernetes.io/cluster/: owned ``` diff --git a/website/content/en/v0.33/concepts/nodepools.md b/website/content/en/v0.33/concepts/nodepools.md index 7ff7f1f64402..607311c5ef6e 100644 --- a/website/content/en/v0.33/concepts/nodepools.md +++ b/website/content/en/v0.33/concepts/nodepools.md @@ -46,6 +46,8 @@ spec: spec: # References the Cloud Provider's NodeClass resource, see your cloud provider specific documentation nodeClassRef: + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass name: default # Provisioned nodes will have these taints @@ -127,12 +129,12 @@ spec: # If using 'WhenUnderutilized', Karpenter will consider all nodes for consolidation and attempt to remove or replace Nodes when it discovers that the Node is underutilized and could be changed to reduce cost # If using `WhenEmpty`, Karpenter will only consider nodes for consolidation that contain no workload pods consolidationPolicy: WhenUnderutilized | WhenEmpty - + # The amount of time Karpenter should wait after discovering a consolidation decision # This value can currently only be set when the consolidationPolicy is 'WhenEmpty' # You can choose to disable consolidation entirely by setting the string value 'Never' here consolidateAfter: 30s - + # The amount of time a Node can live on the cluster before being removed # Avoiding long-running Nodes helps to reduce security vulnerabilities as well as to reduce the chance of issues that can plague Nodes with long uptimes such as file fragmentation or memory leaks from system processes # You can choose to disable expiration entirely by setting the string value 'Never' here @@ -285,7 +287,12 @@ kubelet: Karpenter will automatically configure the system and kube reserved resource requests on the fly on your behalf. These requests are used to configure your node and to make scheduling decisions for your pods. If you have specific requirements or know that you will have additional capacity requirements, you can optionally override the `--system-reserved` configuration defaults with the `.spec.template.spec.kubelet.systemReserved` values and the `--kube-reserved` configuration defaults with the `.spec.template.spec.kubelet.kubeReserved` values. -For more information on the default `--system-reserved` and `--kube-reserved` configuration refer to the [Kubelet Docs](https://kubernetes.io/docs/tasks/administer-cluster/reserve-compute-resources/#kube-reserved) +{{% alert title="Note" color="primary" %}} +Karpenter considers these reserved resources when computing the allocatable ephemeral storage on a given instance type. +If `kubeReserved` is not specified, Karpenter will compute the default reserved [CPU](https://github.com/awslabs/amazon-eks-ami/blob/db28da15d2b696bc08ac3aacc9675694f4a69933/files/bootstrap.sh#L251) and [memory](https://github.com/awslabs/amazon-eks-ami/blob/db28da15d2b696bc08ac3aacc9675694f4a69933/files/bootstrap.sh#L235) resources for the purpose of ephemeral storage computation. +These defaults are based on the defaults on Karpenter's supported AMI families, which are not the same as the kubelet defaults. +You should be aware of the CPU and memory default calculation when using Custom AMI Families. If they don't align, there may be a difference in Karpenter's computed allocatable ephemeral storage and the actually ephemeral storage available on the node. +{{% /alert %}} ### Eviction Thresholds @@ -431,7 +438,7 @@ Review the [Kubernetes core API](https://github.com/kubernetes/api/blob/37748cca Karpenter allows you to describe NodePool preferences through a `weight` mechanism similar to how weight is described with [pod and node affinities](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity). -For more information on weighting NodePools, see the [Weighting NodePools section]({{}}) in the scheduling details. +For more information on weighting NodePools, see the [Weighted NodePools section]({{}}) in the scheduling docs. ## Examples diff --git a/website/content/en/v0.33/concepts/scheduling.md b/website/content/en/v0.33/concepts/scheduling.md index 90339674f24c..2711e7afd983 100755 --- a/website/content/en/v0.33/concepts/scheduling.md +++ b/website/content/en/v0.33/concepts/scheduling.md @@ -214,7 +214,7 @@ All examples below assume that the NodePool doesn't have constraints to prevent - matchExpressions: - key: "topology.kubernetes.io/zone" operator: "In" - values: ["us-west-2a, us-west-2b"] + values: ["us-west-2a", "us-west-2b"] - key: "topology.kubernetes.io/zone" operator: "In" values: ["us-west-2b"] @@ -225,7 +225,7 @@ Changing the second operator to `NotIn` would allow the pod to run in `us-west-2 ```yaml - key: "topology.kubernetes.io/zone" operator: "In" - values: ["us-west-2a, us-west-2b"] + values: ["us-west-2a", "us-west-2b"] - key: "topology.kubernetes.io/zone" operator: "NotIn" values: ["us-west-2b"] @@ -243,7 +243,7 @@ Here, if `us-west-2a` is not available, the second term will cause the pod to ru - matchExpressions: # OR - key: "topology.kubernetes.io/zone" # AND operator: "In" - values: ["us-west-2a, us-west-2b"] + values: ["us-west-2a", "us-west-2b"] - key: "topology.kubernetes.io/zone" # AND operator: "NotIn" values: ["us-west-2b"] @@ -352,6 +352,10 @@ The three supported `topologyKey` values that Karpenter supports are: See [Pod Topology Spread Constraints](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/) for details. +{{% alert title="Note" color="primary" %}} +NodePools do not attempt to balance or rebalance the availability zones for their nodes. Availability zone balancing may be achieved by defining zonal Topology Spread Constraints for Pods that require multi-zone durability, and NodePools will respect these constraints while optimizing for compute costs. +{{% /alert %}} + ## Pod affinity/anti-affinity By using the `podAffinity` and `podAntiAffinity` configuration on a pod spec, you can inform the Karpenter scheduler of your desire for pods to schedule together or apart with respect to different topology domains. For example: diff --git a/website/content/en/v0.33/faq.md b/website/content/en/v0.33/faq.md index bf3102323d98..e757043ce6b1 100644 --- a/website/content/en/v0.33/faq.md +++ b/website/content/en/v0.33/faq.md @@ -14,7 +14,7 @@ See [Configuring NodePools]({{< ref "./concepts/#configuring-nodepools" >}}) for AWS is the first cloud provider supported by Karpenter, although it is designed to be used with other cloud providers as well. ### Can I write my own cloud provider for Karpenter? -Yes, but there is no documentation yet for it. Start with Karpenter's GitHub [cloudprovider](https://github.com/aws/karpenter-core/tree/v0.33.1/pkg/cloudprovider) documentation to see how the AWS provider is built, but there are other sections of the code that will require changes too. +Yes, but there is no documentation yet for it. Start with Karpenter's GitHub [cloudprovider](https://github.com/aws/karpenter-core/tree/v0.33.4/pkg/cloudprovider) documentation to see how the AWS provider is built, but there are other sections of the code that will require changes too. ### What operating system nodes does Karpenter deploy? Karpenter uses the OS defined by the [AMI Family in your EC2NodeClass]({{< ref "./concepts/nodeclasses#specamifamily" >}}). @@ -26,7 +26,7 @@ Karpenter has multiple mechanisms for configuring the [operating system]({{< ref Karpenter is flexible to multi-architecture configurations using [well known labels]({{< ref "./concepts/scheduling/#supported-labels">}}). ### What RBAC access is required? -All the required RBAC rules can be found in the helm chart template. See [clusterrole-core.yaml](https://github.com/aws/karpenter/blob/v0.33.1/charts/karpenter/templates/clusterrole-core.yaml), [clusterrole.yaml](https://github.com/aws/karpenter/blob/v0.33.1/charts/karpenter/templates/clusterrole.yaml), [rolebinding.yaml](https://github.com/aws/karpenter/blob/v0.33.1/charts/karpenter/templates/rolebinding.yaml), and [role.yaml](https://github.com/aws/karpenter/blob/v0.33.1/charts/karpenter/templates/role.yaml) files for details. +All the required RBAC rules can be found in the helm chart template. See [clusterrole-core.yaml](https://github.com/aws/karpenter/blob/v0.33.4/charts/karpenter/templates/clusterrole-core.yaml), [clusterrole.yaml](https://github.com/aws/karpenter/blob/v0.33.4/charts/karpenter/templates/clusterrole.yaml), [rolebinding.yaml](https://github.com/aws/karpenter/blob/v0.33.4/charts/karpenter/templates/rolebinding.yaml), and [role.yaml](https://github.com/aws/karpenter/blob/v0.33.4/charts/karpenter/templates/role.yaml) files for details. ### Can I run Karpenter outside of a Kubernetes cluster? Yes, as long as the controller has network and IAM/RBAC access to the Kubernetes API and your provider API. @@ -179,10 +179,10 @@ Yes, see the [KubeletConfiguration Section in the NodePool docs]({{}} by configuring an `amiSelector` that references the AMI name. +You can specify the [Amazon EKS optimized AMI](https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-windows-ami.html) with Windows Server 2022 Full for Kubernetes 1.28 by configuring an `amiSelector` that references the AMI name. ```yaml amiSelectorTerms: - - name: Windows_Server-2022-English-Full-EKS_Optimized-{{< param "latest_k8s_version" >}}* + - name: Windows_Server-2022-English-Full-EKS_Optimized-1.28* ``` ### Can I use Karpenter to scale my workload's pods? diff --git a/website/content/en/v0.33/getting-started/_index.md b/website/content/en/v0.33/getting-started/_index.md index a70f6bbc422b..cebdf66a99fd 100644 --- a/website/content/en/v0.33/getting-started/_index.md +++ b/website/content/en/v0.33/getting-started/_index.md @@ -21,6 +21,6 @@ Learn more about Karpenter and how to get started below. * [Karpenter EKS Best Practices](https://aws.github.io/aws-eks-best-practices/karpenter/) guide * [EC2 Spot Workshop for Karpenter](https://ec2spotworkshops.com/karpenter.html) * [EKS Karpenter Workshop](https://www.eksworkshop.com/docs/autoscaling/compute/karpenter/) -* [Advanced EKS Immersion Karpenter Workshop](https://catalog.workshops.aws/eks-advanced/karpenter/) +* [Advanced EKS Immersion Karpenter Workshop](https://catalog.us-east-1.prod.workshops.aws/workshops/76a5dd80-3249-4101-8726-9be3eeee09b2/en-US/autoscaling/karpenter) * [Karpenter Blueprints](https://github.com/aws-samples/karpenter-blueprints) * [Tutorial: Run Kubernetes Clusters for Less with Amazon EC2 Spot and Karpenter](https://community.aws/tutorials/run-kubernetes-clusters-for-less-with-amazon-ec2-spot-and-karpenter#step-6-optional-simulate-spot-interruption) diff --git a/website/content/en/v0.33/getting-started/getting-started-with-karpenter/_index.md b/website/content/en/v0.33/getting-started/getting-started-with-karpenter/_index.md index 9c9fbd2cbd18..b27846756420 100644 --- a/website/content/en/v0.33/getting-started/getting-started-with-karpenter/_index.md +++ b/website/content/en/v0.33/getting-started/getting-started-with-karpenter/_index.md @@ -45,8 +45,8 @@ After setting up the tools, set the Karpenter and Kubernetes version: ```bash export KARPENTER_NAMESPACE=kube-system -export KARPENTER_VERSION=v0.33.1 -export K8S_VERSION={{< param "latest_k8s_version" >}} +export KARPENTER_VERSION=v0.33.4 +export K8S_VERSION=1.28 ``` Then set the following environment variable: @@ -58,7 +58,7 @@ If you open a new shell to run steps in this procedure, you need to set some or To remind yourself of these values, type: ```bash -echo $KARPENTER_NAMESPACE $KARPENTER_VERSION $K8S_VERSION $CLUSTER_NAME $AWS_DEFAULT_REGION $AWS_ACCOUNT_ID $TEMPOUT +echo "${KARPENTER_NAMESPACE}" "${KARPENTER_VERSION}" "${K8S_VERSION}" "${CLUSTER_NAME}" "${AWS_DEFAULT_REGION}" "${AWS_ACCOUNT_ID}" "${TEMPOUT}" "${ARM_AMI_ID}" "${AMD_AMI_ID}" "${GPU_AMI_ID}" ``` {{% /alert %}} @@ -188,6 +188,7 @@ com.amazonaws..s3 – For pulling container images com.amazonaws..sts – For IAM roles for service accounts com.amazonaws..ssm - For resolving default AMIs com.amazonaws..sqs - For accessing SQS if using interruption handling +com.amazonaws..eks - For Karpenter to discover the cluster endpoint ``` If you do not currently have these endpoints surfaced in your VPC, you can add the endpoints by running @@ -233,7 +234,7 @@ caused by: Post "https://api.pricing.us-east-1.amazonaws.com/": dial tcp 52.94.2 Kubernetes uses [FlowSchemas](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/#flowschema) and [PriorityLevelConfigurations](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/#prioritylevelconfiguration) to map calls to the API server into buckets which determine each user agent's throttling limits. -By default, Karpenter is installed into the `kube-system` namespace, which leverages the `system-leader-election` and `kube-system-service-accounts` [FlowSchemas] to map calls from the `kube-system` namespace to the `leader-election` and `workload-high` PriorityLevelConfigurations respectively. By putting Karpenter in these PriorityLevelConfigurations, we ensure that Karpenter and other critical cluster components are able to run even if other components on the cluster are throttled in other PriorityLevelConfigurations. +By default, Karpenter is installed into the `kube-system` namespace, which leverages the `system-leader-election` and `kube-system-service-accounts` [FlowSchemas](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/#flowschema) to map calls from the `kube-system` namespace to the `leader-election` and `workload-high` PriorityLevelConfigurations respectively. By putting Karpenter in these PriorityLevelConfigurations, we ensure that Karpenter and other critical cluster components are able to run even if other components on the cluster are throttled in other PriorityLevelConfigurations. If you install Karpenter in a different namespace than the default `kube-system` namespace, Karpenter will not be put into these higher-priority FlowSchemas by default. Instead, you will need to create custom FlowSchemas for the namespace and service account where Karpenter is installed to ensure that requests are put into this higher PriorityLevelConfiguration. diff --git a/website/content/en/v0.33/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json b/website/content/en/v0.33/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json index 9d9cb6ad14a8..d474d01f4e16 100644 --- a/website/content/en/v0.33/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json +++ b/website/content/en/v0.33/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json @@ -484,7 +484,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by ($distribution_filter)(\n karpenter_pods_state{arch=~\"$arch\", capacity_type=~\"$capacity_type\", instance_type=~\"$instance_type\", provisioner=~\"$provisioner\"}\n)", + "expr": "sum by ($distribution_filter)(\n karpenter_pods_state{arch=~\"$arch\", capacity_type=~\"$capacity_type\", instance_type=~\"$instance_type\", nodepool=~\"$nodepool\"}\n)", "legendFormat": "{{label_name}}", "range": true, "refId": "A" @@ -588,7 +588,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "karpenter_provisioner_usage{resource_type=\"cpu\"} / karpenter_provisioner_limit{resource_type=\"cpu\"}", + "expr": "karpenter_nodepool_usage{resource_type=\"cpu\"} / karpenter_nodepool_limit{resource_type=\"cpu\"}", "format": "table", "instant": true, "legendFormat": "CPU Limit Utilization", @@ -602,7 +602,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "count by (provisioner)(karpenter_nodes_allocatable{provisioner!=\"N/A\",resource_type=\"cpu\"}) # Selects a single resource type to get node count", + "expr": "count by (nodepool)(karpenter_nodes_allocatable{nodepool!=\"N/A\",resource_type=\"cpu\"}) # Selects a single resource type to get node count", "format": "table", "hide": false, "instant": true, @@ -616,7 +616,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "karpenter_provisioner_usage{resource_type=\"memory\"} / karpenter_provisioner_limit{resource_type=\"memory\"}", + "expr": "karpenter_nodepool_usage{resource_type=\"memory\"} / karpenter_nodepool_limit{resource_type=\"memory\"}", "format": "table", "hide": false, "instant": true, @@ -631,7 +631,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum by (provisioner)(karpenter_nodes_allocatable{provisioner!=\"N/A\",resource_type=\"cpu\"})", + "expr": "sum by (nodepool)(karpenter_nodes_allocatable{nodepool!=\"N/A\",resource_type=\"cpu\"})", "format": "table", "hide": false, "instant": true, @@ -645,7 +645,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum by (provisioner)(karpenter_nodes_allocatable{provisioner!=\"N/A\",resource_type=\"memory\"})", + "expr": "sum by (nodepool)(karpenter_nodes_allocatable{nodepool!=\"N/A\",resource_type=\"memory\"})", "format": "table", "hide": false, "instant": true, @@ -653,12 +653,12 @@ "refId": "Memory Capacity" } ], - "title": "Provisioner Summary", + "title": "Nodepool Summary", "transformations": [ { "id": "seriesToColumns", "options": { - "byField": "provisioner" + "byField": "nodepool" } }, { @@ -697,7 +697,7 @@ "instance 2": 12, "job 1": 9, "job 2": 13, - "provisioner": 0, + "nodepool": 0, "resource_type 1": 10, "resource_type 2": 14 }, @@ -714,7 +714,7 @@ "instance": "", "instance 1": "", "job": "", - "provisioner": "Provisioner" + "nodepool": "Nodepool" } } } @@ -804,7 +804,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "(count(karpenter_nodes_allocatable{arch=~\"$arch\",capacity_type=\"spot\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"}) or vector(0)) / count(karpenter_nodes_allocatable{arch=~\"$arch\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"})", + "expr": "(count(karpenter_nodes_allocatable{arch=~\"$arch\",capacity_type=\"spot\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}) or vector(0)) / count(karpenter_nodes_allocatable{arch=~\"$arch\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"})", "legendFormat": "Percentage", "range": true, "refId": "A" @@ -946,7 +946,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "((karpenter_nodes_total_daemon_requests{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0) + \n(karpenter_nodes_total_pod_requests{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0)) / \nkarpenter_nodes_allocatable{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"}", + "expr": "((karpenter_nodes_total_daemon_requests{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0) + \n(karpenter_nodes_total_pod_requests{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0)) / \nkarpenter_nodes_allocatable{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}", "format": "table", "hide": false, "instant": true, @@ -961,7 +961,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "((karpenter_nodes_total_daemon_requests{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0) + \n(karpenter_nodes_total_pod_requests{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0)) / \nkarpenter_nodes_allocatable{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"}", + "expr": "((karpenter_nodes_total_daemon_requests{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0) + \n(karpenter_nodes_total_pod_requests{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0)) / \nkarpenter_nodes_allocatable{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}", "format": "table", "hide": false, "instant": true, @@ -976,7 +976,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "karpenter_nodes_total_daemon_requests{resource_type=\"pods\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"} + \nkarpenter_nodes_total_pod_requests{resource_type=\"pods\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"}", + "expr": "karpenter_nodes_total_daemon_requests{resource_type=\"pods\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} + \nkarpenter_nodes_total_pod_requests{resource_type=\"pods\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}", "format": "table", "hide": false, "instant": true, @@ -1091,9 +1091,9 @@ "os 1": true, "os 2": true, "os 3": true, - "provisioner 1": false, - "provisioner 2": true, - "provisioner 3": true, + "nodepool 1": false, + "nodepool 2": true, + "nodepool 3": true, "resource_type": true, "resource_type 1": true, "resource_type 2": true, @@ -1161,9 +1161,9 @@ "os 1": 23, "os 2": 41, "os 3": 61, - "provisioner 1": 2, - "provisioner 2": 42, - "provisioner 3": 62, + "nodepool 1": 2, + "nodepool 2": 42, + "nodepool 3": 62, "resource_type 1": 24, "resource_type 2": 43, "resource_type 3": 63, @@ -1190,7 +1190,7 @@ "instance_type": "Instance Type", "instance_type 1": "Instance Type", "node_name": "Node Name", - "provisioner 1": "Provisioner", + "nodepool 1": "Nodepool", "zone 1": "Zone" } } @@ -1237,14 +1237,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "definition": "label_values(karpenter_nodes_allocatable, provisioner)", + "definition": "label_values(karpenter_nodes_allocatable, nodepool)", "hide": 0, "includeAll": true, "multi": true, - "name": "provisioner", + "name": "nodepool", "options": [], "query": { - "query": "label_values(karpenter_nodes_allocatable, provisioner)", + "query": "label_values(karpenter_nodes_allocatable, nodepool)", "refId": "StandardVariableQuery" }, "refresh": 2, @@ -1376,8 +1376,8 @@ { "current": { "selected": true, - "text": "provisioner", - "value": "provisioner" + "text": "nodepool", + "value": "nodepool" }, "hide": 0, "includeAll": false, @@ -1411,8 +1411,8 @@ }, { "selected": true, - "text": "provisioner", - "value": "provisioner" + "text": "nodepool", + "value": "nodepool" }, { "selected": false, @@ -1420,7 +1420,7 @@ "value": "zone" } ], - "query": "arch,capacity_type,instance_type,namespace,node,provisioner,zone", + "query": "arch,capacity_type,instance_type,namespace,node,nodepool,zone", "queryValue": "", "skipUrlSync": false, "type": "custom" diff --git a/website/content/en/v0.33/getting-started/getting-started-with-karpenter/prometheus-values.yaml b/website/content/en/v0.33/getting-started/getting-started-with-karpenter/prometheus-values.yaml index f7c34fd90dea..588762a8c9dc 100644 --- a/website/content/en/v0.33/getting-started/getting-started-with-karpenter/prometheus-values.yaml +++ b/website/content/en/v0.33/getting-started/getting-started-with-karpenter/prometheus-values.yaml @@ -13,8 +13,10 @@ extraScrapeConfigs: | - role: endpoints namespaces: names: - - kube-system + - $KARPENTER_NAMESPACE relabel_configs: - - source_labels: [__meta_kubernetes_endpoint_port_name] - regex: http-metrics + - source_labels: + - __meta_kubernetes_endpoints_name + - __meta_kubernetes_endpoint_port_name action: keep + regex: karpenter;http-metrics diff --git a/website/content/en/v0.33/getting-started/getting-started-with-karpenter/scripts/add-monitoring.sh b/website/content/en/v0.33/getting-started/getting-started-with-karpenter/scripts/add-monitoring.sh deleted file mode 100755 index 6d2332b06be2..000000000000 --- a/website/content/en/v0.33/getting-started/getting-started-with-karpenter/scripts/add-monitoring.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -set -euo pipefail #fail if one step fails - -if [ "$#" -ne 1 ] -then - echo "Missing required Karpenter version. Usage: add-monitoring.sh v0.0.1" - exit 1 -fi - -export KARPENTER_VERSION=$1 - -declare -a steps=( - step01-config.sh - step09-add-prometheus-grafana.sh - step10-add-grafana-port-forward.sh - step11-grafana-get-password.sh -) - -for step in "${steps[@]}"; do - echo "$step" - source $step -done diff --git a/website/content/en/v0.33/getting-started/getting-started-with-karpenter/scripts/add-provisioner.sh b/website/content/en/v0.33/getting-started/getting-started-with-karpenter/scripts/add-provisioner.sh deleted file mode 100755 index 51f286de0c3c..000000000000 --- a/website/content/en/v0.33/getting-started/getting-started-with-karpenter/scripts/add-provisioner.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -set -euo pipefail #fail if one step fails - -declare -a steps=( - step01-config.sh - step12-add-provisioner.sh - step13-automatic-node-provisioning.sh - step14-automatic-node-termination.sh -) - -for step in "${steps[@]}"; do - echo "$step" - source $step -done diff --git a/website/content/en/v0.33/getting-started/getting-started-with-karpenter/scripts/add-roles.sh b/website/content/en/v0.33/getting-started/getting-started-with-karpenter/scripts/add-roles.sh deleted file mode 100755 index cf0656f3db3d..000000000000 --- a/website/content/en/v0.33/getting-started/getting-started-with-karpenter/scripts/add-roles.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/bash -set -euo pipefail #fail if one step fails - -if [ "$#" -ne 1 ] -then - echo "Missing required Karpenter version. Usage: setup-roles.sh v0.0.1" - exit 1 -fi - -export KARPENTER_VERSION=$1 -SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) - -declare -a steps=( - step03-iam-cloud-formation.sh - step04-grant-access.sh - step05-controller-iam.sh - step06-add-spot-role.sh -) - -for step in "${steps[@]}"; do - echo "$step" - source "$SCRIPT_DIR/$step" -done diff --git a/website/content/en/v0.33/getting-started/getting-started-with-karpenter/scripts/cleanup.sh b/website/content/en/v0.33/getting-started/getting-started-with-karpenter/scripts/cleanup.sh deleted file mode 100755 index 52851e7f975a..000000000000 --- a/website/content/en/v0.33/getting-started/getting-started-with-karpenter/scripts/cleanup.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -declare -a steps=( - step01-config.sh - step16-cleanup.sh -) - -for step in "${steps[@]}"; do - echo "$step" - source $step -done diff --git a/website/content/en/v0.33/getting-started/getting-started-with-karpenter/scripts/install.sh b/website/content/en/v0.33/getting-started/getting-started-with-karpenter/scripts/install.sh deleted file mode 100755 index e0ebfb32e511..000000000000 --- a/website/content/en/v0.33/getting-started/getting-started-with-karpenter/scripts/install.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -set -euo pipefail #fail if one step fails - -if [ "$#" -ne 1 ] -then - echo "Missing required Karpenter version. Usage: install.sh v0.0.1" - exit 1 -fi - -export KARPENTER_VERSION=$1 - -declare -a steps=( - step01-config.sh - step02-create-cluster.sh - step03-iam-cloud-formation.sh - step04-grant-access.sh - step05-controller-iam.sh - step06-add-spot-role.sh - step08-apply-helm-chart.sh -) - -for step in "${steps[@]}"; do - echo "$step" - source $step -done diff --git a/website/content/en/v0.33/getting-started/getting-started-with-karpenter/scripts/step01-config.sh b/website/content/en/v0.33/getting-started/getting-started-with-karpenter/scripts/step01-config.sh index a3af512d02ac..6b948de3c793 100755 --- a/website/content/en/v0.33/getting-started/getting-started-with-karpenter/scripts/step01-config.sh +++ b/website/content/en/v0.33/getting-started/getting-started-with-karpenter/scripts/step01-config.sh @@ -3,3 +3,6 @@ export CLUSTER_NAME="${USER}-karpenter-demo" export AWS_DEFAULT_REGION="us-west-2" export AWS_ACCOUNT_ID="$(aws sts get-caller-identity --query Account --output text)" export TEMPOUT=$(mktemp) +export ARM_AMI_ID="$(aws ssm get-parameter --name /aws/service/eks/optimized-ami/${K8S_VERSION}/amazon-linux-2-arm64/recommended/image_id --query Parameter.Value --output text)" +export AMD_AMI_ID="$(aws ssm get-parameter --name /aws/service/eks/optimized-ami/${K8S_VERSION}/amazon-linux-2/recommended/image_id --query Parameter.Value --output text)" +export GPU_AMI_ID="$(aws ssm get-parameter --name /aws/service/eks/optimized-ami/${K8S_VERSION}/amazon-linux-2-gpu/recommended/image_id --query Parameter.Value --output text)" diff --git a/website/content/en/v0.33/getting-started/getting-started-with-karpenter/scripts/step09-add-prometheus-grafana.sh b/website/content/en/v0.33/getting-started/getting-started-with-karpenter/scripts/step09-add-prometheus-grafana.sh index a539f7491e10..951813a865cc 100755 --- a/website/content/en/v0.33/getting-started/getting-started-with-karpenter/scripts/step09-add-prometheus-grafana.sh +++ b/website/content/en/v0.33/getting-started/getting-started-with-karpenter/scripts/step09-add-prometheus-grafana.sh @@ -4,7 +4,7 @@ helm repo update kubectl create namespace monitoring -curl -fsSL https://raw.githubusercontent.com/aws/karpenter-provider-aws/"${KARPENTER_VERSION}"/website/content/en/preview/getting-started/getting-started-with-karpenter/prometheus-values.yaml | tee prometheus-values.yaml +curl -fsSL https://raw.githubusercontent.com/aws/karpenter-provider-aws/"${KARPENTER_VERSION}"/website/content/en/preview/getting-started/getting-started-with-karpenter/prometheus-values.yaml | envsubst | tee prometheus-values.yaml helm install --namespace monitoring prometheus prometheus-community/prometheus --values prometheus-values.yaml curl -fsSL https://raw.githubusercontent.com/aws/karpenter-provider-aws/"${KARPENTER_VERSION}"/website/content/en/preview/getting-started/getting-started-with-karpenter/grafana-values.yaml | tee grafana-values.yaml diff --git a/website/content/en/v0.33/getting-started/getting-started-with-karpenter/scripts/step12-add-nodepool.sh b/website/content/en/v0.33/getting-started/getting-started-with-karpenter/scripts/step12-add-nodepool.sh index 8c518e9e74b8..33f1cb553b1b 100755 --- a/website/content/en/v0.33/getting-started/getting-started-with-karpenter/scripts/step12-add-nodepool.sh +++ b/website/content/en/v0.33/getting-started/getting-started-with-karpenter/scripts/step12-add-nodepool.sh @@ -23,6 +23,8 @@ spec: operator: Gt values: ["2"] nodeClassRef: + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass name: default limits: cpu: 1000 @@ -43,4 +45,9 @@ spec: securityGroupSelectorTerms: - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + amiSelectorTerms: + - id: "${ARM_AMI_ID}" + - id: "${AMD_AMI_ID}" +# - id: "${GPU_AMI_ID}" # <- GPU Optimized AMD AMI +# - name: "amazon-eks-node-${K8S_VERSION}-*" # <- automatically upgrade when a new AL2 EKS Optimized AMI is released. This is unsafe for production workloads. Validate AMIs in lower environments before deploying them to production. EOF diff --git a/website/content/en/v0.33/getting-started/migrating-from-cas/_index.md b/website/content/en/v0.33/getting-started/migrating-from-cas/_index.md index 52489ae67a5f..7bc114262838 100644 --- a/website/content/en/v0.33/getting-started/migrating-from-cas/_index.md +++ b/website/content/en/v0.33/getting-started/migrating-from-cas/_index.md @@ -92,7 +92,7 @@ One for your Karpenter node role and one for your existing node group. First set the Karpenter release you want to deploy. ```bash -export KARPENTER_VERSION=v0.33.1 +export KARPENTER_VERSION=v0.33.4 ``` We can now generate a full Karpenter deployment yaml from the helm chart. @@ -133,7 +133,7 @@ Now that our deployment is ready we can create the karpenter namespace, create t ## Create default NodePool -We need to create a default NodePool so Karpenter knows what types of nodes we want for unscheduled workloads. You can refer to some of the [example NodePool](https://github.com/aws/karpenter/tree/v0.33.1/examples/v1beta1) for specific needs. +We need to create a default NodePool so Karpenter knows what types of nodes we want for unscheduled workloads. You can refer to some of the [example NodePool](https://github.com/aws/karpenter/tree/v0.33.4/examples/v1beta1) for specific needs. {{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step10-create-nodepool.sh" language="bash" %}} diff --git a/website/content/en/v0.33/getting-started/migrating-from-cas/scripts/step01-env.sh b/website/content/en/v0.33/getting-started/migrating-from-cas/scripts/step01-env.sh index 20645685137b..6276d7aa1439 100644 --- a/website/content/en/v0.33/getting-started/migrating-from-cas/scripts/step01-env.sh +++ b/website/content/en/v0.33/getting-started/migrating-from-cas/scripts/step01-env.sh @@ -4,3 +4,7 @@ OIDC_ENDPOINT="$(aws eks describe-cluster --name ${CLUSTER_NAME} \ --query "cluster.identity.oidc.issuer" --output text)" AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query 'Account' \ --output text) +K8S_VERSION=1.28 +ARM_AMI_ID="$(aws ssm get-parameter --name /aws/service/eks/optimized-ami/${K8S_VERSION}/amazon-linux-2-arm64/recommended/image_id --query Parameter.Value --output text)" +AMD_AMI_ID="$(aws ssm get-parameter --name /aws/service/eks/optimized-ami/${K8S_VERSION}/amazon-linux-2/recommended/image_id --query Parameter.Value --output text)" +GPU_AMI_ID="$(aws ssm get-parameter --name /aws/service/eks/optimized-ami/${K8S_VERSION}/amazon-linux-2-gpu/recommended/image_id --query Parameter.Value --output text)" diff --git a/website/content/en/v0.33/getting-started/migrating-from-cas/scripts/step10-create-nodepool.sh b/website/content/en/v0.33/getting-started/migrating-from-cas/scripts/step10-create-nodepool.sh index 8c518e9e74b8..33f1cb553b1b 100644 --- a/website/content/en/v0.33/getting-started/migrating-from-cas/scripts/step10-create-nodepool.sh +++ b/website/content/en/v0.33/getting-started/migrating-from-cas/scripts/step10-create-nodepool.sh @@ -23,6 +23,8 @@ spec: operator: Gt values: ["2"] nodeClassRef: + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass name: default limits: cpu: 1000 @@ -43,4 +45,9 @@ spec: securityGroupSelectorTerms: - tags: karpenter.sh/discovery: "${CLUSTER_NAME}" # replace with your cluster name + amiSelectorTerms: + - id: "${ARM_AMI_ID}" + - id: "${AMD_AMI_ID}" +# - id: "${GPU_AMI_ID}" # <- GPU Optimized AMD AMI +# - name: "amazon-eks-node-${K8S_VERSION}-*" # <- automatically upgrade when a new AL2 EKS Optimized AMI is released. This is unsafe for production workloads. Validate AMIs in lower environments before deploying them to production. EOF diff --git a/website/content/en/v0.33/reference/cloudformation.md b/website/content/en/v0.33/reference/cloudformation.md index d469ceb2dbac..337e7ff8b866 100644 --- a/website/content/en/v0.33/reference/cloudformation.md +++ b/website/content/en/v0.33/reference/cloudformation.md @@ -17,7 +17,7 @@ These descriptions should allow you to understand: To download a particular version of `cloudformation.yaml`, set the version and use `curl` to pull the file to your local system: ```bash -export KARPENTER_VERSION=v0.33.1 +export KARPENTER_VERSION=v0.33.4 curl https://raw.githubusercontent.com/aws/karpenter-provider-aws/"${KARPENTER_VERSION}"/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml > cloudformation.yaml ``` diff --git a/website/content/en/v0.33/reference/threat-model.md b/website/content/en/v0.33/reference/threat-model.md index 778af396a826..f25f3ab52a11 100644 --- a/website/content/en/v0.33/reference/threat-model.md +++ b/website/content/en/v0.33/reference/threat-model.md @@ -31,11 +31,11 @@ A Cluster Developer has the ability to create pods via `Deployments`, `ReplicaSe Karpenter has permissions to create and manage cloud instances. Karpenter has Kubernetes API permissions to create, update, and remove nodes, as well as evict pods. For a full list of the permissions, see the RBAC rules in the helm chart template. Karpenter also has AWS IAM permissions to create instances with IAM roles. -* [aggregate-clusterrole.yaml](https://github.com/aws/karpenter/blob/v0.33.1/charts/karpenter/templates/aggregate-clusterrole.yaml) -* [clusterrole-core.yaml](https://github.com/aws/karpenter/blob/v0.33.1/charts/karpenter/templates/clusterrole-core.yaml) -* [clusterrole.yaml](https://github.com/aws/karpenter/blob/v0.33.1/charts/karpenter/templates/clusterrole.yaml) -* [rolebinding.yaml](https://github.com/aws/karpenter/blob/v0.33.1/charts/karpenter/templates/rolebinding.yaml) -* [role.yaml](https://github.com/aws/karpenter/blob/v0.33.1/charts/karpenter/templates/role.yaml) +* [aggregate-clusterrole.yaml](https://github.com/aws/karpenter/blob/v0.33.4/charts/karpenter/templates/aggregate-clusterrole.yaml) +* [clusterrole-core.yaml](https://github.com/aws/karpenter/blob/v0.33.4/charts/karpenter/templates/clusterrole-core.yaml) +* [clusterrole.yaml](https://github.com/aws/karpenter/blob/v0.33.4/charts/karpenter/templates/clusterrole.yaml) +* [rolebinding.yaml](https://github.com/aws/karpenter/blob/v0.33.4/charts/karpenter/templates/rolebinding.yaml) +* [role.yaml](https://github.com/aws/karpenter/blob/v0.33.4/charts/karpenter/templates/role.yaml) ## Assumptions diff --git a/website/content/en/v0.33/tasks/_index.md b/website/content/en/v0.33/tasks/_index.md new file mode 100644 index 000000000000..7d4ac8605f4e --- /dev/null +++ b/website/content/en/v0.33/tasks/_index.md @@ -0,0 +1,10 @@ +--- +title: "Tasks" +linkTitle: "Tasks" +weight: 25 +description: > + Tasks to run with Karpenter +cascade: + type: docs +--- + diff --git a/website/content/en/v0.33/tasks/managing-amis.md b/website/content/en/v0.33/tasks/managing-amis.md new file mode 100644 index 000000000000..1ef31f141e08 --- /dev/null +++ b/website/content/en/v0.33/tasks/managing-amis.md @@ -0,0 +1,153 @@ +--- +title: "Managing AMIs" +linkTitle: "Managing AMIs" +weight: 10 +description: > + Task for managing AMIs in Karpenter +--- + +Understanding how Karpenter assigns AMIs to nodes can help ensure that your workloads will run successfully on those nodes and continue to run if the nodes are upgraded to newer AMIs. +Below we describe how Karpenter assigns AMIs to nodes when they are first deployed and how newer AMIs are assigned later when nodes are spun up to replace old ones. +Later, it describes the options you have to assert control over how AMIs are used by Karpenter for your clusters. + +Features for managing AMIs described here should be considered as part of the larger upgrade policies that you have for your clusters. +See [How do I upgrade an EKS Cluster with Karpenter]({{< relref "../faq/#how-do-i-upgrade-an-eks-cluster-with-karpenter" >}}) for details on this process. + +## How Karpenter assigns AMIs to nodes + +Here is how Karpenter assigns AMIs nodes: + +* When you create an `EC2NodeClass`, you are required to set the family of AMIs to use. For example, for the AL2 family, you would set `amiFamily: AL2`. +* With that `amiFamily` set, any time Karpenter spins up a new node, it uses the latest [Amazon EKS optimized Amazon Linux 2 AMIs](https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami.html) release. +* Later, if an existing node needs to be replaced, Karpenter checks to see if a newer AMI in the AL2 family is available and automatically uses the new AMI instead to spin up the new node. In other words, you may automatically get an AMI that you have not tested with your workloads. + +You can manually delete a node managed by Karpenter, which will cause the default behavior just described to take effect. +However, there are situations that will cause node replacements with newer AMIs to happen automatically. +These include: Expiration (if node expiry is set, the node is marked for deletion at a certain time after the node is created), [**Consolidation**]({{< relref "../concepts/disruption/#consolidation" >}}) (if a node is empty of workloads, or deemed to be inefficiently running workloads, nodes can be deleted and more appropriately featured nodes are brought up to consolidate workloads), [Drift]({{< relref "../concepts/disruption/#drift" >}}) (nodes are set for deletion when they drift from the desired state of the `NodeClaims` and new nodes are brought up to replace them), and [Interruption]({{< relref "../concepts/disruption/#interruption" >}}) (nodes are sometimes involuntarily disrupted by things like Spot interruption, health changes, and instance events, requiring new nodes to be deployed). + +See [**Automated Methods**]({{< relref "../concepts/disruption/#automated-methods" >}}) for details on how Karpenter uses these automated actions to replace nodes. + +With these types of automated updates in place, there is some risk that the new AMI being used when replacing instances will introduce some regressions or bugs that cause your workloads to be degraded or fail altogether. +The options described below tell you how to take more control over the ways in which Karpenter selects AMIs for your nodes. + +{{% alert title="Important" color="warning" %}} +If you are new to Karpenter, you should know that the behavior described here is different than you get with Managed Node Groups (MNG). MNG will always use the assigned AMI when it creates a new node and will never automatically upgrade to a new AMI when a new node is required. See [Updating a Managed Node Group](https://docs.aws.amazon.com/eks/latest/userguide/update-managed-node-group.html) to see how you would manually update MNG to use new AMIs. +{{% /alert %}} + +## Choosing AMI options +One of Karpenter's greatest assets is its ability to provide the right node at the right time, with little intervention from the person managing the cluster. +Its default behavior of using a later AMI if one becomes available in the selected family means you automatically get the latest security fixes and features. +However, with this comes the risk that the new AMI could break or degrade your workloads. + +As the Karpenter team looks for new ways to manage AMIs, the options below offer some means of reducing these risks, based on your own security and ease-of-use requirements. +Here are the advantages and challenges of each of the options described below: + +* [Option 1]({{< relref "#option-1-manage-how-amis-are-tested-and-rolled-out" >}}) (Test AMIs): The safest way, and the one we recommend, for ensuring that a new AMI doesn't break your workloads is to test it before putting it into production. This takes the most effort on your part, but most effectively models how your workloads will run in production, allowing you to catch issues ahead of time. Note that you can sometimes get different results from your test environment when you roll a new AMI into production, since issues like scale and other factors can elevate problems you might not see in test. So combining this with other options, that do things like slow rollouts, can allow you to catch problems before they impact your whole cluster. +* [Option 2]({{< relref "#option-2-lock-down-which-amis-are-selected" >}}) (Lock down AMIs): If workloads require a particluar AMI, this option can make sure that it is the only AMI used by Karpenter. This can be used in combination with Option 1, where you lock down the AMI in production, but allow the newest AMIs in a test cluster while you test your workloads before upgrading production. Keep in mind that this makes upgrades a manual process for you. +* [Option 3]({{< relref "#option-3-control-the-pace-of-node-disruptions" >}}) ([Disruption budgets]({{< relref "../concepts/disruption/" >}})): This option can be used as a way of mitigating the scope of impact if a new AMI causes problems with your workloads. With Disruption budgets you can slow the pace of upgrades to nodes with new AMIs or make sure that upgrades only happen during selected dates and times (using `schedule`). This doesn't prevent a bad AMI from being deployed, but it allows you to control when nodes are upgraded, and gives you more time respond to rollout issues. + +## Options + +The following lays out the options you have to impact Karpenter’s behavior as it relates to how nodes are created and AMIs are consumed. + +### Option 1: Manage how AMIs are tested and rolled out + +Instead of just avoiding AMI upgrades, you can set up test clusters where you can try out new AMI releases before they are put into production. +For example, you could have: + +* **Test clusters**: On lower environment clusters, you can run the latest AMIs for your workloads in a safe environment. The `EC2NodeClass` for these clusters could be set with a chosen `amiFamily`, but no `amiSelectorTerms` set. For example, the `NodePool` and `EC2NodeClass` could begin with the following: + + ```yaml + apiVersion: karpenter.sh/v1beta1 + kind: NodePool + metadata: + name: default + spec: + template: + spec: + nodeClassRef: + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass + name: default + --- + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass + metadata: + name: default + spec: + # The latest AMI in this family will be used + amiFamily: AL2 + ``` +* **Production clusters**: After you've confirmed that the AMI works in your lower environments, you can pin the latest AMIs to be deployed in your production clusters to roll out the AMI. One way to do that is to use `amiSelectorTerms` to set the tested AMI to be used in your production cluster. Refer to Option 2 for how to choose a particular AMI by `name` or `id`. Remember that it is still best practice to gradually roll new AMIs into your cluster, even if they have been tested. So consider implementing that for your production clusters as described in Option 3. + +### Option 2: Lock down which AMIs are selected + +Instead of letting Karpenter always run the latest AMI, you can change Karpenter’s default behavior. +When you configure the [**EC2NodeClass**]({{< relref "../concepts/nodeclasses" >}}), you can set a specific AMI that you want Karpenter to always choose, using the `amiSelectorTerms` field. +This prevents a new and potentially untested AMI from replacing existing nodes when those nodes are terminated. + +With the `amiSelectorTerms` field in an `EC2NodeClass`, you can set a specific AMI for Karpenter to use, based on AMI name or id (only one is required). +These examples show two different ways to identify the same AMI: + +```yaml +amiSelectorTerms: +- tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" + environment: prod +- name: al2023-ami-2023.3.20240219.0-kernel-6.1-x86_64 +``` + +or + +```yaml +amiSelectorTerms: +- tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" + environment: prod +- id: ami-052c9ea013e6e3567 +``` + +See the [**spec.amiSelectorTerms**]({{< relref "../concepts/nodeclasses/#specamiselectorterms" >}}) section of the NodeClasses page for details. +Keep in mind, that this could prevent you from getting critical security patches when new AMIs are available, but it does give you control over exactly which AMI is running. + + +### Option 3: Control the pace of node disruptions + +To reduce the risk of entire workloads being immediately degraded when a new AMI is deployed, you can enable Karpenter [**Disruption Budgets**]({{< relref "../concepts/disruption/#disruption-budgets " >}}). +Disruption Budgets limit when and to what extent nodes can be disrupted. +You can prevent disruption based on nodes (a percentage or number of nodes that can be disrupted at a time) and schedule (excluding certain times from disrupting nodes). +You can set Disruption Budgets in a `NodePool` spec. +Here is an example: + +```yaml +disruption: + consolidationPolicy: WhenEmpty + expireAfter: 1440h + budgets: + - nodes: 15% + - nodes: "3" + - nodes: "0" + schedule: "0 7 * * sat-sun" + duration: 12h +``` + +The `disruption` settings define a few fields that indicate the state of a node that should be disrupted. +The `consolidationPolicy` field indicates that a node should be disrupted if the node is either underutilized (`WhenUnderutilized`) or not running any pods (`WhenEmpty`). +With `expireAfter` set to `1440` hours, the node expires after 60 days. +Extending those values causes longer times without disruption. + +Settings for budgets in the above example include the following: + +* **Percentage of nodes**: From the first `nodes` setting, only `15%` of the NodePool’s nodes can be disrupted at a time. +* **Number of nodes**: The second `nodes` setting limits the number of nodes that can be disrupted at a time to `3`. +* **Schedule**: The third `nodes` setting uses schedule to say that zero disruptions (`0`) are allowed starting at 7am on Saturday and Sunday and continues for 12 hours. +The format of the schedule follows the `crontab` format for identifying dates and times. +See the [crontab](https://man7.org/linux/man-pages/man5/crontab.5.html) page for information on the supported values for these fields. + +As with all disruption settings, keep in mind that avoiding updated AMIs for your nodes can result in not getting fixes for known security risks and bugs. +You need to balance that with your desire to not risk breaking the workloads on your cluster. + +## Follow-up + +The Karpenter project continues to add features to give you greater control over AMI upgrades on your clusters. +If you have opinions about features you would like to see to manage AMIs with Karpenter, feel free to enter a Karpenter [New Issue](https://github.com/aws/karpenter-provider-aws/issues/new/choose). diff --git a/website/content/en/v0.33/troubleshooting.md b/website/content/en/v0.33/troubleshooting.md index c33f04264e3b..1c00aac28b3d 100644 --- a/website/content/en/v0.33/troubleshooting.md +++ b/website/content/en/v0.33/troubleshooting.md @@ -105,14 +105,14 @@ Karpenter v0.26.1+ introduced the `karpenter-crd` helm chart. When installing th - In the case of `invalid ownership metadata; label validation error: missing key "app.kubernetes.io/managed-by": must be set to "Helm"` run: ```shell -kubectl label crd awsnodetemplates.karpenter.k8s.aws provisioners.karpenter.sh machines.karpenter.sh app.kubernetes.io/managed-by=Helm --overwrite +kubectl label crd ec2nodeclasses.karpenter.k8s.aws nodepools.karpenter.sh nodeclaims.karpenter.sh app.kubernetes.io/managed-by=Helm --overwrite ``` - In the case of `annotation validation error: missing key "meta.helm.sh/release-namespace": must be set to "karpenter"` run: ```shell -kubectl annotate crd awsnodetemplates.karpenter.k8s.aws provisioners.karpenter.sh machines.karpenter.sh meta.helm.sh/release-name=karpenter-crd --overwrite -kubectl annotate crd awsnodetemplates.karpenter.k8s.aws provisioners.karpenter.sh machines.karpenter.sh meta.helm.sh/release-namespace=karpenter --overwrite +kubectl annotate crd ec2nodeclasses.karpenter.k8s.aws nodepools.karpenter.sh nodeclaims.karpenter.sh meta.helm.sh/release-name=karpenter-crd --overwrite +kubectl annotate crd ec2nodeclasses.karpenter.k8s.aws nodepools.karpenter.sh nodeclaims.karpenter.sh meta.helm.sh/release-namespace=karpenter --overwrite ``` ## Uninstallation @@ -284,7 +284,23 @@ Karpenter does not support [in-tree storage plugins](https://kubernetes.io/blog/ #### Pods were scheduled due to a race condition in Kubernetes -Due to [this race condition in Kubernetes](https://github.com/kubernetes/kubernetes/issues/95911), it's possible that the scheduler and the CSINode can race during node registration such that the scheduler assumes that a node can mount more volumes than the node attachments support. There is currently no solve for this problem other than enforcing `toplogySpreadConstraints` and `podAntiAffinity` on your workloads that use PVCs such that you attempt to reduce the number of PVCs that schedule to a given node. +Due to [this race condition in Kubernetes](https://github.com/kubernetes/kubernetes/issues/95911), it's possible that the scheduler and the CSINode can race during node registration such that the scheduler assumes that a node can mount more volumes than the node attachments support. There is currently no universal solve for this problem other than enforcing `toplogySpreadConstraints` and `podAntiAffinity` on your workloads that use PVCs such that you attempt to reduce the number of PVCs that schedule to a given node. + +The following is a list of known CSI drivers which support a startupTaint to eliminate this issue: +- [aws-ebs-csi-driver](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/master/docs/install.md#configure-node-startup-taint) +- [aws-efs-csi-driver](https://github.com/kubernetes-sigs/aws-efs-csi-driver/tree/master/docs#configure-node-startup-taint) + +These taints should be configured via `startupTaints` on your `NodePool`. For example, to enable this for EBS, add the following to your `NodePool`: +```yaml +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +spec: + template: + spec: + startupTaints: + - key: ebs.csi.aws.com/agent-not-ready + effect: NoExecute +``` ### CNI is unable to allocate IPs to pods @@ -642,7 +658,7 @@ Karpenter [doesn't currently support draining and terminating on spot rebalance These two components do not share information between each other, meaning if you have drain and terminate functionality enabled on NTH, NTH may remove a node for a spot rebalance recommendation. Karpenter will replace the node to fulfill the pod capacity that was being fulfilled by the old node; however, Karpenter won't be aware of the reason that that node was terminated. This means that Karpenter may launch the same instance type that was just deprovisioned, causing a spot rebalance recommendation to be sent again. This can result in very short-lived instances where NTH continually removes nodes and Karpeneter re-launches the same instance type over and over again. -Karpenter doesn't recommend reacting to spot rebalance recommendations when running Karpenter with spot nodes; however, if you absolutely require this functionality, note that the above scenario is possible. +Karpenter doesn't recommend reacting to spot rebalance recommendations when running Karpenter with spot nodes; however, if you absolutely require this functionality, note that the above scenario is possible. Spot instances are time limited and, therefore, interruptible. When a signal is sent by AWS, it triggers actions from NTH and Karpenter, where the former signals a shutdown and the later provisions, creating a recursive situation. This can be mitigated by either completely removing NTH or by setting the following values: diff --git a/website/content/en/v0.33/upgrading/compatibility.md b/website/content/en/v0.33/upgrading/compatibility.md index dccc4caf6aa1..805e107b4f8f 100644 --- a/website/content/en/v0.33/upgrading/compatibility.md +++ b/website/content/en/v0.33/upgrading/compatibility.md @@ -15,9 +15,9 @@ Before you begin upgrading Karpenter, consider Karpenter compatibility issues re [comment]: <> (the content below is generated from hack/docs/compataiblitymetrix_gen_docs.go) -| KUBERNETES | 1.23 | 1.24 | 1.25 | 1.26 | 1.27 | 1.28 | -|------------|---------|---------|---------|---------|---------|---------| -| karpenter | 0.21.x+ | 0.21.x+ | 0.25.x+ | 0.28.x+ | 0.28.x+ | 0.31.x+ | +| KUBERNETES | 1.23 | 1.24 | 1.25 | 1.26 | 1.27 | 1.28 | 1.29 | +|------------|----------|----------|----------|----------|----------|----------|------------| +| karpenter | \>= 0.21 | \>= 0.21 | \>= 0.25 | \>= 0.28 | \>= 0.28 | \>= 0.31 | \>= 0.34.0 | [comment]: <> (end docs generated content from hack/docs/compataiblitymetrix_gen_docs.go) diff --git a/website/content/en/v0.33/upgrading/upgrade-guide.md b/website/content/en/v0.33/upgrading/upgrade-guide.md index d111bf20c2be..0ed871bb9bb3 100644 --- a/website/content/en/v0.33/upgrading/upgrade-guide.md +++ b/website/content/en/v0.33/upgrading/upgrade-guide.md @@ -28,9 +28,9 @@ If you get the error `invalid ownership metadata; label validation error:` while In general, you can reapply the CRDs in the `crds` directory of the Karpenter helm chart: ```shell -kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.33.1/pkg/apis/crds/karpenter.sh_nodepools.yaml -kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.33.1/pkg/apis/crds/karpenter.sh_nodeclaims.yaml -kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.33.1/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml +kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.33.4/pkg/apis/crds/karpenter.sh_nodepools.yaml +kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.33.4/pkg/apis/crds/karpenter.sh_nodeclaims.yaml +kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.33.4/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml ``` ### Upgrading to v0.33.0+ @@ -41,6 +41,7 @@ kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0 v0.33.0+ _only_ supports Karpenter v1beta1 APIs and will not work with existing Provisioner, AWSNodeTemplate or Machine alpha APIs. **Do not** upgrade to v0.33.0+ without first [upgrading to v0.32.x]({{}}). This version supports both the alpha and beta APIs, allowing you to migrate all of your existing APIs to beta APIs without experiencing downtime. {{% /alert %}} +* Karpenter no longer supports using the `karpenter.sh/provisioner-name` label in NodePool labels and requirements or in application node selectors, affinities, or topologySpreadConstraints. If you were previously using this label to target applications to specific Provisioners, you should update your applications to use the `karpenter.sh/nodepool` label instead before upgrading. If you upgrade without changing these labels, you may begin to see pod scheduling failures for these applications. * Karpenter now tags `spot-instances-request` with the same tags that it tags instances, volumes, and primary ENIs. This means that you will now need to add `ec2:CreateTags` permission for `spot-instances-request`. You can also further scope your controller policy for the `ec2:RunInstances` action to require that it launches the `spot-instances-request` with these specific tags. You can view an example of scoping these actions in the [Getting Started Guide's default CloudFormation controller policy](https://github.com/aws/karpenter/blob/v0.33.0/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml#L61). * We now recommend that you set the installation namespace for your Karpenter controllers to `kube-system` to denote Karpenter as a critical cluster component. This ensures that requests from the Karpenter controllers are treated with higher priority by assigning them to a different [PriorityLevelConfiguration](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/#prioritylevelconfiguration) than generic requests from other namespaces. For more details on API Priority and Fairness, read the [Kubernetes API Priority and Fairness Conceptual Docs](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/). Note: Changing the namespace for your Karpenter release will cause the service account namespace to change. If you are using IRSA for authentication with AWS, you will need to change scoping set in the controller's trust policy from `karpenter:karpenter` to `kube-system:karpenter`. * `v0.33.x` disables mutating and validating webhooks by default in favor of using [Common Expression Language for CRD validation](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#validation). The Common Expression Language Validation Feature [is enabled by default on EKS 1.25](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#validation-rules). If you are using Kubernetes version >= 1.25, no further action is required. If you are using a Kubernetes version below 1.25, you now need to set `DISABLE_WEBHOOK=false` in your container environment variables or `--set webhook.enabled=true` if using Helm. View the [Webhook Support Deprecated in Favor of CEL Section of the v1beta1 Migration Guide]({{}}). @@ -57,7 +58,7 @@ Karpenter v0.32.0 introduces v1beta1 APIs, including _significant_ changes to th This version includes **dual support** for both alpha and beta APIs to ensure that you can slowly migrate your existing Provisioner, AWSNodeTemplate, and Machine alpha APIs to the newer NodePool, EC2NodeClass, and NodeClaim beta APIs. -Note that if you are rolling back after upgrading to v0.32.0, note that v0.31.2 and v0.31.3 are the only versions that support handling rollback after you have deployed the v1beta1 APIs to your cluster. +Note that if you are rolling back after upgrading to v0.32.0, note that v0.31.4 is the only version that supports handling rollback after you have deployed the v1beta1 APIs to your cluster. {{% /alert %}} * Karpenter now serves the webhook prometheus metrics server on port `8001`. If this port is already in-use on the pod or you are running in `hostNetworking` mode, you may need to change this port value. You can configure this port value through the `WEBHOOK_METRICS_PORT` environment variable or the `webhook.metrics.port` value if installing via Helm. diff --git a/website/content/en/v0.30/_index.md b/website/content/en/v0.34/_index.md similarity index 81% rename from website/content/en/v0.30/_index.md rename to website/content/en/v0.34/_index.md index 93ddf9368a25..61ab45550ca3 100755 --- a/website/content/en/v0.30/_index.md +++ b/website/content/en/v0.34/_index.md @@ -1,4 +1,3 @@ - --- title: "Documentation" linkTitle: "Docs" @@ -8,23 +7,23 @@ cascade: tags: - preview --- -Karpenter is an open-source node provisioning project built for Kubernetes. +Karpenter is an open-source node lifecycle management project built for Kubernetes. Adding Karpenter to a Kubernetes cluster can dramatically improve the efficiency and cost of running workloads on that cluster. Karpenter works by: * **Watching** for pods that the Kubernetes scheduler has marked as unschedulable * **Evaluating** scheduling constraints (resource requests, nodeselectors, affinities, tolerations, and topology spread constraints) requested by the pods * **Provisioning** nodes that meet the requirements of the pods -* **Removing** the nodes when the nodes are no longer needed +* **Disrupting** the nodes when the nodes are no longer needed As someone using Karpenter, once your Kubernetes cluster and the Karpenter controller are up and running (see [Getting Started]({{}})), you can: -* **Set up provisioners**: By applying a provisioner to Karpenter, you can configure constraints on node provisioning and set timeout values for node expiry or Kubelet configuration values. -Provisioner-level constraints related to Kubernetes and your cloud provider (AWS, for example) include: +* **Set up NodePools**: By applying a NodePool to Karpenter, you can configure constraints on node provisioning and set values for node expiry, node consolidation, or Kubelet configuration values. + NodePool-level constraints related to Kubernetes and your cloud provider (AWS, for example) include: - Taints (`taints`): Identify taints to add to provisioned nodes. If a pod doesn't have a matching toleration for the taint, the effect set by the taint occurs (NoSchedule, PreferNoSchedule, or NoExecute). See Kubernetes [Taints and Tolerations](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) for details. - Labels (`labels`): Apply arbitrary key-value pairs to nodes that can be matched by pods. - - Requirements (`requirements`): Set acceptable (`In`) and unacceptable (`Out`) Kubernetes and Karpenter values for node provisioning based on [Well-Known Labels](https://kubernetes.io/docs/reference/labels-annotations-taints/) and [cloud-specific settings]({{}}). These can include [instance types](https://kubernetes.io/docs/reference/labels-annotations-taints/#nodekubernetesioinstance-type), [zones](https://kubernetes.io/docs/reference/labels-annotations-taints/#topologykubernetesiozone), [computer architecture](https://kubernetes.io/docs/reference/labels-annotations-taints/#kubernetes-io-arch), and [capacity type]({{}}) (such as AWS spot or on-demand). + - Requirements (`requirements`): Set acceptable (`In`) and unacceptable (`Out`) Kubernetes and Karpenter values for node provisioning based on [Well-Known Labels](https://kubernetes.io/docs/reference/labels-annotations-taints/) and [cloud-specific settings]({{}}). These can include [instance types](https://kubernetes.io/docs/reference/labels-annotations-taints/#nodekubernetesioinstance-type), [zones](https://kubernetes.io/docs/reference/labels-annotations-taints/#topologykubernetesiozone), [computer architecture](https://kubernetes.io/docs/reference/labels-annotations-taints/#kubernetes-io-arch), and [capacity type]({{}}) (such as AWS spot or on-demand). - Limits (`limits`): Lets you set limits on the total CPU and Memory that can be used by the cluster, effectively stopping further node provisioning when those limits have been reached. * **Deploy workloads**: When deploying workloads, you can request that scheduling constraints be met to direct which nodes Karpenter provisions for those workloads. Use any of the following Pod spec constraints when you deploy pods: diff --git a/website/content/en/v0.34/concepts/_index.md b/website/content/en/v0.34/concepts/_index.md new file mode 100755 index 000000000000..67d573135369 --- /dev/null +++ b/website/content/en/v0.34/concepts/_index.md @@ -0,0 +1,113 @@ +--- +title: "Concepts" +linkTitle: "Concepts" +weight: 20 +description: > + Understand key concepts of Karpenter +--- + +Users fall under two basic roles: [Kubernetes cluster administrators]({{}}) and [application developers]({{}}). This document describes Karpenter concepts through the lens of those two types of users. + +## Cluster Administrator + +As a Kubernetes cluster administrator, you can engage with Karpenter to: + +* Install Karpenter +* Configure NodePools to set constraints and other features for managing nodes +* Disrupting nodes + +Concepts associated with this role are described below. + + +### Installing Karpenter + +Karpenter is designed to run on a node in your Kubernetes cluster. As part of the installation process, you need credentials from the underlying cloud provider to allow nodes to be started up and added to the cluster as they are needed. + +[Getting Started with Karpenter]({{}}) describes the process of installing Karpenter. Because requests to add and delete nodes and schedule pods are made through Kubernetes, AWS IAM Roles for Service Accounts (IRSA) are needed by your Kubernetes cluster to make privileged requests to AWS. For example, Karpenter uses AWS IRSA roles to grant the permissions needed to describe EC2 instance types and create EC2 instances. + +Once privileges are in place, Karpenter is deployed with a Helm chart. + +### Configuring NodePools + +Karpenter's job is to add nodes to handle unschedulable pods, schedule pods on those nodes, and remove the nodes when they are not needed. To configure Karpenter, you create [NodePools]({{}}) that define how Karpenter manages unschedulable pods and configures nodes. You will also define behaviors for your NodePools, capturing details like how Karpenter handles disruption of nodes and setting limits and weights for each NodePool + +Here are some things to know about Karpenter's NodePools: + +* **Unschedulable pods**: Karpenter only attempts to schedule pods that have a status condition `Unschedulable=True`, which the kube scheduler sets when it fails to schedule the pod to existing capacity. + +* [**Defining Constraints**]({{}}): Karpenter defines a Custom Resource called a NodePool to specify configuration. Each NodePool manages a distinct set of nodes, but pods can be scheduled to any NodePool that supports its scheduling constraints. A NodePool contains constraints that impact the nodes that can be provisioned and attributes of those nodes. See the [NodePools Documentation]({{}}) docs for a description of configuration and NodePool examples. + +* [**Defining Disruption**]({{}}): A NodePool can also include values to indicate when nodes should be disrupted. This includes configuration around concepts like [Consolidation]({{}}), [Drift]({{}}), and [Expiration]({{}}). + +* **Well-known labels**: The NodePool can use well-known Kubernetes labels to allow pods to request only certain instance types, architectures, operating systems, or other attributes when creating nodes. See [Well-Known Labels, Annotations and Taints](https://kubernetes.io/docs/reference/labels-annotations-taints/) for details. Keep in mind that only a subset of these labels are supported in Karpenter, as described later. + +* **Multiple NodePools**: Multiple NodePools can be configured on the same cluster. For example, you might want to configure different teams on the same cluster to run on completely separate capacity. One team could run on nodes using BottleRocket, while another uses EKSOptimizedAMI. + +Although most use cases are addressed with a single NodePool for multiple teams, multiple NodePools are useful to isolate nodes for billing, use different node constraints (such as no GPUs for a team), or use different disruption settings. + +### Disrupting nodes + +Karpenter deletes nodes when they are no longer needed. + +* [**Finalizer**]({{}}): Karpenter places a finalizer bit on each node it creates. +When a request comes in to delete one of those nodes (such as a TTL or a manual `kubectl delete node`), Karpenter will cordon the node, drain all the pods, terminate the EC2 instance, and delete the node object. +Karpenter handles all clean-up work needed to properly delete the node. +* [**Expiration**]({{}}): Karpenter will mark nodes as expired and disrupt them after they have lived a set number of seconds, based on the NodePool's `spec.disruption.expireAfter` value. You can use node expiry to periodically recycle nodes due to security concerns. +* [**Consolidation**]({{}}): Karpenter works to actively reduce cluster cost by identifying when: + * Nodes can be removed because the node is empty + * Nodes can be removed as their workloads will run on other nodes in the cluster. + * Nodes can be replaced with cheaper variants due to a change in the workloads. +* [**Drift**]({{}}): Karpenter will mark nodes as drifted and disrupt nodes that have drifted from their desired specification. See [Drift]({{}}) to see which fields are considered. +* [**Interruption**]({{}}): Karpenter will watch for upcoming interruption events that could affect your nodes (health events, spot interruption, etc.) and will cordon, drain, and terminate the node(s) ahead of the event to reduce workload disruption. + +For more details on how Karpenter deletes nodes, see the [Disruption Documentation]({{}}). + +### Scheduling + +Karpenter launches nodes in response to pods that the Kubernetes scheduler has marked unschedulable. After solving scheduling constraints and launching capacity, Karpenter launches a machine in your chosen cloud provider. + +Once Karpenter brings up a node, that node is available for the Kubernetes scheduler to schedule pods on it as well. + +#### Constraints + +The concept of layered constraints is key to using Karpenter. With no constraints defined in NodePools and none requested from pods being deployed, Karpenter chooses from the entire universe of features available to your cloud provider. Nodes can be created using any instance type and run in any zones. + +An application developer can tighten the constraints defined in a NodePool by the cluster administrator by defining additional scheduling constraints in their pod spec. Refer to the description of Karpenter constraints in the Application Developer section below for details. + +### Cloud Provider + +Karpenter makes requests to provision new nodes to the associated cloud provider. The first supported cloud provider is AWS, although Karpenter is designed to work with other cloud providers. Separating Kubernetes and AWS-specific settings allows Karpenter a clean path to integrating with other cloud providers. + +While using Kubernetes well-known labels, the NodePool can set some values that are specific to the cloud provider. For example, to include a certain instance type, you could use the Kubernetes label `node.kubernetes.io/instance-type`, but set its value to an AWS instance type (such as `m5.large` or `m5.2xlarge`). + +### Kubernetes Cluster Autoscaler + +Like Karpenter, [Kubernetes Cluster Autoscaler](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler) is designed to add nodes when requests come in to run pods that cannot be met by current capacity. Cluster autoscaler is part of the Kubernetes project, with implementations by most major Kubernetes cloud providers. By taking a fresh look at provisioning, Karpenter offers the following improvements: + +* **Designed to handle the full flexibility of the cloud**: Karpenter has the ability to efficiently address the full range of instance types available through AWS. Cluster autoscaler was not originally built with the flexibility to handle hundreds of instance types, zones, and purchase options. + +* **Quick node provisioning**: Karpenter manages each instance directly, without use of additional orchestration mechanisms like node groups. This enables it to retry in milliseconds instead of minutes when capacity is unavailable. It also allows Karpenter to leverage diverse instance types, availability zones, and purchase options without the creation of hundreds of node groups. + +## Application Developer + +As someone deploying pods that might be evaluated by Karpenter, you should know how to request the properties that your pods need of its compute resources. Karpenter's job is to efficiently assess and choose compute assets based on requests from pod deployments. These can include basic Kubernetes features or features that are specific to the cloud provider (such as AWS). + +Layered *constraints* are applied when a pod makes requests for compute resources that cannot be met by current capacity. A pod can specify `nodeAffinity` (to run in a particular zone or instance type) or a `topologySpreadConstraints` spread (to cause a set of pods to be balanced across multiple nodes). +The pod can specify a `nodeSelector` to run only on nodes with a particular label and `resource.requests` to ensure that the node has enough available memory. + +The Kubernetes scheduler tries to match those constraints with available nodes. If the pod is unschedulable, Karpenter creates compute resources that match its needs. When Karpenter tries to provision a node, it analyzes scheduling constraints before choosing the node to create. + +As long as the requests are not outside the NodePool's constraints, Karpenter will look to best match the request, comparing the same well-known labels defined by the pod's scheduling constraints. Note that if the constraints are such that a match is not possible, the pod will remain unscheduled. + +So, what constraints can you use as an application developer deploying pods that could be managed by Karpenter? + +Kubernetes features that Karpenter supports for scheduling pods include nodeAffinity and [nodeSelector](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector). +It also supports [PodDisruptionBudget](https://kubernetes.io/docs/tasks/run-application/configure-pdb/), [topologySpreadConstraints](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/), and [inter-pod affinity and anti-affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity). + +From the Kubernetes [Well-Known Labels, Annotations and Taints](https://kubernetes.io/docs/reference/labels-annotations-taints/) page, you can see a full list of Kubernetes labels, annotations and taints that determine scheduling. Those that are implemented in Karpenter include: + +* **kubernetes.io/arch**: For example, kubernetes.io/arch=amd64 +* **node.kubernetes.io/instance-type**: For example, node.kubernetes.io/instance-type=m3.medium +* **topology.kubernetes.io/zone**: For example, topology.kubernetes.io/zone=us-east-1c + +For more on how, as a developer, you can add constraints to your pod deployment, see [Scheduling](./scheduling/) for details. diff --git a/website/content/en/v0.34/concepts/disruption.md b/website/content/en/v0.34/concepts/disruption.md new file mode 100644 index 000000000000..48e0b479c5f3 --- /dev/null +++ b/website/content/en/v0.34/concepts/disruption.md @@ -0,0 +1,306 @@ +--- +title: "Disruption" +linkTitle: "Disruption" +weight: 4 +description: > + Understand different ways Karpenter disrupts nodes +--- + +## Control Flow + +Karpenter sets a Kubernetes [finalizer](https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/) on each node and node claim it provisions. +The finalizer blocks deletion of the node object while the Termination Controller taints and drains the node, before removing the underlying NodeClaim. Disruption is triggered by the Disruption Controller, by the user through manual disruption, or through an external system that sends a delete request to the node object. + +### Disruption Controller + +Karpenter automatically discovers disruptable nodes and spins up replacements when needed. Karpenter disrupts nodes by executing one [automated method](#automated-methods) at a time, in order of Expiration, Drift, and then Consolidation. Each method varies slightly, but they all follow the standard disruption process. Karpenter uses [disruption budgets]({{}}) to control the speed of disruption. +1. Identify a list of prioritized candidates for the disruption method. + * If there are [pods that cannot be evicted](#pod-eviction) on the node, Karpenter will ignore the node and try disrupting it later. + * If there are no disruptable nodes, continue to the next disruption method. +2. For each disruptable node: + 1. Check if disrupting it would violate its NodePool's disruption budget. + 2. Execute a scheduling simulation with the pods on the node to find if any replacement nodes are needed. +3. Add the `karpenter.sh/disruption:NoSchedule` taint to the node(s) to prevent pods from scheduling to it. +4. Pre-spin any replacement nodes needed as calculated in Step (2), and wait for them to become ready. + * If a replacement node fails to initialize, un-taint the node(s), and restart from Step (1), starting at the first disruption method again. +5. Delete the node(s) and wait for the Termination Controller to gracefully shutdown the node(s). +6. Once the Termination Controller terminates the node, go back to Step (1), starting at the first disruption method again. + +### Termination Controller + +When a Karpenter node is deleted, the Karpenter finalizer will block deletion and the APIServer will set the `DeletionTimestamp` on the node, allowing Karpenter to gracefully shutdown the node, modeled after [Kubernetes Graceful Node Shutdown](https://kubernetes.io/docs/concepts/architecture/nodes/#graceful-node-shutdown). Karpenter's graceful shutdown process will: +1. Add the `karpenter.sh/disruption=disrupting:NoSchedule` taint to the node to prevent pods from scheduling to it. +2. Begin evicting the pods on the node with the [Kubernetes Eviction API](https://kubernetes.io/docs/concepts/scheduling-eviction/api-eviction/) to respect PDBs, while ignoring all [static pods](https://kubernetes.io/docs/tasks/configure-pod-container/static-pod/), pods tolerating the `karpenter.sh/disruption=disrupting:NoSchedule` taint, and succeeded/failed pods. Wait for the node to be fully drained before proceeding to Step (3). + * While waiting, if the underlying NodeClaim for the node no longer exists, remove the finalizer to allow the APIServer to delete the node, completing termination. +3. Terminate the NodeClaim in the Cloud Provider. +4. Remove the finalizer from the node to allow the APIServer to delete the node, completing termination. + +## Manual Methods +* **Node Deletion**: You can use `kubectl` to manually remove a single Karpenter node or nodeclaim. Since each Karpenter node is owned by a NodeClaim, deleting either the node or the nodeclaim will cause cascade deletion of the other: + + ```bash + # Delete a specific nodeclaim + kubectl delete nodeclaim $NODECLAIM_NAME + + # Delete a specific node + kubectl delete node $NODE_NAME + + # Delete all nodeclaims + kubectl delete nodeclaims --all + + # Delete all nodes owned by any nodepool + kubectl delete nodes -l karpenter.sh/nodepool + + # Delete all nodeclaims owned by a specific nodepoolXS + kubectl delete nodeclaims -l karpenter.sh/nodepool=$NODEPOOL_NAME + ``` +* **NodePool Deletion**: NodeClaims are owned by the NodePool through an [owner reference](https://kubernetes.io/docs/concepts/overview/working-with-objects/owners-dependents/#owner-references-in-object-specifications) that launched them. Karpenter will gracefully terminate nodes through cascading deletion when the owning NodePool is deleted. + +{{% alert title="Note" color="primary" %}} +By adding the finalizer, Karpenter improves the default Kubernetes process of node deletion. +When you run `kubectl delete node` on a node without a finalizer, the node is deleted without triggering the finalization logic. The instance will continue running in EC2, even though there is no longer a node object for it. The kubelet isn’t watching for its own existence, so if a node is deleted, the kubelet doesn’t terminate itself. All the pod objects get deleted by a garbage collection process later, because the pods’ node is gone. +{{% /alert %}} + +## Automated Methods + +Automated methods can be rate limited through [NodePool Disruption Budgets]({{}}) + +* **Expiration**: Karpenter will mark nodes as expired and disrupt them after they have lived a set number of seconds, based on the NodePool's `spec.disruption.expireAfter` value. You can use node expiry to periodically recycle nodes due to security concerns. +* [**Consolidation**]({{}}): Karpenter works to actively reduce cluster cost by identifying when: + * Nodes can be removed because the node is empty + * Nodes can be removed as their workloads will run on other nodes in the cluster. + * Nodes can be replaced with lower priced variants due to a change in the workloads. +* [**Drift**]({{}}): Karpenter will mark nodes as drifted and disrupt nodes that have drifted from their desired specification. See [Drift]({{}}) to see which fields are considered. +* [**Interruption**]({{}}): Karpenter will watch for upcoming interruption events that could affect your nodes (health events, spot interruption, etc.) and will taint, drain, and terminate the node(s) ahead of the event to reduce workload disruption. + +{{% alert title="Defaults" color="secondary" %}} +Disruption is configured through the NodePool's disruption block by the `consolidationPolicy`, `expireAfter` and `consolidateAfter` fields. Karpenter will configure these fields with the following values by default if they are not set: + +```yaml +spec: + disruption: + consolidationPolicy: WhenUnderutilized + expireAfter: 720h +``` +{{% /alert %}} + +### Consolidation + +Karpenter has two mechanisms for cluster consolidation: +1. **Deletion** - A node is eligible for deletion if all of its pods can run on free capacity of other nodes in the cluster. +2. **Replace** - A node can be replaced if all of its pods can run on a combination of free capacity of other nodes in the cluster and a single lower price replacement node. + +Consolidation has three mechanisms that are performed in order to attempt to identify a consolidation action: +1. **Empty Node Consolidation** - Delete any entirely empty nodes in parallel +2. **Multi Node Consolidation** - Try to delete two or more nodes in parallel, possibly launching a single replacement whose price is lower than that of all nodes being removed +3. **Single Node Consolidation** - Try to delete any single node, possibly launching a single replacement whose price is lower than that of the node being removed + +It's impractical to examine all possible consolidation options for multi-node consolidation, so Karpenter uses a heuristic to identify a likely set of nodes that can be consolidated. For single-node consolidation we consider each node in the cluster individually. + +When there are multiple nodes that could be potentially deleted or replaced, Karpenter chooses to consolidate the node that overall disrupts your workloads the least by preferring to terminate: + +* Nodes running fewer pods +* Nodes that will expire soon +* Nodes with lower priority pods + +If consolidation is enabled, Karpenter periodically reports events against nodes that indicate why the node can't be consolidated. These events can be used to investigate nodes that you expect to have been consolidated, but still remain in your cluster. + +```bash +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Unconsolidatable 66s karpenter pdb default/inflate-pdb prevents pod evictions + Normal Unconsolidatable 33s (x3 over 30m) karpenter can't replace with a lower-priced node +``` + +{{% alert title="Warning" color="warning" %}} +Using preferred anti-affinity and topology spreads can reduce the effectiveness of consolidation. At node launch, Karpenter attempts to satisfy affinity and topology spread preferences. In order to reduce node churn, consolidation must also attempt to satisfy these constraints to avoid immediately consolidating nodes after they launch. This means that consolidation may not disrupt nodes in order to avoid violating preferences, even if kube-scheduler can fit the host pods elsewhere. Karpenter reports these pods via logging to bring awareness to the possible issues they can cause (e.g. `pod default/inflate-anti-self-55894c5d8b-522jd has a preferred Anti-Affinity which can prevent consolidation`). +{{% /alert %}} + +#### Spot consolidation +For spot nodes, Karpenter has deletion consolidation enabled by default. If you would like to enable replacement with spot consolidation, you need to enable the feature through the [`SpotToSpotConsolidation` feature flag]({{}}). + +Lower priced spot instance types are selected with the [`price-capacity-optimized` strategy](https://aws.amazon.com/blogs/compute/introducing-price-capacity-optimized-allocation-strategy-for-ec2-spot-instances/). Sometimes, the lowest priced spot instance type is not launched due to the likelihood of interruption. As a result, Karpenter uses the number of available instance type options with a price lower than the currently launched spot instance as a heuristic for evaluating whether it should launch a replacement for the current spot node. + +We refer to the number of instances that Karpenter has within its launch decision as a launch's "instance type flexibility." When Karpenter is considering performing a spot-to-spot consolidation replacement, it will check whether replacing the instance type will lead to enough instance type flexibility in the subsequent launch request. As a result, we get the following properties when evaluating for consolidation: +1) We shouldn't continually consolidate down to the lowest priced spot instance which might have very high rates of interruption. +2) We launch with enough instance types that there’s high likelihood that our replacement instance has comparable availability to our current one. + +Karpenter requires a minimum instance type flexibility of 15 instance types when performing single node spot-to-spot consolidations (1 node to 1 node). It does not have the same instance type flexibility requirement for multi-node spot-to-spot consolidations (many nodes to 1 node) since doing so without requiring flexibility won't lead to "race to the bottom" scenarios. + + +### Drift +Drift handles changes to the NodePool/EC2NodeClass. For Drift, values in the NodePool/EC2NodeClass are reflected in the NodeClaimTemplateSpec/EC2NodeClassSpec in the same way that they’re set. A NodeClaim will be detected as drifted if the values in its owning NodePool/EC2NodeClass do not match the values in the NodeClaim. Similar to the upstream `deployment.spec.template` relationship to pods, Karpenter will annotate the owning NodePool and EC2NodeClass with a hash of the NodeClaimTemplateSpec to check for drift. Some special cases will be discovered either from Karpenter or through the CloudProvider interface, triggered by NodeClaim/Instance/NodePool/EC2NodeClass changes. + +#### Special Cases on Drift +In special cases, drift can correspond to multiple values and must be handled differently. Drift on resolved fields can create cases where drift occurs without changes to CRDs, or where CRD changes do not result in drift. For example, if a NodeClaim has `node.kubernetes.io/instance-type: m5.large`, and requirements change from `node.kubernetes.io/instance-type In [m5.large]` to `node.kubernetes.io/instance-type In [m5.large, m5.2xlarge]`, the NodeClaim will not be drifted because its value is still compatible with the new requirements. Conversely, if a NodeClaim is using a NodeClaim image `ami: ami-abc`, but a new image is published, Karpenter's `EC2NodeClass.spec.amiSelectorTerms` will discover that the new correct value is `ami: ami-xyz`, and detect the NodeClaim as drifted. + +##### NodePool +| Fields | +|----------------| +| spec.template.spec.requirements | + +##### EC2NodeClass +| Fields | +|-------------------------------| +| spec.subnetSelectorTerms | +| spec.securityGroupSelectorTerms | +| spec.amiSelectorTerms | + +#### Behavioral Fields +Behavioral Fields are treated as over-arching settings on the NodePool to dictate how Karpenter behaves. These fields don’t correspond to settings on the NodeClaim or instance. They’re set by the user to control Karpenter’s Provisioning and disruption logic. Since these don’t map to a desired state of NodeClaims, __behavioral fields are not considered for Drift__. + +##### NodePool +| Fields | +|---------------------| +| spec.weight | +| spec.limits | +| spec.disruption.* | + +Read the [Drift Design](https://github.com/aws/karpenter-core/blob/main/designs/drift.md) for more. + +To enable the drift feature flag, refer to the [Feature Gates]({{}}). + +Karpenter will add the `Drifted` status condition on NodeClaims if the NodeClaim is drifted from its owning NodePool. Karpenter will also remove the `Drifted` status condition if either: +1. The `Drift` feature gate is not enabled but the NodeClaim is drifted, Karpenter will remove the status condition. +2. The NodeClaim isn't drifted, but has the status condition, Karpenter will remove it. + +### Interruption + +If interruption-handling is enabled, Karpenter will watch for upcoming involuntary interruption events that would cause disruption to your workloads. These interruption events include: + +* Spot Interruption Warnings +* Scheduled Change Health Events (Maintenance Events) +* Instance Terminating Events +* Instance Stopping Events + +When Karpenter detects one of these events will occur to your nodes, it automatically taints, drains, and terminates the node(s) ahead of the interruption event to give the maximum amount of time for workload cleanup prior to compute disruption. This enables scenarios where the `terminationGracePeriod` for your workloads may be long or cleanup for your workloads is critical, and you want enough time to be able to gracefully clean-up your pods. + +For Spot interruptions, the NodePool will start a new node as soon as it sees the Spot interruption warning. Spot interruptions have a __2 minute notice__ before Amazon EC2 reclaims the instance. Karpenter's average node startup time means that, generally, there is sufficient time for the new node to become ready and to move the pods to the new node before the NodeClaim is reclaimed. + +{{% alert title="Note" color="primary" %}} +Karpenter publishes Kubernetes events to the node for all events listed above in addition to [__Spot Rebalance Recommendations__](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/rebalance-recommendations.html). Karpenter does not currently support taint, drain, and terminate logic for Spot Rebalance Recommendations. + +If you require handling for Spot Rebalance Recommendations, you can use the [AWS Node Termination Handler (NTH)](https://github.com/aws/aws-node-termination-handler) alongside Karpenter; however, note that the AWS Node Termination Handler cordons and drains nodes on rebalance recommendations, potentially causing more node churn in the cluster than with interruptions alone. Further information can be found in the [Troubleshooting Guide]({{< ref "../troubleshooting#aws-node-termination-handler-nth-interactions" >}}). +{{% /alert %}} + +Karpenter enables this feature by watching an SQS queue which receives critical events from AWS services which may affect your nodes. Karpenter requires that an SQS queue be provisioned and EventBridge rules and targets be added that forward interruption events from AWS services to the SQS queue. Karpenter provides details for provisioning this infrastructure in the [CloudFormation template in the Getting Started Guide](../../getting-started/getting-started-with-karpenter/#create-the-karpenter-infrastructure-and-iam-roles). + +To enable interruption handling, configure the `--interruption-queue-name` CLI argument with the name of the interruption queue provisioned to handle interruption events. + +## Controls + +### Disruption Budgets + +You can rate limit Karpenter's disruption through the NodePool's `spec.disruption.budgets`. If undefined, Karpenter will default to one budget with `nodes: 10%`. Budgets will consider nodes that are actively being deleted for any reason, and will only block Karpenter from disrupting nodes voluntarily through expiration, drift, emptiness, and consolidation. + +#### Nodes +When calculating if a budget will block nodes from disruption, Karpenter lists the total number of nodes owned by a NodePool, subtracting out the nodes owned by that NodePool that are currently being deleted and nodes that are NotReady. If the number of nodes being deleted by Karpenter or any other processes is greater than the number of allowed disruptions, disruption for this node will not proceed. + +If the budget is configured with a percentage value, such as `20%`, Karpenter will calculate the number of allowed disruptions as `allowed_disruptions = roundup(total * percentage) - total_deleting - total_notready`. If otherwise defined as a non-percentage value, Karpenter will simply subtract the number of nodes from the total `(total - non_percentage_value) - total_deleting - total_notready`. For multiple budgets in a NodePool, Karpenter will take the minimum value (most restrictive) of each of the budgets. + +For example, the following NodePool with three budgets defines the following requirements: +- The first budget will only allow 20% of nodes owned by that NodePool to be disrupted. For instance, if there were 19 nodes owned by the NodePool, 4 disruptions would be allowed, rounding up from `19 * .2 = 3.8`. +- The second budget acts as a ceiling to the previous budget, only allowing 5 disruptions when there are more than 25 nodes. +- The last budget only blocks disruptions during the first 10 minutes of the day, where 0 disruptions are allowed. + +```yaml +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: default +spec: + disruption: + consolidationPolicy: WhenUnderutilized + expireAfter: 720h # 30 * 24h = 720h + budgets: + - nodes: "20%" + - nodes: "5" + - nodes: "0" + schedule: "@daily" + duration: 10m +``` + +#### Schedule +Schedule is a cronjob schedule. Generally, the cron syntax is five space-delimited values with options below, with additional special macros like `@yearly`, `@monthly`, `@weekly`, `@daily`, `@hourly`. +Follow the [Kubernetes documentation](https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/#writing-a-cronjob-spec) for more information on how to follow the cron syntax. + +```bash +# ┌───────────── minute (0 - 59) +# │ ┌───────────── hour (0 - 23) +# │ │ ┌───────────── day of the month (1 - 31) +# │ │ │ ┌───────────── month (1 - 12) +# │ │ │ │ ┌───────────── day of the week (0 - 6) (Sunday to Saturday; +# │ │ │ │ │ 7 is also Sunday on some systems) +# │ │ │ │ │ OR sun, mon, tue, wed, thu, fri, sat +# │ │ │ │ │ +# * * * * * +``` + +{{% alert title="Note" color="primary" %}} +Timezones are not supported. Most images default to UTC, but it is best practice to ensure this is the case when considering how to define your budgets. +{{% /alert %}} + +#### Duration +Duration allows compound durations with minutes and hours values such as `10h5m` or `30m` or `160h`. Since cron syntax does not accept denominations smaller than minutes, users can only define minutes or hours. + +{{% alert title="Note" color="primary" %}} +Duration and Schedule must be defined together. When omitted, the budget is always active. When defined, the schedule determines a starting point where the budget will begin being enforced, and the duration determines how long from that starting point the budget will be enforced. +{{% /alert %}} + +### Pod-Level Controls + +You can block Karpenter from voluntarily choosing to disrupt certain pods by setting the `karpenter.sh/do-not-disrupt: "true"` annotation on the pod. This is useful for pods that you want to run from start to finish without disruption. By opting pods out of this disruption, you are telling Karpenter that it should not voluntarily remove a node containing this pod. + +Examples of pods that you might want to opt-out of disruption include an interactive game that you don't want to interrupt or a long batch job (such as you might have with machine learning) that would need to start over if it were interrupted. + +```yaml +apiVersion: apps/v1 +kind: Deployment +spec: + template: + metadata: + annotations: + karpenter.sh/do-not-disrupt: "true" +``` + +{{% alert title="Note" color="primary" %}} +This annotation will be ignored for [terminating pods](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase) and [terminal pods](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase) (Failed/Succeeded). +{{% /alert %}} + +Examples of voluntary node removal that will be prevented by this annotation include: +- [Consolidation]({{}}) +- [Drift]({{}}) +- Expiration + +{{% alert title="Note" color="primary" %}} +Voluntary node removal does not include [Interruption]({{}}) or manual deletion initiated through `kubectl delete node`. Both of these are considered involuntary events, since node removal cannot be delayed. +{{% /alert %}} + +### Node-Level Controls + +You can block Karpenter from voluntarily choosing to disrupt certain nodes by setting the `karpenter.sh/do-not-disrupt: "true"` annotation on the node. This will prevent disruption actions on the node. + +```yaml +apiVersion: v1 +kind: Node +metadata: + annotations: + karpenter.sh/do-not-disrupt: "true" +``` + +#### Example: Disable Disruption on a NodePool + +NodePool `.spec.annotations` allow you to set annotations that will be applied to all nodes launched by this NodePool. By setting the annotation `karpenter.sh/do-not-disrupt: "true"` on the NodePool, you will selectively prevent all nodes launched by this NodePool from being considered in disruption actions. + +```yaml +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: default +spec: + template: + metadata: + annotations: # will be applied to all nodes + karpenter.sh/do-not-disrupt: "true" +``` diff --git a/website/content/en/v0.34/concepts/nodeclasses.md b/website/content/en/v0.34/concepts/nodeclasses.md new file mode 100644 index 000000000000..bc2cac8545ad --- /dev/null +++ b/website/content/en/v0.34/concepts/nodeclasses.md @@ -0,0 +1,1001 @@ +--- +title: "NodeClasses" +linkTitle: "NodeClasses" +weight: 2 +description: > + Configure AWS-specific settings with EC2NodeClasses +--- + +Node Classes enable configuration of AWS specific settings. +Each NodePool must reference an EC2NodeClass using `spec.template.spec.nodeClassRef`. +Multiple NodePools may point to the same EC2NodeClass. + +```yaml +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: default +spec: + template: + spec: + nodeClassRef: + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass + name: default +--- +apiVersion: karpenter.k8s.aws/v1beta1 +kind: EC2NodeClass +metadata: + name: default +spec: + # Required, resolves a default ami and userdata + amiFamily: AL2 + + # Required, discovers subnets to attach to instances + # Each term in the array of subnetSelectorTerms is ORed together + # Within a single term, all conditions are ANDed + subnetSelectorTerms: + # Select on any subnet that has the "karpenter.sh/discovery: ${CLUSTER_NAME}" + # AND the "environment: test" tag OR any subnet with ID "subnet-09fa4a0a8f233a921" + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" + environment: test + - id: subnet-09fa4a0a8f233a921 + + # Required, discovers security groups to attach to instances + # Each term in the array of securityGroupSelectorTerms is ORed together + # Within a single term, all conditions are ANDed + securityGroupSelectorTerms: + # Select on any security group that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any security group with the "my-security-group" name + # OR any security group with ID "sg-063d7acfb4b06c82c" + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" + environment: test + - name: my-security-group + - id: sg-063d7acfb4b06c82c + + # Optional, IAM role to use for the node identity. + # The "role" field is immutable after EC2NodeClass creation. This may change in the + # future, but this restriction is currently in place today to ensure that Karpenter + # avoids leaking managed instance profiles in your account. + # Must specify one of "role" or "instanceProfile" for Karpenter to launch nodes + role: "KarpenterNodeRole-${CLUSTER_NAME}" + + # Optional, IAM instance profile to use for the node identity. + # Must specify one of "role" or "instanceProfile" for Karpenter to launch nodes + instanceProfile: "KarpenterNodeInstanceProfile-${CLUSTER_NAME}" + + # Optional, discovers amis to override the amiFamily's default amis + # Each term in the array of amiSelectorTerms is ORed together + # Within a single term, all conditions are ANDed + amiSelectorTerms: + # Select on any AMI that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any AMI with the "my-ami" name + # OR any AMI with ID "ami-123" + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" + environment: test + - name: my-ami + - id: ami-123 + + # Optional, use instance-store volumes for node ephemeral-storage + instanceStorePolicy: RAID0 + + # Optional, overrides autogenerated userdata with a merge semantic + userData: | + echo "Hello world" + + # Optional, propagates tags to underlying EC2 resources + tags: + team: team-a + app: team-a-app + + # Optional, configures IMDS for the instance + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 2 + httpTokens: required + + # Optional, configures storage devices for the instance + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 100Gi + volumeType: gp3 + iops: 10000 + encrypted: true + kmsKeyID: "1234abcd-12ab-34cd-56ef-1234567890ab" + deleteOnTermination: true + throughput: 125 + snapshotID: snap-0123456789 + + # Optional, configures detailed monitoring for the instance + detailedMonitoring: true +status: + # Resolved subnets + subnets: + - id: subnet-0a462d98193ff9fac + zone: us-east-2b + - id: subnet-0322dfafd76a609b6 + zone: us-east-2c + - id: subnet-0727ef01daf4ac9fe + zone: us-east-2b + - id: subnet-00c99aeafe2a70304 + zone: us-east-2a + - id: subnet-023b232fd5eb0028e + zone: us-east-2c + - id: subnet-03941e7ad6afeaa72 + zone: us-east-2a + + # Resolved security groups + securityGroups: + - id: sg-041513b454818610b + name: ClusterSharedNodeSecurityGroup + - id: sg-0286715698b894bca + name: ControlPlaneSecurityGroup-1AQ073TSAAPW + + # Resolved AMIs + amis: + - id: ami-01234567890123456 + name: custom-ami-amd64 + requirements: + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - id: ami-01234567890123456 + name: custom-ami-arm64 + requirements: + - key: kubernetes.io/arch + operator: In + values: + - arm64 + + # Generated instance profile name from "role" + instanceProfile: "${CLUSTER_NAME}-0123456778901234567789" +``` +Refer to the [NodePool docs]({{}}) for settings applicable to all providers. To explore various `EC2NodeClass` configurations, refer to the examples provided [in the Karpenter Github repository](https://github.com/aws/karpenter/blob/main/examples/v1beta1/). + +## spec.amiFamily + +AMIFamily is a required field, dictating both the default bootstrapping logic for nodes provisioned through this `EC2NodeClass` but also selecting a group of recommended, latest AMIs by default. Currently, Karpenter supports `amiFamily` values `AL2`, `Bottlerocket`, `Ubuntu`, `Windows2019`, `Windows2022` and `Custom`. GPUs are only supported by default with `AL2` and `Bottlerocket`. The `AL2` amiFamily does not support ARM64 GPU instance types unless you specify custom [`amiSelectorTerms`]({{}}). Default bootstrapping logic is shown below for each of the supported families. + +### AL2 + +```bash +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="//" + +--// +Content-Type: text/x-shellscript; charset="us-ascii" + +#!/bin/bash -xe +exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 +/etc/eks/bootstrap.sh 'test-cluster' --apiserver-endpoint 'https://test-cluster' --b64-cluster-ca 'ca-bundle' \ +--dns-cluster-ip '10.100.0.10' \ +--use-max-pods false \ +--kubelet-extra-args '--node-labels=karpenter.sh/capacity-type=on-demand,karpenter.sh/nodepool=test --max-pods=110' +--//-- +``` + +### Bottlerocket + +```toml +[settings] +[settings.kubernetes] +api-server = 'https://test-cluster' +cluster-certificate = 'ca-bundle' +cluster-name = 'test-cluster' +cluster-dns-ip = '10.100.0.10' +max-pods = 110 + +[settings.kubernetes.node-labels] +'karpenter.sh/capacity-type' = 'on-demand' +'karpenter.sh/nodepool' = 'test' +``` + +### Ubuntu + +```bash +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="//" + +--// +Content-Type: text/x-shellscript; charset="us-ascii" + +#!/bin/bash -xe +exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 +/etc/eks/bootstrap.sh 'test-cluster' --apiserver-endpoint 'https://test-cluster' --b64-cluster-ca 'ca-bundle' \ +--dns-cluster-ip '10.100.0.10' \ +--use-max-pods false \ +--kubelet-extra-args '--node-labels="karpenter.sh/capacity-type=on-demand,karpenter.sh/nodepool=test" --max-pods=110' +--//-- +``` + +### Windows2019 + +```powershell + +[string]$EKSBootstrapScriptFile = "$env:ProgramFiles\Amazon\EKS\Start-EKSBootstrap.ps1" +& $EKSBootstrapScriptFile -EKSClusterName 'test-cluster' -APIServerEndpoint 'https://test-cluster' -Base64ClusterCA 'ca-bundle' -KubeletExtraArgs '--node-labels="karpenter.sh/capacity-type=on-demand,karpenter.sh/nodepool=test" --max-pods=110' -DNSClusterIP '10.100.0.10' + +``` + +### Windows2022 + +```powershell + +[string]$EKSBootstrapScriptFile = "$env:ProgramFiles\Amazon\EKS\Start-EKSBootstrap.ps1" +& $EKSBootstrapScriptFile -EKSClusterName 'test-cluster' -APIServerEndpoint 'https://test-cluster' -Base64ClusterCA 'ca-bundle' -KubeletExtraArgs '--node-labels="karpenter.sh/capacity-type=on-demand,karpenter.sh/nodepool=test" --max-pods=110' -DNSClusterIP '10.100.0.10' + +``` + +{{% alert title="Note" color="primary" %}} +Karpenter will automatically query for the appropriate [EKS optimized AMI](https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-amis.html) via AWS Systems Manager (SSM). In the case of the `Custom` AMIFamily, no default AMIs are defined. As a result, `amiSelectorTerms` must be specified to inform Karpenter on which custom AMIs are to be used. +{{% /alert %}} + +### Custom + +The `Custom` AMIFamily ships without any default userData to allow you to configure custom bootstrapping for control planes or images that don't support the default methods from the other families. + +## spec.subnetSelectorTerms + +Subnet Selector Terms allow you to specify selection logic for a set of subnet options that Karpenter can choose from when launching an instance from the `EC2NodeClass`. Karpenter discovers subnets through the `EC2NodeClass` using ids or [tags](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html). When launching nodes, a subnet is automatically chosen that matches the desired zone. If multiple subnets exist for a zone, the one with the most available IP addresses will be used. + +This selection logic is modeled as terms, where each term contains multiple conditions that must all be satisfied for the selector to match. Effectively, all requirements within a single term are ANDed together. It's possible that you may want to select on two different subnets that have unrelated requirements. In this case, you can specify multiple terms which will be ORed together to form your selection logic. The example below shows how this selection logic is fulfilled. + +```yaml +subnetSelectorTerms: + # Select on any subnet that has the "karpenter.sh/discovery: ${CLUSTER_NAME}" + # AND the "environment: test" tag OR any subnet with ID "subnet-09fa4a0a8f233a921" + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" + environment: test + - id: subnet-09fa4a0a8f233a921 +``` + +{{% alert title="Tip" color="secondary" %}} +Subnets may be specified by any tag, including `Name`. Selecting tag values using wildcards (`*`) is supported. +{{% /alert %}} + +#### Examples + +Select all with a specified tag key: +```yaml +spec: + subnetSelectorTerms: + - tags: + karpenter.sh/discovery/MyClusterName: '*' +``` + +Select by name and tag (all criteria must match): +```yaml +spec: + subnetSelectorTerms: + - tags: + Name: my-subnet + MyTag: '' # matches all resources with the tag +``` + +Select using multiple tag terms: +```yaml +spec: + subnetSelectorTerms: + - tags: + Name: "my-subnet-1" + - tags: + Name: "my-subnet-2" +``` + +Select using wildcards: +```yaml +spec: + subnetSelectorTerms: + - tags: + Name: "*Public*" + +``` + +Select using ids: +```yaml +spec: + subnetSelectorTerms: + - id: "subnet-09fa4a0a8f233a921" + - id: "subnet-0471ca205b8a129ae" +``` + +## spec.securityGroupSelectorTerms + +Security Group Selector Terms allow you to specify selection logic for all security groups that will be attached to an instance launched from the `EC2NodeClass`. The security group of an instance is comparable to a set of firewall rules. +[EKS creates at least two security groups by default](https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html). + +This selection logic is modeled as terms, where each term contains multiple conditions that must all be satisfied for the selector to match. Effectively, all requirements within a single term are ANDed together. It's possible that you may want to select on two different security groups that have unrelated requirements. In this case, you can specify multiple terms which will be ORed together to form your selection logic. The example below shows how this selection logic is fulfilled. + +```yaml +securityGroupSelectorTerms: + # Select on any security group that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any security group with the "my-security-group" name + # OR any security group with ID "sg-063d7acfb4b06c82c" + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" + environment: test + - name: my-security-group + - id: sg-063d7acfb4b06c82c +``` + +{{% alert title="Tip" color="secondary" %}} +Security groups may be specified by any tag, including "Name". Selecting tags using wildcards (`*`) is supported. +{{% /alert %}} + +{{% alert title="Note" color="primary" %}} +When launching nodes, Karpenter uses all the security groups that match the selector. If you choose to use the `kubernetes.io/cluster/$CLUSTER_NAME` tag for discovery, note that this may result in failures using the AWS Load Balancer controller. The Load Balancer controller only supports a single security group having that tag key. See [this issue](https://github.com/kubernetes-sigs/aws-load-balancer-controller/issues/2367) for more details. + +To verify if this restriction affects you, run the following commands. +```bash +CLUSTER_VPC_ID="$(aws eks describe-cluster --name $CLUSTER_NAME --query cluster.resourcesVpcConfig.vpcId --output text)" + +aws ec2 describe-security-groups --filters Name=vpc-id,Values=$CLUSTER_VPC_ID Name=tag-key,Values=kubernetes.io/cluster/$CLUSTER_NAME --query 'SecurityGroups[].[GroupName]' --output text +``` + +If multiple securityGroups are printed, you will need more specific securityGroupSelectorTerms. We generally recommend that you use the `karpenter.sh/discovery: $CLUSTER_NAME` tag selector instead. +{{% /alert %}} + +#### Examples + +Select all assigned to a cluster: +```yaml +spec: + securityGroupSelectorTerms: + - tags: + kubernetes.io/cluster/$CLUSTER_NAME: "owned" +``` + +Select all with a specified tag key: +```yaml +spec: + securityGroupSelectorTerms: + - tags: + MyTag: '*' +``` + +Select by name and tag (all criteria must match): +```yaml +spec: + securityGroupSelectorTerms: + - name: my-security-group + tags: + MyTag: '*' # matches all resources with the tag +``` + +Select using multiple tag terms: +```yaml +spec: + securityGroupSelectorTerms: + - tags: + Name: "my-security-group-1" + - tags: + Name: "my-security-group-2" +``` + +Select by name using a wildcard: +```yaml +spec: + securityGroupSelectorTerms: + - name: "*Public*" +``` + +Select using ids: +```yaml +spec: + securityGroupSelectorTerms: + - id: "sg-063d7acfb4b06c82c" + - id: "sg-06e0cf9c198874591" +``` + +## spec.amiSelectorTerms + +AMI Selector Terms are used to configure custom AMIs for Karpenter to use, where the AMIs are discovered through ids, owners, name, and [tags](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html). **When you specify `amiSelectorTerms`, you fully override the default AMIs that are selected on by your EC2NodeClass [`amiFamily`]({{< ref "#specamifamily" >}}).** + +This selection logic is modeled as terms, where each term contains multiple conditions that must all be satisfied for the selector to match. Effectively, all requirements within a single term are ANDed together. It's possible that you may want to select on two different AMIs that have unrelated requirements. In this case, you can specify multiple terms which will be ORed together to form your selection logic. The example below shows how this selection logic is fulfilled. + +```yaml +amiSelectorTerms: + # Select on any AMI that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any AMI with the "my-ami" name + # OR any AMI with ID "ami-123" + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" + environment: test + - name: my-ami + - id: ami-123 +``` + +This field is optional, and Karpenter will use the latest EKS-optimized AMIs for the AMIFamily if no amiSelectorTerms are specified. To select an AMI by name, use the `name` field in the selector term. To select an AMI by id, use the `id` field in the selector term. To ensure that AMIs are owned by the expected owner, use the `owner` field - you can use a combination of account aliases (e.g. `self` `amazon`, `your-aws-account-name`) and account IDs. + +If owner is not set for `name`, it defaults to `self,amazon`, preventing Karpenter from inadvertently selecting an AMI that is owned by a different account. Tags don't require an owner as tags can only be discovered by the user who created them. + +{{% alert title="Tip" color="secondary" %}} +AMIs may be specified by any AWS tag, including `Name`. Selecting by tag or by name using wildcards (`*`) is supported. +{{% /alert %}} + +{{% alert title="Note" color="primary" %}} +If `amiSelectorTerms` match more than one AMI, Karpenter will automatically determine which AMI best fits the workloads on the launched worker node under the following constraints: + +* When launching nodes, Karpenter automatically determines which architecture a custom AMI is compatible with and will use images that match an instanceType's requirements. + * Note that Karpenter **cannot** detect any requirement other than architecture. If you need to specify different AMIs for different kind of nodes (e.g. accelerated GPU AMIs), you should use a separate `EC2NodeClass`. +* If multiple AMIs are found that can be used, Karpenter will choose the latest one. +* If no AMIs are found that can be used, then no nodes will be provisioned. +{{% /alert %}} + +#### Examples + +Select all with a specified tag: +```yaml + amiSelectorTerms: + - tags: + karpenter.sh/discovery/MyClusterName: '*' +``` + +Select by name: +```yaml + amiSelectorTerms: + - name: my-ami +``` + +Select by `Name` tag: +```yaml + amiSelectorTerms: + - tags: + Name: my-ami +``` + +Select by name and owner: +```yaml + amiSelectorTerms: + - name: my-ami + owner: self + - name: my-ami + owner: 0123456789 +``` + +Select by name using a wildcard: +```yaml +spec: + amiSelectorTerms: + - name: "*EKS*" +``` + +Select by all under an owner: +```yaml +spec: + amiSelectorTerms: + - name: "*" + owner: self +``` + +Specify using ids: +```yaml + amiSelectorTerms: + - id: "ami-123" + - id: "ami-456" +``` + +## spec.role + +`Role` is an optional field and tells Karpenter which IAM identity nodes should assume. You must specify one of `role` or `instanceProfile` when creating a Karpenter `EC2NodeClass`. If using the [Karpenter Getting Started Guide]({{}}) to deploy Karpenter, you can use the `KarpenterNodeRole-$CLUSTER_NAME` role provisioned by that process. + +```yaml +spec: + role: "KarpenterNodeRole-$CLUSTER_NAME" +``` + +## spec.instanceProfile + +`InstanceProfile` is an optional field and tells Karpenter which IAM identity nodes should assume. You must specify one of `role` or `instanceProfile` when creating a Karpenter `EC2NodeClass`. If you use the `instanceProfile` field instead of `role`, Karpenter will not manage the InstanceProfile on your behalf; instead, it expects that you have pre-provisioned an IAM instance profile and assigned it a role. + +You can provision and assign a role to an IAM instance profile using [CloudFormation](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iam-instanceprofile.html) or by using the [`aws iam create-instance-profile`](https://docs.aws.amazon.com/cli/latest/reference/iam/create-instance-profile.html) and [`aws iam add-role-to-instance-profile`](https://docs.aws.amazon.com/cli/latest/reference/iam/add-role-to-instance-profile.html) commands in the CLI. + +{{% alert title="Note" color="primary" %}} + +For [private clusters](https://docs.aws.amazon.com/eks/latest/userguide/private-clusters.html) that do not have access to the public internet, using `spec.instanceProfile` is required. `spec.role` cannot be used since Karpenter needs to access IAM endpoints to manage a generated instance profile. IAM [doesn't support private endpoints](https://docs.aws.amazon.com/vpc/latest/privatelink/aws-services-privatelink-support.html) to enable accessing the service without going to the public internet. + +{{% /alert %}} + +## spec.tags + +Karpenter adds tags to all resources it creates, including EC2 Instances, EBS volumes, and Launch Templates. The default set of tags are listed below. + +```yaml +Name: +karpenter.sh/nodeclaim: +karpenter.sh/nodepool: +karpenter.k8s.aws/ec2nodeclass: +kubernetes.io/cluster/: owned +``` + +Additional tags can be added in the tags section, which will be merged with the default tags specified above. +```yaml +spec: + tags: + InternalAccountingTag: 1234 + dev.corp.net/app: Calculator + dev.corp.net/team: MyTeam +``` + +{{% alert title="Note" color="primary" %}} +Karpenter allows overrides of the default "Name" tag but does not allow overrides to restricted domains (such as "karpenter.sh", "karpenter.k8s.aws", and "kubernetes.io/cluster"). This ensures that Karpenter is able to correctly auto-discover nodes that it owns. +{{% /alert %}} + +## spec.metadataOptions + +Control the exposure of [Instance Metadata Service](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) on EC2 Instances launched by this EC2NodeClass using a generated launch template. + +Refer to [recommended, security best practices](https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node) for limiting exposure of Instance Metadata and User Data to pods. + +If metadataOptions are omitted from this EC2NodeClass, the following default settings are applied: + +```yaml +spec: + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 2 + httpTokens: required +``` + +## spec.blockDeviceMappings + +The `blockDeviceMappings` field in an `EC2NodeClass` can be used to control the [Elastic Block Storage (EBS) volumes](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/block-device-mapping-concepts.html#instance-block-device-mapping) that Karpenter attaches to provisioned nodes. Karpenter uses default block device mappings for the AMIFamily specified. For example, the `Bottlerocket` AMI Family defaults with two block device mappings, one for Bottlerocket's control volume and the other for container resources such as images and logs. + +```yaml +spec: + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 100Gi + volumeType: gp3 + iops: 10000 + encrypted: true + kmsKeyID: "1234abcd-12ab-34cd-56ef-1234567890ab" + deleteOnTermination: true + throughput: 125 + snapshotID: snap-0123456789 +``` + +The following blockDeviceMapping defaults are used for each `AMIFamily` if no `blockDeviceMapping` overrides are specified in the `EC2NodeClass` + +### AL2 +```yaml +spec: + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 20Gi + volumeType: gp3 + encrypted: true +``` + +### Bottlerocket +```yaml +spec: + blockDeviceMappings: + # Root device + - deviceName: /dev/xvda + ebs: + volumeSize: 4Gi + volumeType: gp3 + encrypted: true + # Data device: Container resources such as images and logs + - deviceName: /dev/xvdb + ebs: + volumeSize: 20Gi + volumeType: gp3 + encrypted: true +``` + +### Ubuntu +```yaml +spec: + blockDeviceMappings: + - deviceName: /dev/sda1 + ebs: + volumeSize: 20Gi + volumeType: gp3 + encrypted: true +``` + +### Windows2019/Windows2022 +```yaml +spec: + blockDeviceMappings: + - deviceName: /dev/sda1 + ebs: + volumeSize: 50Gi + volumeType: gp3 + encrypted: true +``` + +### Custom + +The `Custom` AMIFamily ships without any default `blockDeviceMappings`. + +## spec.instanceStorePolicy + +The `instanceStorePolicy` field controls how [instance-store](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/InstanceStorage.html) volumes are handled. By default, Karpenter and Kubernetes will simply ignore them. + +### RAID0 + +If you intend to use these volumes for faster node ephemeral-storage, set `instanceStorePolicy` to `RAID0`: + +```yaml +spec: + instanceStorePolicy: RAID0 +``` + +This will set the allocatable ephemeral-storage of each node to the total size of the instance-store volume(s). + +The disks must be formatted & mounted in a RAID0 and be the underlying filesystem for the Kubelet & Containerd. Instructions for each AMI family are listed below: + +#### AL2 + +On AL2, Karpenter automatically configures the disks through an additional boostrap argument (`--local-disks raid0`). The device name is `/dev/md/0` and its mount point is `/mnt/k8s-disks/0`. You should ensure any additional disk setup does not interfere with these. + +#### Others + +For all other AMI families, you must configure the disks yourself. Check out the [`setup-local-disks`](https://github.com/awslabs/amazon-eks-ami/blob/master/files/bin/setup-local-disks) script in [amazon-eks-ami](https://github.com/awslabs/amazon-eks-ami) to see how this is done for AL2. + +{{% alert title="Tip" color="secondary" %}} +Since the Kubelet & Containerd will be using the instance-store filesystem, you may consider using a more minimal root volume size. +{{% /alert %}} + +## spec.userData + +You can control the UserData that is applied to your worker nodes via this field. This allows you to run custom scripts or pass-through custom configuration to Karpenter instances on start-up. + +```yaml +apiVersion: karpenter.k8s.aws/v1beta1 +kind: EC2NodeClass +metadata: + name: bottlerocket-example +spec: + ... + amiFamily: Bottlerocket + userData: | + [settings.kubernetes] + "kube-api-qps" = 30 + "shutdown-grace-period" = "30s" + "shutdown-grace-period-for-critical-pods" = "30s" + [settings.kubernetes.eviction-hard] + "memory.available" = "20%" +``` + +This example adds SSH keys to allow remote login to the node (replace *my-authorized_keys* with your key file): + +{{% alert title="Note" color="primary" %}} +Instead of using SSH as set up in this example, you can use Session Manager (SSM) or EC2 Instance Connect to gain shell access to Karpenter nodes. +See [Node NotReady]({{< ref "../troubleshooting/#node-notready" >}}) troubleshooting for an example of starting an SSM session from the command line or [EC2 Instance Connect](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-connect-set-up.html) documentation to connect to nodes using SSH. +{{% /alert %}} + +```yaml +apiVersion: karpenter.k8s.aws/v1beta1 +kind: EC2NodeClass +metadata: + name: al2-example +spec: + ... + amiFamily: AL2 + userData: | + #!/bin/bash + mkdir -p ~ec2-user/.ssh/ + touch ~ec2-user/.ssh/authorized_keys + cat >> ~ec2-user/.ssh/authorized_keys <}}) section for more details on these defaults. View the sections below to understand the different merge strategies for each AMIFamily. + +### AL2/Ubuntu + +* Your UserData can be in the [MIME multi part archive](https://cloudinit.readthedocs.io/en/latest/topics/format.html#mime-multi-part-archive) format. +* Karpenter will transform your custom user-data as a MIME part, if necessary, and then merge a final MIME part to the end of your UserData parts which will bootstrap the worker node. Karpenter will have full control over all the parameters being passed to the bootstrap script. + * Karpenter will continue to set MaxPods, ClusterDNS and all other parameters defined in `spec.kubeletConfiguration` as before. + +Consider the following example to understand how your custom UserData will be merged - + +#### Passed-in UserData (bash) + +```bash +#!/bin/bash +echo "Running custom user data script" +``` + +#### Passed-in UserData (MIME) + +```bash +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="BOUNDARY" + +--BOUNDARY +Content-Type: text/x-shellscript; charset="us-ascii" + +#!/bin/bash +echo "Running custom user data script" + +--BOUNDARY-- +``` + +#### Merged UserData + +```bash +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="//" + +--// +Content-Type: text/x-shellscript; charset="us-ascii" + +#!/bin/bash +echo "Running custom user data script" + +--// +Content-Type: text/x-shellscript; charset="us-ascii" + +#!/bin/bash -xe +exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 +/etc/eks/bootstrap.sh 'test-cluster' --apiserver-endpoint 'https://test-cluster' --b64-cluster-ca 'ca-bundle' \ +--use-max-pods false \ +--kubelet-extra-args '--node-labels=karpenter.sh/capacity-type=on-demand,karpenter.sh/nodepool=test --max-pods=110' +--//-- +``` + +{{% alert title="Note" color="primary" %}} +You can also set kubelet-config properties by modifying the kubelet-config.json file before the EKS bootstrap script starts the kubelet: + +```yaml +apiVersion: karpenter.k8s.aws/v1beta1 +kind: EC2NodeClass +metadata: + name: kubelet-config-example +spec: + ... + amiFamily: AL2 + userData: | + #!/bin/bash + echo "$(jq '.kubeAPIQPS=50' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json +``` +{{% /alert %}} + +### Bottlerocket + +* Your UserData must be valid TOML. +* Karpenter will automatically merge settings to ensure successful bootstrap including `cluster-name`, `api-server` and `cluster-certificate`. Any labels and taints that need to be set based on pod requirements will also be specified in the final merged UserData. + * All Kubelet settings that Karpenter applies will override the corresponding settings in the provided UserData. For example, if you've specified `settings.kubernetes.cluster-name`, it will be overridden. + * If MaxPods is specified via the binary arg to Karpenter, the value will override anything specified in the UserData. + * If ClusterDNS is specified via `spec.kubeletConfiguration`, then that value will override anything specified in the UserData. +* Unknown TOML fields will be ignored when the final merged UserData is generated by Karpenter. + +Consider the following example to understand how your custom UserData settings will be merged in. + +#### Passed-in UserData + +```toml +[settings.kubernetes.eviction-hard] +"memory.available" = "12%" +[settings.kubernetes] +"unknown-setting" = "unknown" +[settings.kubernetes.node-labels] +'field.controlled.by/karpenter' = 'will-be-overridden' +``` + +#### Merged UserData + +```toml +[settings] +[settings.kubernetes] +api-server = 'https://cluster' +cluster-certificate = 'ca-bundle' +cluster-name = 'cluster' + +[settings.kubernetes.node-labels] +'karpenter.sh/capacity-type' = 'on-demand' +'karpenter.sh/nodepool' = 'default' + +[settings.kubernetes.node-taints] + +[settings.kubernetes.eviction-hard] +'memory.available' = '12%%' +``` + +### Windows2019/Windows2022 + +* Your UserData must be specified as PowerShell commands. +* The UserData specified will be prepended to a Karpenter managed section that will bootstrap the kubelet. +* Karpenter will continue to set ClusterDNS and all other parameters defined in spec.kubeletConfiguration as before. + +Consider the following example to understand how your custom UserData settings will be merged in. + +#### Passed-in UserData + +```powershell +Write-Host "Running custom user data script" +``` + +#### Merged UserData + +```powershell + +Write-Host "Running custom user data script" +[string]$EKSBootstrapScriptFile = "$env:ProgramFiles\Amazon\EKS\Start-EKSBootstrap.ps1" +& $EKSBootstrapScriptFile -EKSClusterName 'test-cluster' -APIServerEndpoint 'https://test-cluster' -Base64ClusterCA 'ca-bundle' -KubeletExtraArgs '--node-labels="karpenter.sh/capacity-type=spot,karpenter.sh/nodepool=windows2022" --max-pods=110' -DNSClusterIP '10.0.100.10' + +``` + +{{% alert title="Windows Support Notice" color="warning" %}} +Currently, Karpenter does not specify `-ServiceCIDR` to [EKS Windows AMI Bootstrap script](https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-windows-ami.html#bootstrap-script-configuration-parameters). +Windows worker nodes will use `172.20.0.0/16` or `10.100.0.0/16` for Kubernetes service IP address ranges based on the IP address of the primary interface. +The effective ServiceCIDR can be verified at `$env:ProgramData\Amazon\EKS\cni\config\vpc-bridge.conf` on the worker node. + +Support for the Windows ServiceCIDR argument can be tracked in a [Karpenter Github Issue](https://github.com/aws/karpenter/issues/4088). Currently, if the effective ServiceCIDR is incorrect for your windows worker nodes, you can add the following userData as a workaround. + +```yaml +spec: + userData: | + $global:EKSCluster = Get-EKSCluster -Name my-cluster +``` +{{% /alert %}} + +### Custom + +* No merging is performed, your UserData must perform all setup required of the node to allow it to join the cluster. + +## spec.detailedMonitoring + +Enabling detailed monitoring controls the [EC2 detailed monitoring](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-cloudwatch-new.html) feature. If you enable this option, the Amazon EC2 console displays monitoring graphs with a 1-minute period for the instances that Karpenter launches. + +```yaml +spec: + detailedMonitoring: true +``` + +## status.subnets +[`status.subnets`]({{< ref "#statussubnets" >}}) contains the resolved `id` and `zone` of the subnets that were selected by the [`spec.subnetSelectorTerms`]({{< ref "#specsubnetselectorterms" >}}) for the node class. The subnets will be sorted by the available IP address count in decreasing order. + +#### Examples + +```yaml +spec: + subnetSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" +status: + subnets: + - id: subnet-0a462d98193ff9fac + zone: us-east-2b + - id: subnet-0322dfafd76a609b6 + zone: us-east-2c + - id: subnet-0727ef01daf4ac9fe + zone: us-east-2b + - id: subnet-00c99aeafe2a70304 + zone: us-east-2a + - id: subnet-023b232fd5eb0028e + zone: us-east-2c + - id: subnet-03941e7ad6afeaa72 + zone: us-east-2a +``` + +## status.securityGroups + +[`status.securityGroups`]({{< ref "#statussecuritygroups" >}}) contains the resolved `id` and `name` of the security groups that were selected by the [`spec.securityGroupSelectorTerms`]({{< ref "#specsecuritygroupselectorterms" >}}) for the node class. The subnets will be sorted by the available IP address count in decreasing order. + +#### Examples + +```yaml +spec: + securityGroupSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" +status: + securityGroups: + - id: sg-041513b454818610b + name: ClusterSharedNodeSecurityGroup + - id: sg-0286715698b894bca + name: ControlPlaneSecurityGroup-1AQ073TSAAPW +``` + +## status.amis + +[`status.amis`]({{< ref "#statusamis" >}}) contains the resolved `id`, `name`, and `requirements` of either the default AMIs for the [`spec.amiFamily`]({{< ref "#specamifamily" >}}) or the AMIs selected by the [`spec.amiSelectorTerms`]({{< ref "#specamiselectorterms" >}}) if this field is specified. + +#### Examples + +Default AMIs resolved from the AL2 AMIFamily: + +```yaml +spec: + amiFamily: AL2 +status: + amis: + - id: ami-03c3a3dcda64f5b75 + name: amazon-linux-2-gpu + requirements: + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: karpenter.k8s.aws/instance-gpu-count + operator: Exists + - id: ami-03c3a3dcda64f5b75 + name: amazon-linux-2-gpu + requirements: + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: karpenter.k8s.aws/instance-accelerator-count + operator: Exists + - id: ami-06afb2d101cc4b8bd + name: amazon-linux-2-arm64 + requirements: + - key: kubernetes.io/arch + operator: In + values: + - arm64 + - key: karpenter.k8s.aws/instance-gpu-count + operator: DoesNotExist + - key: karpenter.k8s.aws/instance-accelerator-count + operator: DoesNotExist + - id: ami-0e28b76d768af234e + name: amazon-linux-2 + requirements: + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: karpenter.k8s.aws/instance-gpu-count + operator: DoesNotExist + - key: karpenter.k8s.aws/instance-accelerator-count + operator: DoesNotExist +``` + +AMIs resolved from [`spec.amiSelectorTerms`]({{< ref "#specamiselectorterms" >}}): + +```yaml +spec: + amiFamily: AL2 + amiSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" +status: + amis: + - id: ami-01234567890123456 + name: custom-ami-amd64 + requirements: + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - id: ami-01234567890123456 + name: custom-ami-arm64 + requirements: + - key: kubernetes.io/arch + operator: In + values: + - arm64 +``` + +## status.instanceProfile + +[`status.instanceProfile`]({{< ref "#statusinstanceprofile" >}}) contains the resolved instance profile generated by Karpenter from the [`spec.role`]({{< ref "#specrole" >}}) + +```yaml +spec: + role: "KarpenterNodeRole-${CLUSTER_NAME}" +status: + instanceProfile: "${CLUSTER_NAME}-0123456778901234567789" +``` diff --git a/website/content/en/v0.34/concepts/nodepools.md b/website/content/en/v0.34/concepts/nodepools.md new file mode 100644 index 000000000000..cb85d5988475 --- /dev/null +++ b/website/content/en/v0.34/concepts/nodepools.md @@ -0,0 +1,499 @@ +--- +title: "NodePools" +linkTitle: "NodePools" +weight: 1 +description: > + Configure Karpenter with NodePools +--- + +When you first installed Karpenter, you set up a default NodePool. The NodePool sets constraints on the nodes that can be created by Karpenter and the pods that can run on those nodes. The NodePool can be set to do things like: + +* Define taints to limit the pods that can run on nodes Karpenter creates +* Define any startup taints to inform Karpenter that it should taint the node initially, but that the taint is temporary. +* Limit node creation to certain zones, instance types, and computer architectures +* Set defaults for node expiration + +You can change your NodePool or add other NodePools to Karpenter. +Here are things you should know about NodePools: + +* Karpenter won't do anything if there is not at least one NodePool configured. +* Each NodePool that is configured is looped through by Karpenter. +* If Karpenter encounters a taint in the NodePool that is not tolerated by a Pod, Karpenter won't use that NodePool to provision the pod. +* If Karpenter encounters a startup taint in the NodePool it will be applied to nodes that are provisioned, but pods do not need to tolerate the taint. Karpenter assumes that the taint is temporary and some other system will remove the taint. +* It is recommended to create NodePools that are mutually exclusive. So no Pod should match multiple NodePools. If multiple NodePools are matched, Karpenter will use the NodePool with the highest [weight](#specweight). + +For some example `NodePool` configurations, see the [examples in the Karpenter GitHub repository](https://github.com/aws/karpenter/blob/main/examples/v1beta1/). + +```yaml +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: default +spec: + # Template section that describes how to template out NodeClaim resources that Karpenter will provision + # Karpenter will consider this template to be the minimum requirements needed to provision a Node using this NodePool + # It will overlay this NodePool with Pods that need to schedule to further constrain the NodeClaims + # Karpenter will provision to launch new Nodes for the cluster + template: + metadata: + # Labels are arbitrary key-values that are applied to all nodes + labels: + billing-team: my-team + + # Annotations are arbitrary key-values that are applied to all nodes + annotations: + example.com/owner: "my-team" + spec: + # References the Cloud Provider's NodeClass resource, see your cloud provider specific documentation + nodeClassRef: + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass + name: default + + # Provisioned nodes will have these taints + # Taints may prevent pods from scheduling if they are not tolerated by the pod. + taints: + - key: example.com/special-taint + effect: NoSchedule + + # Provisioned nodes will have these taints, but pods do not need to tolerate these taints to be provisioned by this + # NodePool. These taints are expected to be temporary and some other entity (e.g. a DaemonSet) is responsible for + # removing the taint after it has finished initializing the node. + startupTaints: + - key: example.com/another-taint + effect: NoSchedule + + # Requirements that constrain the parameters of provisioned nodes. + # These requirements are combined with pod.spec.topologySpreadConstraints, pod.spec.affinity.nodeAffinity, pod.spec.affinity.podAffinity, and pod.spec.nodeSelector rules. + # Operators { In, NotIn, Exists, DoesNotExist, Gt, and Lt } are supported. + # https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#operators + requirements: + - key: "karpenter.k8s.aws/instance-category" + operator: In + values: ["c", "m", "r"] + - key: "karpenter.k8s.aws/instance-cpu" + operator: In + values: ["4", "8", "16", "32"] + - key: "karpenter.k8s.aws/instance-hypervisor" + operator: In + values: ["nitro"] + - key: "karpenter.k8s.aws/instance-generation" + operator: Gt + values: ["2"] + - key: "topology.kubernetes.io/zone" + operator: In + values: ["us-west-2a", "us-west-2b"] + - key: "kubernetes.io/arch" + operator: In + values: ["arm64", "amd64"] + - key: "karpenter.sh/capacity-type" + operator: In + values: ["spot", "on-demand"] + + # Karpenter provides the ability to specify a few additional Kubelet args. + # These are all optional and provide support for additional customization and use cases. + kubelet: + clusterDNS: ["10.0.1.100"] + systemReserved: + cpu: 100m + memory: 100Mi + ephemeral-storage: 1Gi + kubeReserved: + cpu: 200m + memory: 100Mi + ephemeral-storage: 3Gi + evictionHard: + memory.available: 5% + nodefs.available: 10% + nodefs.inodesFree: 10% + evictionSoft: + memory.available: 500Mi + nodefs.available: 15% + nodefs.inodesFree: 15% + evictionSoftGracePeriod: + memory.available: 1m + nodefs.available: 1m30s + nodefs.inodesFree: 2m + evictionMaxPodGracePeriod: 60 + imageGCHighThresholdPercent: 85 + imageGCLowThresholdPercent: 80 + cpuCFSQuota: true + podsPerCore: 2 + maxPods: 20 + + # Disruption section which describes the ways in which Karpenter can disrupt and replace Nodes + # Configuration in this section constrains how aggressive Karpenter can be with performing operations + # like rolling Nodes due to them hitting their maximum lifetime (expiry) or scaling down nodes to reduce cluster cost + disruption: + # Describes which types of Nodes Karpenter should consider for consolidation + # If using 'WhenUnderutilized', Karpenter will consider all nodes for consolidation and attempt to remove or replace Nodes when it discovers that the Node is underutilized and could be changed to reduce cost + # If using `WhenEmpty`, Karpenter will only consider nodes for consolidation that contain no workload pods + consolidationPolicy: WhenUnderutilized | WhenEmpty + + # The amount of time Karpenter should wait after discovering a consolidation decision + # This value can currently only be set when the consolidationPolicy is 'WhenEmpty' + # You can choose to disable consolidation entirely by setting the string value 'Never' here + consolidateAfter: 30s + + # The amount of time a Node can live on the cluster before being removed + # Avoiding long-running Nodes helps to reduce security vulnerabilities as well as to reduce the chance of issues that can plague Nodes with long uptimes such as file fragmentation or memory leaks from system processes + # You can choose to disable expiration entirely by setting the string value 'Never' here + expireAfter: 720h + + # Budgets control the speed Karpenter can scale down nodes. + # Karpenter will respect the minimum of the currently active budgets, and will round up + # when considering percentages. Duration and Schedule must be set together. + budgets: + - nodes: 10% + # On Weekdays during business hours, don't do any deprovisioning. + - schedule: "0 9 * * mon-fri" + duration: 8h + nodes: "0" + + # Resource limits constrain the total size of the cluster. + # Limits prevent Karpenter from creating new instances once the limit is exceeded. + limits: + cpu: "1000" + memory: 1000Gi + + # Priority given to the NodePool when the scheduler considers which NodePool + # to select. Higher weights indicate higher priority when comparing NodePools. + # Specifying no weight is equivalent to specifying a weight of 0. + weight: 10 +``` + +## spec.template.spec.requirements + +Kubernetes defines the following [Well-Known Labels](https://kubernetes.io/docs/reference/labels-annotations-taints/), and cloud providers (e.g., AWS) implement them. They are defined at the "spec.requirements" section of the NodePool API. + +In addition to the well-known labels from Kubernetes, Karpenter supports AWS-specific labels for more advanced scheduling. See the full list [here](../scheduling/#well-known-labels). + +These well-known labels may be specified at the NodePool level, or in a workload definition (e.g., nodeSelector on a pod.spec). Nodes are chosen using both the NodePool's and pod's requirements. If there is no overlap, nodes will not be launched. In other words, a pod's requirements must be within the NodePool's requirements. If a requirement is not defined for a well known label, any value available to the cloud provider may be chosen. + +For example, an instance type may be specified using a nodeSelector in a pod spec. If the instance type requested is not included in the NodePool list and the NodePool has instance type requirements, Karpenter will not create a node or schedule the pod. + +### Instance Types + +- key: `node.kubernetes.io/instance-type` +- key: `karpenter.k8s.aws/instance-family` +- key: `karpenter.k8s.aws/instance-category` +- key: `karpenter.k8s.aws/instance-generation` + +Generally, instance types should be a list and not a single value. Leaving these requirements undefined is recommended, as it maximizes choices for efficiently placing pods. + +Review [AWS instance types](../instance-types). Most instance types are supported with the exclusion of [non-HVM](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/virtualization_types.html). + +### Availability Zones + +- key: `topology.kubernetes.io/zone` +- value example: `us-east-1c` +- value list: `aws ec2 describe-availability-zones --region ` + +Karpenter can be configured to create nodes in a particular zone. Note that the Availability Zone `us-east-1a` for your AWS account might not have the same location as `us-east-1a` for another AWS account. + +[Learn more about Availability Zone +IDs.](https://docs.aws.amazon.com/ram/latest/userguide/working-with-az-ids.html) + +### Architecture + +- key: `kubernetes.io/arch` +- values + - `amd64` + - `arm64` + +Karpenter supports `amd64` nodes, and `arm64` nodes. + +### Operating System + - key: `kubernetes.io/os` + - values + - `linux` + - `windows` + +Karpenter supports `linux` and `windows` operating systems. + +### Capacity Type + +- key: `karpenter.sh/capacity-type` +- values + - `spot` + - `on-demand` + +Karpenter supports specifying capacity type, which is analogous to [EC2 purchase options](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-purchasing-options.html). + +Karpenter prioritizes Spot offerings if the NodePool allows Spot and on-demand instances. If the provider API (e.g. EC2 Fleet's API) indicates Spot capacity is unavailable, Karpenter caches that result across all attempts to provision EC2 capacity for that instance type and zone for the next 45 seconds. If there are no other possible offerings available for Spot, Karpenter will attempt to provision on-demand instances, generally within milliseconds. + +Karpenter also allows `karpenter.sh/capacity-type` to be used as a topology key for enforcing topology-spread. + +{{% alert title="Recommended" color="primary" %}} +Karpenter allows you to be extremely flexible with your NodePools by only constraining your instance types in ways that are absolutely necessary for your cluster. By default, Karpenter will enforce that you specify the `spec.template.spec.requirements` field, but will not enforce that you specify any requirements within the field. If you choose to specify `requirements: []`, this means that you will completely flexible to _all_ instance types that your cloud provider supports. + +Though Karpenter doesn't enforce these defaults, for most use-cases, we recommend that you specify _some_ requirements to avoid odd behavior or exotic instance types. Below, is a high-level recommendation for requirements that should fit the majority of use-cases for generic workloads + +```yaml +spec: + template: + spec: + requirements: + - key: kubernetes.io/arch + operator: In + values: ["amd64"] + - key: kubernetes.io/os + operator: In + values: ["linux"] + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand"] + - key: karpenter.k8s.aws/instance-category + operator: In + values: ["c", "m", "r"] + - key: karpenter.k8s.aws/instance-generation + operator: Gt + values: ["2"] +``` + +{{% /alert %}} + +## spec.template.spec.nodeClassRef + +This field points to the Cloud Provider NodeClass resource. Learn more about [EC2NodeClasses]({{}}). + +## spec.template.spec.kubelet + +Karpenter provides the ability to specify a few additional Kubelet args. These are all optional and provide support for +additional customization and use cases. Adjust these only if you know you need to do so. For more details on kubelet configuration arguments, [see the KubeletConfiguration API specification docs](https://kubernetes.io/docs/reference/config-api/kubelet-config.v1beta1/). The implemented fields are a subset of the full list of upstream kubelet configuration arguments. Please cut an issue if you'd like to see another field implemented. + +```yaml +kubelet: + clusterDNS: ["10.0.1.100"] + systemReserved: + cpu: 100m + memory: 100Mi + ephemeral-storage: 1Gi + kubeReserved: + cpu: 200m + memory: 100Mi + ephemeral-storage: 3Gi + evictionHard: + memory.available: 5% + nodefs.available: 10% + nodefs.inodesFree: 10% + evictionSoft: + memory.available: 500Mi + nodefs.available: 15% + nodefs.inodesFree: 15% + evictionSoftGracePeriod: + memory.available: 1m + nodefs.available: 1m30s + nodefs.inodesFree: 2m + evictionMaxPodGracePeriod: 60 + imageGCHighThresholdPercent: 85 + imageGCLowThresholdPercent: 80 + cpuCFSQuota: true + podsPerCore: 2 + maxPods: 20 +``` + +### Reserved Resources + +Karpenter will automatically configure the system and kube reserved resource requests on the fly on your behalf. These requests are used to configure your node and to make scheduling decisions for your pods. If you have specific requirements or know that you will have additional capacity requirements, you can optionally override the `--system-reserved` configuration defaults with the `.spec.template.spec.kubelet.systemReserved` values and the `--kube-reserved` configuration defaults with the `.spec.template.spec.kubelet.kubeReserved` values. + +{{% alert title="Note" color="primary" %}} +Karpenter considers these reserved resources when computing the allocatable ephemeral storage on a given instance type. +If `kubeReserved` is not specified, Karpenter will compute the default reserved [CPU](https://github.com/awslabs/amazon-eks-ami/blob/db28da15d2b696bc08ac3aacc9675694f4a69933/files/bootstrap.sh#L251) and [memory](https://github.com/awslabs/amazon-eks-ami/blob/db28da15d2b696bc08ac3aacc9675694f4a69933/files/bootstrap.sh#L235) resources for the purpose of ephemeral storage computation. +These defaults are based on the defaults on Karpenter's supported AMI families, which are not the same as the kubelet defaults. +You should be aware of the CPU and memory default calculation when using Custom AMI Families. If they don't align, there may be a difference in Karpenter's computed allocatable ephemeral storage and the actually ephemeral storage available on the node. +{{% /alert %}} + +### Eviction Thresholds + +The kubelet supports eviction thresholds by default. When enough memory or file system pressure is exerted on the node, the kubelet will begin to evict pods to ensure that system daemons and other system processes can continue to run in a healthy manner. + +Kubelet has the notion of [hard evictions](https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#hard-eviction-thresholds) and [soft evictions](https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#soft-eviction-thresholds). In hard evictions, pods are evicted as soon as a threshold is met, with no grace period to terminate. Soft evictions, on the other hand, provide an opportunity for pods to be terminated gracefully. They do so by sending a termination signal to pods that are planning to be evicted and allowing those pods to terminate up to their grace period. + +Karpenter supports [hard evictions](https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#hard-eviction-thresholds) through the `.spec.template.spec.kubelet.evictionHard` field and [soft evictions](https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#soft-eviction-thresholds) through the `.spec.template.spec.kubelet.evictionSoft` field. `evictionHard` and `evictionSoft` are configured by listing [signal names](https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#eviction-signals) with either percentage values or resource values. + +```yaml +kubelet: + evictionHard: + memory.available: 500Mi + nodefs.available: 10% + nodefs.inodesFree: 10% + imagefs.available: 5% + imagefs.inodesFree: 5% + pid.available: 7% + evictionSoft: + memory.available: 1Gi + nodefs.available: 15% + nodefs.inodesFree: 15% + imagefs.available: 10% + imagefs.inodesFree: 10% + pid.available: 10% +``` + +#### Supported Eviction Signals + +| Eviction Signal | Description | +|--------------------|---------------------------------------------------------------------------------| +| memory.available | memory.available := node.status.capacity[memory] - node.stats.memory.workingSet | +| nodefs.available | nodefs.available := node.stats.fs.available | +| nodefs.inodesFree | nodefs.inodesFree := node.stats.fs.inodesFree | +| imagefs.available | imagefs.available := node.stats.runtime.imagefs.available | +| imagefs.inodesFree | imagefs.inodesFree := node.stats.runtime.imagefs.inodesFree | +| pid.available | pid.available := node.stats.rlimit.maxpid - node.stats.rlimit.curproc | + +For more information on eviction thresholds, view the [Node-pressure Eviction](https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction) section of the official Kubernetes docs. + +#### Soft Eviction Grace Periods + +Soft eviction pairs an eviction threshold with a specified grace period. With soft eviction thresholds, the kubelet will only begin evicting pods when the node exceeds its soft eviction threshold over the entire duration of its grace period. For example, if you specify `evictionSoft[memory.available]` of `500Mi` and a `evictionSoftGracePeriod[memory.available]` of `1m30`, the node must have less than `500Mi` of available memory over a minute and a half in order for the kubelet to begin evicting pods. + +Optionally, you can specify an `evictionMaxPodGracePeriod` which defines the administrator-specified maximum pod termination grace period to use during soft eviction. If a namespace-owner had specified a pod `terminationGracePeriodInSeconds` on pods in their namespace, the minimum of `evictionPodGracePeriod` and `terminationGracePeriodInSeconds` would be used. + +```yaml +kubelet: + evictionSoftGracePeriod: + memory.available: 1m + nodefs.available: 1m30s + nodefs.inodesFree: 2m + imagefs.available: 1m30s + imagefs.inodesFree: 2m + pid.available: 2m + evictionMaxPodGracePeriod: 60 +``` + +### Pod Density + +By default, the number of pods on a node is limited by both the number of networking interfaces (ENIs) that may be attached to an instance type and the number of IP addresses that can be assigned to each ENI. See [IP addresses per network interface per instance type](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-eni.html#AvailableIpPerENI) for a more detailed information on these instance types' limits. + +{{% alert title="Note" color="primary" %}} +By default, the VPC CNI allocates IPs for a node and pods from the same subnet. With [VPC CNI Custom Networking](https://aws.github.io/aws-eks-best-practices/networking/custom-networking), the pods will receive IP addresses from another subnet dedicated to pod IPs. This approach makes it easier to manage IP addresses and allows for separate Network Access Control Lists (NACLs) applied to your pods. VPC CNI Custom Networking reduces the pod density of a node since one of the ENI attachments will be used for the node and cannot share the allocated IPs on the interface to pods. Karpenter supports VPC CNI Custom Networking and similar CNI setups where the primary node interface is separated from the pods interfaces through a global [setting](./settings.md#configmap) within the karpenter-global-settings configmap: `aws.reservedENIs`. In the common case, `aws.reservedENIs` should be set to `"1"` if using Custom Networking. +{{% /alert %}} + +{{% alert title="Windows Support Notice" color="warning" %}} +It's currently not possible to specify custom networking with Windows nodes. +{{% /alert %}} + +#### Max Pods + +For small instances that require an increased pod density or large instances that require a reduced pod density, you can override this default value with `.spec.template.spec.kubelet.maxPods`. This value will be used during Karpenter pod scheduling and passed through to `--max-pods` on kubelet startup. + +{{% alert title="Note" color="primary" %}} +When using small instance types, it may be necessary to enable [prefix assignment mode](https://aws.amazon.com/blogs/containers/amazon-vpc-cni-increases-pods-per-node-limits/) in the AWS VPC CNI plugin to support a higher pod density per node. Prefix assignment mode was introduced in AWS VPC CNI v1.9 and allows ENIs to manage a broader set of IP addresses. Much higher pod densities are supported as a result. +{{% /alert %}} + +{{% alert title="Windows Support Notice" color="warning" %}} +Presently, Windows worker nodes do not support using more than one ENI. +As a consequence, the number of IP addresses, and subsequently, the number of pods that a Windows worker node can support is limited by the number of IPv4 addresses available on the primary ENI. +Currently, Karpenter will only consider individual secondary IP addresses when calculating the pod density limit. +{{% /alert %}} + +#### Pods Per Core + +An alternative way to dynamically set the maximum density of pods on a node is to use the `.spec.template.spec.kubelet.podsPerCore` value. Karpenter will calculate the pod density during scheduling by multiplying this value by the number of logical cores (vCPUs) on an instance type. This value will also be passed through to the `--pods-per-core` value on kubelet startup to configure the number of allocatable pods the kubelet can assign to the node instance. + +The value generated from `podsPerCore` cannot exceed `maxPods`, meaning, if both are set, the minimum of the `podsPerCore` dynamic pod density and the static `maxPods` value will be used for scheduling. + +{{% alert title="Note" color="primary" %}} +`maxPods` may not be set in the `kubelet` of a NodePool, but may still be restricted by the `ENI_LIMITED_POD_DENSITY` value. You may want to ensure that the `podsPerCore` value that will be used for instance families associated with the NodePool will not cause unexpected behavior by exceeding the `maxPods` value. +{{% /alert %}} + +{{% alert title="Pods Per Core on Bottlerocket" color="warning" %}} +Bottlerocket AMIFamily currently does not support `podsPerCore` configuration. If a NodePool contains a `provider` or `providerRef` to a node template that will launch a Bottlerocket instance, the `podsPerCore` value will be ignored for scheduling and for configuring the kubelet. +{{% /alert %}} + +## spec.disruption + +You can configure Karpenter to disrupt Nodes through your NodePool in multiple ways. You can use `spec.disruption.consolidationPolicy`, `spec.disruption.consolidateAfter` or `spec.disruption.expireAfter`. Read [Disruption]({{}}) for more. + +## spec.limits + +The NodePool spec includes a limits section (`spec.limits`), which constrains the maximum amount of resources that the NodePool will manage. + +Karpenter supports limits of any resource type reported by your cloudprovider. It limits instance types when scheduling to those that will not exceed the specified limits. If a limit has been exceeded, nodes provisioning is prevented until some nodes have been terminated. + +```yaml +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: default +spec: + template: + spec: + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["spot"] + limits: + cpu: 1000 + memory: 1000Gi + nvidia.com/gpu: 2 +``` + +{{% alert title="Note" color="primary" %}} +Karpenter provisioning is highly parallel. Because of this, limit checking is eventually consistent, which can result in overrun during rapid scale outs. +{{% /alert %}} + +CPU limits are described with a `DecimalSI` value. Note that the Kubernetes API will coerce this into a string, so we recommend against using integers to avoid GitOps skew. + +Memory limits are described with a [`BinarySI` value, such as 1000Gi.](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#meaning-of-memory) + +You can view the current consumption of cpu and memory on your cluster by running: +``` +kubectl get nodepool -o=jsonpath='{.items[0].status}' +``` + +Review the [Kubernetes core API](https://github.com/kubernetes/api/blob/37748cca582229600a3599b40e9a82a951d8bbbf/core/v1/resource.go#L23) (`k8s.io/api/core/v1`) for more information on `resources`. + +## spec.weight + +Karpenter allows you to describe NodePool preferences through a `weight` mechanism similar to how weight is described with [pod and node affinities](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity). + +For more information on weighting NodePools, see the [Weighted NodePools section]({{}}) in the scheduling docs. + +## Examples + +### Isolating Expensive Hardware + +A NodePool can be set up to only provision nodes on particular processor types. +The following example sets a taint that only allows pods with tolerations for Nvidia GPUs to be scheduled: + +```yaml +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: gpu +spec: + disruption: + consolidationPolicy: WhenUnderutilized + template: + spec: + requirements: + - key: node.kubernetes.io/instance-type + operator: In + values: ["p3.8xlarge", "p3.16xlarge"] + taints: + - key: nvidia.com/gpu + value: "true" + effect: NoSchedule +``` +In order for a pod to run on a node defined in this NodePool, it must tolerate `nvidia.com/gpu` in its pod spec. + +### Cilium Startup Taint + +Per the Cilium [docs](https://docs.cilium.io/en/stable/installation/taints/#taint-effects), it's recommended to place a taint of `node.cilium.io/agent-not-ready=true:NoExecute` on nodes to allow Cilium to configure networking prior to other pods starting. This can be accomplished via the use of Karpenter `startupTaints`. These taints are placed on the node, but pods aren't required to tolerate these taints to be considered for provisioning. + +```yaml +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: cilium-startup +spec: + disruption: + consolidationPolicy: WhenUnderutilized + template: + spec: + startupTaints: + - key: node.cilium.io/agent-not-ready + value: "true" + effect: NoExecute +``` diff --git a/website/content/en/v0.31/concepts/scheduling.md b/website/content/en/v0.34/concepts/scheduling.md similarity index 79% rename from website/content/en/v0.31/concepts/scheduling.md rename to website/content/en/v0.34/concepts/scheduling.md index 0f70f1a0463f..5352f0706fb9 100755 --- a/website/content/en/v0.31/concepts/scheduling.md +++ b/website/content/en/v0.34/concepts/scheduling.md @@ -15,10 +15,10 @@ Reasons for constraining where your pods run could include: * Wanting to use techniques like topology spread to help ensure high availability Your Cloud Provider defines the first layer of constraints, including all instance types, architectures, zones, and purchase types available to its cloud. -The cluster administrator adds the next layer of constraints by creating one or more provisioners. +The cluster administrator adds the next layer of constraints by creating one or more NodePools. The final layer comes from you adding specifications to your Kubernetes pod deployments. -Pod scheduling constraints must fall within a provisioner's constraints or the pods will not deploy. -For example, if the provisioner sets limits that allow only a particular zone to be used, and a pod asks for a different zone, it will not be scheduled. +Pod scheduling constraints must fall within a NodePool's constraints or the pods will not deploy. +For example, if the NodePool sets limits that allow only a particular zone to be used, and a pod asks for a different zone, it will not be scheduled. Constraints you can request include: @@ -62,7 +62,7 @@ Its limits are set to 256MiB of memory and 1 CPU. Instance type selection math only uses `requests`, but `limits` may be configured to enable resource oversubscription. -See [Managing Resources for Containers](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) for details on resource types supported by Kubernetes, [Specify a memory request and a memory limit](https://kubernetes.io/docs/tasks/configure-pod-container/assign-memory-resource/#specify-a-memory-request-and-a-memory-limit) for examples of memory requests, and [Provisioner Configuration]({{}}) for a list of supported resources. +See [Managing Resources for Containers](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) for details on resource types supported by Kubernetes, [Specify a memory request and a memory limit](https://kubernetes.io/docs/tasks/configure-pod-container/assign-memory-resource/#specify-a-memory-request-and-a-memory-limit) for examples of memory requests, and [NodePools]({{}}) for a list of supported resources. ### Accelerators/GPU Resources @@ -129,7 +129,7 @@ This can include well-known labels or custom labels you create yourself. You can use `affinity` to define more complicated constraints, see [Node Affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity) for the complete specification. ### Labels -Well-known labels may be specified as provisioner requirements or pod scheduling constraints. You can also define your own custom labels by specifying `requirements` or `labels` on your Provisioner and select them using `nodeAffinity` or `nodeSelectors` on your Pods. +Well-known labels may be specified as NodePool requirements or pod scheduling constraints. You can also define your own custom labels by specifying `requirements` or `labels` on your NodePool and select them using `nodeAffinity` or `nodeSelectors` on your Pods. {{% alert title="Warning" color="warning" %}} Take care to ensure the label domains are correct. A well known label like `karpenter.k8s.aws/instance-family` will enforce node properties, but may be confused with `node.kubernetes.io/instance-family`, which is unknown to Karpenter, and treated as a custom label which will not enforce node properties. @@ -167,7 +167,7 @@ Karpenter translates the following deprecated labels to their stable equivalents #### User-Defined Labels -Karpenter is aware of several well-known labels, deriving them from instance type details. If you specify a `nodeSelector` or a required `nodeAffinity` using a label that is not well-known to Karpenter, it will not launch nodes with these labels and pods will remain pending. For Karpenter to become aware that it can schedule for these labels, you must specify the label in the Provisioner requirements with the `Exists` operator: +Karpenter is aware of several well-known labels, deriving them from instance type details. If you specify a `nodeSelector` or a required `nodeAffinity` using a label that is not well-known to Karpenter, it will not launch nodes with these labels and pods will remain pending. For Karpenter to become aware that it can schedule for these labels, you must specify the label in the NodePool requirements with the `Exists` operator: ```yaml requirements: @@ -186,7 +186,7 @@ nodeSelector: ``` This example features a well-known label (`topology.kubernetes.io/zone`) and a label that is well known to Karpenter (`karpenter.sh/capacity-type`). -If you want to create a custom label, you should do that at the provisioner level. +If you want to create a custom label, you should do that at the NodePool level. Then the pod can declare that custom label. @@ -204,8 +204,7 @@ When setting rules, the following Node affinity types define how hard or soft ea The `IgnoredDuringExecution` part of each tells the pod to keep running, even if conditions change on the node so the rules no longer matched. You can think of these concepts as `required` and `preferred`, since Kubernetes never implemented other variants of these rules. -All examples below assume that the provisioner doesn't have constraints to prevent those zones from being used. -The first constraint says you could use `us-west-2a` or `us-west-2b`, the second constraint makes it so only `us-west-2b` can be used. +All examples below assume that the NodePool doesn't have constraints to prevent those zones from being used. The first constraint says you could use `us-west-2a` or `us-west-2b`, the second constraint makes it so only `us-west-2b` can be used. ```yaml affinity: @@ -215,7 +214,7 @@ The first constraint says you could use `us-west-2a` or `us-west-2b`, the second - matchExpressions: - key: "topology.kubernetes.io/zone" operator: "In" - values: ["us-west-2a, us-west-2b"] + values: ["us-west-2a", "us-west-2b"] - key: "topology.kubernetes.io/zone" operator: "In" values: ["us-west-2b"] @@ -226,7 +225,7 @@ Changing the second operator to `NotIn` would allow the pod to run in `us-west-2 ```yaml - key: "topology.kubernetes.io/zone" operator: "In" - values: ["us-west-2a, us-west-2b"] + values: ["us-west-2a", "us-west-2b"] - key: "topology.kubernetes.io/zone" operator: "NotIn" values: ["us-west-2b"] @@ -244,7 +243,7 @@ Here, if `us-west-2a` is not available, the second term will cause the pod to ru - matchExpressions: # OR - key: "topology.kubernetes.io/zone" # AND operator: "In" - values: ["us-west-2a, us-west-2b"] + values: ["us-west-2a", "us-west-2b"] - key: "topology.kubernetes.io/zone" # AND operator: "NotIn" values: ["us-west-2b"] @@ -265,28 +264,28 @@ So if capacity becomes available, it will schedule the pod without user interven ## Taints and tolerations Taints are the opposite of affinity. -Setting a taint on a node tells the scheduler to not run a pod on it unless the pod has explicitly said it can tolerate that taint. -This example shows a Provisioner that was set up with a taint for only running pods that require a GPU, such as the following: - +Setting a taint on a node tells the scheduler to not run a pod on it unless the pod has explicitly said it can tolerate that taint. This example shows a NodePool that was set up with a taint for only running pods that require a GPU, such as the following: ```yaml -apiVersion: karpenter.sh/v1alpha5 -kind: Provisioner +apiVersion: karpenter.sh/v1beta1 +kind: NodePool metadata: name: gpu spec: - requirements: - - key: karpenter.k8s.aws/instance-family - operator: In - values: - - p3 - taints: - - key: nvidia.com/gpu - value: true - effect: "NoSchedule" + template: + spec: + requirements: + - key: karpenter.k8s.aws/instance-family + operator: In + values: + - p3 + taints: + - key: nvidia.com/gpu + value: true + effect: "NoSchedule" ``` -For a pod to request to run on a node that has provisioner, it could set a toleration as follows: +For a pod to request to run on a node that has this NodePool, it could set a toleration as follows: ```yaml apiVersion: v1 @@ -311,8 +310,7 @@ See [Taints and Tolerations](https://kubernetes.io/docs/concepts/scheduling-evic ## Topology Spread -By using the Kubernetes `topologySpreadConstraints` you can ask the provisioner to have pods push away from each other to limit the blast radius of an outage. -Think of it as the Kubernetes evolution for pod affinity: it lets you relate pods with respect to nodes while still allowing spread. +By using the Kubernetes `topologySpreadConstraints` you can ask the NodePool to have pods push away from each other to limit the blast radius of an outage. Think of it as the Kubernetes evolution for pod affinity: it lets you relate pods with respect to nodes while still allowing spread. For example: ```yaml @@ -351,12 +349,15 @@ The three supported `topologyKey` values that Karpenter supports are: - `kubernetes.io/hostname` - `karpenter.sh/capacity-type` - See [Pod Topology Spread Constraints](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/) for details. +{{% alert title="Note" color="primary" %}} +NodePools do not attempt to balance or rebalance the availability zones for their nodes. Availability zone balancing may be achieved by defining zonal Topology Spread Constraints for Pods that require multi-zone durability, and NodePools will respect these constraints while optimizing for compute costs. +{{% /alert %}} + ## Pod affinity/anti-affinity -By using the `podAffinity` and `podAntiAffinity` configuration on a pod spec, you can inform the provisioner of your desire for pods to schedule together or apart with respect to different topology domains. For example: +By using the `podAffinity` and `podAntiAffinity` configuration on a pod spec, you can inform the Karpenter scheduler of your desire for pods to schedule together or apart with respect to different topology domains. For example: ```yaml spec: @@ -441,167 +442,174 @@ The EBS CSI driver uses `topology.ebs.csi.aws.com/zone` instead of the standard The topology key `topology.kubernetes.io/region` is not supported. Legacy in-tree CSI providers specify this label. Instead, install an out-of-tree CSI provider. [Learn more about moving to CSI providers.](https://kubernetes.io/blog/2021/12/10/storage-in-tree-to-csi-migration-status-update/#quick-recap-what-is-csi-migration-and-why-migrate) {{% /alert %}} -## Weighting Provisioners +## Weighted NodePools -Karpenter allows you to order your provisioners using the `.spec.weight` field so that the node scheduler will deterministically attempt to schedule with one provisioner before another. Below are a few example use-cases that are now supported with the provisioner weighting semantic. +Karpenter allows you to order your NodePools using the `.spec.weight` field so that the Karpenter scheduler will attempt to schedule one NodePool before another. ### Savings Plans and Reserved Instances If you have purchased a [Savings Plan](https://aws.amazon.com/savingsplans/) or [Reserved Instances](https://aws.amazon.com/ec2/pricing/reserved-instances/), you may want to tell Karpenter to prioritize this reserved capacity ahead of other instance types. -To enable this, you will need to tell the Karpenter controllers which instance types to prioritize and what is the maximum amount of capacity that should be provisioned using those instance types. We can set the `.spec.limits` on the provisioner to limit the capacity that can be launched by this provisioner. Combined with the `.spec.weight` value, we can tell Karpenter to pull from instance types in the reserved provisioner before defaulting to generic instance types. - -#### Reserved Instance Provisioner +To enable this, you will need to tell the Karpenter controllers which instance types to prioritize and what is the maximum amount of capacity that should be provisioned using those instance types. We can set the `.spec.limits` field on the NodePool to limit the capacity that can be launched by this NodePool. Combined with the `.spec.weight` value, we can tell Karpenter to pull from instance types in the reserved NodePool before defaulting to generic instance types. ```yaml -apiVersion: karpenter.sh/v1alpha5 -kind: Provisioner +apiVersion: karpenter.sh/v1beta1 +kind: NodePool metadata: name: reserved-instance spec: weight: 50 - requirements: - - key: "node.kubernetes.io/instance-type" - operator: In - values: ["c4.large"] limits: - resources: - cpu: 100 -``` - -#### Default Provisioner - -```yaml -apiVersion: karpenter.sh/v1alpha5 -kind: Provisioner + cpu: 100 + template: + spec: + requirements: + - key: "node.kubernetes.io/instance-type" + operator: In + values: ["c4.large"] +--- +apiVersion: karpenter.sh/v1beta1 +kind: NodePool metadata: name: default spec: - requirements: - - key: karpenter.sh/capacity-type - operator: In - values: ["spot", "on-demand"] - - key: kubernetes.io/arch - operator: In - values: ["amd64"] + template: + spec: + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["spot", "on-demand"] + - key: kubernetes.io/arch + operator: In + values: ["amd64"] ``` -### Default Node Configuration +### Fallback Pods that do not specify node selectors or affinities can potentially be assigned to any node with any configuration. There may be cases where you require these pods to schedule to a specific capacity type or architecture but assigning the relevant node selectors or affinities to all these workload pods may be too tedious or infeasible. Instead, we want to define a cluster-wide default configuration for nodes launched using Karpenter. -By assigning a higher `.spec.weight` value and restricting a provisioner to a specific capacity type or architecture, we can set default configuration for the nodes launched by pods that don't have node configuration restrictions. - -#### Default Provisioner +By assigning a higher `.spec.weight` value and restricting a NodePool to a specific capacity type or architecture, we can set default configuration for the nodes launched by pods that don't have node configuration restrictions. ```yaml -apiVersion: karpenter.sh/v1alpha5 -kind: Provisioner +apiVersion: karpenter.sh/v1beta1 +kind: NodePool metadata: name: default spec: weight: 50 - requirements: - - key: karpenter.sh/capacity-type - operator: In - values: ["spot", "on-demand"] - - key: kubernetes.io/arch - operator: In - values: ["amd64"] -``` - -#### ARM-64 Specific Provisioner - -```yaml -apiVersion: karpenter.sh/v1alpha5 -kind: Provisioner + template: + spec: + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["spot", "on-demand"] + - key: kubernetes.io/arch + operator: In + values: ["amd64"] +--- +apiVersion: karpenter.sh/v1beta1 +kind: NodePool metadata: name: arm64-specific spec: - requirements: - - key: karpenter.sh/capacity-type - operator: In - values: ["spot", "on-demand"] - - key: kubernetes.io/arch - operator: In - values: ["arm64"] - - key: node.kubernetes.io/instance-type - operator: In - values: ["a1.large", "a1.xlarge"] + template: + spec: + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["spot", "on-demand"] + - key: kubernetes.io/arch + operator: In + values: ["arm64"] + - key: node.kubernetes.io/instance-type + operator: In + values: ["a1.large", "a1.xlarge"] ``` {{% alert title="Note" color="primary" %}} -Based on the way that Karpenter performs pod batching and bin packing, it is not guaranteed that Karpenter will always choose the highest priority provisioner given specific requirements. For example, if a pod can't be scheduled with the highest priority provisioner it will force creation of a node using a lower priority provisioner which may allow other pods from that batch to also schedule on that node. The behavior may also occur if existing capacity is available, as the kube-scheduler will schedule the pods instead of allowing Karpenter to provision a new node. +Based on the way that Karpenter performs pod batching and bin packing, it is not guaranteed that Karpenter will always choose the highest priority NodePool given specific requirements. For example, if a pod can't be scheduled with the highest priority NodePool, it will force creation of a node using a lower priority NodePool, allowing other pods from that batch to also schedule on that node. The behavior may also occur if existing capacity is available, as the kube-scheduler will schedule the pods instead of allowing Karpenter to provision a new node. {{% /alert %}} ## Advanced Scheduling Techniques ### `Exists` Operator -The `Exists` operator can be used on a provisioner to provide workload segregation across nodes. +The `Exists` operator can be used on a NodePool to provide workload segregation across nodes. ```yaml ... - requirements: - - key: company.com/team - operator: Exists +requirements: +- key: company.com/team + operator: Exists ... ``` -With the requirement on the provisioner in place, workloads can optionally specify a custom value as a required node affinity or node selector. Karpenter will then label the nodes it launches for these pods which prevents `kube-scheduler` from scheduling conflicting pods to those nodes. This provides a way to more dynamically isolate workloads without requiring a unique provisioner for each workload subset. +With the requirement on the NodePool, workloads can optionally specify a custom value as a required node affinity or node selector. Karpenter will then label the nodes it launches for these pods which prevents `kube-scheduler` from scheduling conflicting pods to those nodes. This provides a way to more dynamically isolate workloads without requiring a unique NodePool for each workload subset. ```yaml - nodeSelector: - company.com/team: team-a +nodeSelector: + company.com/team: team-a ``` {{% alert title="Note" color="primary" %}} -If a workload matches the provisioner but doesn't specify a label, Karpenter will generate a random label for the node. +If a workload matches the NodePool but doesn't specify a label, Karpenter will generate a random label for the node. {{% /alert %}} ### On-Demand/Spot Ratio Split Taking advantage of Karpenter's ability to assign labels to node and using a topology spread across those labels enables a crude method for splitting a workload across on-demand and spot instances in a desired ratio. -To do this, we create a provisioner each for spot and on-demand with disjoint values for a unique new label called `capacity-spread`. In the example below, we provide four unique values for the spot provisioner and one value for the on-demand provisioner. When we spread across our new label evenly, we'll end up with a ratio of 4:1 spot to on-demand nodes. +To do this, we create one NodePool each for spot and on-demand with disjoint values for a unique new label called `capacity-spread`. In the example below, we provide four unique values for the spot NodePool and one value for the on-demand NodePool. When we spread across our new label evenly, we'll end up with a ratio of 4:1 spot to on-demand nodes. {{% alert title="Warning" color="warning" %}} This is not identical to a topology spread with a specified ratio. We are constructing 'virtual domains' to spread evenly across and the ratio of those 'virtual domains' to spot and on-demand happen to coincide with the desired spot to on-demand ratio. As an example, if you launch pods using the provided example, Karpenter will launch nodes with `capacity-spread` labels of 1, 2, 3, 4, and 5. `kube-scheduler` will then schedule evenly across those nodes to give the desired ratio. {{% /alert %}} -#### Spot Provisioner -```yaml - requirements: - - key: "karpenter.sh/capacity-type" - operator: In - values: [ "spot"] - - key: capacity-spread - operator: In - values: - - "2" - - "3" - - "4" - - "5" -``` +#### NodePools -#### On-Demand Provisioner ```yaml - requirements: - - key: "karpenter.sh/capacity-type" - operator: In - values: [ "on-demand"] - - key: capacity-spread - operator: In - values: - - "1" +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: spot +spec: + template: + spec: + requirements: + - key: "karpenter.sh/capacity-type" + operator: In + values: ["spot"] + - key: capacity-spread + operator: In + values: + - "2" + - "3" + - "4" + - "5" +--- +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: on-demand +spec: + template: + spec: + requirements: + - key: "karpenter.sh/capacity-type" + operator: In + values: ["on-demand"] + - key: capacity-spread + operator: In + values: + - "1" ``` #### Workload Topology Spread Constraint ```yaml topologySpreadConstraints: - - maxSkew: 1 - topologyKey: capacity-spread - whenUnsatisfiable: DoNotSchedule - labelSelector: - ... +- maxSkew: 1 + topologyKey: capacity-spread + whenUnsatisfiable: DoNotSchedule + labelSelector: + ... ``` diff --git a/website/content/en/v0.31/contributing/_index.md b/website/content/en/v0.34/contributing/_index.md similarity index 90% rename from website/content/en/v0.31/contributing/_index.md rename to website/content/en/v0.34/contributing/_index.md index 10bb749d39dc..6ec2c3df504e 100644 --- a/website/content/en/v0.31/contributing/_index.md +++ b/website/content/en/v0.34/contributing/_index.md @@ -1,7 +1,7 @@ --- title: "Contributing" linkTitle: "Contributing" -weight: 100 +weight: 40 description: > Learn about how to contribute to Karpenter --- diff --git a/website/content/en/v0.30/contributing/design-guide.md b/website/content/en/v0.34/contributing/design-guide.md similarity index 100% rename from website/content/en/v0.30/contributing/design-guide.md rename to website/content/en/v0.34/contributing/design-guide.md diff --git a/website/content/en/v0.30/contributing/development-guide.md b/website/content/en/v0.34/contributing/development-guide.md similarity index 95% rename from website/content/en/v0.30/contributing/development-guide.md rename to website/content/en/v0.34/contributing/development-guide.md index 24ac17efe270..e803169d6c5a 100644 --- a/website/content/en/v0.30/contributing/development-guide.md +++ b/website/content/en/v0.34/contributing/development-guide.md @@ -73,9 +73,9 @@ make test # E2E correctness tests ### Change Log Level +By default, `make apply` will set the log level to debug. You can change the log level by setting the log level in your helm values. ```bash -kubectl patch configmap config-logging -n karpenter --patch '{"data":{"loglevel.controller":"debug"}}' # Debug Level -kubectl patch configmap config-logging -n karpenter --patch '{"data":{"loglevel.controller":"info"}}' # Info Level +--set logLevel=debug ``` ### Debugging Metrics diff --git a/website/content/en/v0.30/contributing/working-group.md b/website/content/en/v0.34/contributing/working-group.md similarity index 100% rename from website/content/en/v0.30/contributing/working-group.md rename to website/content/en/v0.34/contributing/working-group.md diff --git a/website/content/en/v0.31/faq.md b/website/content/en/v0.34/faq.md similarity index 51% rename from website/content/en/v0.31/faq.md rename to website/content/en/v0.34/faq.md index 81027f2061ff..5096c34f69db 100644 --- a/website/content/en/v0.31/faq.md +++ b/website/content/en/v0.34/faq.md @@ -1,101 +1,85 @@ --- title: "FAQs" linkTitle: "FAQs" -weight: 90 +weight: 60 description: > Review Karpenter Frequently Asked Questions --- ## General -### How does a provisioner decide to manage a particular node? -See [Configuring provisioners]({{< ref "./concepts/#configuring-provisioners" >}}) for information on how Karpenter provisions and manages nodes. +### How does a NodePool decide to manage a particular node? +See [Configuring NodePools]({{< ref "./concepts/#configuring-nodepools" >}}) for information on how Karpenter configures and manages nodes. ### What cloud providers are supported? AWS is the first cloud provider supported by Karpenter, although it is designed to be used with other cloud providers as well. ### Can I write my own cloud provider for Karpenter? -Yes, but there is no documentation yet for it. -Start with Karpenter's GitHub [cloudprovider](https://github.com/aws/karpenter-core/tree/v0.31.3/pkg/cloudprovider) documentation to see how the AWS provider is built, but there are other sections of the code that will require changes too. +Yes, but there is no documentation yet for it. Start with Karpenter's GitHub [cloudprovider](https://github.com/aws/karpenter-core/tree/v0.34.5/pkg/cloudprovider) documentation to see how the AWS provider is built, but there are other sections of the code that will require changes too. ### What operating system nodes does Karpenter deploy? -By default, Karpenter uses Amazon Linux 2 images. +Karpenter uses the OS defined by the [AMI Family in your EC2NodeClass]({{< ref "./concepts/nodeclasses#specamifamily" >}}). ### Can I provide my own custom operating system images? -Karpenter has multiple mechanisms for configuring the [operating system]({{< ref "./concepts/node-templates/#spec-amiselector" >}}) for your nodes. +Karpenter has multiple mechanisms for configuring the [operating system]({{< ref "./concepts/nodeclasses/#specamiselectorterms" >}}) for your nodes. ### Can Karpenter deal with workloads for mixed architecture cluster (arm vs. amd)? -Karpenter is flexible to multi architecture configurations using [well known labels]({{< ref "./concepts/scheduling/#supported-labels">}}). +Karpenter is flexible to multi-architecture configurations using [well known labels]({{< ref "./concepts/scheduling/#supported-labels">}}). ### What RBAC access is required? -All of the required RBAC rules can be found in the helm chart template. -See [clusterrolebinding.yaml](https://github.com/aws/karpenter/blob/v0.31.3/charts/karpenter/templates/clusterrolebinding.yaml), [clusterrole.yaml](https://github.com/aws/karpenter/blob/v0.31.3/charts/karpenter/templates/clusterrole.yaml), [rolebinding.yaml](https://github.com/aws/karpenter/blob/v0.31.3/charts/karpenter/templates/rolebinding.yaml), and [role.yaml](https://github.com/aws/karpenter/blob/v0.31.3/charts/karpenter/templates/role.yaml) files for details. +All the required RBAC rules can be found in the helm chart template. See [clusterrole-core.yaml](https://github.com/aws/karpenter/blob/v0.34.5/charts/karpenter/templates/clusterrole-core.yaml), [clusterrole.yaml](https://github.com/aws/karpenter/blob/v0.34.5/charts/karpenter/templates/clusterrole.yaml), [rolebinding.yaml](https://github.com/aws/karpenter/blob/v0.34.5/charts/karpenter/templates/rolebinding.yaml), and [role.yaml](https://github.com/aws/karpenter/blob/v0.34.5/charts/karpenter/templates/role.yaml) files for details. ### Can I run Karpenter outside of a Kubernetes cluster? Yes, as long as the controller has network and IAM/RBAC access to the Kubernetes API and your provider API. +### What do I do if I encounter a security issue with Karpenter? +Refer to [Reporting Security Issues](https://github.com/aws/karpenter/security/policy) for information on how to report Karpenter security issues. Do not create a public GitHub issue. + ## Compatibility ### Which versions of Kubernetes does Karpenter support? -See the [Compatibility Matrix in the Upgrade Guide]({{< ref "./upgrade-guide#compatibility-matrix" >}}) to view the supported Kubernetes versions per Karpenter released version. +See the [Compatibility Matrix in the Upgrade Section]({{< ref "./upgrading/compatibility#compatibility-matrix" >}}) to view the supported Kubernetes versions per Karpenter released version. ### What Kubernetes distributions are supported? -Karpenter documents integration with a fresh or existing install of the latest AWS Elastic Kubernetes Service (EKS). -Other Kubernetes distributions (KOPs, etc.) can be used, but setting up cloud provider permissions for those distributions has not been documented. +Karpenter documents integration with a fresh or existing installation of the latest AWS Elastic Kubernetes Service (EKS). Other Kubernetes distributions (KOPs, etc.) can be used, but setting up cloud provider permissions for those distributions has not been documented. ### How does Karpenter interact with AWS node group features? -Provisioners are designed to work alongside static capacity management solutions like EKS Managed Node Groups and EC2 Auto Scaling Groups. -You can manage all capacity using provisioners, use a mixed model with dynamic and statically managed capacity, or use a fully static approach. -We expect most users will use a mixed approach in the near term and provisioner-managed in the long term. +NodePools are designed to work alongside static capacity management solutions like EKS Managed Node Groups and EC2 Auto Scaling Groups. You can manage all capacity using NodePools, use a mixed model with dynamic and statically managed capacity, or use a fully static approach. We expect most users will use a mixed approach in the near term and NodePool-managed in the long term. ### How does Karpenter interact with Kubernetes features? -* Kubernetes Cluster Autoscaler: Karpenter can work alongside cluster autoscaler. -See [Kubernetes cluster autoscaler]({{< ref "./concepts/#kubernetes-cluster-autoscaler" >}}) for details. -* Kubernetes Scheduler: Karpenter focuses on scheduling pods that the Kubernetes scheduler has marked as unschedulable. -See [Scheduling]({{< ref "./concepts/scheduling" >}}) for details on how Karpenter interacts with the Kubernetes scheduler. +* Kubernetes Cluster Autoscaler: Karpenter can work alongside Cluster Autoscaler. See [Kubernetes Cluster Autoscaler]({{< ref "./concepts/#kubernetes-cluster-autoscaler" >}}) for details. +* Kubernetes Scheduler: Karpenter focuses on scheduling pods that the Kubernetes scheduler has marked as unschedulable. See [Scheduling]({{< ref "./concepts/scheduling" >}}) for details on how Karpenter interacts with the Kubernetes scheduler. ## Provisioning -### What features does the Karpenter provisioner support? -See [Provisioner API]({{< ref "./concepts/provisioners" >}}) for provisioner examples and descriptions of features. +### What features does the Karpenter NodePool support? +See the [NodePool API docs]({{< ref "./concepts/nodepools" >}}) for NodePool examples and descriptions of features. -### Can I create multiple (team-based) provisioners on a cluster? -Yes, provisioners can identify multiple teams based on labels. -See [Provisioner API]({{< ref "./concepts/provisioners" >}}) for details. +### Can I create multiple (team-based) NodePools on a cluster? +Yes, NodePools can identify multiple teams based on labels. See the [NodePool API docs]({{< ref "./concepts/nodepools" >}}) for details. -### If multiple provisioners are defined, which will my pod use? +### If multiple NodePools are defined, which will my pod use? -Pending pods will be handled by any Provisioner that matches the requirements of the pod. -There is no ordering guarantee if multiple provisioners match pod requirements. -We recommend that Provisioners are setup to be mutually exclusive. -Read more about this recommendation in the [EKS Best Practices Guide for Karpenter](https://aws.github.io/aws-eks-best-practices/karpenter/#create-provisioners-that-are-mutually-exclusive). -To select a specific provisioner, use the node selector `karpenter.sh/provisioner-name: my-provisioner`. +Pending pods will be handled by any NodePools that matches the requirements of the pod. There is no ordering guarantee if multiple NodePools match pod requirements. We recommend that NodePools are set-up to be mutually exclusive. To select a specific NodePool, use the node selector `karpenter.sh/nodepool: my-nodepool`. ### How can I configure Karpenter to only provision pods for a particular namespace? -There is no native support for namespaced based provisioning. -Karpenter can be configured to provision a subset of pods based on a combination of taints/tolerations and node selectors. -This allows Karpenter to work in concert with the `kube-scheduler` in that the same mechanisms that `kube-scheduler` uses to determine if a pod can schedule to an existing node are also used for provisioning new nodes. -This avoids scenarios where pods are bound to nodes that were provisioned by Karpenter which Karpenter would not have bound itself. -If this were to occur, a node could remain non-empty and have its lifetime extended due to a pod that wouldn't have caused the node to be provisioned had the pod been unschedulable. +There is no native support for namespaced-based provisioning. Karpenter can be configured to provision a subset of pods based on a combination of taints/tolerations and node selectors. This allows Karpenter to work in concert with the `kube-scheduler` using the same mechanisms to determine if a pod can schedule to an existing node are also used for provisioning new nodes. This avoids scenarios where pods are bound to nodes that were provisioned by Karpenter which Karpenter would not have bound itself. If this were to occur, a node could remain non-empty and have its lifetime extended due to a pod that wouldn't have caused the node to be provisioned had the pod been unschedulable. -We recommend using Kubernetes native scheduling constraints to achieve namespace based scheduling segregation. Using native scheduling constraints ensures that Karpenter, `kube-scheduler` and any other scheduling or auto-provisioning mechanism all have an identical understanding of which pods can be scheduled on which nodes. This can be enforced via policy agents, an example of which can be seen [here](https://blog.mikesir87.io/2022/01/creating-tenant-node-pools-with-karpenter/). +We recommend using Kubernetes native scheduling constraints to achieve namespace-based scheduling segregation. Using native scheduling constraints ensures that Karpenter, `kube-scheduler` and any other scheduling or auto-provisioning mechanism all have an identical understanding of which pods can be scheduled on which nodes. This can be enforced via policy agents, an example of which can be seen [here](https://blog.mikesir87.io/2022/01/creating-tenant-node-pools-with-karpenter/). -### Can I add SSH keys to a provisioner? +### Can I add SSH keys to a NodePool? -Karpenter does not offer a way to add SSH keys via provisioners or secrets to the nodes it manages. -However, you can use Session Manager (SSM) or EC2 Instance Connect to gain shell access to Karpenter nodes. -See [Node NotReady]({{< ref "./troubleshooting/#node-notready" >}}) troubleshooting for an example of starting an SSM session from the command line or [EC2 Instance Connect](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-connect-set-up.html) documentation to connect to nodes using SSH. +Karpenter does not offer a way to add SSH keys via NodePools or secrets to the nodes it manages. +However, you can use Session Manager (SSM) or EC2 Instance Connect to gain shell access to Karpenter nodes. See [Node NotReady]({{< ref "./troubleshooting/#node-notready" >}}) troubleshooting for an example of starting an SSM session from the command line or [EC2 Instance Connect](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-connect-set-up.html) documentation to connect to nodes using SSH. -Though not recommended, if you need to access Karpenter-managed nodes without AWS credentials, you can add SSH keys using AWSNodeTemplate. -See [Custom User Data]({{< ref "./concepts/node-templates/#spec-userdata" >}}) for details. +Though not recommended, if you need to access Karpenter-managed nodes without AWS credentials, you can add SSH keys using EC2NodeClass User Data. See the [User Data section in the EC2NodeClass documentation]({{< ref "./concepts/nodeclasses/#specuserdata" >}}) for details. -### Can I set total limits of CPU and memory for a provisioner? -Yes, the setting is provider-specific. -See examples in [Accelerators, GPU]({{< ref "./concepts/scheduling/#accelerators-gpu-resources" >}}) Karpenter documentation. +### Can I set limits of CPU and memory for a NodePool? +Yes. View the [NodePool API docs]({{< ref "./concepts/nodepools#speclimits" >}}) for NodePool examples and descriptions of how to configure limits. ### Can I mix spot and on-demand EC2 run types? -Yes, see [Provisioning]({{< ref "./concepts/provisioners#examples" >}}) for an example. +Yes, see the [NodePool API docs]({{< ref "./concepts/nodepools#examples" >}}) for an example. ### Can I restrict EC2 instance types? @@ -104,53 +88,48 @@ Yes, see [Provisioning]({{< ref "./concepts/provisioners#examples" >}}) for an e ### Can I use Bare Metal instance types? -Yes, Karpenter supports provisioning metal instance types when a Provisioner's `node.kubernetes.io/instance-type` Requirements only include `metal` instance types. If other instance types fulfill pod requirements, then Karpenter will prioritize all non-metal instance types before metal ones are provisioned. +Yes, Karpenter supports provisioning metal instance types when a NodePool's `node.kubernetes.io/instance-type` Requirements only include `metal` instance types. If other instance types fulfill pod requirements, then Karpenter will prioritize all non-metal instance types before metal ones are provisioned. ### How does Karpenter dynamically select instance types? -Karpenter batches pending pods and then binpacks them based on CPU, memory, and GPUs required, taking into account node overhead, VPC CNI resources required, and daemonsets that will be packed when bringing up a new node. -By default Karpenter uses C, M, and R >= Gen 3 instance types, but it can be constrained in the provisioner spec with the [instance-type](https://kubernetes.io/docs/reference/labels-annotations-taints/#nodekubernetesioinstance-type) well-known label in the requirements section. +Karpenter batches pending pods and then binpacks them based on CPU, memory, and GPUs required, taking into account node overhead, VPC CNI resources required, and daemonsets that will be packed when bringing up a new node. Karpenter [recommends the use of C, M, and R >= Gen 3 instance types]({{< ref "./concepts/nodepools#spectemplatespecrequirements" >}}) for most generic workloads, but it can be constrained in the NodePool spec with the [instance-type](https://kubernetes.io/docs/reference/labels-annotations-taints/#nodekubernetesioinstance-type) well-known label in the requirements section. + After the pods are binpacked on the most efficient instance type (i.e. the smallest instance type that can fit the pod batch), Karpenter takes 59 other instance types that are larger than the most efficient packing, and passes all 60 instance type options to an API called Amazon EC2 Fleet. -The EC2 fleet API attempts to provision the instance type based on an allocation strategy. -If you are using the on-demand capacity type, then Karpenter uses the `lowest-price` allocation strategy. -So fleet will provision the lowest priced instance type it can get from the 60 instance types Karpenter passed to the EC2 fleet API. -If the instance type is unavailable for some reason, then fleet will move on to the next cheapest instance type. -If you are using the spot capacity type, Karpenter uses the price-capacity-optimized allocation strategy. This tells fleet to find the instance type that EC2 has the most capacity for while also considering price. This allocation strategy will balance cost and decrease the probability of a spot interruption happening in the near term. -See [Choose the appropriate allocation strategy](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-fleet-allocation-strategy.html#ec2-fleet-allocation-use-cases) for information on fleet optimization. + + +The EC2 fleet API attempts to provision the instance type based on the [Price Capacity Optimized allocation strategy](https://aws.amazon.com/blogs/compute/introducing-price-capacity-optimized-allocation-strategy-for-ec2-spot-instances/). For the on-demand capacity type, this is effectively equivalent to the `lowest-price` allocation strategy. For the spot capacity type, Fleet will determine an instance type that has both the lowest price combined with the lowest chance of being interrupted. Note that this may not give you the instance type with the strictly lowest price for spot. ### How does Karpenter calculate the resource usage of Daemonsets when simulating scheduling? -Karpenter currently calculates the applicable daemonsets at the provisioner level with label selectors/taints, etc. It does not look to see if there are requirements on the daemonsets that would exclude it from running on particular instances that the provisioner could or couldn't launch. -The recommendation for now is to use multiple provisioners with taints/tolerations or label selectors to limit daemonsets to only nodes launched from specific provisioners. +Karpenter currently calculates the applicable daemonsets at the NodePool level with label selectors/taints, etc. It does not look to see if there are requirements on the daemonsets that would exclude it from running on particular instances that the NodePool could or couldn't launch. +The recommendation for now is to use multiple NodePools with taints/tolerations or label selectors to limit daemonsets to only nodes launched from specific NodePoools. ### What if there is no Spot capacity? Will Karpenter use On-Demand? -The best defense against running out of Spot capacity is to allow Karpenter to provision as many different instance types as possible. -Even instance types that have higher specs, e.g. vCPU, memory, etc., than what you need can still be cheaper in the Spot market than using On-Demand instances. -When Spot capacity is constrained, On-Demand capacity can also be constrained since Spot is fundamentally spare On-Demand capacity. -Allowing Karpenter to provision nodes from a large, diverse set of instance types will help you to stay on Spot longer and lower your costs due to Spot’s discounted pricing. -Moreover, if Spot capacity becomes constrained, this diversity will also increase the chances that you’ll be able to continue to launch On-Demand capacity for your workloads. +The best defense against running out of Spot capacity is to allow Karpenter to provision as many distinct instance types as possible. Even instance types that have higher specs (e.g. vCPU, memory, etc.) than what you need can still be cheaper in the Spot market than using On-Demand instances. When Spot capacity is constrained, On-Demand capacity can also be constrained since Spot is fundamentally spare On-Demand capacity. -If your Karpenter Provisioner specifies flexibility to both Spot and On-Demand capacity, Karpenter will attempt to provision On-Demand capacity if there is no Spot capacity available. -However, it’s strongly recommended that you specify at least 20 instance types in your Provisioner (or none and allow Karpenter to pick the best instance types) as our research indicates that this additional diversity increases the chances that your workloads will not need to launch On-Demand capacity at all. -Today, Karpenter will warn you if the number of instances in your Provisioner isn’t sufficiently diverse. +Allowing Karpenter to provision nodes from a large, diverse set of instance types will help you to stay on Spot longer and lower your costs due to Spot’s discounted pricing. Moreover, if Spot capacity becomes constrained, this diversity will also increase the chances that you’ll be able to continue to launch On-Demand capacity for your workloads. -Technically, Karpenter has a concept of an “offering” for each instance type, which is a combination of zone and capacity type (equivalent in the AWS cloud provider to an EC2 purchase option – Spot or On-Demand). -Whenever the Fleet API returns an insufficient capacity error for Spot instances, those particular offerings are temporarily removed from consideration (across the entire provisioner) so that Karpenter can make forward progress with different options. +If your Karpenter NodePool specifies allows both Spot and On-Demand capacity, Karpenter will fallback to provision On-Demand capacity if there is no Spot capacity available. However, it’s strongly recommended that you allow at least 20 instance types in your NodePool since this additional diversity increases the chances that your workloads will not need to launch On-Demand capacity at all. + +Karpenter has a concept of an “offering” for each instance type, which is a combination of zone and capacity type. Whenever the Fleet API returns an insufficient capacity error for Spot instances, those particular offerings are temporarily removed from consideration (across the entire NodePool) so that Karpenter can make forward progress with different options. ### Does Karpenter support IPv6? -Yes! Karpenter dynamically discovers if you are running in an IPv6 cluster by checking the kube-dns service's cluster-ip. When using an AMI Family such as `AL2`, Karpenter will automatically configure the EKS Bootstrap script for IPv6. Some EC2 instance types do not support IPv6 and the Amazon VPC CNI only supports instance types that run on the Nitro hypervisor. It's best to add a requirement to your Provisioner to only allow Nitro instance types: +Yes! Karpenter dynamically discovers if you are running in an IPv6 cluster by checking the kube-dns service's cluster-ip. When using an AMI Family such as `AL2`, Karpenter will automatically configure the EKS Bootstrap script for IPv6. Some EC2 instance types do not support IPv6 and the Amazon VPC CNI only supports instance types that run on the Nitro hypervisor. It's best to add a requirement to your NodePool to only allow Nitro instance types: ``` -kind: Provisioner +apiVersion: karpenter.sh/v1beta1 +kind: NodePool ... spec: - requirements: - - key: karpenter.k8s.aws/instance-hypervisor - operator: In - values: - - nitro + template: + spec: + requirements: + - key: karpenter.k8s.aws/instance-hypervisor + operator: In + values: + - nitro ``` For more documentation on enabling IPv6 with the Amazon VPC CNI, see the [docs](https://docs.aws.amazon.com/eks/latest/userguide/cni-ipv6.html). @@ -165,7 +144,7 @@ Windows nodes do not support IPv6. `kube-scheduler` is responsible for the scheduling of pods, while Karpenter launches the capacity. When using any sort of preferred scheduling constraint, `kube-scheduler` will schedule pods to nodes anytime it is possible. -As an example, suppose you scale up a deployment with a preferred zonal topology spread and none of the newly created pods can run on your existing cluster. Karpenter will then launch multiple nodes to satisfy that preference. If a) one of the nodes becomes ready slightly faster than other nodes and b) has enough capacity for multiple pods, `kube-scheduler` will schedule as many pods as possible to the single ready node so they won't remain unschedulable. It doesn't consider the in-flight capacity that will be ready in a few seconds. If all of the pods fit on the single node, the remaining nodes that Karpenter has launched aren't needed when they become ready and consolidation will delete them. +As an example, suppose you scale up a deployment with a preferred zonal topology spread and none of the newly created pods can run on your existing cluster. Karpenter will then launch multiple nodes to satisfy that preference. If a) one of the nodes becomes ready slightly faster than other nodes and b) has enough capacity for multiple pods, `kube-scheduler` will schedule as many pods as possible to the single ready node, so they won't remain unschedulable. It doesn't consider the in-flight capacity that will be ready in a few seconds. If all the pods fit on the single node, the remaining nodes that Karpenter has launched aren't needed when they become ready and consolidation will delete them. ### When deploying an additional DaemonSet to my cluster, why does Karpenter not scale-up my nodes to support the extra DaemonSet? @@ -194,21 +173,26 @@ See [Application developer]({{< ref "./concepts/#application-developer" >}}) for Yes. See [Persistent Volume Topology]({{< ref "./concepts/scheduling#persistent-volume-topology" >}}) for details. ### Can I set `--max-pods` on my nodes? -Yes, see the [KubeletConfiguration Section in the Provisioners Documentation]({{}}) to learn more. +Yes, see the [KubeletConfiguration Section in the NodePool docs]({{}}) to learn more. ### Why do the Windows2019 and Windows2022 AMI families only support Windows Server Core? The difference between the Core and Full variants is that Core is a minimal OS with less components and no graphic user interface (GUI) or desktop experience. `Windows2019` and `Windows2022` AMI families use the Windows Server Core option for simplicity, but if required, you can specify a custom AMI to run Windows Server Full. -You can specify the [Amazon EKS optimized AMI](https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-windows-ami.html) with Windows Server 2022 Full for Kubernetes 1.27 by configuring an `amiSelector` that references the AMI name. -``` -amiSelector: - aws::name: Windows_Server-2022-English-Full-EKS_Optimized-1.27* +You can specify the [Amazon EKS optimized AMI](https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-windows-ami.html) with Windows Server 2022 Full for Kubernetes 1.29 by configuring an `amiSelector` that references the AMI name. +```yaml +amiSelectorTerms: + - name: Windows_Server-2022-English-Full-EKS_Optimized-1.29* ``` +### Can I use Karpenter to scale my workload's pods? +Karpenter is a node autoscaler which will create new nodes in response to unschedulable pods. Scaling the pods themselves is outside of its scope. +This is the realm of pod autoscalers such as the [Vertical Pod Autoscaler](https://github.com/kubernetes/autoscaler/tree/master/vertical-pod-autoscaler) (for scaling an individual pod's resources) or the [Horizontal Pod Autoscaler](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) (for scaling replicas). +We also recommend taking a look at [Keda](https://keda.sh/) if you're looking for more advanced autoscaling capabilities for pods. + ## Deprovisioning ### How does Karpenter deprovision nodes? -See [Deprovisioning nodes]({{< ref "./concepts/deprovisioning" >}}) for information on how Karpenter deprovisions nodes. +See [Deprovisioning nodes]({{< ref "./concepts/disruption" >}}) for information on how Karpenter deprovisions nodes. ## Upgrading Karpenter @@ -221,23 +205,21 @@ To upgrade Karpenter to version `$VERSION`, make sure that the `KarpenterNode IA Next, locate `KarpenterController IAM Role` ARN (i.e., ARN of the resource created in [Create the KarpenterController IAM Role](../getting-started/getting-started-with-karpenter/#create-the-karpentercontroller-iam-role)) and pass them to the helm upgrade command. {{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step08-apply-helm-chart.sh" language="bash"%}} -For information on upgrading Karpenter, see the [Upgrade Guide]({{< ref "./upgrade-guide/" >}}). - -### Why do I get an `unknown field "startupTaints"` error when creating a provisioner with startupTaints? - -```bash -error: error validating "provisioner.yaml": error validating data: ValidationError(Provisioner.spec): unknown field "startupTaints" in sh.karpenter.v1alpha5.Provisioner.spec; if you choose to ignore these errors, turn validation off with --validate=false -``` - -The `startupTaints` parameter was added in v0.10.0. Helm upgrades do not upgrade the CRD describing the provisioner, so it must be done manually. For specific details, see the [Upgrade Guide]({{< ref "./upgrade-guide/#upgrading-to-v0100" >}}) +For information on upgrading Karpenter, see the [Upgrade Guide]({{< ref "./upgrading/upgrade-guide/" >}}). ## Upgrading Kubernetes Cluster ### How do I upgrade an EKS Cluster with Karpenter? -When upgrading an Amazon EKS cluster, [Karpenter's Drift feature]({{}}) can automatically upgrade the Karpenter-provisioned nodes to stay in-sync with the EKS control plane. Karpenter Drift currently needs to be enabled using a [feature gate]({{}}). Karpenter's default [AWSNodeTemplate `amiFamily` configuration]({{}}) uses the latest EKS Optimized AL2 AMI for the same major and minor version as the EKS cluster's control plane. Karpenter's AWSNodeTemplate can be configured to not use the EKS optimized AL2 AMI in favor of a custom AMI by configuring the [`amiSelector`]({{}}). If using a custom AMI, you will need to trigger the rollout of this new worker node image through the publication of a new AMI with tags matching the [`amiSelector`]({{}}), or a change to the [`amiSelector`]({{}}) field. +When upgrading an Amazon EKS cluster, [Karpenter's Drift feature]({{}}) can automatically upgrade the Karpenter-provisioned nodes to stay in-sync with the EKS control plane. Karpenter Drift is enabled by default starting v0.33.x. + +{{% alert title="Note" color="primary" %}} +Karpenter's default [EC2NodeClass `amiFamily` configuration]({{}}) uses the latest EKS Optimized AL2 AMI for the same major and minor version as the EKS cluster's control plane, meaning that an upgrade of the control plane will cause Karpenter to auto-discover the new AMIs for that version. + +If using a custom AMI, you will need to trigger the rollout of this new worker node image through the publication of a new AMI with tags matching the [`amiSelector`]({{}}), or a change to the [`amiSelector`]({{}}) field. +{{% /alert %}} -Start by [upgrading the EKS Cluster control plane](https://docs.aws.amazon.com/eks/latest/userguide/update-cluster.html). After the EKS Cluster upgrade completes, Karpenter's Drift feature will detect that the Karpenter-provisioned nodes are using EKS Optimized AMIs for the previous cluster version, and [automatically cordon, drain, and replace those nodes]({{}}). To support pods moving to new nodes, follow Kubernetes best practices by setting appropriate pod [Resource Quotas](https://kubernetes.io/docs/concepts/policy/resource-quotas/), and using [Pod Disruption Budgets](https://kubernetes.io/docs/concepts/workloads/pods/disruptions/) (PDB). Karpenter's Drift feature will spin up replacement nodes based on the pod resource requests, and will respect the PDBs when deprovisioning nodes. +Start by [upgrading the EKS Cluster control plane](https://docs.aws.amazon.com/eks/latest/userguide/update-cluster.html). After the EKS Cluster upgrade completes, Karpenter's Drift feature will detect that the Karpenter-provisioned nodes are using EKS Optimized AMIs for the previous cluster version, and [automatically cordon, drain, and replace those nodes]({{}}). To support pods moving to new nodes, follow Kubernetes best practices by setting appropriate pod [Resource Quotas](https://kubernetes.io/docs/concepts/policy/resource-quotas/), and using [Pod Disruption Budgets](https://kubernetes.io/docs/concepts/workloads/pods/disruptions/) (PDB). Karpenter's Drift feature will spin up replacement nodes based on the pod resource requests, and will respect the PDBs when deprovisioning nodes. ## Interruption Handling @@ -249,10 +231,10 @@ Karpenter's native interruption handling offers two main benefits over the stand 1. You don't have to manage and maintain a separate component to exclusively handle interruption events. 2. Karpenter's native interruption handling coordinates with other deprovisioning so that consolidation, expiration, etc. can be aware of interruption events and vice-versa. -### Why am I receiving QueueNotFound errors when I set `aws.interruptionQueueName`? +### Why am I receiving QueueNotFound errors when I set `--interruption-queue-name`? Karpenter requires a queue to exist that receives event messages from EC2 and health services in order to handle interruption messages properly for nodes. -Details on the types of events that Karpenter handles can be found in the [Interruption Handling Docs]({{< ref "./concepts/deprovisioning/#interruption" >}}). +Details on the types of events that Karpenter handles can be found in the [Interruption Handling Docs]({{< ref "./concepts/disruption/#interruption" >}}). Details on provisioning the SQS queue and EventBridge rules can be found in the [Getting Started Guide]({{< ref "./getting-started/getting-started-with-karpenter/#create-the-karpenter-infrastructure-and-iam-roles" >}}). diff --git a/website/content/en/v0.31/getting-started/_index.md b/website/content/en/v0.34/getting-started/_index.md similarity index 75% rename from website/content/en/v0.31/getting-started/_index.md rename to website/content/en/v0.34/getting-started/_index.md index c159f68f5e48..cebdf66a99fd 100644 --- a/website/content/en/v0.31/getting-started/_index.md +++ b/website/content/en/v0.34/getting-started/_index.md @@ -1,18 +1,16 @@ --- title: "Getting Started" linkTitle: "Getting Started" -weight: 1 +weight: 10 description: > Choose from different methods to get started with Karpenter -cascade: - type: docs --- To get started with Karpenter, the [Getting Started with Karpenter]({{< relref "getting-started-with-karpenter" >}}) guide provides an end-to-end procedure for creating a cluster (with `eksctl`) and adding Karpenter. If you prefer, the following instructions use Terraform to create a cluster and add Karpenter: -* [Amazon EKS Blueprints for Terraform](https://aws-ia.github.io/terraform-aws-eks-blueprints): Follow a basic [Getting Started](https://aws-ia.github.io/terraform-aws-eks-blueprints/v4.18.0/getting-started/) guide and also add modules and add-ons. This includes a [Karpenter](https://aws-ia.github.io/terraform-aws-eks-blueprints/v4.18.0/add-ons/karpenter/) add-on that lets you bypass the instructions in this guide for setting up Karpenter. +* [Amazon EKS Blueprints for Terraform](https://aws-ia.github.io/terraform-aws-eks-blueprints): Follow a basic [Getting Started](https://aws-ia.github.io/terraform-aws-eks-blueprints/getting-started/) guide and also add modules and add-ons. This includes a [Karpenter](https://aws-ia.github.io/terraform-aws-eks-blueprints/patterns/karpenter/) add-on that lets you bypass the instructions in this guide for setting up Karpenter. Although not supported, you could also try Karpenter on other Kubernetes distributions running on AWS. For example: @@ -23,6 +21,6 @@ Learn more about Karpenter and how to get started below. * [Karpenter EKS Best Practices](https://aws.github.io/aws-eks-best-practices/karpenter/) guide * [EC2 Spot Workshop for Karpenter](https://ec2spotworkshops.com/karpenter.html) * [EKS Karpenter Workshop](https://www.eksworkshop.com/docs/autoscaling/compute/karpenter/) -* [Advanced EKS Immersion Karpenter Workshop](https://catalog.workshops.aws/eks-advanced/karpenter/) +* [Advanced EKS Immersion Karpenter Workshop](https://catalog.us-east-1.prod.workshops.aws/workshops/76a5dd80-3249-4101-8726-9be3eeee09b2/en-US/autoscaling/karpenter) * [Karpenter Blueprints](https://github.com/aws-samples/karpenter-blueprints) * [Tutorial: Run Kubernetes Clusters for Less with Amazon EC2 Spot and Karpenter](https://community.aws/tutorials/run-kubernetes-clusters-for-less-with-amazon-ec2-spot-and-karpenter#step-6-optional-simulate-spot-interruption) diff --git a/website/content/en/v0.34/getting-started/getting-started-with-karpenter/_index.md b/website/content/en/v0.34/getting-started/getting-started-with-karpenter/_index.md new file mode 100644 index 000000000000..490f1ebac6ec --- /dev/null +++ b/website/content/en/v0.34/getting-started/getting-started-with-karpenter/_index.md @@ -0,0 +1,247 @@ + +--- +title: "Getting Started with Karpenter" +linkTitle: "Getting Started with Karpenter" +weight: 10 +description: > + Set up a cluster and add Karpenter +--- + +Karpenter automatically provisions new nodes in response to unschedulable pods. Karpenter does this by observing events within the Kubernetes cluster, and then sending commands to the underlying cloud provider. + +This guide shows how to get started with Karpenter by creating a Kubernetes cluster and installing Karpenter. +To use Karpenter, you must be running a supported Kubernetes cluster on a supported cloud provider. +Currently, only EKS on AWS is supported. + +## Create a cluster and add Karpenter + +This guide uses `eksctl` to create the cluster. +It should take less than 1 hour to complete, and cost less than $0.25. +Follow the clean-up instructions to reduce any charges. + +### 1. Install utilities + +Karpenter is installed in clusters with a Helm chart. + +Karpenter requires cloud provider permissions to provision nodes, for AWS IAM +Roles for Service Accounts (IRSA) should be used. IRSA permits Karpenter +(within the cluster) to make privileged requests to AWS (as the cloud provider) +via a ServiceAccount. + +Install these tools before proceeding: + +1. [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2-linux.html) +2. `kubectl` - [the Kubernetes CLI](https://kubernetes.io/docs/tasks/tools/install-kubectl-linux/) +3. `eksctl` (>= v0.169.0) - [the CLI for AWS EKS](https://docs.aws.amazon.com/eks/latest/userguide/eksctl.html) +4. `helm` - [the package manager for Kubernetes](https://helm.sh/docs/intro/install/) + +[Configure the AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-quickstart.html) +with a user that has sufficient privileges to create an EKS cluster. Verify that the CLI can +authenticate properly by running `aws sts get-caller-identity`. + +### 2. Set environment variables + +After setting up the tools, set the Karpenter and Kubernetes version: + +```bash +export KARPENTER_NAMESPACE=kube-system +export KARPENTER_VERSION=v0.34.5 +export K8S_VERSION=1.29 +``` + +Then set the following environment variable: + +{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step01-config.sh" language="bash"%}} + +{{% alert title="Warning" color="warning" %}} +If you open a new shell to run steps in this procedure, you need to set some or all of the environment variables again. +To remind yourself of these values, type: + +```bash +echo "${KARPENTER_NAMESPACE}" "${KARPENTER_VERSION}" "${K8S_VERSION}" "${CLUSTER_NAME}" "${AWS_DEFAULT_REGION}" "${AWS_ACCOUNT_ID}" "${TEMPOUT}" "${ARM_AMI_ID}" "${AMD_AMI_ID}" "${GPU_AMI_ID}" +``` + +{{% /alert %}} + + +### 3. Create a Cluster + +Create a basic cluster with `eksctl`. +The following cluster configuration will: + +* Use CloudFormation to set up the infrastructure needed by the EKS cluster. See [CloudFormation]({{< relref "../../reference/cloudformation/" >}}) for a complete description of what `cloudformation.yaml` does for Karpenter. +* Create a Kubernetes service account and AWS IAM Role, and associate them using IRSA to let Karpenter launch instances. +* Add the Karpenter node role to the aws-auth configmap to allow nodes to connect. +* Use [AWS EKS managed node groups](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html) for the kube-system and karpenter namespaces. Uncomment fargateProfiles settings (and comment out managedNodeGroups settings) to use Fargate for both namespaces instead. +* Set KARPENTER_IAM_ROLE_ARN variables. +* Create a role to allow spot instances. +* Run helm to install karpenter + +{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step02-create-cluster.sh" language="bash"%}} + +{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step06-add-spot-role.sh" language="bash"%}} + +{{% alert title="Windows Support Notice" color="warning" %}} +In order to run Windows workloads, Windows support should be enabled in your EKS Cluster. +See [Enabling Windows support](https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html#enable-windows-support) to learn more. +{{% /alert %}} + +### 4. Install Karpenter + +{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step08-apply-helm-chart.sh" language="bash"%}} + +{{% alert title="DNS Policy Notice" color="warning" %}} +Karpenter uses the `ClusterFirst` pod DNS policy by default. This is the Kubernetes cluster default and this ensures that Karpetner can reach-out to internal Kubernetes services during its lifetime. There may be cases where you do not have the DNS service that you are using on your cluster up-and-running before Karpenter starts up. The most common case of this is you want Karpenter to manage the node capacity where your DNS service pods are running. + +If you need Karpenter to manage the DNS service pods' capacity, this means that DNS won't be running when Karpenter starts-up. In this case, you will need to set the pod DNS policy to `Default` with `--set dnsPolicy=Default`. This will tell Karpenter to use the host's DNS resolution instead of the internal DNS resolution, ensuring that you don't have a dependency on the DNS service pods to run. More details on this issue can be found in the following Github issues: [#2186](https://github.com/aws/karpenter-provider-aws/issues/2186) and [#4947](https://github.com/aws/karpenter-provider-aws/issues/4947). +{{% /alert %}} + +{{% alert title="Common Expression Language/Webhooks Notice" color="warning" %}} +Karpenter supports using [Kubernetes Common Expression Language](https://kubernetes.io/docs/reference/using-api/cel/) for validating its Custom Resource Definitions out-of-the-box; however, this feature is not supported on versions of Kubernetes < 1.25. If you are running an earlier version of Kubernetes, you will need to use the Karpenter admission webhooks for validation instead. You can enable these webhooks with `--set webhook.enabled=true` when applying the Karpenter helm chart. +{{% /alert %}} + +{{% alert title="Pod Identity Supports Notice" color="warning" %}} +Karpenter now supports using [Pod Identity](https://docs.aws.amazon.com/eks/latest/userguide/pod-identities.html) to authenticate AWS SDK to make API requests to AWS services using AWS Identity and Access Management (IAM) permissions. This feature not supported on versions of Kubernetes < 1.24. If you are running an earlier version of Kubernetes, you will need to use the [IAM Roles for Service Accounts(IRSA)](https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/setting-up-enable-IAM.html) for pod authentication instead. You can enable these IRSA with `--set "serviceAccount.annotations.eks\.amazonaws\.com/role-arn=${KARPENTER_IAM_ROLE_ARN}"` when applying the Karpenter helm chart. +{{% /alert %}} + +{{% alert title="Warning" color="warning" %}} +Karpenter creates a mapping between CloudProvider machines and CustomResources in the cluster for capacity tracking. To ensure this mapping is consistent, Karpenter utilizes the following tag keys: + +* `karpenter.sh/managed-by` +* `karpenter.sh/nodepool` +* `kubernetes.io/cluster/${CLUSTER_NAME}` + +Because Karpenter takes this dependency, any user that has the ability to Create/Delete these tags on CloudProvider machines will have the ability to orchestrate Karpenter to Create/Delete CloudProvider machines as a side effect. We recommend that you [enforce tag-based IAM policies](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_tags.html) on these tags against any EC2 instance resource (`i-*`) for any users that might have [CreateTags](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateTags.html)/[DeleteTags](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DeleteTags.html) permissions but should not have [RunInstances](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_RunInstances.html)/[TerminateInstances](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_TerminateInstances.html) permissions. +{{% /alert %}} + +### 5. Create NodePool + +A single Karpenter NodePool is capable of handling many different pod shapes. Karpenter makes scheduling and provisioning decisions based on pod attributes such as labels and affinity. In other words, Karpenter eliminates the need to manage many different node groups. + +Create a default NodePool using the command below. This NodePool uses `securityGroupSelectorTerms` and `subnetSelectorTerms` to discover resources used to launch nodes. We applied the tag `karpenter.sh/discovery` in the `eksctl` command above. Depending on how these resources are shared between clusters, you may need to use different tagging schemes. + +The `consolidationPolicy` set to `WhenUnderutilized` in the `disruption` block configures Karpenter to reduce cluster cost by removing and replacing nodes. As a result, consolidation will terminate any empty nodes on the cluster. This behavior can be disabled by setting `consolidateAfter` to `Never`, telling Karpenter that it should never consolidate nodes. Review the [NodePool API docs]({{}}) for more information. + +Note: This NodePool will create capacity as long as the sum of all created capacity is less than the specified limit. + +{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step12-add-nodepool.sh" language="bash"%}} + +Karpenter is now active and ready to begin provisioning nodes. + +### 6. Scale up deployment + +This deployment uses the [pause image](https://www.ianlewis.org/en/almighty-pause-container) and starts with zero replicas. + +{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step13-automatic-node-provisioning.sh" language="bash"%}} + +### 7. Scale down deployment + +Now, delete the deployment. After a short amount of time, Karpenter should terminate the empty nodes due to consolidation. + +{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step14-deprovisioning.sh" language="bash"%}} + +### 8. Delete Karpenter nodes manually + +If you delete a node with kubectl, Karpenter will gracefully cordon, drain, +and shutdown the corresponding instance. Under the hood, Karpenter adds a +finalizer to the node object, which blocks deletion until all pods are +drained and the instance is terminated. Keep in mind, this only works for +nodes provisioned by Karpenter. + +{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step16-delete-node.sh" language="bash"%}} + +### 9. Delete the cluster +To avoid additional charges, remove the demo infrastructure from your AWS account. + +{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step17-cleanup.sh" language="bash"%}} + +## Monitoring with Grafana (optional) + +This section describes optional ways to configure Karpenter to enhance its capabilities. +In particular, the following commands deploy a Prometheus and Grafana stack that is suitable for this guide but does not include persistent storage or other configurations that would be necessary for monitoring a production deployment of Karpenter. +This deployment includes two Karpenter dashboards that are automatically onboarded to Grafana. They provide a variety of visualization examples on Karpenter metrics. + +{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step09-add-prometheus-grafana.sh" language="bash"%}} + +The Grafana instance may be accessed using port forwarding. + +{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step10-add-grafana-port-forward.sh" language="bash"%}} + +The new stack has only one user, `admin`, and the password is stored in a secret. The following command will retrieve the password. + +{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step11-grafana-get-password.sh" language="bash"%}} + +## Advanced Installation + +The section below covers advanced installation techniques for installing Karpenter. This includes things such as running Karpenter on a cluster without public internet access or ensuring that Karpenter avoids getting throttled by other components in your cluster. + +### Private Clusters + +You can optionally install Karpenter on a [private cluster](https://docs.aws.amazon.com/eks/latest/userguide/private-clusters.html#private-cluster-requirements) using the `eksctl` installation by setting `privateCluster.enabled` to true in your [ClusterConfig](https://eksctl.io/usage/eks-private-cluster/#eks-fully-private-cluster) and by setting `--set settings.isolatedVPC=true` when installing the `karpenter` helm chart. + +```bash +privateCluster: + enabled: true +``` + +Private clusters have no outbound access to the internet. This means that in order for Karpenter to reach out to the services that it needs to access, you need to enable specific VPC private endpoints. Below shows the endpoints that you need to enable to successfully run Karpenter in a private cluster: + +```text +com.amazonaws..ec2 +com.amazonaws..ecr.api +com.amazonaws..ecr.dkr +com.amazonaws..s3 – For pulling container images +com.amazonaws..sts – For IAM roles for service accounts +com.amazonaws..ssm - For resolving default AMIs +com.amazonaws..sqs - For accessing SQS if using interruption handling +com.amazonaws..eks - For Karpenter to discover the cluster endpoint +``` + +If you do not currently have these endpoints surfaced in your VPC, you can add the endpoints by running + +```bash +aws ec2 create-vpc-endpoint --vpc-id ${VPC_ID} --service-name ${SERVICE_NAME} --vpc-endpoint-type Interface --subnet-ids ${SUBNET_IDS} --security-group-ids ${SECURITY_GROUP_IDS} +``` + +{{% alert title="Note" color="primary" %}} + +Karpenter (controller and webhook deployment) container images must be in or copied to Amazon ECR private or to another private registry accessible from inside the VPC. If these are not available from within the VPC, or from networks peered with the VPC, you will get Image pull errors when Kubernetes tries to pull these images from ECR public. + +{{% /alert %}} + +{{% alert title="Note" color="primary" %}} + +There is currently no VPC private endpoint for the [IAM API](https://docs.aws.amazon.com/IAM/latest/APIReference/welcome.html). As a result, you cannot use the default `spec.role` field in your `EC2NodeClass`. Instead, you need to provision and manage an instance profile manually and then specify Karpenter to use this instance profile through the `spec.instanceProfile` field. + +You can provision an instance profile manually and assign a Node role to it by calling the following command + +```bash +aws iam create-instance-profile --instance-profile-name "KarpenterNodeInstanceProfile-${CLUSTER_NAME}" +aws iam add-role-to-instance-profile --instance-profile-name "KarpenterNodeInstanceProfile-${CLUSTER_NAME}" --role-name "KarpenterNodeRole-${CLUSTER_NAME}" +``` + +{{% /alert %}} + +{{% alert title="Note" color="primary" %}} + +There is currently no VPC private endpoint for the [Price List Query API](https://docs.aws.amazon.com/awsaccountbilling/latest/aboutv2/using-price-list-query-api.html). As a result, pricing data can go stale over time. By default, Karpenter ships a static price list that is updated when each binary is released. + +Failed requests for pricing data will result in the following error messages + +```bash +ERROR controller.aws.pricing updating on-demand pricing, RequestError: send request failed +caused by: Post "https://api.pricing.us-east-1.amazonaws.com/": dial tcp 52.94.231.236:443: i/o timeout; RequestError: send request failed +caused by: Post "https://api.pricing.us-east-1.amazonaws.com/": dial tcp 52.94.231.236:443: i/o timeout, using existing pricing data from 2022-08-17T00:19:52Z {"commit": "4b5f953"} +``` + +{{% /alert %}} + +### Preventing APIServer Request Throttling + +Kubernetes uses [FlowSchemas](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/#flowschema) and [PriorityLevelConfigurations](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/#prioritylevelconfiguration) to map calls to the API server into buckets which determine each user agent's throttling limits. + +By default, Karpenter is installed into the `kube-system` namespace, which leverages the `system-leader-election` and `kube-system-service-accounts` [FlowSchemas](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/#flowschema) to map calls from the `kube-system` namespace to the `leader-election` and `workload-high` PriorityLevelConfigurations respectively. By putting Karpenter in these PriorityLevelConfigurations, we ensure that Karpenter and other critical cluster components are able to run even if other components on the cluster are throttled in other PriorityLevelConfigurations. + +If you install Karpenter in a different namespace than the default `kube-system` namespace, Karpenter will not be put into these higher-priority FlowSchemas by default. Instead, you will need to create custom FlowSchemas for the namespace and service account where Karpenter is installed to ensure that requests are put into this higher PriorityLevelConfiguration. + +{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step15-apply-flowschemas.sh" language="bash"%}} diff --git a/website/content/en/v0.31/getting-started/getting-started-with-karpenter/cloudformation.yaml b/website/content/en/v0.34/getting-started/getting-started-with-karpenter/cloudformation.yaml similarity index 71% rename from website/content/en/v0.31/getting-started/getting-started-with-karpenter/cloudformation.yaml rename to website/content/en/v0.34/getting-started/getting-started-with-karpenter/cloudformation.yaml index b8c8d64614e7..207761007d2d 100644 --- a/website/content/en/v0.31/getting-started/getting-started-with-karpenter/cloudformation.yaml +++ b/website/content/en/v0.34/getting-started/getting-started-with-karpenter/cloudformation.yaml @@ -5,13 +5,6 @@ Parameters: Type: String Description: "EKS cluster name" Resources: - KarpenterNodeInstanceProfile: - Type: "AWS::IAM::InstanceProfile" - Properties: - InstanceProfileName: !Sub "KarpenterNodeInstanceProfile-${ClusterName}" - Path: "/" - Roles: - - !Ref "KarpenterNodeRole" KarpenterNodeRole: Type: "AWS::IAM::Role" Properties: @@ -42,21 +35,36 @@ Resources: "Version": "2012-10-17", "Statement": [ { - "Sid": "AllowScopedEC2InstanceActions", + "Sid": "AllowScopedEC2InstanceAccessActions", "Effect": "Allow", "Resource": [ "arn:${AWS::Partition}:ec2:${AWS::Region}::image/*", "arn:${AWS::Partition}:ec2:${AWS::Region}::snapshot/*", - "arn:${AWS::Partition}:ec2:${AWS::Region}:*:spot-instances-request/*", "arn:${AWS::Partition}:ec2:${AWS::Region}:*:security-group/*", - "arn:${AWS::Partition}:ec2:${AWS::Region}:*:subnet/*", - "arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*" + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:subnet/*" ], "Action": [ "ec2:RunInstances", "ec2:CreateFleet" ] }, + { + "Sid": "AllowScopedEC2LaunchTemplateAccessActions", + "Effect": "Allow", + "Resource": "arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*", + "Action": [ + "ec2:RunInstances", + "ec2:CreateFleet" + ], + "Condition": { + "StringEquals": { + "aws:ResourceTag/kubernetes.io/cluster/${ClusterName}": "owned" + }, + "StringLike": { + "aws:ResourceTag/karpenter.sh/nodepool": "*" + } + } + }, { "Sid": "AllowScopedEC2InstanceActionsWithTags", "Effect": "Allow", @@ -65,7 +73,8 @@ Resources: "arn:${AWS::Partition}:ec2:${AWS::Region}:*:instance/*", "arn:${AWS::Partition}:ec2:${AWS::Region}:*:volume/*", "arn:${AWS::Partition}:ec2:${AWS::Region}:*:network-interface/*", - "arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*" + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*", + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:spot-instances-request/*" ], "Action": [ "ec2:RunInstances", @@ -77,7 +86,7 @@ Resources: "aws:RequestTag/kubernetes.io/cluster/${ClusterName}": "owned" }, "StringLike": { - "aws:RequestTag/karpenter.sh/provisioner-name": "*" + "aws:RequestTag/karpenter.sh/nodepool": "*" } } }, @@ -89,7 +98,8 @@ Resources: "arn:${AWS::Partition}:ec2:${AWS::Region}:*:instance/*", "arn:${AWS::Partition}:ec2:${AWS::Region}:*:volume/*", "arn:${AWS::Partition}:ec2:${AWS::Region}:*:network-interface/*", - "arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*" + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*", + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:spot-instances-request/*" ], "Action": "ec2:CreateTags", "Condition": { @@ -102,27 +112,26 @@ Resources: ] }, "StringLike": { - "aws:RequestTag/karpenter.sh/provisioner-name": "*" + "aws:RequestTag/karpenter.sh/nodepool": "*" } } }, { - "Sid": "AllowMachineMigrationTagging", + "Sid": "AllowScopedResourceTagging", "Effect": "Allow", "Resource": "arn:${AWS::Partition}:ec2:${AWS::Region}:*:instance/*", "Action": "ec2:CreateTags", "Condition": { "StringEquals": { - "aws:ResourceTag/kubernetes.io/cluster/${ClusterName}": "owned", - "aws:RequestTag/karpenter.sh/managed-by": "${ClusterName}" + "aws:ResourceTag/kubernetes.io/cluster/${ClusterName}": "owned" }, "StringLike": { - "aws:RequestTag/karpenter.sh/provisioner-name": "*" + "aws:ResourceTag/karpenter.sh/nodepool": "*" }, "ForAllValues:StringEquals": { "aws:TagKeys": [ - "karpenter.sh/provisioner-name", - "karpenter.sh/managed-by" + "karpenter.sh/nodeclaim", + "Name" ] } } @@ -143,7 +152,7 @@ Resources: "aws:ResourceTag/kubernetes.io/cluster/${ClusterName}": "owned" }, "StringLike": { - "aws:ResourceTag/karpenter.sh/provisioner-name": "*" + "aws:ResourceTag/karpenter.sh/nodepool": "*" } } }, @@ -186,7 +195,6 @@ Resources: "Resource": "${KarpenterInterruptionQueue.Arn}", "Action": [ "sqs:DeleteMessage", - "sqs:GetQueueAttributes", "sqs:GetQueueUrl", "sqs:ReceiveMessage" ] @@ -202,6 +210,68 @@ Resources: } } }, + { + "Sid": "AllowScopedInstanceProfileCreationActions", + "Effect": "Allow", + "Resource": "*", + "Action": [ + "iam:CreateInstanceProfile" + ], + "Condition": { + "StringEquals": { + "aws:RequestTag/kubernetes.io/cluster/${ClusterName}": "owned", + "aws:RequestTag/topology.kubernetes.io/region": "${AWS::Region}" + }, + "StringLike": { + "aws:RequestTag/karpenter.k8s.aws/ec2nodeclass": "*" + } + } + }, + { + "Sid": "AllowScopedInstanceProfileTagActions", + "Effect": "Allow", + "Resource": "*", + "Action": [ + "iam:TagInstanceProfile" + ], + "Condition": { + "StringEquals": { + "aws:ResourceTag/kubernetes.io/cluster/${ClusterName}": "owned", + "aws:ResourceTag/topology.kubernetes.io/region": "${AWS::Region}", + "aws:RequestTag/kubernetes.io/cluster/${ClusterName}": "owned", + "aws:RequestTag/topology.kubernetes.io/region": "${AWS::Region}" + }, + "StringLike": { + "aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass": "*", + "aws:RequestTag/karpenter.k8s.aws/ec2nodeclass": "*" + } + } + }, + { + "Sid": "AllowScopedInstanceProfileActions", + "Effect": "Allow", + "Resource": "*", + "Action": [ + "iam:AddRoleToInstanceProfile", + "iam:RemoveRoleFromInstanceProfile", + "iam:DeleteInstanceProfile" + ], + "Condition": { + "StringEquals": { + "aws:ResourceTag/kubernetes.io/cluster/${ClusterName}": "owned", + "aws:ResourceTag/topology.kubernetes.io/region": "${AWS::Region}" + }, + "StringLike": { + "aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass": "*" + } + } + }, + { + "Sid": "AllowInstanceProfileReadActions", + "Effect": "Allow", + "Resource": "*", + "Action": "iam:GetInstanceProfile" + }, { "Sid": "AllowAPIServerEndpointDiscovery", "Effect": "Allow", diff --git a/website/content/en/v0.30/getting-started/getting-started-with-karpenter/grafana-values.yaml b/website/content/en/v0.34/getting-started/getting-started-with-karpenter/grafana-values.yaml similarity index 83% rename from website/content/en/v0.30/getting-started/getting-started-with-karpenter/grafana-values.yaml rename to website/content/en/v0.34/getting-started/getting-started-with-karpenter/grafana-values.yaml index 29a27a0f6fd3..b68a986cf99b 100644 --- a/website/content/en/v0.30/getting-started/getting-started-with-karpenter/grafana-values.yaml +++ b/website/content/en/v0.34/getting-started/getting-started-with-karpenter/grafana-values.yaml @@ -22,6 +22,6 @@ dashboardProviders: dashboards: default: capacity-dashboard: - url: https://karpenter.sh/v0.30/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json + url: https://karpenter.sh/v0.34/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json performance-dashboard: - url: https://karpenter.sh/v0.30/getting-started/getting-started-with-karpenter/karpenter-performance-dashboard.json + url: https://karpenter.sh/v0.34/getting-started/getting-started-with-karpenter/karpenter-performance-dashboard.json diff --git a/website/content/en/v0.30/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json b/website/content/en/v0.34/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json similarity index 94% rename from website/content/en/v0.30/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json rename to website/content/en/v0.34/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json index 9d9cb6ad14a8..d474d01f4e16 100644 --- a/website/content/en/v0.30/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json +++ b/website/content/en/v0.34/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json @@ -484,7 +484,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by ($distribution_filter)(\n karpenter_pods_state{arch=~\"$arch\", capacity_type=~\"$capacity_type\", instance_type=~\"$instance_type\", provisioner=~\"$provisioner\"}\n)", + "expr": "sum by ($distribution_filter)(\n karpenter_pods_state{arch=~\"$arch\", capacity_type=~\"$capacity_type\", instance_type=~\"$instance_type\", nodepool=~\"$nodepool\"}\n)", "legendFormat": "{{label_name}}", "range": true, "refId": "A" @@ -588,7 +588,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "karpenter_provisioner_usage{resource_type=\"cpu\"} / karpenter_provisioner_limit{resource_type=\"cpu\"}", + "expr": "karpenter_nodepool_usage{resource_type=\"cpu\"} / karpenter_nodepool_limit{resource_type=\"cpu\"}", "format": "table", "instant": true, "legendFormat": "CPU Limit Utilization", @@ -602,7 +602,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "count by (provisioner)(karpenter_nodes_allocatable{provisioner!=\"N/A\",resource_type=\"cpu\"}) # Selects a single resource type to get node count", + "expr": "count by (nodepool)(karpenter_nodes_allocatable{nodepool!=\"N/A\",resource_type=\"cpu\"}) # Selects a single resource type to get node count", "format": "table", "hide": false, "instant": true, @@ -616,7 +616,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "karpenter_provisioner_usage{resource_type=\"memory\"} / karpenter_provisioner_limit{resource_type=\"memory\"}", + "expr": "karpenter_nodepool_usage{resource_type=\"memory\"} / karpenter_nodepool_limit{resource_type=\"memory\"}", "format": "table", "hide": false, "instant": true, @@ -631,7 +631,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum by (provisioner)(karpenter_nodes_allocatable{provisioner!=\"N/A\",resource_type=\"cpu\"})", + "expr": "sum by (nodepool)(karpenter_nodes_allocatable{nodepool!=\"N/A\",resource_type=\"cpu\"})", "format": "table", "hide": false, "instant": true, @@ -645,7 +645,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum by (provisioner)(karpenter_nodes_allocatable{provisioner!=\"N/A\",resource_type=\"memory\"})", + "expr": "sum by (nodepool)(karpenter_nodes_allocatable{nodepool!=\"N/A\",resource_type=\"memory\"})", "format": "table", "hide": false, "instant": true, @@ -653,12 +653,12 @@ "refId": "Memory Capacity" } ], - "title": "Provisioner Summary", + "title": "Nodepool Summary", "transformations": [ { "id": "seriesToColumns", "options": { - "byField": "provisioner" + "byField": "nodepool" } }, { @@ -697,7 +697,7 @@ "instance 2": 12, "job 1": 9, "job 2": 13, - "provisioner": 0, + "nodepool": 0, "resource_type 1": 10, "resource_type 2": 14 }, @@ -714,7 +714,7 @@ "instance": "", "instance 1": "", "job": "", - "provisioner": "Provisioner" + "nodepool": "Nodepool" } } } @@ -804,7 +804,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "(count(karpenter_nodes_allocatable{arch=~\"$arch\",capacity_type=\"spot\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"}) or vector(0)) / count(karpenter_nodes_allocatable{arch=~\"$arch\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"})", + "expr": "(count(karpenter_nodes_allocatable{arch=~\"$arch\",capacity_type=\"spot\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}) or vector(0)) / count(karpenter_nodes_allocatable{arch=~\"$arch\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"})", "legendFormat": "Percentage", "range": true, "refId": "A" @@ -946,7 +946,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "((karpenter_nodes_total_daemon_requests{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0) + \n(karpenter_nodes_total_pod_requests{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0)) / \nkarpenter_nodes_allocatable{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"}", + "expr": "((karpenter_nodes_total_daemon_requests{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0) + \n(karpenter_nodes_total_pod_requests{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0)) / \nkarpenter_nodes_allocatable{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}", "format": "table", "hide": false, "instant": true, @@ -961,7 +961,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "((karpenter_nodes_total_daemon_requests{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0) + \n(karpenter_nodes_total_pod_requests{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0)) / \nkarpenter_nodes_allocatable{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"}", + "expr": "((karpenter_nodes_total_daemon_requests{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0) + \n(karpenter_nodes_total_pod_requests{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0)) / \nkarpenter_nodes_allocatable{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}", "format": "table", "hide": false, "instant": true, @@ -976,7 +976,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "karpenter_nodes_total_daemon_requests{resource_type=\"pods\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"} + \nkarpenter_nodes_total_pod_requests{resource_type=\"pods\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"}", + "expr": "karpenter_nodes_total_daemon_requests{resource_type=\"pods\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} + \nkarpenter_nodes_total_pod_requests{resource_type=\"pods\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}", "format": "table", "hide": false, "instant": true, @@ -1091,9 +1091,9 @@ "os 1": true, "os 2": true, "os 3": true, - "provisioner 1": false, - "provisioner 2": true, - "provisioner 3": true, + "nodepool 1": false, + "nodepool 2": true, + "nodepool 3": true, "resource_type": true, "resource_type 1": true, "resource_type 2": true, @@ -1161,9 +1161,9 @@ "os 1": 23, "os 2": 41, "os 3": 61, - "provisioner 1": 2, - "provisioner 2": 42, - "provisioner 3": 62, + "nodepool 1": 2, + "nodepool 2": 42, + "nodepool 3": 62, "resource_type 1": 24, "resource_type 2": 43, "resource_type 3": 63, @@ -1190,7 +1190,7 @@ "instance_type": "Instance Type", "instance_type 1": "Instance Type", "node_name": "Node Name", - "provisioner 1": "Provisioner", + "nodepool 1": "Nodepool", "zone 1": "Zone" } } @@ -1237,14 +1237,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "definition": "label_values(karpenter_nodes_allocatable, provisioner)", + "definition": "label_values(karpenter_nodes_allocatable, nodepool)", "hide": 0, "includeAll": true, "multi": true, - "name": "provisioner", + "name": "nodepool", "options": [], "query": { - "query": "label_values(karpenter_nodes_allocatable, provisioner)", + "query": "label_values(karpenter_nodes_allocatable, nodepool)", "refId": "StandardVariableQuery" }, "refresh": 2, @@ -1376,8 +1376,8 @@ { "current": { "selected": true, - "text": "provisioner", - "value": "provisioner" + "text": "nodepool", + "value": "nodepool" }, "hide": 0, "includeAll": false, @@ -1411,8 +1411,8 @@ }, { "selected": true, - "text": "provisioner", - "value": "provisioner" + "text": "nodepool", + "value": "nodepool" }, { "selected": false, @@ -1420,7 +1420,7 @@ "value": "zone" } ], - "query": "arch,capacity_type,instance_type,namespace,node,provisioner,zone", + "query": "arch,capacity_type,instance_type,namespace,node,nodepool,zone", "queryValue": "", "skipUrlSync": false, "type": "custom" diff --git a/website/content/en/v0.30/getting-started/getting-started-with-karpenter/karpenter-controllers-allocation.json b/website/content/en/v0.34/getting-started/getting-started-with-karpenter/karpenter-controllers-allocation.json similarity index 100% rename from website/content/en/v0.30/getting-started/getting-started-with-karpenter/karpenter-controllers-allocation.json rename to website/content/en/v0.34/getting-started/getting-started-with-karpenter/karpenter-controllers-allocation.json diff --git a/website/content/en/v0.30/getting-started/getting-started-with-karpenter/karpenter-controllers.json b/website/content/en/v0.34/getting-started/getting-started-with-karpenter/karpenter-controllers.json similarity index 100% rename from website/content/en/v0.30/getting-started/getting-started-with-karpenter/karpenter-controllers.json rename to website/content/en/v0.34/getting-started/getting-started-with-karpenter/karpenter-controllers.json diff --git a/website/content/en/v0.30/getting-started/getting-started-with-karpenter/karpenter-performance-dashboard.json b/website/content/en/v0.34/getting-started/getting-started-with-karpenter/karpenter-performance-dashboard.json similarity index 100% rename from website/content/en/v0.30/getting-started/getting-started-with-karpenter/karpenter-performance-dashboard.json rename to website/content/en/v0.34/getting-started/getting-started-with-karpenter/karpenter-performance-dashboard.json diff --git a/website/content/en/v0.30/getting-started/getting-started-with-karpenter/prometheus-values.yaml b/website/content/en/v0.34/getting-started/getting-started-with-karpenter/prometheus-values.yaml similarity index 63% rename from website/content/en/v0.30/getting-started/getting-started-with-karpenter/prometheus-values.yaml rename to website/content/en/v0.34/getting-started/getting-started-with-karpenter/prometheus-values.yaml index 4cd78495351b..588762a8c9dc 100644 --- a/website/content/en/v0.30/getting-started/getting-started-with-karpenter/prometheus-values.yaml +++ b/website/content/en/v0.34/getting-started/getting-started-with-karpenter/prometheus-values.yaml @@ -13,8 +13,10 @@ extraScrapeConfigs: | - role: endpoints namespaces: names: - - karpenter + - $KARPENTER_NAMESPACE relabel_configs: - - source_labels: [__meta_kubernetes_endpoint_port_name] - regex: http-metrics + - source_labels: + - __meta_kubernetes_endpoints_name + - __meta_kubernetes_endpoint_port_name action: keep + regex: karpenter;http-metrics diff --git a/website/content/en/v0.34/getting-started/getting-started-with-karpenter/scripts/step01-config.sh b/website/content/en/v0.34/getting-started/getting-started-with-karpenter/scripts/step01-config.sh new file mode 100755 index 000000000000..6b948de3c793 --- /dev/null +++ b/website/content/en/v0.34/getting-started/getting-started-with-karpenter/scripts/step01-config.sh @@ -0,0 +1,8 @@ +export AWS_PARTITION="aws" # if you are not using standard partitions, you may need to configure to aws-cn / aws-us-gov +export CLUSTER_NAME="${USER}-karpenter-demo" +export AWS_DEFAULT_REGION="us-west-2" +export AWS_ACCOUNT_ID="$(aws sts get-caller-identity --query Account --output text)" +export TEMPOUT=$(mktemp) +export ARM_AMI_ID="$(aws ssm get-parameter --name /aws/service/eks/optimized-ami/${K8S_VERSION}/amazon-linux-2-arm64/recommended/image_id --query Parameter.Value --output text)" +export AMD_AMI_ID="$(aws ssm get-parameter --name /aws/service/eks/optimized-ami/${K8S_VERSION}/amazon-linux-2/recommended/image_id --query Parameter.Value --output text)" +export GPU_AMI_ID="$(aws ssm get-parameter --name /aws/service/eks/optimized-ami/${K8S_VERSION}/amazon-linux-2-gpu/recommended/image_id --query Parameter.Value --output text)" diff --git a/website/content/en/v0.30/getting-started/getting-started-with-karpenter/scripts/step02-create-cluster-fargate.sh b/website/content/en/v0.34/getting-started/getting-started-with-karpenter/scripts/step02-create-cluster-fargate.sh similarity index 80% rename from website/content/en/v0.30/getting-started/getting-started-with-karpenter/scripts/step02-create-cluster-fargate.sh rename to website/content/en/v0.34/getting-started/getting-started-with-karpenter/scripts/step02-create-cluster-fargate.sh index d5ca27944cf1..e957e364acd7 100755 --- a/website/content/en/v0.30/getting-started/getting-started-with-karpenter/scripts/step02-create-cluster-fargate.sh +++ b/website/content/en/v0.34/getting-started/getting-started-with-karpenter/scripts/step02-create-cluster-fargate.sh @@ -5,13 +5,13 @@ kind: ClusterConfig metadata: name: ${CLUSTER_NAME} region: ${AWS_DEFAULT_REGION} - version: "1.24" + version: "${K8S_VERSION}" tags: karpenter.sh/discovery: ${CLUSTER_NAME} fargateProfiles: - name: karpenter selectors: - - namespace: karpenter + - namespace: "${KARPENTER_NAMESPACE}" iam: withOIDC: true EOF diff --git a/website/content/en/v0.31/getting-started/getting-started-with-karpenter/scripts/step02-create-cluster.sh b/website/content/en/v0.34/getting-started/getting-started-with-karpenter/scripts/step02-create-cluster.sh similarity index 70% rename from website/content/en/v0.31/getting-started/getting-started-with-karpenter/scripts/step02-create-cluster.sh rename to website/content/en/v0.34/getting-started/getting-started-with-karpenter/scripts/step02-create-cluster.sh index 7e7d70ce77b0..7a7b57c05a4c 100755 --- a/website/content/en/v0.31/getting-started/getting-started-with-karpenter/scripts/step02-create-cluster.sh +++ b/website/content/en/v0.34/getting-started/getting-started-with-karpenter/scripts/step02-create-cluster.sh @@ -12,20 +12,32 @@ kind: ClusterConfig metadata: name: ${CLUSTER_NAME} region: ${AWS_DEFAULT_REGION} - version: "1.27" + version: "${K8S_VERSION}" tags: karpenter.sh/discovery: ${CLUSTER_NAME} iam: withOIDC: true - serviceAccounts: - - metadata: - name: karpenter - namespace: karpenter + podIdentityAssociations: + - namespace: "${KARPENTER_NAMESPACE}" + serviceAccountName: karpenter roleName: ${CLUSTER_NAME}-karpenter - attachPolicyARNs: + permissionPolicyARNs: - arn:${AWS_PARTITION}:iam::${AWS_ACCOUNT_ID}:policy/KarpenterControllerPolicy-${CLUSTER_NAME} - roleOnly: true + +## Optionally run on fargate or on k8s 1.23 +# Pod Identity is not available on fargate +# https://docs.aws.amazon.com/eks/latest/userguide/pod-identities.html +# iam: +# withOIDC: true +# serviceAccounts: +# - metadata: +# name: karpenter +# namespace: "${KARPENTER_NAMESPACE}" +# roleName: ${CLUSTER_NAME}-karpenter +# attachPolicyARNs: +# - arn:${AWS_PARTITION}:iam::${AWS_ACCOUNT_ID}:policy/KarpenterControllerPolicy-${CLUSTER_NAME} +# roleOnly: true iamIdentityMappings: - arn: "arn:${AWS_PARTITION}:iam::${AWS_ACCOUNT_ID}:role/KarpenterNodeRole-${CLUSTER_NAME}" @@ -45,11 +57,14 @@ managedNodeGroups: minSize: 1 maxSize: 10 +addons: +- name: eks-pod-identity-agent + ## Optionally run on fargate # fargateProfiles: # - name: karpenter # selectors: -# - namespace: karpenter +# - namespace: "${KARPENTER_NAMESPACE}" EOF export CLUSTER_ENDPOINT="$(aws eks describe-cluster --name ${CLUSTER_NAME} --query "cluster.endpoint" --output text)" diff --git a/website/content/en/v0.30/getting-started/getting-started-with-karpenter/scripts/step03-iam-cloud-formation.sh b/website/content/en/v0.34/getting-started/getting-started-with-karpenter/scripts/step03-iam-cloud-formation.sh similarity index 100% rename from website/content/en/v0.30/getting-started/getting-started-with-karpenter/scripts/step03-iam-cloud-formation.sh rename to website/content/en/v0.34/getting-started/getting-started-with-karpenter/scripts/step03-iam-cloud-formation.sh diff --git a/website/content/en/v0.30/getting-started/getting-started-with-karpenter/scripts/step04-grant-access.sh b/website/content/en/v0.34/getting-started/getting-started-with-karpenter/scripts/step04-grant-access.sh similarity index 100% rename from website/content/en/v0.30/getting-started/getting-started-with-karpenter/scripts/step04-grant-access.sh rename to website/content/en/v0.34/getting-started/getting-started-with-karpenter/scripts/step04-grant-access.sh diff --git a/website/content/en/v0.30/getting-started/getting-started-with-karpenter/scripts/step05-controller-iam.sh b/website/content/en/v0.34/getting-started/getting-started-with-karpenter/scripts/step05-controller-iam.sh similarity index 79% rename from website/content/en/v0.30/getting-started/getting-started-with-karpenter/scripts/step05-controller-iam.sh rename to website/content/en/v0.34/getting-started/getting-started-with-karpenter/scripts/step05-controller-iam.sh index 923317b0b93f..6975d31ea9e4 100755 --- a/website/content/en/v0.30/getting-started/getting-started-with-karpenter/scripts/step05-controller-iam.sh +++ b/website/content/en/v0.34/getting-started/getting-started-with-karpenter/scripts/step05-controller-iam.sh @@ -1,5 +1,5 @@ eksctl create iamserviceaccount \ - --cluster "${CLUSTER_NAME}" --name karpenter --namespace karpenter \ + --cluster "${CLUSTER_NAME}" --name karpenter --namespace "${KARPENTER_NAMESPACE}" \ --role-name "${CLUSTER_NAME}-karpenter" \ --attach-policy-arn "arn:${AWS_PARTITION}:iam::${AWS_ACCOUNT_ID}:policy/KarpenterControllerPolicy-${CLUSTER_NAME}" \ --role-only \ diff --git a/website/content/en/v0.30/getting-started/getting-started-with-karpenter/scripts/step06-add-spot-role.sh b/website/content/en/v0.34/getting-started/getting-started-with-karpenter/scripts/step06-add-spot-role.sh similarity index 100% rename from website/content/en/v0.30/getting-started/getting-started-with-karpenter/scripts/step06-add-spot-role.sh rename to website/content/en/v0.34/getting-started/getting-started-with-karpenter/scripts/step06-add-spot-role.sh diff --git a/website/content/en/v0.30/getting-started/getting-started-with-karpenter/scripts/step08-apply-helm-chart.sh b/website/content/en/v0.34/getting-started/getting-started-with-karpenter/scripts/step08-apply-helm-chart.sh similarity index 50% rename from website/content/en/v0.30/getting-started/getting-started-with-karpenter/scripts/step08-apply-helm-chart.sh rename to website/content/en/v0.34/getting-started/getting-started-with-karpenter/scripts/step08-apply-helm-chart.sh index 99fa927b12b6..aca3c191d684 100755 --- a/website/content/en/v0.30/getting-started/getting-started-with-karpenter/scripts/step08-apply-helm-chart.sh +++ b/website/content/en/v0.34/getting-started/getting-started-with-karpenter/scripts/step08-apply-helm-chart.sh @@ -1,11 +1,9 @@ # Logout of helm registry to perform an unauthenticated pull against the public ECR helm registry logout public.ecr.aws -helm upgrade --install karpenter oci://public.ecr.aws/karpenter/karpenter --version ${KARPENTER_VERSION} --namespace karpenter --create-namespace \ - --set serviceAccount.annotations."eks\.amazonaws\.com/role-arn"=${KARPENTER_IAM_ROLE_ARN} \ - --set settings.aws.clusterName=${CLUSTER_NAME} \ - --set settings.aws.defaultInstanceProfile=KarpenterNodeInstanceProfile-${CLUSTER_NAME} \ - --set settings.aws.interruptionQueueName=${CLUSTER_NAME} \ +helm upgrade --install karpenter oci://public.ecr.aws/karpenter/karpenter --version "${KARPENTER_VERSION}" --namespace "${KARPENTER_NAMESPACE}" --create-namespace \ + --set "settings.clusterName=${CLUSTER_NAME}" \ + --set "settings.interruptionQueue=${CLUSTER_NAME}" \ --set controller.resources.requests.cpu=1 \ --set controller.resources.requests.memory=1Gi \ --set controller.resources.limits.cpu=1 \ diff --git a/website/content/en/v0.30/getting-started/getting-started-with-karpenter/scripts/step09-add-prometheus-grafana.sh b/website/content/en/v0.34/getting-started/getting-started-with-karpenter/scripts/step09-add-prometheus-grafana.sh similarity index 95% rename from website/content/en/v0.30/getting-started/getting-started-with-karpenter/scripts/step09-add-prometheus-grafana.sh rename to website/content/en/v0.34/getting-started/getting-started-with-karpenter/scripts/step09-add-prometheus-grafana.sh index a539f7491e10..a7fe55c64ed5 100755 --- a/website/content/en/v0.30/getting-started/getting-started-with-karpenter/scripts/step09-add-prometheus-grafana.sh +++ b/website/content/en/v0.34/getting-started/getting-started-with-karpenter/scripts/step09-add-prometheus-grafana.sh @@ -2,9 +2,10 @@ helm repo add grafana-charts https://grafana.github.io/helm-charts helm repo add prometheus-community https://prometheus-community.github.io/helm-charts helm repo update + kubectl create namespace monitoring -curl -fsSL https://raw.githubusercontent.com/aws/karpenter-provider-aws/"${KARPENTER_VERSION}"/website/content/en/preview/getting-started/getting-started-with-karpenter/prometheus-values.yaml | tee prometheus-values.yaml +curl -fsSL https://raw.githubusercontent.com/aws/karpenter-provider-aws/"${KARPENTER_VERSION}"/website/content/en/preview/getting-started/getting-started-with-karpenter/prometheus-values.yaml | envsubst | tee prometheus-values.yaml helm install --namespace monitoring prometheus prometheus-community/prometheus --values prometheus-values.yaml curl -fsSL https://raw.githubusercontent.com/aws/karpenter-provider-aws/"${KARPENTER_VERSION}"/website/content/en/preview/getting-started/getting-started-with-karpenter/grafana-values.yaml | tee grafana-values.yaml diff --git a/website/content/en/v0.30/getting-started/getting-started-with-karpenter/scripts/step10-add-grafana-port-forward.sh b/website/content/en/v0.34/getting-started/getting-started-with-karpenter/scripts/step10-add-grafana-port-forward.sh similarity index 100% rename from website/content/en/v0.30/getting-started/getting-started-with-karpenter/scripts/step10-add-grafana-port-forward.sh rename to website/content/en/v0.34/getting-started/getting-started-with-karpenter/scripts/step10-add-grafana-port-forward.sh diff --git a/website/content/en/v0.30/getting-started/getting-started-with-karpenter/scripts/step11-grafana-get-password.sh b/website/content/en/v0.34/getting-started/getting-started-with-karpenter/scripts/step11-grafana-get-password.sh similarity index 100% rename from website/content/en/v0.30/getting-started/getting-started-with-karpenter/scripts/step11-grafana-get-password.sh rename to website/content/en/v0.34/getting-started/getting-started-with-karpenter/scripts/step11-grafana-get-password.sh diff --git a/website/content/en/v0.34/getting-started/getting-started-with-karpenter/scripts/step12-add-nodepool.sh b/website/content/en/v0.34/getting-started/getting-started-with-karpenter/scripts/step12-add-nodepool.sh new file mode 100755 index 000000000000..33f1cb553b1b --- /dev/null +++ b/website/content/en/v0.34/getting-started/getting-started-with-karpenter/scripts/step12-add-nodepool.sh @@ -0,0 +1,53 @@ +cat < ``` @@ -43,35 +44,31 @@ Now attach the required policies to the role {{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step03-node-policies.sh" language="bash" %}} -Attach the IAM role to an EC2 instance profile. - -{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step04-instance-profile.sh" language="bash" %}} - Now we need to create an IAM role that the Karpenter controller will use to provision new instances. The controller will be using [IAM Roles for Service Accounts (IRSA)](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html) which requires an OIDC endpoint. If you have another option for using IAM credentials with workloads (e.g. [kube2iam](https://github.com/jtblin/kube2iam)) your steps will be different. -{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step05-controller-iam.sh" language="bash" %}} +{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step04-controller-iam.sh" language="bash" %}} ## Add tags to subnets and security groups We need to add tags to our nodegroup subnets so Karpenter will know which subnets to use. -{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step06-tag-subnets.sh" language="bash" %}} +{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step05-tag-subnets.sh" language="bash" %}} Add tags to our security groups. This command only tags the security groups for the first nodegroup in the cluster. If you have multiple nodegroups or multiple security groups you will need to decide which one Karpenter should use. -{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step07-tag-security-groups.sh" language="bash" %}} +{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step06-tag-security-groups.sh" language="bash" %}} ## Update aws-auth ConfigMap We need to allow nodes that are using the node IAM role we just created to join the cluster. To do that we have to modify the `aws-auth` ConfigMap in the cluster. -{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step08-edit-aws-auth.sh" language="bash" %}} +{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step07-edit-aws-auth.sh" language="bash" %}} You will need to add a section to the mapRoles that looks something like this. Replace the `${AWS_PARTITION}` variable with the account partition, `${AWS_ACCOUNT_ID}` variable with your account ID, and `${CLUSTER_NAME}` variable with the cluster name, but do not replace the `{{EC2PrivateDNSName}}`. @@ -95,12 +92,12 @@ One for your Karpenter node role and one for your existing node group. First set the Karpenter release you want to deploy. ```bash -export KARPENTER_VERSION=v0.30.0 +export KARPENTER_VERSION=v0.34.5 ``` We can now generate a full Karpenter deployment yaml from the helm chart. -{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step09-generate-chart.sh" language="bash" %}} +{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step08-generate-chart.sh" language="bash" %}} Modify the following lines in the karpenter.yaml file. @@ -118,7 +115,7 @@ affinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - - key: karpenter.sh/provisioner-name + - key: karpenter.sh/nodepool operator: DoesNotExist - matchExpressions: - key: eks.amazonaws.com/nodegroup @@ -130,16 +127,15 @@ affinity: - topologyKey: "kubernetes.io/hostname" ``` -Now that our deployment is ready we can create the karpenter namespace, create the provisioner CRD, and then deploy the rest of the karpenter resources. +Now that our deployment is ready we can create the karpenter namespace, create the NodePool CRD, and then deploy the rest of the karpenter resources. -{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step10-deploy.sh" language="bash" %}} +{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step09-deploy.sh" language="bash" %}} -## Create default provisioner +## Create default NodePool -We need to create a default provisioner so Karpenter knows what types of nodes we want for unscheduled workloads. -You can refer to some of the [example provisioners](https://github.com/aws/karpenter/tree/v0.30.0/examples/provisioner) for specific needs. +We need to create a default NodePool so Karpenter knows what types of nodes we want for unscheduled workloads. You can refer to some of the [example NodePool](https://github.com/aws/karpenter/tree/v0.34.5/examples/v1beta1) for specific needs. -{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step11-create-provisioner.sh" language="bash" %}} +{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step10-create-nodepool.sh" language="bash" %}} ## Set nodeAffinity for critical workloads (optional) @@ -170,7 +166,7 @@ affinity: Now that karpenter is running we can disable the cluster autoscaler. To do that we will scale the number of replicas to zero. -{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step12-scale-cas.sh" language="bash" %}} +{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step11-scale-cas.sh" language="bash" %}} To get rid of the instances that were added from the node group we can scale our nodegroup down to a minimum size to support Karpenter and other critical services. @@ -178,11 +174,11 @@ To get rid of the instances that were added from the node group we can scale our If you have a single multi-AZ node group, we suggest a minimum of 2 instances. -{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step13-scale-single-ng.sh" language="bash" %}} +{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step12-scale-single-ng.sh" language="bash" %}} Or, if you have multiple single-AZ node groups, we suggest a minimum of 1 instance each. -{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step13-scale-multiple-ng.sh" language="bash" %}} +{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step12-scale-multiple-ng.sh" language="bash" %}} {{% alert title="Note" color="warning" %}} If you have a lot of nodes or workloads you may want to slowly scale down your node groups by a few instances at a time. It is recommended to watch the transition carefully for workloads that may not have enough replicas running or disruption budgets configured. diff --git a/website/content/en/v0.34/getting-started/migrating-from-cas/scripts/step01-env.sh b/website/content/en/v0.34/getting-started/migrating-from-cas/scripts/step01-env.sh new file mode 100644 index 000000000000..6276d7aa1439 --- /dev/null +++ b/website/content/en/v0.34/getting-started/migrating-from-cas/scripts/step01-env.sh @@ -0,0 +1,10 @@ +AWS_PARTITION="aws" # if you are not using standard partitions, you may need to configure to aws-cn / aws-us-gov +AWS_REGION="$(aws configure list | grep region | tr -s " " | cut -d" " -f3)" +OIDC_ENDPOINT="$(aws eks describe-cluster --name ${CLUSTER_NAME} \ + --query "cluster.identity.oidc.issuer" --output text)" +AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query 'Account' \ + --output text) +K8S_VERSION=1.28 +ARM_AMI_ID="$(aws ssm get-parameter --name /aws/service/eks/optimized-ami/${K8S_VERSION}/amazon-linux-2-arm64/recommended/image_id --query Parameter.Value --output text)" +AMD_AMI_ID="$(aws ssm get-parameter --name /aws/service/eks/optimized-ami/${K8S_VERSION}/amazon-linux-2/recommended/image_id --query Parameter.Value --output text)" +GPU_AMI_ID="$(aws ssm get-parameter --name /aws/service/eks/optimized-ami/${K8S_VERSION}/amazon-linux-2-gpu/recommended/image_id --query Parameter.Value --output text)" diff --git a/website/content/en/v0.30/getting-started/migrating-from-cas/scripts/step02-node-iam.sh b/website/content/en/v0.34/getting-started/migrating-from-cas/scripts/step02-node-iam.sh similarity index 100% rename from website/content/en/v0.30/getting-started/migrating-from-cas/scripts/step02-node-iam.sh rename to website/content/en/v0.34/getting-started/migrating-from-cas/scripts/step02-node-iam.sh diff --git a/website/content/en/v0.30/getting-started/migrating-from-cas/scripts/step03-node-policies.sh b/website/content/en/v0.34/getting-started/migrating-from-cas/scripts/step03-node-policies.sh similarity index 100% rename from website/content/en/v0.30/getting-started/migrating-from-cas/scripts/step03-node-policies.sh rename to website/content/en/v0.34/getting-started/migrating-from-cas/scripts/step03-node-policies.sh diff --git a/website/content/en/v0.30/getting-started/migrating-from-cas/scripts/step05-controller-iam.sh b/website/content/en/v0.34/getting-started/migrating-from-cas/scripts/step04-controller-iam.sh similarity index 52% rename from website/content/en/v0.30/getting-started/migrating-from-cas/scripts/step05-controller-iam.sh rename to website/content/en/v0.34/getting-started/migrating-from-cas/scripts/step04-controller-iam.sh index c3487b6feae5..70c59e826393 100644 --- a/website/content/en/v0.30/getting-started/migrating-from-cas/scripts/step05-controller-iam.sh +++ b/website/content/en/v0.34/getting-started/migrating-from-cas/scripts/step04-controller-iam.sh @@ -11,7 +11,7 @@ cat << EOF > controller-trust-policy.json "Condition": { "StringEquals": { "${OIDC_ENDPOINT#*//}:aud": "sts.amazonaws.com", - "${OIDC_ENDPOINT#*//}:sub": "system:serviceaccount:karpenter:karpenter" + "${OIDC_ENDPOINT#*//}:sub": "system:serviceaccount:${KARPENTER_NAMESPACE}:karpenter" } } } @@ -52,7 +52,7 @@ cat << EOF > controller-policy.json "Action": "ec2:TerminateInstances", "Condition": { "StringLike": { - "ec2:ResourceTag/karpenter.sh/provisioner-name": "*" + "ec2:ResourceTag/karpenter.sh/nodepool": "*" } }, "Effect": "Allow", @@ -70,6 +70,68 @@ cat << EOF > controller-policy.json "Action": "eks:DescribeCluster", "Resource": "arn:${AWS_PARTITION}:eks:${AWS_REGION}:${AWS_ACCOUNT_ID}:cluster/${CLUSTER_NAME}", "Sid": "EKSClusterEndpointLookup" + }, + { + "Sid": "AllowScopedInstanceProfileCreationActions", + "Effect": "Allow", + "Resource": "*", + "Action": [ + "iam:CreateInstanceProfile" + ], + "Condition": { + "StringEquals": { + "aws:RequestTag/kubernetes.io/cluster/${CLUSTER_NAME}": "owned", + "aws:RequestTag/topology.kubernetes.io/region": "${AWS_REGION}" + }, + "StringLike": { + "aws:RequestTag/karpenter.k8s.aws/ec2nodeclass": "*" + } + } + }, + { + "Sid": "AllowScopedInstanceProfileTagActions", + "Effect": "Allow", + "Resource": "*", + "Action": [ + "iam:TagInstanceProfile" + ], + "Condition": { + "StringEquals": { + "aws:ResourceTag/kubernetes.io/cluster/${CLUSTER_NAME}": "owned", + "aws:ResourceTag/topology.kubernetes.io/region": "${AWS_REGION}", + "aws:RequestTag/kubernetes.io/cluster/${CLUSTER_NAME}": "owned", + "aws:RequestTag/topology.kubernetes.io/region": "${AWS_REGION}" + }, + "StringLike": { + "aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass": "*", + "aws:RequestTag/karpenter.k8s.aws/ec2nodeclass": "*" + } + } + }, + { + "Sid": "AllowScopedInstanceProfileActions", + "Effect": "Allow", + "Resource": "*", + "Action": [ + "iam:AddRoleToInstanceProfile", + "iam:RemoveRoleFromInstanceProfile", + "iam:DeleteInstanceProfile" + ], + "Condition": { + "StringEquals": { + "aws:ResourceTag/kubernetes.io/cluster/${CLUSTER_NAME}": "owned", + "aws:ResourceTag/topology.kubernetes.io/region": "${AWS_REGION}" + }, + "StringLike": { + "aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass": "*" + } + } + }, + { + "Sid": "AllowInstanceProfileReadActions", + "Effect": "Allow", + "Resource": "*", + "Action": "iam:GetInstanceProfile" } ], "Version": "2012-10-17" diff --git a/website/content/en/v0.30/getting-started/migrating-from-cas/scripts/step06-tag-subnets.sh b/website/content/en/v0.34/getting-started/migrating-from-cas/scripts/step05-tag-subnets.sh similarity index 100% rename from website/content/en/v0.30/getting-started/migrating-from-cas/scripts/step06-tag-subnets.sh rename to website/content/en/v0.34/getting-started/migrating-from-cas/scripts/step05-tag-subnets.sh diff --git a/website/content/en/v0.30/getting-started/migrating-from-cas/scripts/step07-tag-security-groups.sh b/website/content/en/v0.34/getting-started/migrating-from-cas/scripts/step06-tag-security-groups.sh similarity index 100% rename from website/content/en/v0.30/getting-started/migrating-from-cas/scripts/step07-tag-security-groups.sh rename to website/content/en/v0.34/getting-started/migrating-from-cas/scripts/step06-tag-security-groups.sh diff --git a/website/content/en/v0.30/getting-started/migrating-from-cas/scripts/step08-edit-aws-auth.sh b/website/content/en/v0.34/getting-started/migrating-from-cas/scripts/step07-edit-aws-auth.sh similarity index 100% rename from website/content/en/v0.30/getting-started/migrating-from-cas/scripts/step08-edit-aws-auth.sh rename to website/content/en/v0.34/getting-started/migrating-from-cas/scripts/step07-edit-aws-auth.sh diff --git a/website/content/en/v0.34/getting-started/migrating-from-cas/scripts/step08-generate-chart.sh b/website/content/en/v0.34/getting-started/migrating-from-cas/scripts/step08-generate-chart.sh new file mode 100644 index 000000000000..f81ac0b90cc8 --- /dev/null +++ b/website/content/en/v0.34/getting-started/migrating-from-cas/scripts/step08-generate-chart.sh @@ -0,0 +1,7 @@ +helm template karpenter oci://public.ecr.aws/karpenter/karpenter --version "${KARPENTER_VERSION}" --namespace "${KARPENTER_NAMESPACE}" \ + --set "settings.clusterName=${CLUSTER_NAME}" \ + --set "serviceAccount.annotations.eks\.amazonaws\.com/role-arn=arn:${AWS_PARTITION}:iam::${AWS_ACCOUNT_ID}:role/KarpenterControllerRole-${CLUSTER_NAME}" \ + --set controller.resources.requests.cpu=1 \ + --set controller.resources.requests.memory=1Gi \ + --set controller.resources.limits.cpu=1 \ + --set controller.resources.limits.memory=1Gi > karpenter.yaml diff --git a/website/content/en/v0.30/getting-started/migrating-from-cas/scripts/step10-deploy.sh b/website/content/en/v0.34/getting-started/migrating-from-cas/scripts/step09-deploy.sh similarity index 52% rename from website/content/en/v0.30/getting-started/migrating-from-cas/scripts/step10-deploy.sh rename to website/content/en/v0.34/getting-started/migrating-from-cas/scripts/step09-deploy.sh index d66941f52ed0..51714d78f6dd 100644 --- a/website/content/en/v0.30/getting-started/migrating-from-cas/scripts/step10-deploy.sh +++ b/website/content/en/v0.34/getting-started/migrating-from-cas/scripts/step09-deploy.sh @@ -1,8 +1,8 @@ -kubectl create namespace karpenter +kubectl create namespace "${KARPENTER_NAMESPACE}" || true kubectl create -f \ - https://raw.githubusercontent.com/aws/karpenter-provider-aws/${KARPENTER_VERSION}/pkg/apis/crds/karpenter.sh_provisioners.yaml + https://raw.githubusercontent.com/aws/karpenter-provider-aws/${KARPENTER_VERSION}/pkg/apis/crds/karpenter.sh_nodepools.yaml kubectl create -f \ - https://raw.githubusercontent.com/aws/karpenter-provider-aws/${KARPENTER_VERSION}/pkg/apis/crds/karpenter.k8s.aws_awsnodetemplates.yaml + https://raw.githubusercontent.com/aws/karpenter-provider-aws/${KARPENTER_VERSION}/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml kubectl create -f \ - https://raw.githubusercontent.com/aws/karpenter-provider-aws/${KARPENTER_VERSION}/pkg/apis/crds/karpenter.sh_machines.yaml + https://raw.githubusercontent.com/aws/karpenter-provider-aws/${KARPENTER_VERSION}/pkg/apis/crds/karpenter.sh_nodeclaims.yaml kubectl apply -f karpenter.yaml diff --git a/website/content/en/v0.34/getting-started/migrating-from-cas/scripts/step10-create-nodepool.sh b/website/content/en/v0.34/getting-started/migrating-from-cas/scripts/step10-create-nodepool.sh new file mode 100644 index 000000000000..33f1cb553b1b --- /dev/null +++ b/website/content/en/v0.34/getting-started/migrating-from-cas/scripts/step10-create-nodepool.sh @@ -0,0 +1,53 @@ +cat < + Reference documentation for Karpenter +--- \ No newline at end of file diff --git a/website/content/en/v0.34/reference/cloudformation.md b/website/content/en/v0.34/reference/cloudformation.md new file mode 100644 index 000000000000..012e25a9d323 --- /dev/null +++ b/website/content/en/v0.34/reference/cloudformation.md @@ -0,0 +1,614 @@ +--- +title: "CloudFormation" +linkTitle: "CloudFormation" +weight: 5 +description: > + A description of the Getting Started CloudFormation file and permissions +--- +The [Getting Started with Karpenter]({{< relref "../getting-started/getting-started-with-karpenter" >}}) guide uses CloudFormation to bootstrap the cluster to enable Karpenter to create and manage nodes, as well as to allow Karpenter to respond to interruption events. +This document describes the `cloudformation.yaml` file used in that guide. +These descriptions should allow you to understand: + +* What Karpenter is authorized to do with your EKS cluster and AWS resources when using the `cloudformation.yaml` file +* What permissions you need to set up if you are adding Karpenter to an existing cluster + +## Overview + +To download a particular version of `cloudformation.yaml`, set the version and use `curl` to pull the file to your local system: + +```bash +export KARPENTER_VERSION=v0.34.5 +curl https://raw.githubusercontent.com/aws/karpenter-provider-aws/"${KARPENTER_VERSION}"/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml > cloudformation.yaml +``` + +Following some header information, the rest of the `cloudformation.yaml` file describes the resources that CloudFormation deploys. +The sections of that file can be grouped together under the following general headings: + +* [**Node Authorization**]({{< relref "#node-authorization" >}}): Creates a NodeInstanceProfile, attaches a NodeRole to it, and connects it to an IAM Identity Mapping used to authorize nodes to the cluster. This defines the permissions each node managed by Karpenter has to access EC2 and other AWS resources. This doesn't actually create the IAM Identity Mapping. That part is orchestrated by `eksctl` in the Getting Started guide. +* [**Controller Authorization**]({{< relref "#controller-authorization" >}}): Creates the `KarpenterControllerPolicy` that is attached to the service account. +Again, the actual service account creation (`karpenter`), that is combined with the `KarpenterControllerPolicy`, is orchestrated by `eksctl` in the Getting Started guide. +* [**Interruption Handling**]({{< relref "#interruption-handling" >}}): Allows the Karpenter controller to see and respond to interruptions that occur with the nodes that Karpenter is managing. See the [Interruption]({{< relref "../concepts/disruption#interruption" >}}) section of the Disruption page for details. + +A lot of the object naming that is done by `cloudformation.yaml` is based on the following: + +* Cluster name: With a username of `bob` the Getting Started Guide would name your cluster `bob-karpenter-demo` +That name would then be appended to any name below where `${ClusterName}` is included. + +* Partition: Any time an ARN is used, it includes the [partition name](https://docs.aws.amazon.com/whitepapers/latest/aws-fault-isolation-boundaries/partitions.html) to identify where the object is found. In most cases, that partition name is `aws`. However, it could also be `aws-cn` (for China Regions) or `aws-us-gov` (for AWS GovCloud US Regions). + +## Node Authorization + +The following sections of the `cloudformation.yaml` file set up IAM permissions for Kubernetes nodes created by Karpenter. +In particular, this involves setting up a node role that can be attached and passed to instance profiles that Karpenter generates at runtime: + +* KarpenterNodeRole + +### KarpenterNodeRole + +This section of the template defines the IAM role attached to generated instance profiles. +Given a cluster name of `bob-karpenter-demo`, this role would end up being named `"KarpenterNodeRole-bob-karpenter-demo`. + +```yaml +KarpenterNodeRole: + Type: "AWS::IAM::Role" + Properties: + RoleName: !Sub "KarpenterNodeRole-${ClusterName}" + Path: / + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + Service: + !Sub "ec2.${AWS::URLSuffix}" + Action: + - "sts:AssumeRole" + ManagedPolicyArns: + - !Sub "arn:${AWS::Partition}:iam::aws:policy/AmazonEKS_CNI_Policy" + - !Sub "arn:${AWS::Partition}:iam::aws:policy/AmazonEKSWorkerNodePolicy" + - !Sub "arn:${AWS::Partition}:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" + - !Sub "arn:${AWS::Partition}:iam::aws:policy/AmazonSSMManagedInstanceCore" +``` + +The role created here includes several AWS managed policies, which are designed to provide permissions for specific uses needed by the nodes to work with EC2 and other AWS resources. These include: + +* [AmazonEKS_CNI_Policy](https://docs.aws.amazon.com/aws-managed-policy/latest/reference/AmazonEKS_CNI_Policy.html): Provides the permissions that the Amazon VPC CNI Plugin needs to configure EKS worker nodes. +* [AmazonEKSWorkerNodePolicy](https://docs.aws.amazon.com/aws-managed-policy/latest/reference/AmazonEKSWorkerNodePolicy.html): Lets Amazon EKS worker nodes connect to EKS Clusters. +* [AmazonEC2ContainerRegistryReadOnly](https://docs.aws.amazon.com/aws-managed-policy/latest/reference/AmazonEC2ContainerRegistryReadOnly.html): Allows read-only access to repositories in the Amazon EC2 Container Registry. +* [AmazonSSMManagedInstanceCore](https://docs.aws.amazon.com/aws-managed-policy/latest/reference/AmazonSSMManagedInstanceCore.html): Adds AWS Systems Manager service core functions for Amazon EC2. + +If you were to use a node role from an existing cluster, you could skip this provisioning step and pass this node role to any EC2NodeClasses that you create. Additionally, you would ensure that the [Controller Policy]({{< relref "#controllerpolicy" >}}) has `iam:PassRole` permission to the role attached to the generated instance profiles. + +## Controller Authorization + +This section sets the AWS permissions for the Karpenter Controller. When used in the Getting Started guide, `eksctl` uses these permissions to create a service account (karpenter) that is combined with the KarpenterControllerPolicy. + +The resources defined in this section are associated with: + +* KarpenterControllerPolicy + +Because the scope of the KarpenterControllerPolicy is an AWS region, the cluster's AWS region is included in the `AllowScopedEC2InstanceAccessActions`. + +### KarpenterControllerPolicy + +A `KarpenterControllerPolicy` object sets the name of the policy, then defines a set of resources and actions allowed for those resources. +For our example, the KarpenterControllerPolicy would be named: `KarpenterControllerPolicy-bob-karpenter-demo` + +```yaml +KarpenterControllerPolicy: + Type: AWS::IAM::ManagedPolicy + Properties: + ManagedPolicyName: !Sub "KarpenterControllerPolicy-${ClusterName}" + # The PolicyDocument must be in JSON string format because we use a StringEquals condition that uses an interpolated + # value in one of its key parameters which isn't natively supported by CloudFormation + PolicyDocument: !Sub | + { + "Version": "2012-10-17", + "Statement": [ +``` + +Someone wanting to add Karpenter to an existing cluster, instead of using `cloudformation.yaml`, would need to create the IAM policy directly and assign that policy to the role leveraged by the service account using IRSA. + +#### AllowScopedEC2InstanceAccessActions + +The AllowScopedEC2InstanceAccessActions statement ID (Sid) identifies a set of EC2 resources that are allowed to be accessed with +[RunInstances](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_RunInstances.html) and [CreateFleet](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html) actions. +For `RunInstances` and `CreateFleet` actions, the Karpenter controller can read (but not create) `image`, `snapshot`, `security-group`, `subnet` and `launch-template` EC2 resources, scoped for the particular AWS partition and region. + +```json +{ + "Sid": "AllowScopedEC2InstanceAccessActions", + "Effect": "Allow", + "Resource": [ + "arn:${AWS::Partition}:ec2:${AWS::Region}::image/*", + "arn:${AWS::Partition}:ec2:${AWS::Region}::snapshot/*", + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:security-group/*", + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:subnet/*" + ], + "Action": [ + "ec2:RunInstances", + "ec2:CreateFleet" + ] +} +``` + +#### AllowScopedEC2LaunchTemplateAccessActions + +The AllowScopedEC2InstanceAccessActions statement ID (Sid) identifies launch templates that are allowed to be accessed with +[RunInstances](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_RunInstances.html) and [CreateFleet](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html) actions. +For `RunInstances` and `CreateFleet` actions, the Karpenter controller can read (but not create) `launch-template` EC2 resources that have the `kubernetes.io/cluster/${ClusterName}` tag be set to `owned` and a `karpenter.sh/nodepool` tag, scoped for the particular AWS partition and region. This ensures that an instance launch can't access launch templates that weren't provisioned by Karpenter. + +```json +{ + "Sid": "AllowScopedEC2LaunchTemplateAccessActions", + "Effect": "Allow", + "Resource": "arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*", + "Action": [ + "ec2:RunInstances", + "ec2:CreateFleet" + ], + "Condition": { + "StringEquals": { + "aws:ResourceTag/kubernetes.io/cluster/${ClusterName}": "owned" + }, + "StringLike": { + "aws:ResourceTag/karpenter.sh/nodepool": "*" + } + } +} +``` + +#### AllowScopedEC2InstanceActionsWithTags + +The AllowScopedEC2InstanceActionsWithTags Sid allows the +[RunInstances](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_RunInstances.html), [CreateFleet](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html), and [CreateLaunchTemplate](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateLaunchTemplate.html) +actions requested by the Karpenter controller to create all `fleet`, `instance`, `volume`, `network-interface`, `launch-template` or `spot-instances-request` EC2 resources (for the partition and region), and requires that the `kubernetes.io/cluster/${ClusterName}` tag be set to `owned` and a `karpenter.sh/nodepool` tag be set to any value. This ensures that Karpenter is only allowed to create instances for a single EKS cluster. + +```json +{ + "Sid": "AllowScopedEC2InstanceActionsWithTags", + "Effect": "Allow", + "Resource": [ + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:fleet/*", + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:instance/*", + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:volume/*", + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:network-interface/*", + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*", + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:spot-instances-request/*" + ], + "Action": [ + "ec2:RunInstances", + "ec2:CreateFleet", + "ec2:CreateLaunchTemplate" + ], + "Condition": { + "StringEquals": { + "aws:RequestTag/kubernetes.io/cluster/${ClusterName}": "owned" + }, + "StringLike": { + "aws:RequestTag/karpenter.sh/nodepool": "*" + } + } +} +``` + +#### AllowScopedResourceCreationTagging + +The AllowScopedResourceCreationTagging Sid allows EC2 [CreateTags](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateTags.html) +actions on `fleet`, `instance`, `volume`, `network-interface`, `launch-template` and `spot-instances-request` resources, While making `RunInstance`, `CreateFleet`, or `CreateLaunchTemplate` calls. Additionally, this ensures that resources can't be tagged arbitrarily by Karpenter after they are created. + +```json +{ + "Sid": "AllowScopedResourceCreationTagging", + "Effect": "Allow", + "Resource": [ + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:fleet/*", + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:instance/*", + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:volume/*", + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:network-interface/*", + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*", + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:spot-instances-request/*" + ], + "Action": "ec2:CreateTags", + "Condition": { + "StringEquals": { + "aws:RequestTag/kubernetes.io/cluster/${ClusterName}": "owned", + "ec2:CreateAction": [ + "RunInstances", + "CreateFleet", + "CreateLaunchTemplate" + ] + }, + "StringLike": { + "aws:RequestTag/karpenter.sh/nodepool": "*" + } + } +} +``` + +#### AllowScopedResourceTagging + +The AllowScopedResourceTagging Sid allows EC2 [CreateTags](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateTags.html) actions on all instances created by Karpenter after their creation. It enforces that Karpenter is only able to update the tags on cluster instances it is operating on through the `kubernetes.io/cluster/${ClusterName}`" and `karpenter.sh/nodepool` tags. +```json +{ + "Sid": "AllowScopedResourceTagging", + "Effect": "Allow", + "Resource": "arn:${AWS::Partition}:ec2:${AWS::Region}:*:instance/*", + "Action": "ec2:CreateTags", + "Condition": { + "StringEquals": { + "aws:ResourceTag/kubernetes.io/cluster/${ClusterName}": "owned" + }, + "StringLike": { + "aws:ResourceTag/karpenter.sh/nodepool": "*" + }, + "ForAllValues:StringEquals": { + "aws:TagKeys": [ + "karpenter.sh/nodeclaim", + "Name" + ] + } + } +} +``` + +#### AllowScopedDeletion + +The AllowScopedDeletion Sid allows [TerminateInstances](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_TerminateInstances.html) and [DeleteLaunchTemplate](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DeleteLaunchTemplate.html) actions to delete instance and launch-template resources, provided that `karpenter.sh/nodepool` and `kubernetes.io/cluster/${ClusterName}` tags are set. These tags must be present on all resources that Karpenter is going to delete. This ensures that Karpenter can only delete instances and launch templates that are associated with it. + +```json +{ + "Sid": "AllowScopedDeletion", + "Effect": "Allow", + "Resource": [ + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:instance/*", + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*" + ], + "Action": [ + "ec2:TerminateInstances", + "ec2:DeleteLaunchTemplate" + ], + "Condition": { + "StringEquals": { + "aws:ResourceTag/kubernetes.io/cluster/${ClusterName}": "owned" + }, + "StringLike": { + "aws:ResourceTag/karpenter.sh/nodepool": "*" + } + } +} +``` + +#### AllowRegionalReadActions + +The AllowRegionalReadActions Sid allows [DescribeAvailabilityZones](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeAvailabilityZones.html), [DescribeImages](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeImages.html), [DescribeInstances](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeInstances.html), [DescribeInstanceTypeOfferings](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeInstanceTypeOfferings.html), [DescribeInstanceTypes](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeInstanceTypes.html), [DescribeLaunchTemplates](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeLaunchTemplates.html), [DescribeSecurityGroups](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeSecurityGroups.html), [DescribeSpotPriceHistory](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeSpotPriceHistory.html), and [DescribeSubnets](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeSubnets.html) actions for the current AWS region. +This allows the Karpenter controller to do any of those read-only actions across all related resources for that AWS region. + +```json +{ + "Sid": "AllowRegionalReadActions", + "Effect": "Allow", + "Resource": "*", + "Action": [ + "ec2:DescribeAvailabilityZones", + "ec2:DescribeImages", + "ec2:DescribeInstances", + "ec2:DescribeInstanceTypeOfferings", + "ec2:DescribeInstanceTypes", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSpotPriceHistory", + "ec2:DescribeSubnets" + ], + "Condition": { + "StringEquals": { + "aws:RequestedRegion": "${AWS::Region}" + } + } +} +``` + +#### AllowSSMReadActions + +The AllowSSMReadActions Sid allows the Karpenter controller to read SSM parameters (`ssm:GetParameter`) from the current region for SSM parameters generated by ASW services. + +**NOTE**: If potentially sensitive information is stored in SSM parameters, you could consider restricting access to these messages further. +```json +{ + "Sid": "AllowSSMReadActions", + "Effect": "Allow", + "Resource": "arn:${AWS::Partition}:ssm:${AWS::Region}::parameter/aws/service/*", + "Action": "ssm:GetParameter" +} +``` + +#### AllowPricingReadActions + +Because pricing information does not exist in every region at the moment, the AllowPricingReadActions Sid allows the Karpenter controller to get product pricing information (`pricing:GetProducts`) for all related resources across all regions. + +```json +{ + "Sid": "AllowPricingReadActions", + "Effect": "Allow", + "Resource": "*", + "Action": "pricing:GetProducts" +} +``` + +#### AllowInterruptionQueueActions + +Karpenter supports interruption queues, that you can create as described in the [Interruption]({{< relref "../concepts/disruption#interruption" >}}) section of the Disruption page. +This section of the cloudformation.yaml template can give Karpenter permission to access those queues by specifying the resource ARN. +For the interruption queue you created (`${KarpenterInterruptionQueue.Arn}`), the AllowInterruptionQueueActions Sid lets the Karpenter controller have permission to delete messages ([DeleteMessage](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_DeleteMessage.html)), get queue URL ([GetQueueUrl](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_GetQueueUrl.html)), and receive messages ([ReceiveMessage](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_ReceiveMessage.html)). + +```json +{ + "Sid": "AllowInterruptionQueueActions", + "Effect": "Allow", + "Resource": "${KarpenterInterruptionQueue.Arn}", + "Action": [ + "sqs:DeleteMessage", + "sqs:GetQueueUrl", + "sqs:ReceiveMessage" + ] +} +``` + +#### AllowPassingInstanceRole + +The AllowPassingInstanceRole Sid gives the Karpenter controller permission to pass (`iam:PassRole`) the node role (`KarpenterNodeRole-${ClusterName}`) to generated instance profiles. +This gives EC2 permission explicit permission to use the `KarpenterNodeRole-${ClusterName}` when assigning permissions to generated instance profiles while launching nodes. + +```json +{ + "Sid": "AllowPassingInstanceRole", + "Effect": "Allow", + "Resource": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/KarpenterNodeRole-${ClusterName}", + "Action": "iam:PassRole", + "Condition": { + "StringEquals": { + "iam:PassedToService": "ec2.amazonaws.com" + } + } +} +``` + +#### AllowScopedInstanceProfileCreationActions + +The AllowScopedInstanceProfileCreationActions Sid gives the Karpenter controller permission to create a new instance profile with [`iam:CreateInstanceProfile`](https://docs.aws.amazon.com/IAM/latest/APIReference/API_CreateInstanceProfile.html), +provided that the request is made to a cluster with `kubernetes.io/cluster/${ClusterName` set to owned and is made in the current region. +Also, `karpenter.k8s.aws/ec2nodeclass` must be set to some value. This ensures that Karpenter can generate instance profiles on your behalf based on roles specified in your `EC2NodeClasses` that you use to configure Karpenter. + +```json +{ + "Sid": "AllowScopedInstanceProfileCreationActions", + "Effect": "Allow", + "Resource": "*", + "Action": [ + "iam:CreateInstanceProfile" + ], + "Condition": { + "StringEquals": { + "aws:RequestTag/kubernetes.io/cluster/${ClusterName}": "owned", + "aws:RequestTag/topology.kubernetes.io/region": "${AWS::Region}" + }, + "StringLike": { + "aws:RequestTag/karpenter.k8s.aws/ec2nodeclass": "*" + } + } +} +``` + +#### AllowScopedInstanceProfileTagActions + +The AllowScopedInstanceProfileTagActions Sid gives the Karpenter controller permission to tag an instance profile with [`iam:TagInstanceProfile`](https://docs.aws.amazon.com/IAM/latest/APIReference/API_TagInstanceProfile.html), based on the values shown below, +Also, `karpenter.k8s.aws/ec2nodeclass` must be set to some value. This ensures that Karpenter is only able to act on instance profiles that it provisions for this cluster. + +```json +{ + "Sid": "AllowScopedInstanceProfileTagActions", + "Effect": "Allow", + "Resource": "*", + "Action": [ + "iam:TagInstanceProfile" + ], + "Condition": { + "StringEquals": { + "aws:ResourceTag/kubernetes.io/cluster/${ClusterName}": "owned", + "aws:ResourceTag/topology.kubernetes.io/region": "${AWS::Region}", + "aws:RequestTag/kubernetes.io/cluster/${ClusterName}": "owned", + "aws:RequestTag/topology.kubernetes.io/region": "${AWS::Region}" + }, + "StringLike": { + "aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass": "*", + "aws:RequestTag/karpenter.k8s.aws/ec2nodeclass": "*" + } + } +} +``` + + +#### AllowScopedInstanceProfileActions + +The AllowScopedInstanceProfileActions Sid gives the Karpenter controller permission to perform [`iam:AddRoleToInstanceProfile`](https://docs.aws.amazon.com/IAM/latest/APIReference/API_AddRoleToInstanceProfile.html), [`iam:RemoveRoleFromInstanceProfile`](https://docs.aws.amazon.com/IAM/latest/APIReference/API_RemoveRoleFromInstanceProfile.html), and [`iam:DeleteInstanceProfile`](https://docs.aws.amazon.com/IAM/latest/APIReference/API_DeleteInstanceProfile.html) actions, +provided that the request is made to a cluster with `kubernetes.io/cluster/${ClusterName` set to owned and is made in the current region. +Also, `karpenter.k8s.aws/ec2nodeclass` must be set to some value. This permission is further enforced by the `iam:PassRole` permission. If Karpenter attempts to add a role to an instance profile that it doesn't have `iam:PassRole` permission on, that call will fail. Therefore, if you configure Karpenter to use a new role through the `EC2NodeClass`, ensure that you also specify that role within your `iam:PassRole` permission. + +```json +{ + "Sid": "AllowScopedInstanceProfileActions", + "Effect": "Allow", + "Resource": "*", + "Action": [ + "iam:AddRoleToInstanceProfile", + "iam:RemoveRoleFromInstanceProfile", + "iam:DeleteInstanceProfile" + ], + "Condition": { + "StringEquals": { + "aws:ResourceTag/kubernetes.io/cluster/${ClusterName}": "owned", + "aws:ResourceTag/topology.kubernetes.io/region": "${AWS::Region}" + }, + "StringLike": { + "aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass": "*" + } + } +} +``` + +#### AllowInstanceProfileActions + +The AllowInstanceProfileActions Sid gives the Karpenter controller permission to perform [`iam:GetInstanceProfile`](https://docs.aws.amazon.com/IAM/latest/APIReference/API_GetInstanceProfile.html) actions to retrieve information about a specified instance profile, including understanding if an instance profile has been provisioned for an `EC2NodeClass` or needs to be re-provisioned. + +```json +{ + "Sid": "AllowInstanceProfileReadActions", + "Effect": "Allow", + "Resource": "*", + "Action": "iam:GetInstanceProfile" +} +``` + +#### AllowAPIServerEndpointDiscovery + +You can optionally allow the Karpenter controller to discover the Kubernetes cluster's external API endpoint to enable EC2 nodes to successfully join the EKS cluster. + +> **Note**: If you are not using an EKS control plane, you will have to specify this endpoint explicitly. See the description of the `aws.clusterEndpoint` setting in the [ConfigMap](.settings/#configmap) documentation for details. + +The AllowAPIServerEndpointDiscovery Sid allows the Karpenter controller to get that information (`eks:DescribeCluster`) for the cluster (`cluster/${ClusterName}`). +```json +{ + "Sid": "AllowAPIServerEndpointDiscovery", + "Effect": "Allow", + "Resource": "arn:${AWS::Partition}:eks:${AWS::Region}:${AWS::AccountId}:cluster/${ClusterName}", + "Action": "eks:DescribeCluster" +} +``` + +## Interruption Handling + +Settings in this section allow the Karpenter controller to stand-up an interruption queue to receive notification messages from other AWS services about the health and status of instances. For example, this interruption queue allows Karpenter to be aware of spot instance interruptions that are sent 2 minutes before spot instances are reclaimed by EC2. Adding this queue allows Karpenter to be proactive in migrating workloads to new nodes. +See the [Interruption]({{< relref "../concepts/disruption#interruption" >}}) section of the Disruption page for details. + +Defining the `KarpenterInterruptionQueuePolicy` allows Karpenter to see and respond to the following: + +* AWS health events +* Spot interruptions +* Spot rebalance recommendations +* Instance state changes + +The resources defined in this section include: + +* KarpenterInterruptionQueue +* KarpenterInterruptionQueuePolicy +* ScheduledChangeRule +* SpotInterruptionRule +* RebalanceRule +* InstanceStateChangeRule + +### KarpenterInterruptionQueue + +The [AWS::SQS::Queue](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-sqs-queue.html) resource is used to create an Amazon SQS standard queue. +Properties of that resource set the `QueueName` to the name of your cluster, the time for which SQS retains each message (`MessageRetentionPeriod`) to 300 seconds, and enabling serverside-side encryption using SQS owned encryption keys (`SqsManagedSseEnabled`) to `true`. +See [SetQueueAttributes](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_SetQueueAttributes.html) for descriptions of some of these attributes. + +```yaml +KarpenterInterruptionQueue: + Type: AWS::SQS::Queue + Properties: + QueueName: !Sub "${ClusterName}" + MessageRetentionPeriod: 300 + SqsManagedSseEnabled: true +``` + +### KarpenterInterruptionQueuePolicy + +The Karpenter interruption queue policy is created to allow AWS services that we want to receive instance notifications from to push notification messages to the queue. +The [AWS::SQS::QueuePolicy](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-sqs-queuepolicy.html) resource here applies `EC2InterruptionPolicy` to the `KarpenterInterruptionQueue`. The policy allows [sqs:SendMessage](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_SendMessage.html) actions to `events.amazonaws.com` and `sqs.amazonaws.com` services. It also allows the `GetAtt` function to get attributes from `KarpenterInterruptionQueue.Arn`. + +```yaml +KarpenterInterruptionQueuePolicy: + Type: AWS::SQS::QueuePolicy + Properties: + Queues: + - !Ref KarpenterInterruptionQueue + PolicyDocument: + Id: EC2InterruptionPolicy + Statement: + - Effect: Allow + Principal: + Service: + - events.amazonaws.com + - sqs.amazonaws.com + Action: sqs:SendMessage + Resource: !GetAtt KarpenterInterruptionQueue.Arn +``` + +### Rules + +This section allows Karpenter to gather [AWS Health Events](https://docs.aws.amazon.com/health/latest/ug/cloudwatch-events-health.html#about-public-events) and direct them to a queue where they can be consumed by Karpenter. +These rules include: + +* ScheduledChangeRule: The [AWS::Events::Rule](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-events-rule.html) creates a rule where the [EventPattern](https://docs.aws.amazon.com/eventbridge/latest/userguide/eb-event-patterns.html) is set to send events from the `aws.health` source to `KarpenterInterruptionQueue`. + + ```yaml + ScheduledChangeRule: + Type: 'AWS::Events::Rule' + Properties: + EventPattern: + source: + - aws.health + detail-type: + - AWS Health Event + Targets: + - Id: KarpenterInterruptionQueueTarget + Arn: !GetAtt KarpenterInterruptionQueue.Arn + ``` + +* SpotInterruptionRule: An EC2 Spot Instance Interruption warning tells you that AWS is about to reclaim a Spot instance you are using. This rule allows Karpenter to gather [EC2 Spot Instance Interruption Warning](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-interruptions.html) events and direct them to a queue where they can be consumed by Karpenter. In particular, the [AWS::Events::Rule](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-events-rule.html) here creates a rule where the [EventPattern](https://docs.aws.amazon.com/eventbridge/latest/userguide/eb-event-patterns.html) is set to send events from the `aws.ec2` source to `KarpenterInterruptionQueue`. + + ```yaml + SpotInterruptionRule: + Type: 'AWS::Events::Rule' + Properties: + EventPattern: + source: + - aws.ec2 + detail-type: + - EC2 Spot Instance Interruption Warning + Targets: + - Id: KarpenterInterruptionQueueTarget + Arn: !GetAtt KarpenterInterruptionQueue.Arn + ``` + +* RebalanceRule: An EC2 Instance Rebalance Recommendation signal tells you that a Spot instance is at a heightened risk of being interrupted, allowing Karpenter to get new instances or simply rebalance workloads. This rule allows Karpenter to gather [EC2 Instance Rebalance Recommendation](https://docs.aws.amazon.com/AWSEC2/latest/WindowsGuide/rebalance-recommendations.html) signals and direct them to a queue where they can be consumed by Karpenter. In particular, the [AWS::Events::Rule](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-events-rule.html) here creates a rule where the [EventPattern](https://docs.aws.amazon.com/eventbridge/latest/userguide/eb-event-patterns.html) is set to send events from the `aws.ec2` source to `KarpenterInterruptionQueue`. + + ```yaml + RebalanceRule: + Type: 'AWS::Events::Rule' + Properties: + EventPattern: + source: + - aws.ec2 + detail-type: + - EC2 Instance Rebalance Recommendation + Targets: + - Id: KarpenterInterruptionQueueTarget + Arn: !GetAtt KarpenterInterruptionQueue.Arn + ``` + +* InstanceStateChangeRule: An EC2 Instance State-change Notification signal tells you that the state of an instance has changed to one of the following states: pending, running, stopping, stopped, shutting-down, or terminated. This rule allows Karpenter to gather [EC2 Instance State-change](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/monitoring-instance-state-changes.html) signals and direct them to a queue where they can be consumed by Karpenter. In particular, the [AWS::Events::Rule](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-events-rule.html) here creates a rule where the [EventPattern](https://docs.aws.amazon.com/eventbridge/latest/userguide/eb-event-patterns.html) is set to send events from the `aws.ec2` source to `KarpenterInterruptionQueue`. + + ```yaml + InstanceStateChangeRule: + Type: 'AWS::Events::Rule' + Properties: + EventPattern: + source: + - aws.ec2 + detail-type: + - EC2 Instance State-change Notification + Targets: + - Id: KarpenterInterruptionQueueTarget + Arn: !GetAtt KarpenterInterruptionQueue.Arn + ``` diff --git a/website/content/en/v0.31/concepts/instance-types.md b/website/content/en/v0.34/reference/instance-types.md similarity index 93% rename from website/content/en/v0.31/concepts/instance-types.md rename to website/content/en/v0.34/reference/instance-types.md index 95752c33498f..64aeaf1f1174 100644 --- a/website/content/en/v0.31/concepts/instance-types.md +++ b/website/content/en/v0.34/reference/instance-types.md @@ -25,7 +25,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|2048| |karpenter.k8s.aws/instance-network-bandwidth|500| - |karpenter.k8s.aws/instance-pods|8| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -50,7 +49,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|4096| |karpenter.k8s.aws/instance-network-bandwidth|750| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -75,7 +73,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -100,7 +97,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -125,7 +121,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -150,7 +145,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -175,7 +169,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|1| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|1740| - |karpenter.k8s.aws/instance-pods|12| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -198,7 +191,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|1| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|7168| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -222,7 +214,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|3| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|3840| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -245,7 +236,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|3| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|7680| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -268,7 +258,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|3| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|15360| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -291,7 +280,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|3| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|30720| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -315,7 +303,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|61440| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -339,7 +326,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|4| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|3840| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -362,7 +348,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|4| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|7680| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -385,7 +370,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|4| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|15360| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -408,7 +392,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|4| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|30720| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -432,7 +415,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|61440| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -457,7 +439,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|4096| |karpenter.k8s.aws/instance-network-bandwidth|750| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -482,7 +463,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -507,7 +487,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -532,7 +511,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -557,7 +535,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|73728| |karpenter.k8s.aws/instance-network-bandwidth|12000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|9xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -582,7 +559,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|98304| |karpenter.k8s.aws/instance-network-bandwidth|12000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -607,7 +583,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|147456| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|18xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -632,7 +607,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -657,7 +631,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -683,7 +656,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|4096| |karpenter.k8s.aws/instance-network-bandwidth|750| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -708,7 +680,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -733,7 +704,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -758,7 +728,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -783,7 +752,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -808,7 +776,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|98304| |karpenter.k8s.aws/instance-network-bandwidth|12000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -833,7 +800,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|20000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -858,7 +824,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|20000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -885,7 +850,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|75| |karpenter.k8s.aws/instance-memory|4096| |karpenter.k8s.aws/instance-network-bandwidth|750| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -911,7 +875,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|150| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -937,7 +900,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|300| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -963,7 +925,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|600| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -989,7 +950,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1200| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1015,7 +975,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1800| |karpenter.k8s.aws/instance-memory|98304| |karpenter.k8s.aws/instance-network-bandwidth|12000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1041,7 +1000,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|2400| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|20000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1067,7 +1025,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|20000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1094,7 +1051,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|50| |karpenter.k8s.aws/instance-memory|4096| |karpenter.k8s.aws/instance-network-bandwidth|750| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1120,7 +1076,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|100| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1146,7 +1101,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|200| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1172,7 +1126,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|400| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1198,7 +1151,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|900| |karpenter.k8s.aws/instance-memory|73728| |karpenter.k8s.aws/instance-network-bandwidth|12000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|9xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1224,7 +1176,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1800| |karpenter.k8s.aws/instance-memory|98304| |karpenter.k8s.aws/instance-network-bandwidth|12000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1250,7 +1201,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1800| |karpenter.k8s.aws/instance-memory|147456| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|18xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1276,7 +1226,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3600| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1302,7 +1251,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3600| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1328,7 +1276,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|5376| |karpenter.k8s.aws/instance-network-bandwidth|3000| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1353,7 +1300,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|10752| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1378,7 +1324,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|21504| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1403,7 +1348,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|43008| |karpenter.k8s.aws/instance-network-bandwidth|15000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1428,7 +1372,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|98304| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|9xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1440,6 +1383,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|88002Mi| |pods|234| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|54| ### `c5n.18xlarge` #### Labels @@ -1453,7 +1397,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|18xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1465,6 +1408,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|173400Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ### `c5n.metal` #### Labels @@ -1478,7 +1422,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1490,6 +1433,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|173400Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## c6a Family ### `c6a.large` @@ -1504,7 +1448,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|4096| |karpenter.k8s.aws/instance-network-bandwidth|781| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1529,7 +1472,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|1562| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1554,7 +1496,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|3125| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1579,7 +1520,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|6250| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1604,7 +1544,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1629,7 +1568,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|98304| |karpenter.k8s.aws/instance-network-bandwidth|18750| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1654,7 +1592,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1679,7 +1616,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|37500| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1704,7 +1640,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|32xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1729,7 +1664,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|48xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1741,6 +1675,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|355262Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ### `c6a.metal` #### Labels @@ -1754,7 +1689,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1766,6 +1700,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|355262Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## c6g Family ### `c6g.medium` @@ -1780,7 +1715,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|2048| |karpenter.k8s.aws/instance-network-bandwidth|500| - |karpenter.k8s.aws/instance-pods|8| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -1805,7 +1739,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|4096| |karpenter.k8s.aws/instance-network-bandwidth|750| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -1830,7 +1763,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -1855,7 +1787,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -1880,7 +1811,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -1905,7 +1835,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|12000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -1930,7 +1859,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|98304| |karpenter.k8s.aws/instance-network-bandwidth|20000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -1955,7 +1883,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -1980,7 +1907,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -2007,7 +1933,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|59| |karpenter.k8s.aws/instance-memory|2048| |karpenter.k8s.aws/instance-network-bandwidth|500| - |karpenter.k8s.aws/instance-pods|8| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -2033,7 +1958,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|118| |karpenter.k8s.aws/instance-memory|4096| |karpenter.k8s.aws/instance-network-bandwidth|750| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -2059,7 +1983,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|237| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -2085,7 +2008,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|474| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -2111,7 +2033,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|950| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -2137,7 +2058,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1900| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|12000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -2163,7 +2083,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|2850| |karpenter.k8s.aws/instance-memory|98304| |karpenter.k8s.aws/instance-network-bandwidth|20000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -2189,7 +2108,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -2215,7 +2133,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -2241,7 +2158,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|2048| |karpenter.k8s.aws/instance-network-bandwidth|1600| - |karpenter.k8s.aws/instance-pods|8| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -2266,7 +2182,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|4096| |karpenter.k8s.aws/instance-network-bandwidth|3000| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -2291,7 +2206,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|6300| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -2316,7 +2230,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -2340,8 +2253,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|6| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| - |karpenter.k8s.aws/instance-network-bandwidth|15000| - |karpenter.k8s.aws/instance-pods|234| + |karpenter.k8s.aws/instance-network-bandwidth|25000| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -2366,7 +2278,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -2391,7 +2302,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|98304| |karpenter.k8s.aws/instance-network-bandwidth|75000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -2416,7 +2326,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -2428,6 +2337,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|112720Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## c6i Family ### `c6i.large` @@ -2442,7 +2352,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|4096| |karpenter.k8s.aws/instance-network-bandwidth|781| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -2467,7 +2376,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|1562| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -2492,7 +2400,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|3125| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -2517,7 +2424,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|6250| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -2542,7 +2448,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -2567,7 +2472,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|98304| |karpenter.k8s.aws/instance-network-bandwidth|18750| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -2592,7 +2496,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -2617,7 +2520,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|37500| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -2642,7 +2544,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|32xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -2654,6 +2555,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|234021Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ### `c6i.metal` #### Labels @@ -2667,7 +2569,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -2679,6 +2580,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|234021Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## c6id Family ### `c6id.large` @@ -2694,7 +2596,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|118| |karpenter.k8s.aws/instance-memory|4096| |karpenter.k8s.aws/instance-network-bandwidth|781| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -2720,7 +2621,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|237| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|1562| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -2746,7 +2646,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|474| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|3125| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -2772,7 +2671,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|950| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|6250| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -2798,7 +2696,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1900| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -2824,7 +2721,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|2850| |karpenter.k8s.aws/instance-memory|98304| |karpenter.k8s.aws/instance-network-bandwidth|18750| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -2850,7 +2746,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -2876,7 +2771,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|5700| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|37500| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -2902,7 +2796,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|7600| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|32xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -2914,6 +2807,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|234021Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ### `c6id.metal` #### Labels @@ -2928,7 +2822,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|7600| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -2940,6 +2833,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|234021Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## c6in Family ### `c6in.large` @@ -2954,7 +2848,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|4096| |karpenter.k8s.aws/instance-network-bandwidth|3125| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -2979,7 +2872,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|6250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3004,7 +2896,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3029,7 +2920,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3054,7 +2944,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3079,7 +2968,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|98304| |karpenter.k8s.aws/instance-network-bandwidth|75000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3104,7 +2992,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3129,7 +3016,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|150000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3154,7 +3040,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|200000| - |karpenter.k8s.aws/instance-pods|345| |karpenter.k8s.aws/instance-size|32xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3166,6 +3051,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|238333Mi| |pods|345| + |vpc.amazonaws.com/efa|2| |vpc.amazonaws.com/pod-eni|108| ### `c6in.metal` #### Labels @@ -3179,7 +3065,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|200000| - |karpenter.k8s.aws/instance-pods|345| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3191,7 +3076,299 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|238333Mi| |pods|345| + |vpc.amazonaws.com/efa|2| |vpc.amazonaws.com/pod-eni|108| +## c7a Family +### `c7a.medium` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|1| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7a| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|2048| + |karpenter.k8s.aws/instance-network-bandwidth|390| + |karpenter.k8s.aws/instance-size|medium| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7a.medium| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|940m| + |ephemeral-storage|17Gi| + |memory|1451Mi| + |pods|8| + |vpc.amazonaws.com/pod-eni|4| +### `c7a.large` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7a| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|4096| + |karpenter.k8s.aws/instance-network-bandwidth|781| + |karpenter.k8s.aws/instance-size|large| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7a.large| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|1930m| + |ephemeral-storage|17Gi| + |memory|3114Mi| + |pods|29| + |vpc.amazonaws.com/pod-eni|9| +### `c7a.xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7a| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|8192| + |karpenter.k8s.aws/instance-network-bandwidth|1562| + |karpenter.k8s.aws/instance-size|xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7a.xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|3920m| + |ephemeral-storage|17Gi| + |memory|6584Mi| + |pods|58| + |vpc.amazonaws.com/pod-eni|18| +### `c7a.2xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7a| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|16384| + |karpenter.k8s.aws/instance-network-bandwidth|3125| + |karpenter.k8s.aws/instance-size|2xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7a.2xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|7910m| + |ephemeral-storage|17Gi| + |memory|14162Mi| + |pods|58| + |vpc.amazonaws.com/pod-eni|38| +### `c7a.4xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7a| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|32768| + |karpenter.k8s.aws/instance-network-bandwidth|6250| + |karpenter.k8s.aws/instance-size|4xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7a.4xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|15890m| + |ephemeral-storage|17Gi| + |memory|27381Mi| + |pods|234| + |vpc.amazonaws.com/pod-eni|54| +### `c7a.8xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7a| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|65536| + |karpenter.k8s.aws/instance-network-bandwidth|12500| + |karpenter.k8s.aws/instance-size|8xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7a.8xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|31850m| + |ephemeral-storage|17Gi| + |memory|57691Mi| + |pods|234| + |vpc.amazonaws.com/pod-eni|84| +### `c7a.12xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7a| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|98304| + |karpenter.k8s.aws/instance-network-bandwidth|18750| + |karpenter.k8s.aws/instance-size|12xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7a.12xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|47810m| + |ephemeral-storage|17Gi| + |memory|88002Mi| + |pods|234| + |vpc.amazonaws.com/pod-eni|114| +### `c7a.16xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7a| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|131072| + |karpenter.k8s.aws/instance-network-bandwidth|25000| + |karpenter.k8s.aws/instance-size|16xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7a.16xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|63770m| + |ephemeral-storage|17Gi| + |memory|112779Mi| + |pods|737| + |vpc.amazonaws.com/pod-eni|107| +### `c7a.24xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7a| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|196608| + |karpenter.k8s.aws/instance-network-bandwidth|37500| + |karpenter.k8s.aws/instance-size|24xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7a.24xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|95690m| + |ephemeral-storage|17Gi| + |memory|173400Mi| + |pods|737| + |vpc.amazonaws.com/pod-eni|107| +### `c7a.32xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7a| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|262144| + |karpenter.k8s.aws/instance-network-bandwidth|50000| + |karpenter.k8s.aws/instance-size|32xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7a.32xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|127610m| + |ephemeral-storage|17Gi| + |memory|234021Mi| + |pods|737| + |vpc.amazonaws.com/pod-eni|107| +### `c7a.48xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|192| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7a| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|393216| + |karpenter.k8s.aws/instance-network-bandwidth|50000| + |karpenter.k8s.aws/instance-size|48xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7a.48xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|191450m| + |ephemeral-storage|17Gi| + |memory|355262Mi| + |pods|737| + |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| +### `c7a.metal-48xl` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|192| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7a| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|| + |karpenter.k8s.aws/instance-memory|393216| + |karpenter.k8s.aws/instance-network-bandwidth|50000| + |karpenter.k8s.aws/instance-size|metal-48xl| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7a.metal-48xl| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|191450m| + |ephemeral-storage|17Gi| + |memory|355262Mi| + |pods|737| + |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| ## c7g Family ### `c7g.medium` #### Labels @@ -3205,7 +3382,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|2048| |karpenter.k8s.aws/instance-network-bandwidth|520| - |karpenter.k8s.aws/instance-pods|8| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3230,7 +3406,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|4096| |karpenter.k8s.aws/instance-network-bandwidth|937| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3255,7 +3430,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|1876| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3280,7 +3454,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|3750| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3305,7 +3478,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|7500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3330,7 +3502,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|15000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3355,7 +3526,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|98304| |karpenter.k8s.aws/instance-network-bandwidth|22500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3380,7 +3550,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|30000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3392,6 +3561,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|112720Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ### `c7g.metal` #### Labels @@ -3405,7 +3575,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|30000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3417,6 +3586,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|112720Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## c7gd Family ### `c7gd.medium` @@ -3432,7 +3602,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|59| |karpenter.k8s.aws/instance-memory|2048| |karpenter.k8s.aws/instance-network-bandwidth|520| - |karpenter.k8s.aws/instance-pods|8| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3458,7 +3627,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|118| |karpenter.k8s.aws/instance-memory|4096| |karpenter.k8s.aws/instance-network-bandwidth|937| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3484,7 +3652,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|237| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|1876| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3510,7 +3677,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|474| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|3750| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3536,7 +3702,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|950| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|7500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3562,7 +3727,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1900| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|15000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3588,7 +3752,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|2850| |karpenter.k8s.aws/instance-memory|98304| |karpenter.k8s.aws/instance-network-bandwidth|22500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3614,7 +3777,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|30000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3626,6 +3788,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|112720Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## c7gn Family ### `c7gn.medium` @@ -3640,7 +3803,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|2048| |karpenter.k8s.aws/instance-network-bandwidth|3125| - |karpenter.k8s.aws/instance-pods|8| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3665,7 +3827,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|4096| |karpenter.k8s.aws/instance-network-bandwidth|6250| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3690,7 +3851,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3715,7 +3875,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3740,7 +3899,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3765,7 +3923,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3790,7 +3947,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|98304| |karpenter.k8s.aws/instance-network-bandwidth|150000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3815,7 +3971,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|200000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3827,6 +3982,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|112720Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## c7i Family ### `c7i.large` @@ -3841,7 +3997,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|4096| |karpenter.k8s.aws/instance-network-bandwidth|781| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3853,6 +4008,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|3114Mi| |pods|29| + |vpc.amazonaws.com/pod-eni|9| ### `c7i.xlarge` #### Labels | Label | Value | @@ -3865,7 +4021,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|1562| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3877,6 +4032,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|6584Mi| |pods|58| + |vpc.amazonaws.com/pod-eni|18| ### `c7i.2xlarge` #### Labels | Label | Value | @@ -3889,7 +4045,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|3125| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3901,6 +4056,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|14162Mi| |pods|58| + |vpc.amazonaws.com/pod-eni|38| ### `c7i.4xlarge` #### Labels | Label | Value | @@ -3913,7 +4069,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|6250| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3925,6 +4080,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|27381Mi| |pods|234| + |vpc.amazonaws.com/pod-eni|54| ### `c7i.8xlarge` #### Labels | Label | Value | @@ -3937,7 +4093,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3949,6 +4104,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|57691Mi| |pods|234| + |vpc.amazonaws.com/pod-eni|84| ### `c7i.12xlarge` #### Labels | Label | Value | @@ -3961,7 +4117,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|98304| |karpenter.k8s.aws/instance-network-bandwidth|18750| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3973,6 +4128,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|88002Mi| |pods|234| + |vpc.amazonaws.com/pod-eni|114| ### `c7i.16xlarge` #### Labels | Label | Value | @@ -3985,7 +4141,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3997,6 +4152,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|112779Mi| |pods|737| + |vpc.amazonaws.com/pod-eni|107| ### `c7i.24xlarge` #### Labels | Label | Value | @@ -4009,7 +4165,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|37500| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4021,6 +4176,31 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|173400Mi| |pods|737| + |vpc.amazonaws.com/pod-eni|107| +### `c7i.metal-24xl` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7i| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|| + |karpenter.k8s.aws/instance-memory|196608| + |karpenter.k8s.aws/instance-network-bandwidth|37500| + |karpenter.k8s.aws/instance-size|metal-24xl| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7i.metal-24xl| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|95690m| + |ephemeral-storage|17Gi| + |memory|173400Mi| + |pods|737| + |vpc.amazonaws.com/pod-eni|107| ### `c7i.48xlarge` #### Labels | Label | Value | @@ -4033,7 +4213,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|48xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4045,6 +4224,33 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|355262Mi| |pods|737| + |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| +### `c7i.metal-48xl` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|192| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7i| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|| + |karpenter.k8s.aws/instance-memory|393216| + |karpenter.k8s.aws/instance-network-bandwidth|50000| + |karpenter.k8s.aws/instance-size|metal-48xl| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7i.metal-48xl| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|191450m| + |ephemeral-storage|17Gi| + |memory|355262Mi| + |pods|737| + |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| ## d2 Family ### `d2.xlarge` #### Labels @@ -4057,7 +4263,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|2| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|31232| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4080,7 +4285,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|2| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|62464| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4103,7 +4307,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|2| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|124928| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4127,7 +4330,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|249856| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4153,7 +4355,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|5940| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|3000| - |karpenter.k8s.aws/instance-pods|10| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4179,7 +4380,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|11880| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|6000| - |karpenter.k8s.aws/instance-pods|18| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4205,7 +4405,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|23760| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|38| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4231,7 +4430,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|47520| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|59| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4258,7 +4456,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|27960| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|6000| - |karpenter.k8s.aws/instance-pods|10| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4284,7 +4481,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|55920| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|18| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4310,7 +4506,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|111840| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|38| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4336,7 +4531,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|167760| |karpenter.k8s.aws/instance-memory|98304| |karpenter.k8s.aws/instance-network-bandwidth|40000| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|6xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4362,7 +4556,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|223680| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|78| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4388,7 +4581,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|335520| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|75000| - |karpenter.k8s.aws/instance-pods|89| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4401,6 +4593,38 @@ below are the resources available with some assumptions and after the instance o |memory|180528Mi| |pods|89| |vpc.amazonaws.com/pod-eni|119| +## dl1 Family +### `dl1.24xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|dl| + |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|dl1| + |karpenter.k8s.aws/instance-generation|1| + |karpenter.k8s.aws/instance-gpu-count|8| + |karpenter.k8s.aws/instance-gpu-manufacturer|habana| + |karpenter.k8s.aws/instance-gpu-memory|32768| + |karpenter.k8s.aws/instance-gpu-name|gaudi-hl-205| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-local-nvme|4000| + |karpenter.k8s.aws/instance-memory|786432| + |karpenter.k8s.aws/instance-network-bandwidth|400000| + |karpenter.k8s.aws/instance-size|24xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|dl1.24xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|95690m| + |ephemeral-storage|17Gi| + |habana.ai/gaudi|8| + |memory|718987Mi| + |pods|737| + |vpc.amazonaws.com/efa|4| + |vpc.amazonaws.com/pod-eni|62| ## f1 Family ### `f1.2xlarge` #### Labels @@ -4415,7 +4639,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|470| |karpenter.k8s.aws/instance-memory|124928| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4440,7 +4663,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|940| |karpenter.k8s.aws/instance-memory|249856| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4465,7 +4687,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3760| |karpenter.k8s.aws/instance-memory|999424| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|394| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4477,63 +4698,6 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|919778Mi| |pods|394| -## g2 Family -### `g2.2xlarge` -#### Labels - | Label | Value | - |--|--| - |karpenter.k8s.aws/instance-category|g| - |karpenter.k8s.aws/instance-cpu|8| - |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| - |karpenter.k8s.aws/instance-family|g2| - |karpenter.k8s.aws/instance-generation|2| - |karpenter.k8s.aws/instance-gpu-count|1| - |karpenter.k8s.aws/instance-gpu-manufacturer|nvidia| - |karpenter.k8s.aws/instance-gpu-memory|4096| - |karpenter.k8s.aws/instance-gpu-name|k520| - |karpenter.k8s.aws/instance-hypervisor|xen| - |karpenter.k8s.aws/instance-memory|15360| - |karpenter.k8s.aws/instance-pods|58| - |karpenter.k8s.aws/instance-size|2xlarge| - |kubernetes.io/arch|amd64| - |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|g2.2xlarge| -#### Resources - | Resource | Quantity | - |--|--| - |cpu|7910m| - |ephemeral-storage|17Gi| - |memory|13215Mi| - |nvidia.com/gpu|1| - |pods|58| -### `g2.8xlarge` -#### Labels - | Label | Value | - |--|--| - |karpenter.k8s.aws/instance-category|g| - |karpenter.k8s.aws/instance-cpu|32| - |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| - |karpenter.k8s.aws/instance-family|g2| - |karpenter.k8s.aws/instance-generation|2| - |karpenter.k8s.aws/instance-gpu-count|4| - |karpenter.k8s.aws/instance-gpu-manufacturer|nvidia| - |karpenter.k8s.aws/instance-gpu-memory|4096| - |karpenter.k8s.aws/instance-gpu-name|k520| - |karpenter.k8s.aws/instance-hypervisor|xen| - |karpenter.k8s.aws/instance-memory|61440| - |karpenter.k8s.aws/instance-pods|234| - |karpenter.k8s.aws/instance-size|8xlarge| - |kubernetes.io/arch|amd64| - |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|g2.8xlarge| -#### Resources - | Resource | Quantity | - |--|--| - |cpu|31850m| - |ephemeral-storage|17Gi| - |memory|53903Mi| - |nvidia.com/gpu|4| - |pods|234| ## g3 Family ### `g3.4xlarge` #### Labels @@ -4551,7 +4715,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|124928| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4580,7 +4743,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|249856| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4609,7 +4771,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|499712| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4638,7 +4799,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-gpu-name|m60| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|31232| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4669,7 +4829,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|150| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|2000| - |karpenter.k8s.aws/instance-pods|8| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4700,7 +4859,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|300| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|4167| - |karpenter.k8s.aws/instance-pods|8| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4731,7 +4889,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|600| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|8333| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4762,7 +4919,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1200| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|15000| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4793,7 +4949,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|2400| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4825,7 +4980,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|125| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4856,7 +5010,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|225| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4887,7 +5040,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|225| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|20000| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4918,7 +5070,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|900| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4931,6 +5082,7 @@ below are the resources available with some assumptions and after the instance o |memory|120248Mi| |nvidia.com/gpu|1| |pods|58| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|58| ### `g4dn.12xlarge` #### Labels @@ -4949,7 +5101,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|900| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4962,6 +5113,7 @@ below are the resources available with some assumptions and after the instance o |memory|178933Mi| |nvidia.com/gpu|4| |pods|234| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|54| ### `g4dn.16xlarge` #### Labels @@ -4980,7 +5132,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|900| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4993,6 +5144,7 @@ below are the resources available with some assumptions and after the instance o |memory|241490Mi| |nvidia.com/gpu|1| |pods|58| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|118| ### `g4dn.metal` #### Labels @@ -5011,7 +5163,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1800| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5024,6 +5175,7 @@ below are the resources available with some assumptions and after the instance o |memory|355262Mi| |nvidia.com/gpu|8| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## g5 Family ### `g5.xlarge` @@ -5043,7 +5195,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|250| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5074,7 +5225,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|450| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5105,7 +5255,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|600| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5136,7 +5285,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|900| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5149,6 +5297,7 @@ below are the resources available with some assumptions and after the instance o |memory|118312Mi| |nvidia.com/gpu|1| |pods|234| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|84| ### `g5.12xlarge` #### Labels @@ -5167,7 +5316,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|40000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5180,6 +5328,7 @@ below are the resources available with some assumptions and after the instance o |memory|173400Mi| |nvidia.com/gpu|4| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ### `g5.16xlarge` #### Labels @@ -5198,7 +5347,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1900| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5211,6 +5359,7 @@ below are the resources available with some assumptions and after the instance o |memory|239554Mi| |nvidia.com/gpu|1| |pods|234| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|114| ### `g5.24xlarge` #### Labels @@ -5229,7 +5378,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5242,6 +5390,7 @@ below are the resources available with some assumptions and after the instance o |memory|355262Mi| |nvidia.com/gpu|4| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ### `g5.48xlarge` #### Labels @@ -5260,7 +5409,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|7600| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|345| |karpenter.k8s.aws/instance-size|48xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5273,6 +5421,7 @@ below are the resources available with some assumptions and after the instance o |memory|723299Mi| |nvidia.com/gpu|8| |pods|345| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|115| ## g5g Family ### `g5g.xlarge` @@ -5291,7 +5440,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -5321,7 +5469,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -5351,7 +5498,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -5381,7 +5527,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|12000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -5411,7 +5556,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -5441,7 +5585,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -5468,7 +5611,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5492,7 +5634,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5516,7 +5657,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5540,7 +5680,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5550,8 +5689,81 @@ below are the resources available with some assumptions and after the instance o |--|--| |cpu|63770m| |ephemeral-storage|17Gi| - |memory|234021Mi| - |pods|737| + |memory|237794Mi| + |pods|394| +## hpc7g Family +### `hpc7g.4xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|hpc| + |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|hpc7g| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|131072| + |karpenter.k8s.aws/instance-network-bandwidth|200000| + |karpenter.k8s.aws/instance-size|4xlarge| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|hpc7g.4xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|15890m| + |ephemeral-storage|17Gi| + |memory|118649Mi| + |pods|198| + |vpc.amazonaws.com/efa|1| +### `hpc7g.8xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|hpc| + |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|hpc7g| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|131072| + |karpenter.k8s.aws/instance-network-bandwidth|200000| + |karpenter.k8s.aws/instance-size|8xlarge| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|hpc7g.8xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|31850m| + |ephemeral-storage|17Gi| + |memory|118649Mi| + |pods|198| + |vpc.amazonaws.com/efa|1| +### `hpc7g.16xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|hpc| + |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|hpc7g| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|131072| + |karpenter.k8s.aws/instance-network-bandwidth|200000| + |karpenter.k8s.aws/instance-size|16xlarge| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|hpc7g.16xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|63770m| + |ephemeral-storage|17Gi| + |memory|118649Mi| + |pods|198| + |vpc.amazonaws.com/efa|1| ## i2 Family ### `i2.xlarge` #### Labels @@ -5564,7 +5776,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|2| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|31232| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5587,7 +5798,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|2| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|62464| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5610,7 +5820,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|2| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|124928| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5634,7 +5843,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|249856| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5660,7 +5868,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|475| |karpenter.k8s.aws/instance-memory|15616| |karpenter.k8s.aws/instance-network-bandwidth|750| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5685,7 +5892,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|950| |karpenter.k8s.aws/instance-memory|31232| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5710,7 +5916,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1900| |karpenter.k8s.aws/instance-memory|62464| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5735,7 +5940,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|124928| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5760,7 +5964,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|7600| |karpenter.k8s.aws/instance-memory|249856| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5785,7 +5988,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|15200| |karpenter.k8s.aws/instance-memory|499712| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5810,7 +6012,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|15200| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5837,7 +6038,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1250| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|2100| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5863,7 +6063,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|2500| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|4200| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5889,7 +6088,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|5000| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|8400| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5915,7 +6113,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|7500| |karpenter.k8s.aws/instance-memory|98304| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|3xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5941,7 +6138,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|15000| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|6xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5967,7 +6163,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|30000| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5979,6 +6174,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|360795Mi| |pods|234| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|54| ### `i3en.24xlarge` #### Labels @@ -5993,7 +6189,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|60000| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6005,6 +6200,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|718987Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ### `i3en.metal` #### Labels @@ -6019,7 +6215,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|60000| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6031,6 +6226,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|718987Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## i4g Family ### `i4g.large` @@ -6046,7 +6242,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|468| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|781| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -6072,7 +6267,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|937| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|1875| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -6098,7 +6292,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1875| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|4687| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -6124,7 +6317,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3750| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|9375| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -6150,7 +6342,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|7500| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|18750| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -6176,7 +6367,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|15000| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|37500| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -6188,6 +6378,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|476445Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## i4i Family ### `i4i.large` @@ -6203,7 +6394,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|468| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|781| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6228,7 +6418,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|937| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|1875| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6254,7 +6443,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1875| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|4687| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6280,7 +6468,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3750| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|9375| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6306,7 +6493,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|7500| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|18750| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6319,6 +6505,31 @@ below are the resources available with some assumptions and after the instance o |memory|239554Mi| |pods|234| |vpc.amazonaws.com/pod-eni|112| +### `i4i.12xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|i| + |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|i4i| + |karpenter.k8s.aws/instance-generation|4| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-local-nvme|11250| + |karpenter.k8s.aws/instance-memory|393216| + |karpenter.k8s.aws/instance-network-bandwidth|28120| + |karpenter.k8s.aws/instance-size|12xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|i4i.12xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|47810m| + |ephemeral-storage|17Gi| + |memory|360795Mi| + |pods|234| + |vpc.amazonaws.com/pod-eni|112| ### `i4i.16xlarge` #### Labels | Label | Value | @@ -6332,7 +6543,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|15000| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|37500| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6345,6 +6555,31 @@ below are the resources available with some assumptions and after the instance o |memory|476504Mi| |pods|737| |vpc.amazonaws.com/pod-eni|120| +### `i4i.24xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|i| + |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|i4i| + |karpenter.k8s.aws/instance-generation|4| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-local-nvme|22500| + |karpenter.k8s.aws/instance-memory|786432| + |karpenter.k8s.aws/instance-network-bandwidth|56250| + |karpenter.k8s.aws/instance-size|24xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|i4i.24xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|95690m| + |ephemeral-storage|17Gi| + |memory|722287Mi| + |pods|437| + |vpc.amazonaws.com/pod-eni|105| ### `i4i.32xlarge` #### Labels | Label | Value | @@ -6358,7 +6593,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|30000| |karpenter.k8s.aws/instance-memory|1048576| |karpenter.k8s.aws/instance-network-bandwidth|75000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|32xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6370,6 +6604,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|961470Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|120| ### `i4i.metal` #### Labels @@ -6384,7 +6619,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|30000| |karpenter.k8s.aws/instance-memory|1048576| |karpenter.k8s.aws/instance-network-bandwidth|75000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6396,6 +6630,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|961470Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|120| ## im4gn Family ### `im4gn.large` @@ -6411,7 +6646,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|937| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|3125| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -6437,7 +6671,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1875| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|6250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -6463,7 +6696,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3750| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -6489,7 +6721,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|7500| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -6515,7 +6746,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|15000| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -6541,7 +6771,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|30000| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -6553,6 +6782,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|233962Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## inf1 Family ### `inf1.xlarge` @@ -6570,7 +6800,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|38| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6599,7 +6828,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|38| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6628,7 +6856,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|49152| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|6xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6657,7 +6884,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|321| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6670,6 +6896,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|177976Mi| |pods|321| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|111| ## inf2 Family ### `inf2.xlarge` @@ -6687,7 +6914,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|2083| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6716,7 +6942,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|16667| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6745,7 +6970,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6774,7 +6998,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|48xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6802,7 +7025,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|937| |karpenter.k8s.aws/instance-memory|6144| |karpenter.k8s.aws/instance-network-bandwidth|1562| - |karpenter.k8s.aws/instance-pods|8| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -6828,7 +7050,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1875| |karpenter.k8s.aws/instance-memory|12288| |karpenter.k8s.aws/instance-network-bandwidth|3125| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -6854,7 +7075,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3750| |karpenter.k8s.aws/instance-memory|24576| |karpenter.k8s.aws/instance-network-bandwidth|6250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -6880,7 +7100,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|7500| |karpenter.k8s.aws/instance-memory|49152| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -6906,7 +7125,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|15000| |karpenter.k8s.aws/instance-memory|98304| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -6932,7 +7150,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|30000| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -6957,7 +7174,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|1| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|1740| - |karpenter.k8s.aws/instance-pods|8| |karpenter.k8s.aws/instance-size|small| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6980,7 +7196,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|1| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|3788| - |karpenter.k8s.aws/instance-pods|12| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7003,7 +7218,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|1| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|7680| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7026,7 +7240,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|1| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|15360| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7050,7 +7263,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|2| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|17510| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7073,7 +7285,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|2| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|35020| - |karpenter.k8s.aws/instance-pods|118| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7096,7 +7307,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|2| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|70041| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7120,7 +7330,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|3| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|3840| - |karpenter.k8s.aws/instance-pods|12| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7143,7 +7352,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|3| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|7680| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7166,7 +7374,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|3| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|15360| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7189,7 +7396,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|3| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|30720| - |karpenter.k8s.aws/instance-pods|118| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7213,7 +7419,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|4| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|8192| - |karpenter.k8s.aws/instance-pods|20| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7236,7 +7441,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|4| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|16384| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7259,7 +7463,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|4| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|32768| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7282,7 +7485,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|4| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|65536| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7306,7 +7508,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|163840| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|10xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7330,7 +7531,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7355,7 +7555,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|750| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7380,7 +7579,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7405,7 +7603,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7430,7 +7627,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7455,7 +7651,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7480,7 +7675,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|12000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7505,7 +7699,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|20000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7530,7 +7723,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7555,7 +7747,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7581,7 +7772,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|750| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7606,7 +7796,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7631,7 +7820,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7656,7 +7844,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7681,7 +7868,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|7500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7706,7 +7892,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7731,7 +7916,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|12000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7756,7 +7940,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|20000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7783,7 +7966,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|75| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|750| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7809,7 +7991,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|150| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7835,7 +8016,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|300| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7861,7 +8041,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|600| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7887,7 +8066,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1200| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|7500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7913,7 +8091,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1800| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7939,7 +8116,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|2400| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|12000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7965,7 +8141,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3600| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|20000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7992,7 +8167,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|75| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|750| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8018,7 +8192,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|150| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8044,7 +8217,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|300| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8070,7 +8242,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|600| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8096,7 +8267,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1200| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8122,7 +8292,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1800| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|12000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8148,7 +8317,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|2400| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|20000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8174,7 +8342,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3600| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8200,7 +8367,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3600| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8227,7 +8393,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|75| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|2100| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8253,7 +8418,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|150| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|4100| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8279,7 +8443,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|300| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|8125| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8305,7 +8468,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|600| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|16250| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8331,7 +8493,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1200| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8357,7 +8518,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1800| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8383,7 +8543,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|2400| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|75000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8409,7 +8568,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3600| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8421,6 +8579,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|355262Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ### `m5dn.metal` #### Labels @@ -8435,7 +8594,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3600| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8447,6 +8605,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|355262Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## m5n Family ### `m5n.large` @@ -8461,7 +8620,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|2100| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8486,7 +8644,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|4100| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8511,7 +8668,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|8125| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8536,7 +8692,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|16250| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8561,7 +8716,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8586,7 +8740,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8611,7 +8764,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|75000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8636,7 +8788,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8648,6 +8799,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|355262Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ### `m5n.metal` #### Labels @@ -8661,7 +8813,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8673,6 +8824,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|355262Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## m5zn Family ### `m5zn.large` @@ -8687,7 +8839,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|3000| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8712,7 +8863,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8737,7 +8887,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8762,7 +8911,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|49152| |karpenter.k8s.aws/instance-network-bandwidth|15000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|3xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8787,7 +8935,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|98304| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|6xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8812,7 +8959,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8824,6 +8970,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|173400Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ### `m5zn.metal` #### Labels @@ -8837,7 +8984,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8849,6 +8995,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|173400Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## m6a Family ### `m6a.large` @@ -8863,7 +9010,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|781| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8888,7 +9034,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|1562| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8913,7 +9058,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|3125| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8938,7 +9082,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|6250| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8963,7 +9106,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8988,7 +9130,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|18750| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -9013,7 +9154,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -9038,7 +9178,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|37500| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -9063,7 +9202,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|32xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -9088,7 +9226,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|48xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -9100,6 +9237,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|718987Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ### `m6a.metal` #### Labels @@ -9113,7 +9251,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -9125,6 +9262,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|718987Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## m6g Family ### `m6g.medium` @@ -9139,7 +9277,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|4096| |karpenter.k8s.aws/instance-network-bandwidth|500| - |karpenter.k8s.aws/instance-pods|8| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -9164,7 +9301,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|750| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -9189,7 +9325,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -9214,7 +9349,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -9239,7 +9373,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -9264,7 +9397,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|12000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -9289,7 +9421,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|20000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -9314,7 +9445,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -9339,7 +9469,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -9366,7 +9495,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|59| |karpenter.k8s.aws/instance-memory|4096| |karpenter.k8s.aws/instance-network-bandwidth|500| - |karpenter.k8s.aws/instance-pods|8| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -9392,7 +9520,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|118| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|750| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -9418,7 +9545,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|237| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -9444,7 +9570,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|474| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -9470,7 +9595,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|950| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -9496,7 +9620,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1900| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|12000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -9522,7 +9645,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|2850| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|20000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -9548,7 +9670,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -9574,7 +9695,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -9600,7 +9720,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|781| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -9625,7 +9744,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|1562| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -9650,7 +9768,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|3125| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -9675,7 +9792,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|6250| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -9700,7 +9816,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -9725,7 +9840,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|18750| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -9750,7 +9864,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -9775,7 +9888,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|37500| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -9800,7 +9912,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|32xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -9812,6 +9923,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|476504Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ### `m6i.metal` #### Labels @@ -9825,7 +9937,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -9837,6 +9948,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|476504Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## m6id Family ### `m6id.large` @@ -9852,7 +9964,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|118| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|781| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -9878,7 +9989,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|237| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|1562| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -9904,7 +10014,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|474| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|3125| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -9930,7 +10039,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|950| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|6250| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -9956,7 +10064,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1900| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -9982,7 +10089,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|2850| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|18750| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10008,7 +10114,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10034,7 +10139,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|5700| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|37500| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10060,7 +10164,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|7600| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|32xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10072,6 +10175,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|476504Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ### `m6id.metal` #### Labels @@ -10086,7 +10190,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|7600| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10098,6 +10201,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|476504Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## m6idn Family ### `m6idn.large` @@ -10113,7 +10217,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|118| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|3125| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10139,7 +10242,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|237| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|6250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10165,7 +10267,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|474| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10191,7 +10292,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|950| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10217,7 +10317,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1900| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10243,7 +10342,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|2850| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|75000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10269,7 +10367,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10295,7 +10392,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|5700| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|150000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10321,7 +10417,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|7600| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|200000| - |karpenter.k8s.aws/instance-pods|345| |karpenter.k8s.aws/instance-size|32xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10333,6 +10428,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|480816Mi| |pods|345| + |vpc.amazonaws.com/efa|2| |vpc.amazonaws.com/pod-eni|108| ### `m6idn.metal` #### Labels @@ -10347,7 +10443,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|7600| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|200000| - |karpenter.k8s.aws/instance-pods|345| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10359,6 +10454,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|480816Mi| |pods|345| + |vpc.amazonaws.com/efa|2| |vpc.amazonaws.com/pod-eni|108| ## m6in Family ### `m6in.large` @@ -10373,7 +10469,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|3125| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10398,7 +10493,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|6250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10423,7 +10517,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10448,7 +10541,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10473,7 +10565,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10498,7 +10589,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|75000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10523,7 +10613,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10548,7 +10637,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|150000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10573,7 +10661,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|200000| - |karpenter.k8s.aws/instance-pods|345| |karpenter.k8s.aws/instance-size|32xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10585,6 +10672,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|480816Mi| |pods|345| + |vpc.amazonaws.com/efa|2| |vpc.amazonaws.com/pod-eni|108| ### `m6in.metal` #### Labels @@ -10598,7 +10686,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|200000| - |karpenter.k8s.aws/instance-pods|345| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10610,6 +10697,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|480816Mi| |pods|345| + |vpc.amazonaws.com/efa|2| |vpc.amazonaws.com/pod-eni|108| ## m7a Family ### `m7a.medium` @@ -10624,7 +10712,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|4096| |karpenter.k8s.aws/instance-network-bandwidth|390| - |karpenter.k8s.aws/instance-pods|8| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10649,7 +10736,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|781| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10674,7 +10760,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|1562| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10699,7 +10784,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|3125| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10724,7 +10808,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|6250| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10749,7 +10832,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10774,7 +10856,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|18750| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10799,7 +10880,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10824,7 +10904,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|37500| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10849,7 +10928,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|32xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10874,7 +10952,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|48xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10886,6 +10963,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|718987Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ### `m7a.metal-48xl` #### Labels @@ -10899,7 +10977,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal-48xl| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10911,6 +10988,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|718987Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## m7g Family ### `m7g.medium` @@ -10925,7 +11003,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|4096| |karpenter.k8s.aws/instance-network-bandwidth|520| - |karpenter.k8s.aws/instance-pods|8| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -10950,7 +11027,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|937| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -10975,7 +11051,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|1876| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -11000,7 +11075,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|3750| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -11025,7 +11099,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|7500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -11050,7 +11123,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|15000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -11075,7 +11147,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|22500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -11100,7 +11171,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|30000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -11112,6 +11182,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|233962Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ### `m7g.metal` #### Labels @@ -11125,7 +11196,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|30000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -11137,6 +11207,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|233962Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## m7gd Family ### `m7gd.medium` @@ -11152,7 +11223,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|59| |karpenter.k8s.aws/instance-memory|4096| |karpenter.k8s.aws/instance-network-bandwidth|520| - |karpenter.k8s.aws/instance-pods|8| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -11178,7 +11248,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|118| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|937| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -11204,7 +11273,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|237| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|1876| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -11230,7 +11298,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|474| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|3750| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -11256,7 +11323,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|950| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|7500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -11282,7 +11348,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1900| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|15000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -11308,7 +11373,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|2850| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|22500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -11334,7 +11398,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|30000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -11346,6 +11409,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|233962Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## m7i Family ### `m7i.large` @@ -11360,7 +11424,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|781| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11385,7 +11448,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|1562| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11410,7 +11472,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|3125| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11435,7 +11496,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|6250| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11460,7 +11520,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11485,7 +11544,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|18750| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11510,7 +11568,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11535,7 +11592,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|37500| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11548,6 +11604,30 @@ below are the resources available with some assumptions and after the instance o |memory|355262Mi| |pods|737| |vpc.amazonaws.com/pod-eni|107| +### `m7i.metal-24xl` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|m| + |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|m7i| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|| + |karpenter.k8s.aws/instance-memory|393216| + |karpenter.k8s.aws/instance-network-bandwidth|37500| + |karpenter.k8s.aws/instance-size|metal-24xl| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|m7i.metal-24xl| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|95690m| + |ephemeral-storage|17Gi| + |memory|355262Mi| + |pods|737| + |vpc.amazonaws.com/pod-eni|107| ### `m7i.48xlarge` #### Labels | Label | Value | @@ -11560,7 +11640,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|48xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11572,6 +11651,32 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|718987Mi| |pods|737| + |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| +### `m7i.metal-48xl` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|m| + |karpenter.k8s.aws/instance-cpu|192| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|m7i| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|| + |karpenter.k8s.aws/instance-memory|786432| + |karpenter.k8s.aws/instance-network-bandwidth|50000| + |karpenter.k8s.aws/instance-size|metal-48xl| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|m7i.metal-48xl| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|191450m| + |ephemeral-storage|17Gi| + |memory|718987Mi| + |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## m7i-flex Family ### `m7i-flex.large` @@ -11586,7 +11691,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|390| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11611,7 +11715,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|781| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11636,7 +11739,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|1562| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11661,7 +11763,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|3125| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11686,7 +11787,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|6250| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11715,7 +11815,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-gpu-name|k80| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|62464| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11744,7 +11843,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|499712| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11773,7 +11871,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|749568| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11786,19 +11883,198 @@ below are the resources available with some assumptions and after the instance o |memory|690421Mi| |nvidia.com/gpu|16| |pods|234| -## r3 Family -### `r3.large` +## p3 Family +### `p3.2xlarge` #### Labels | Label | Value | |--|--| - |karpenter.k8s.aws/instance-category|r| - |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-category|p| + |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| + |karpenter.k8s.aws/instance-family|p3| + |karpenter.k8s.aws/instance-generation|3| + |karpenter.k8s.aws/instance-gpu-count|1| + |karpenter.k8s.aws/instance-gpu-manufacturer|nvidia| + |karpenter.k8s.aws/instance-gpu-memory|16384| + |karpenter.k8s.aws/instance-gpu-name|v100| + |karpenter.k8s.aws/instance-hypervisor|xen| + |karpenter.k8s.aws/instance-memory|62464| + |karpenter.k8s.aws/instance-size|2xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|p3.2xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|7910m| + |ephemeral-storage|17Gi| + |memory|56786Mi| + |nvidia.com/gpu|1| + |pods|58| +### `p3.8xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|p| + |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| + |karpenter.k8s.aws/instance-family|p3| + |karpenter.k8s.aws/instance-generation|3| + |karpenter.k8s.aws/instance-gpu-count|4| + |karpenter.k8s.aws/instance-gpu-manufacturer|nvidia| + |karpenter.k8s.aws/instance-gpu-memory|16384| + |karpenter.k8s.aws/instance-gpu-name|v100| + |karpenter.k8s.aws/instance-hypervisor|xen| + |karpenter.k8s.aws/instance-memory|249856| + |karpenter.k8s.aws/instance-network-bandwidth|10000| + |karpenter.k8s.aws/instance-size|8xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|p3.8xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|31850m| + |ephemeral-storage|17Gi| + |memory|228187Mi| + |nvidia.com/gpu|4| + |pods|234| +### `p3.16xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|p| + |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| + |karpenter.k8s.aws/instance-family|p3| + |karpenter.k8s.aws/instance-generation|3| + |karpenter.k8s.aws/instance-gpu-count|8| + |karpenter.k8s.aws/instance-gpu-manufacturer|nvidia| + |karpenter.k8s.aws/instance-gpu-memory|16384| + |karpenter.k8s.aws/instance-gpu-name|v100| + |karpenter.k8s.aws/instance-hypervisor|xen| + |karpenter.k8s.aws/instance-memory|499712| + |karpenter.k8s.aws/instance-network-bandwidth|25000| + |karpenter.k8s.aws/instance-size|16xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|p3.16xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|63770m| + |ephemeral-storage|17Gi| + |memory|459304Mi| + |nvidia.com/gpu|8| + |pods|234| +## p3dn Family +### `p3dn.24xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|p| + |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|p3dn| + |karpenter.k8s.aws/instance-generation|3| + |karpenter.k8s.aws/instance-gpu-count|8| + |karpenter.k8s.aws/instance-gpu-manufacturer|nvidia| + |karpenter.k8s.aws/instance-gpu-memory|32768| + |karpenter.k8s.aws/instance-gpu-name|v100| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-local-nvme|1800| + |karpenter.k8s.aws/instance-memory|786432| + |karpenter.k8s.aws/instance-network-bandwidth|100000| + |karpenter.k8s.aws/instance-size|24xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|p3dn.24xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|95690m| + |ephemeral-storage|17Gi| + |memory|718987Mi| + |nvidia.com/gpu|8| + |pods|737| + |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| +## p4d Family +### `p4d.24xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|p| + |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|p4d| + |karpenter.k8s.aws/instance-generation|4| + |karpenter.k8s.aws/instance-gpu-count|8| + |karpenter.k8s.aws/instance-gpu-manufacturer|nvidia| + |karpenter.k8s.aws/instance-gpu-memory|40960| + |karpenter.k8s.aws/instance-gpu-name|a100| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-local-nvme|8000| + |karpenter.k8s.aws/instance-memory|1179648| + |karpenter.k8s.aws/instance-network-bandwidth|400000| + |karpenter.k8s.aws/instance-size|24xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|p4d.24xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|95690m| + |ephemeral-storage|17Gi| + |memory|1082712Mi| + |nvidia.com/gpu|8| + |pods|737| + |vpc.amazonaws.com/efa|4| + |vpc.amazonaws.com/pod-eni|62| +## p5 Family +### `p5.48xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|p| + |karpenter.k8s.aws/instance-cpu|192| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|p5| + |karpenter.k8s.aws/instance-generation|5| + |karpenter.k8s.aws/instance-gpu-count|8| + |karpenter.k8s.aws/instance-gpu-manufacturer|nvidia| + |karpenter.k8s.aws/instance-gpu-memory|81920| + |karpenter.k8s.aws/instance-gpu-name|h100| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-local-nvme|30400| + |karpenter.k8s.aws/instance-memory|2097152| + |karpenter.k8s.aws/instance-network-bandwidth|3200000| + |karpenter.k8s.aws/instance-size|48xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|p5.48xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|191450m| + |ephemeral-storage|17Gi| + |memory|1938410Mi| + |nvidia.com/gpu|8| + |pods|100| + |vpc.amazonaws.com/efa|32| + |vpc.amazonaws.com/pod-eni|120| +## r3 Family +### `r3.large` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|r3| |karpenter.k8s.aws/instance-generation|3| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|15360| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11821,7 +12097,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|3| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|31232| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11844,7 +12119,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|3| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|62464| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11867,7 +12141,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|3| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|124928| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11891,7 +12164,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|249856| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11916,7 +12188,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|15616| |karpenter.k8s.aws/instance-network-bandwidth|750| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11940,7 +12211,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|31232| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11964,7 +12234,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|62464| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11988,7 +12257,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|124928| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12012,7 +12280,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|249856| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12036,7 +12303,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|499712| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12061,7 +12327,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|750| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12086,7 +12351,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12111,7 +12375,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12136,7 +12399,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12161,7 +12423,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12186,7 +12447,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|12000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12211,7 +12471,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|20000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12236,7 +12495,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12261,7 +12519,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12287,7 +12544,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|750| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12312,7 +12568,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12337,7 +12592,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12362,7 +12616,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12387,7 +12640,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|7500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12412,7 +12664,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12437,7 +12688,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|12000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12462,7 +12712,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|20000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12489,7 +12738,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|75| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|750| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12515,7 +12763,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|150| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12541,7 +12788,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|300| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12567,7 +12813,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|600| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12593,7 +12838,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1200| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|7500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12619,7 +12863,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1800| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12645,7 +12888,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|2400| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|12000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12671,7 +12913,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3600| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|20000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12697,7 +12938,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|750| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12722,7 +12962,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12747,7 +12986,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12772,7 +13010,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12797,7 +13034,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12822,7 +13058,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|12000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12847,7 +13082,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|20000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12872,7 +13106,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12897,7 +13130,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12924,7 +13156,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|75| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|750| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12950,7 +13181,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|150| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12976,7 +13206,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|300| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13002,7 +13231,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|600| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13028,7 +13256,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1200| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13054,7 +13281,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1800| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|12000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13080,7 +13306,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|2400| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|20000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13106,7 +13331,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3600| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13132,7 +13356,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3600| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13159,7 +13382,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|75| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|2100| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13185,7 +13407,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|150| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|4100| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13211,7 +13432,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|300| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|8125| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13237,7 +13457,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|600| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|16250| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13263,7 +13482,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1200| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13289,7 +13507,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1800| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13315,7 +13532,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|2400| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|75000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13341,7 +13557,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3600| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13353,6 +13568,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|718987Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ### `r5dn.metal` #### Labels @@ -13367,7 +13583,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3600| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13379,6 +13594,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|718987Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## r5n Family ### `r5n.large` @@ -13393,7 +13609,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|2100| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13418,7 +13633,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|4100| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13443,7 +13657,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|8125| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13468,7 +13681,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|16250| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13493,7 +13705,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13518,7 +13729,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13543,7 +13753,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|75000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13568,7 +13777,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13580,6 +13788,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|718987Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ### `r5n.metal` #### Labels @@ -13593,7 +13802,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13605,6 +13813,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|718987Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## r6a Family ### `r6a.large` @@ -13619,7 +13828,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|781| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13644,7 +13852,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|1562| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13669,7 +13876,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|3125| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13694,7 +13900,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|6250| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13719,7 +13924,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13744,7 +13948,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|18750| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13769,7 +13972,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13794,7 +13996,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|37500| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13819,7 +14020,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|1048576| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|32xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13844,7 +14044,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|1572864| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|48xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13856,6 +14055,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|1446437Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ### `r6a.metal` #### Labels @@ -13869,7 +14069,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|1572864| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13881,6 +14080,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|1446437Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## r6g Family ### `r6g.medium` @@ -13895,7 +14095,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|500| - |karpenter.k8s.aws/instance-pods|8| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -13920,7 +14119,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|750| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -13945,7 +14143,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -13970,7 +14167,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -13995,7 +14191,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -14020,7 +14215,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|12000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -14045,7 +14239,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|20000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -14070,7 +14263,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -14095,7 +14287,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -14122,7 +14313,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|59| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|500| - |karpenter.k8s.aws/instance-pods|8| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -14148,7 +14338,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|118| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|750| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -14174,7 +14363,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|237| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -14200,7 +14388,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|474| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -14226,7 +14413,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|950| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -14252,7 +14438,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1900| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|12000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -14278,7 +14463,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|2850| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|20000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -14304,7 +14488,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -14330,7 +14513,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -14356,7 +14538,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|781| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -14381,7 +14562,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|1562| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -14406,7 +14586,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|3125| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -14431,7 +14610,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|6250| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -14456,7 +14634,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -14481,7 +14658,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|18750| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -14506,7 +14682,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -14531,7 +14706,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|37500| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -14556,7 +14730,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|1048576| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|32xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -14568,6 +14741,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|961470Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ### `r6i.metal` #### Labels @@ -14581,7 +14755,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|1048576| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -14593,6 +14766,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|961470Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## r6id Family ### `r6id.large` @@ -14608,7 +14782,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|118| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|781| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -14634,7 +14807,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|237| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|1562| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -14660,7 +14832,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|474| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|3125| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -14686,7 +14857,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|950| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|6250| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -14712,7 +14882,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1900| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -14738,7 +14907,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|2850| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|18750| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -14764,7 +14932,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -14790,7 +14957,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|5700| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|37500| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -14816,7 +14982,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|7600| |karpenter.k8s.aws/instance-memory|1048576| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|32xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -14828,6 +14993,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|961470Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ### `r6id.metal` #### Labels @@ -14842,7 +15008,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|7600| |karpenter.k8s.aws/instance-memory|1048576| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -14854,6 +15019,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|961470Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## r6idn Family ### `r6idn.large` @@ -14869,7 +15035,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|118| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|3125| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -14895,7 +15060,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|237| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|6250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -14921,7 +15085,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|474| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -14947,7 +15110,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|950| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -14973,7 +15135,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1900| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -14999,7 +15160,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|2850| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|75000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15025,7 +15185,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15051,7 +15210,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|5700| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|150000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15077,7 +15235,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|7600| |karpenter.k8s.aws/instance-memory|1048576| |karpenter.k8s.aws/instance-network-bandwidth|200000| - |karpenter.k8s.aws/instance-pods|345| |karpenter.k8s.aws/instance-size|32xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15089,6 +15246,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|965782Mi| |pods|345| + |vpc.amazonaws.com/efa|2| |vpc.amazonaws.com/pod-eni|108| ### `r6idn.metal` #### Labels @@ -15103,7 +15261,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|7600| |karpenter.k8s.aws/instance-memory|1048576| |karpenter.k8s.aws/instance-network-bandwidth|200000| - |karpenter.k8s.aws/instance-pods|345| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15115,6 +15272,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|965782Mi| |pods|345| + |vpc.amazonaws.com/efa|2| |vpc.amazonaws.com/pod-eni|108| ## r6in Family ### `r6in.large` @@ -15129,7 +15287,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|3125| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15154,7 +15311,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|6250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15179,7 +15335,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15204,7 +15359,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15229,7 +15383,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15254,7 +15407,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|75000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15279,7 +15431,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15304,7 +15455,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|150000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15329,7 +15479,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|1048576| |karpenter.k8s.aws/instance-network-bandwidth|200000| - |karpenter.k8s.aws/instance-pods|345| |karpenter.k8s.aws/instance-size|32xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15341,6 +15490,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|965782Mi| |pods|345| + |vpc.amazonaws.com/efa|2| |vpc.amazonaws.com/pod-eni|108| ### `r6in.metal` #### Labels @@ -15354,7 +15504,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|1048576| |karpenter.k8s.aws/instance-network-bandwidth|200000| - |karpenter.k8s.aws/instance-pods|345| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15366,6 +15515,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|965782Mi| |pods|345| + |vpc.amazonaws.com/efa|2| |vpc.amazonaws.com/pod-eni|108| ## r7a Family ### `r7a.medium` @@ -15380,7 +15530,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|390| - |karpenter.k8s.aws/instance-pods|8| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15392,6 +15541,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|7134Mi| |pods|8| + |vpc.amazonaws.com/pod-eni|4| ### `r7a.large` #### Labels | Label | Value | @@ -15404,7 +15554,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|781| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15416,6 +15565,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|14481Mi| |pods|29| + |vpc.amazonaws.com/pod-eni|9| ### `r7a.xlarge` #### Labels | Label | Value | @@ -15428,7 +15578,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|1562| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15440,6 +15589,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|29317Mi| |pods|58| + |vpc.amazonaws.com/pod-eni|18| ### `r7a.2xlarge` #### Labels | Label | Value | @@ -15452,7 +15602,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|3125| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15464,6 +15613,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|59627Mi| |pods|58| + |vpc.amazonaws.com/pod-eni|38| ### `r7a.4xlarge` #### Labels | Label | Value | @@ -15476,7 +15626,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|6250| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15488,6 +15637,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|118312Mi| |pods|234| + |vpc.amazonaws.com/pod-eni|54| ### `r7a.8xlarge` #### Labels | Label | Value | @@ -15500,7 +15650,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15512,6 +15661,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|239554Mi| |pods|234| + |vpc.amazonaws.com/pod-eni|84| ### `r7a.12xlarge` #### Labels | Label | Value | @@ -15524,7 +15674,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|18750| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15536,6 +15685,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|360795Mi| |pods|234| + |vpc.amazonaws.com/pod-eni|114| ### `r7a.16xlarge` #### Labels | Label | Value | @@ -15548,7 +15698,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15560,6 +15709,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|476504Mi| |pods|737| + |vpc.amazonaws.com/pod-eni|107| ### `r7a.24xlarge` #### Labels | Label | Value | @@ -15572,7 +15722,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|37500| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15584,6 +15733,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|718987Mi| |pods|737| + |vpc.amazonaws.com/pod-eni|107| ### `r7a.32xlarge` #### Labels | Label | Value | @@ -15596,7 +15746,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|1048576| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|32xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15608,6 +15757,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|961470Mi| |pods|737| + |vpc.amazonaws.com/pod-eni|107| ### `r7a.48xlarge` #### Labels | Label | Value | @@ -15620,7 +15770,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|1572864| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|48xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15632,6 +15781,33 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|1446437Mi| |pods|737| + |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| +### `r7a.metal-48xl` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|192| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7a| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|| + |karpenter.k8s.aws/instance-memory|1572864| + |karpenter.k8s.aws/instance-network-bandwidth|50000| + |karpenter.k8s.aws/instance-size|metal-48xl| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7a.metal-48xl| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|191450m| + |ephemeral-storage|17Gi| + |memory|1446437Mi| + |pods|737| + |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| ## r7g Family ### `r7g.medium` #### Labels @@ -15645,7 +15821,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|520| - |karpenter.k8s.aws/instance-pods|8| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -15670,7 +15845,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|937| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -15695,7 +15869,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|1876| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -15720,7 +15893,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|3750| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -15745,7 +15917,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|7500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -15770,7 +15941,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|15000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -15795,7 +15965,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|22500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -15820,7 +15989,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|30000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -15832,6 +16000,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|476445Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ### `r7g.metal` #### Labels @@ -15845,7 +16014,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|30000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -15857,6 +16025,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|476445Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## r7gd Family ### `r7gd.medium` @@ -15872,7 +16041,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|59| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|520| - |karpenter.k8s.aws/instance-pods|8| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -15898,7 +16066,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|118| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|937| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -15924,7 +16091,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|237| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|1876| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -15950,7 +16116,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|474| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|3750| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -15976,7 +16141,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|950| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|7500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -16000,9 +16164,8 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-local-nvme|1900| - |karpenter.k8s.aws/instance-memory|196608| + |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|15000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -16012,7 +16175,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |cpu|31850m| |ephemeral-storage|17Gi| - |memory|178874Mi| + |memory|239495Mi| |pods|234| |vpc.amazonaws.com/pod-eni|54| ### `r7gd.12xlarge` @@ -16026,9 +16189,8 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-local-nvme|2850| - |karpenter.k8s.aws/instance-memory|262144| + |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|22500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -16038,7 +16200,7 @@ below are the resources available with some assumptions and after the instance o |--|--| |cpu|47810m| |ephemeral-storage|17Gi| - |memory|239495Mi| + |memory|360736Mi| |pods|234| |vpc.amazonaws.com/pod-eni|54| ### `r7gd.16xlarge` @@ -16054,7 +16216,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|30000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -16066,25 +16227,25 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|476445Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| -## r7iz Family -### `r7iz.large` +## r7i Family +### `r7i.large` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| - |karpenter.k8s.aws/instance-family|r7iz| + |karpenter.k8s.aws/instance-family|r7i| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|781| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|r7iz.large| + |node.kubernetes.io/instance-type|r7i.large| #### Resources | Resource | Quantity | |--|--| @@ -16092,23 +16253,23 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|14481Mi| |pods|29| -### `r7iz.xlarge` + |vpc.amazonaws.com/pod-eni|9| +### `r7i.xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| - |karpenter.k8s.aws/instance-family|r7iz| + |karpenter.k8s.aws/instance-family|r7i| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|1562| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|r7iz.xlarge| + |node.kubernetes.io/instance-type|r7i.xlarge| #### Resources | Resource | Quantity | |--|--| @@ -16116,23 +16277,23 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|29317Mi| |pods|58| -### `r7iz.2xlarge` + |vpc.amazonaws.com/pod-eni|18| +### `r7i.2xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| - |karpenter.k8s.aws/instance-family|r7iz| + |karpenter.k8s.aws/instance-family|r7i| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|3125| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|r7iz.2xlarge| + |node.kubernetes.io/instance-type|r7i.2xlarge| #### Resources | Resource | Quantity | |--|--| @@ -16140,23 +16301,23 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|59627Mi| |pods|58| -### `r7iz.4xlarge` + |vpc.amazonaws.com/pod-eni|38| +### `r7i.4xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| - |karpenter.k8s.aws/instance-family|r7iz| + |karpenter.k8s.aws/instance-family|r7i| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|6250| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|r7iz.4xlarge| + |node.kubernetes.io/instance-type|r7i.4xlarge| #### Resources | Resource | Quantity | |--|--| @@ -16164,23 +16325,23 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|118312Mi| |pods|234| -### `r7iz.8xlarge` + |vpc.amazonaws.com/pod-eni|54| +### `r7i.8xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| - |karpenter.k8s.aws/instance-family|r7iz| + |karpenter.k8s.aws/instance-family|r7i| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|r7iz.8xlarge| + |node.kubernetes.io/instance-type|r7i.8xlarge| #### Resources | Resource | Quantity | |--|--| @@ -16188,23 +16349,23 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|239554Mi| |pods|234| -### `r7iz.12xlarge` + |vpc.amazonaws.com/pod-eni|84| +### `r7i.12xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| - |karpenter.k8s.aws/instance-family|r7iz| + |karpenter.k8s.aws/instance-family|r7i| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|393216| - |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|234| + |karpenter.k8s.aws/instance-network-bandwidth|18750| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|r7iz.12xlarge| + |node.kubernetes.io/instance-type|r7i.12xlarge| #### Resources | Resource | Quantity | |--|--| @@ -16212,23 +16373,23 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|360795Mi| |pods|234| -### `r7iz.16xlarge` + |vpc.amazonaws.com/pod-eni|114| +### `r7i.16xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| - |karpenter.k8s.aws/instance-family|r7iz| + |karpenter.k8s.aws/instance-family|r7i| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|r7iz.16xlarge| + |node.kubernetes.io/instance-type|r7i.16xlarge| #### Resources | Resource | Quantity | |--|--| @@ -16236,67 +16397,383 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|476504Mi| |pods|737| -### `r7iz.32xlarge` + |vpc.amazonaws.com/pod-eni|107| +### `r7i.24xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| - |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| - |karpenter.k8s.aws/instance-family|r7iz| + |karpenter.k8s.aws/instance-family|r7i| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| - |karpenter.k8s.aws/instance-memory|1048576| - |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| - |karpenter.k8s.aws/instance-size|32xlarge| + |karpenter.k8s.aws/instance-memory|786432| + |karpenter.k8s.aws/instance-network-bandwidth|37500| + |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|r7iz.32xlarge| + |node.kubernetes.io/instance-type|r7i.24xlarge| #### Resources | Resource | Quantity | |--|--| - |cpu|127610m| + |cpu|95690m| |ephemeral-storage|17Gi| - |memory|961470Mi| + |memory|718987Mi| |pods|737| -## t1 Family -### `t1.micro` + |vpc.amazonaws.com/pod-eni|107| +### `r7i.metal-24xl` #### Labels | Label | Value | |--|--| - |karpenter.k8s.aws/instance-category|t| - |karpenter.k8s.aws/instance-cpu|1| - |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| - |karpenter.k8s.aws/instance-family|t1| - |karpenter.k8s.aws/instance-generation|1| - |karpenter.k8s.aws/instance-hypervisor|xen| - |karpenter.k8s.aws/instance-memory|627| - |karpenter.k8s.aws/instance-pods|4| - |karpenter.k8s.aws/instance-size|micro| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7i| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|| + |karpenter.k8s.aws/instance-memory|786432| + |karpenter.k8s.aws/instance-network-bandwidth|37500| + |karpenter.k8s.aws/instance-size|metal-24xl| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|t1.micro| + |node.kubernetes.io/instance-type|r7i.metal-24xl| #### Resources | Resource | Quantity | |--|--| - |cpu|940m| + |cpu|95690m| |ephemeral-storage|17Gi| - |memory|180Mi| - |pods|4| -## t2 Family -### `t2.nano` + |memory|718987Mi| + |pods|737| + |vpc.amazonaws.com/pod-eni|107| +### `r7i.48xlarge` #### Labels | Label | Value | |--|--| - |karpenter.k8s.aws/instance-category|t| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|192| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7i| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|1572864| + |karpenter.k8s.aws/instance-network-bandwidth|50000| + |karpenter.k8s.aws/instance-size|48xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7i.48xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|191450m| + |ephemeral-storage|17Gi| + |memory|1446437Mi| + |pods|737| + |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| +### `r7i.metal-48xl` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|192| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7i| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|| + |karpenter.k8s.aws/instance-memory|1572864| + |karpenter.k8s.aws/instance-network-bandwidth|50000| + |karpenter.k8s.aws/instance-size|metal-48xl| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7i.metal-48xl| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|191450m| + |ephemeral-storage|17Gi| + |memory|1446437Mi| + |pods|737| + |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| +## r7iz Family +### `r7iz.large` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7iz| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|16384| + |karpenter.k8s.aws/instance-network-bandwidth|781| + |karpenter.k8s.aws/instance-size|large| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7iz.large| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|1930m| + |ephemeral-storage|17Gi| + |memory|14481Mi| + |pods|29| + |vpc.amazonaws.com/pod-eni|9| +### `r7iz.xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7iz| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|32768| + |karpenter.k8s.aws/instance-network-bandwidth|1562| + |karpenter.k8s.aws/instance-size|xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7iz.xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|3920m| + |ephemeral-storage|17Gi| + |memory|29317Mi| + |pods|58| + |vpc.amazonaws.com/pod-eni|18| +### `r7iz.2xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7iz| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|65536| + |karpenter.k8s.aws/instance-network-bandwidth|3125| + |karpenter.k8s.aws/instance-size|2xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7iz.2xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|7910m| + |ephemeral-storage|17Gi| + |memory|59627Mi| + |pods|58| + |vpc.amazonaws.com/pod-eni|38| +### `r7iz.4xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7iz| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|131072| + |karpenter.k8s.aws/instance-network-bandwidth|6250| + |karpenter.k8s.aws/instance-size|4xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7iz.4xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|15890m| + |ephemeral-storage|17Gi| + |memory|118312Mi| + |pods|234| + |vpc.amazonaws.com/pod-eni|54| +### `r7iz.8xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7iz| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|262144| + |karpenter.k8s.aws/instance-network-bandwidth|12500| + |karpenter.k8s.aws/instance-size|8xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7iz.8xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|31850m| + |ephemeral-storage|17Gi| + |memory|239554Mi| + |pods|234| + |vpc.amazonaws.com/pod-eni|84| +### `r7iz.12xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7iz| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|393216| + |karpenter.k8s.aws/instance-network-bandwidth|25000| + |karpenter.k8s.aws/instance-size|12xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7iz.12xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|47810m| + |ephemeral-storage|17Gi| + |memory|360795Mi| + |pods|234| + |vpc.amazonaws.com/pod-eni|114| +### `r7iz.16xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7iz| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|524288| + |karpenter.k8s.aws/instance-network-bandwidth|25000| + |karpenter.k8s.aws/instance-size|16xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7iz.16xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|63770m| + |ephemeral-storage|17Gi| + |memory|476504Mi| + |pods|737| + |vpc.amazonaws.com/pod-eni|107| +### `r7iz.metal-16xl` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7iz| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|| + |karpenter.k8s.aws/instance-memory|524288| + |karpenter.k8s.aws/instance-network-bandwidth|25000| + |karpenter.k8s.aws/instance-size|metal-16xl| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7iz.metal-16xl| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|63770m| + |ephemeral-storage|17Gi| + |memory|476504Mi| + |pods|737| + |vpc.amazonaws.com/pod-eni|107| +### `r7iz.32xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7iz| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|1048576| + |karpenter.k8s.aws/instance-network-bandwidth|50000| + |karpenter.k8s.aws/instance-size|32xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7iz.32xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|127610m| + |ephemeral-storage|17Gi| + |memory|961470Mi| + |pods|737| + |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| +### `r7iz.metal-32xl` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7iz| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|| + |karpenter.k8s.aws/instance-memory|1048576| + |karpenter.k8s.aws/instance-network-bandwidth|50000| + |karpenter.k8s.aws/instance-size|metal-32xl| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7iz.metal-32xl| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|127610m| + |ephemeral-storage|17Gi| + |memory|961470Mi| + |pods|737| + |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| +## t1 Family +### `t1.micro` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|t| + |karpenter.k8s.aws/instance-cpu|1| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| + |karpenter.k8s.aws/instance-family|t1| + |karpenter.k8s.aws/instance-generation|1| + |karpenter.k8s.aws/instance-hypervisor|xen| + |karpenter.k8s.aws/instance-memory|627| + |karpenter.k8s.aws/instance-size|micro| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|t1.micro| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|940m| + |ephemeral-storage|17Gi| + |memory|180Mi| + |pods|4| +## t2 Family +### `t2.nano` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|t| |karpenter.k8s.aws/instance-cpu|1| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| |karpenter.k8s.aws/instance-family|t2| |karpenter.k8s.aws/instance-generation|2| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|512| - |karpenter.k8s.aws/instance-pods|4| |karpenter.k8s.aws/instance-size|nano| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16319,7 +16796,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|2| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|1024| - |karpenter.k8s.aws/instance-pods|4| |karpenter.k8s.aws/instance-size|micro| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16342,7 +16818,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|2| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|2048| - |karpenter.k8s.aws/instance-pods|11| |karpenter.k8s.aws/instance-size|small| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16365,7 +16840,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|2| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|4096| - |karpenter.k8s.aws/instance-pods|17| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16388,7 +16862,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|2| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|8192| - |karpenter.k8s.aws/instance-pods|35| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16411,7 +16884,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|2| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|16384| - |karpenter.k8s.aws/instance-pods|44| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16434,7 +16906,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|2| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|32768| - |karpenter.k8s.aws/instance-pods|44| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16459,7 +16930,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|512| |karpenter.k8s.aws/instance-network-bandwidth|32| - |karpenter.k8s.aws/instance-pods|4| |karpenter.k8s.aws/instance-size|nano| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16483,7 +16953,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|1024| |karpenter.k8s.aws/instance-network-bandwidth|64| - |karpenter.k8s.aws/instance-pods|4| |karpenter.k8s.aws/instance-size|micro| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16507,7 +16976,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|2048| |karpenter.k8s.aws/instance-network-bandwidth|128| - |karpenter.k8s.aws/instance-pods|11| |karpenter.k8s.aws/instance-size|small| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16531,7 +16999,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|4096| |karpenter.k8s.aws/instance-network-bandwidth|256| - |karpenter.k8s.aws/instance-pods|17| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16555,7 +17022,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|512| - |karpenter.k8s.aws/instance-pods|35| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16579,7 +17045,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|1024| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16603,7 +17068,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|2048| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16628,7 +17092,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|512| |karpenter.k8s.aws/instance-network-bandwidth|32| - |karpenter.k8s.aws/instance-pods|4| |karpenter.k8s.aws/instance-size|nano| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16652,7 +17115,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|1024| |karpenter.k8s.aws/instance-network-bandwidth|64| - |karpenter.k8s.aws/instance-pods|4| |karpenter.k8s.aws/instance-size|micro| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16676,7 +17138,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|2048| |karpenter.k8s.aws/instance-network-bandwidth|128| - |karpenter.k8s.aws/instance-pods|8| |karpenter.k8s.aws/instance-size|small| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16700,7 +17161,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|4096| |karpenter.k8s.aws/instance-network-bandwidth|256| - |karpenter.k8s.aws/instance-pods|17| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16724,7 +17184,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|512| - |karpenter.k8s.aws/instance-pods|35| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16748,7 +17207,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|1024| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16772,7 +17230,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|2048| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16797,7 +17254,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|512| |karpenter.k8s.aws/instance-network-bandwidth|32| - |karpenter.k8s.aws/instance-pods|4| |karpenter.k8s.aws/instance-size|nano| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -16821,7 +17277,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|1024| |karpenter.k8s.aws/instance-network-bandwidth|64| - |karpenter.k8s.aws/instance-pods|4| |karpenter.k8s.aws/instance-size|micro| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -16845,7 +17300,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|2048| |karpenter.k8s.aws/instance-network-bandwidth|128| - |karpenter.k8s.aws/instance-pods|11| |karpenter.k8s.aws/instance-size|small| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -16869,7 +17323,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|4096| |karpenter.k8s.aws/instance-network-bandwidth|256| - |karpenter.k8s.aws/instance-pods|17| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -16893,7 +17346,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|512| - |karpenter.k8s.aws/instance-pods|35| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -16917,7 +17369,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|1024| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -16941,7 +17392,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|2048| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -16954,6 +17404,35 @@ below are the resources available with some assumptions and after the instance o |memory|29258Mi| |pods|58| ## trn1 Family +### `trn1.2xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-accelerator-count|1| + |karpenter.k8s.aws/instance-accelerator-manufacturer|aws| + |karpenter.k8s.aws/instance-accelerator-name|inferentia| + |karpenter.k8s.aws/instance-category|trn| + |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|trn1| + |karpenter.k8s.aws/instance-generation|1| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-local-nvme|474| + |karpenter.k8s.aws/instance-memory|32768| + |karpenter.k8s.aws/instance-network-bandwidth|3125| + |karpenter.k8s.aws/instance-size|2xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|trn1.2xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |aws.amazon.com/neuron|1| + |cpu|7910m| + |ephemeral-storage|17Gi| + |memory|29317Mi| + |pods|58| + |vpc.amazonaws.com/pod-eni|17| ### `trn1.32xlarge` #### Labels | Label | Value | @@ -16970,7 +17449,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|7600| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|800000| - |karpenter.k8s.aws/instance-pods|247| |karpenter.k8s.aws/instance-size|32xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16983,7 +17461,39 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|481894Mi| |pods|247| + |vpc.amazonaws.com/efa|8| |vpc.amazonaws.com/pod-eni|82| +## trn1n Family +### `trn1n.32xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-accelerator-count|16| + |karpenter.k8s.aws/instance-accelerator-manufacturer|aws| + |karpenter.k8s.aws/instance-accelerator-name|inferentia| + |karpenter.k8s.aws/instance-category|trn| + |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|trn1n| + |karpenter.k8s.aws/instance-generation|1| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-local-nvme|7600| + |karpenter.k8s.aws/instance-memory|524288| + |karpenter.k8s.aws/instance-network-bandwidth|1600000| + |karpenter.k8s.aws/instance-size|32xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|trn1n.32xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |aws.amazon.com/neuron|16| + |cpu|127610m| + |ephemeral-storage|17Gi| + |memory|481894Mi| + |pods|247| + |vpc.amazonaws.com/efa|16| + |vpc.amazonaws.com/pod-eni|120| ## u-12tb1 Family ### `u-12tb1.112xlarge` #### Labels @@ -16997,7 +17507,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|12582912| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|112xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17022,7 +17531,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|18874368| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|112xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17047,7 +17555,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|25165824| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|112xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17072,7 +17579,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|3145728| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|56xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17098,7 +17604,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|6291456| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|56xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17122,7 +17627,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|6291456| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|112xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17147,7 +17651,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|9437184| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|112xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17159,6 +17662,80 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|8720933Mi| |pods|737| +## vt1 Family +### `vt1.3xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|vt| + |karpenter.k8s.aws/instance-cpu|12| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|vt1| + |karpenter.k8s.aws/instance-generation|1| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|24576| + |karpenter.k8s.aws/instance-network-bandwidth|3120| + |karpenter.k8s.aws/instance-size|3xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|vt1.3xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|11900m| + |ephemeral-storage|17Gi| + |memory|21739Mi| + |pods|58| + |vpc.amazonaws.com/pod-eni|38| +### `vt1.6xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|vt| + |karpenter.k8s.aws/instance-cpu|24| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|vt1| + |karpenter.k8s.aws/instance-generation|1| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|49152| + |karpenter.k8s.aws/instance-network-bandwidth|6250| + |karpenter.k8s.aws/instance-size|6xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|vt1.6xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|23870m| + |ephemeral-storage|17Gi| + |memory|42536Mi| + |pods|234| + |vpc.amazonaws.com/pod-eni|54| +### `vt1.24xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|vt| + |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|vt1| + |karpenter.k8s.aws/instance-generation|1| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|196608| + |karpenter.k8s.aws/instance-network-bandwidth|25000| + |karpenter.k8s.aws/instance-size|24xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|vt1.24xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|95690m| + |ephemeral-storage|17Gi| + |memory|173400Mi| + |pods|737| + |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| ## x1 Family ### `x1.16xlarge` #### Labels @@ -17172,7 +17749,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|999424| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17196,7 +17772,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|1998848| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|32xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17221,7 +17796,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|124928| |karpenter.k8s.aws/instance-network-bandwidth|625| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17245,7 +17819,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|249856| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17269,7 +17842,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|499712| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17293,7 +17865,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|999424| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17317,7 +17888,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|1998848| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17341,7 +17911,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|3997696| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|32xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17367,7 +17936,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|59| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|500| - |karpenter.k8s.aws/instance-pods|8| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -17393,7 +17961,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|118| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|750| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -17419,7 +17986,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|237| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -17445,7 +18011,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|475| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -17471,7 +18036,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|950| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -17497,7 +18061,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1900| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|12000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -17523,7 +18086,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|2850| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|20000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -17549,7 +18111,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|1048576| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -17575,7 +18136,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|1048576| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -17602,7 +18162,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1900| |karpenter.k8s.aws/instance-memory|1048576| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17628,7 +18187,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|2850| |karpenter.k8s.aws/instance-memory|1572864| |karpenter.k8s.aws/instance-network-bandwidth|75000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17654,7 +18212,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|2097152| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|32xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17666,6 +18223,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|1931403Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ### `x2idn.metal` #### Labels @@ -17680,7 +18238,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|2097152| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17692,6 +18249,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|1931403Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## x2iedn Family ### `x2iedn.xlarge` @@ -17707,7 +18265,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|118| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|1875| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17733,7 +18290,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|237| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17759,7 +18315,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|475| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17785,7 +18340,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|950| |karpenter.k8s.aws/instance-memory|1048576| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17811,7 +18365,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1900| |karpenter.k8s.aws/instance-memory|2097152| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17837,7 +18390,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|2850| |karpenter.k8s.aws/instance-memory|3145728| |karpenter.k8s.aws/instance-network-bandwidth|75000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17863,7 +18415,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|4194304| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|32xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17875,6 +18426,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|3871269Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ### `x2iedn.metal` #### Labels @@ -17889,7 +18441,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|4194304| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17901,6 +18452,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|3871269Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## x2iezn Family ### `x2iezn.2xlarge` @@ -17915,7 +18467,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17940,7 +18491,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|15000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17965,7 +18515,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|6xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17990,7 +18539,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|1048576| |karpenter.k8s.aws/instance-network-bandwidth|75000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -18015,7 +18563,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|1572864| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -18027,6 +18574,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|1446437Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ### `x2iezn.metal` #### Labels @@ -18040,7 +18588,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|1572864| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -18052,6 +18599,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|1446437Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## z1d Family ### `z1d.large` @@ -18067,7 +18615,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|75| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|750| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -18093,7 +18640,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|150| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -18119,7 +18665,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|300| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -18145,7 +18690,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|450| |karpenter.k8s.aws/instance-memory|98304| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|3xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -18171,7 +18715,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|900| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|12000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|6xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -18197,7 +18740,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1800| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -18223,7 +18765,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1800| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| diff --git a/website/content/en/v0.31/concepts/metrics.md b/website/content/en/v0.34/reference/metrics.md similarity index 61% rename from website/content/en/v0.31/concepts/metrics.md rename to website/content/en/v0.34/reference/metrics.md index 112f63b0fe9c..7ec8e338711d 100644 --- a/website/content/en/v0.31/concepts/metrics.md +++ b/website/content/en/v0.34/reference/metrics.md @@ -8,164 +8,172 @@ description: > --- Karpenter makes several metrics available in Prometheus format to allow monitoring cluster provisioning status. These metrics are available by default at `karpenter.karpenter.svc.cluster.local:8000/metrics` configurable via the `METRICS_PORT` environment variable documented [here](../settings) -## Controller Runtime Metrics +### `karpenter_build_info` +A metric with a constant '1' value labeled by version from which karpenter was built. -### `controller_runtime_active_workers` -Number of currently used workers per controller +## Nodepool Metrics -### `controller_runtime_max_concurrent_reconciles` -Maximum number of concurrent reconciles per controller +### `karpenter_nodepool_usage` +The nodepool usage is the amount of resources that have been provisioned by a particular nodepool. Labeled by nodepool name and resource type. -### `controller_runtime_reconcile_errors_total` -Total number of reconciliation errors per controller +### `karpenter_nodepool_limit` +The nodepool limits are the limits specified on the nodepool that restrict the quantity of resources provisioned. Labeled by nodepool name and resource type. -### `controller_runtime_reconcile_time_seconds` -Length of time per reconciliation per controller - -### `controller_runtime_reconcile_total` -Total number of reconciliations per controller - -## Consistency Metrics - -### `karpenter_consistency_errors` -Number of consistency checks that have failed. +## Nodes Metrics -## Deprovisioning Metrics +### `karpenter_nodes_total_pod_requests` +Node total pod requests are the resources requested by non-DaemonSet pods bound to nodes. -### `karpenter_deprovisioning_actions_performed` -Number of deprovisioning actions performed. Labeled by deprovisioner. +### `karpenter_nodes_total_pod_limits` +Node total pod limits are the resources specified by non-DaemonSet pod limits. -### `karpenter_deprovisioning_consolidation_timeouts` -Number of times the Consolidation algorithm has reached a timeout. Labeled by consolidation type. +### `karpenter_nodes_total_daemon_requests` +Node total daemon requests are the resource requested by DaemonSet pods bound to nodes. -### `karpenter_deprovisioning_eligible_machines` -Number of machines eligible for deprovisioning by Karpenter. Labeled by deprovisioner +### `karpenter_nodes_total_daemon_limits` +Node total daemon limits are the resources specified by DaemonSet pod limits. -### `karpenter_deprovisioning_evaluation_duration_seconds` -Duration of the deprovisioning evaluation process in seconds. +### `karpenter_nodes_termination_time_seconds` +The time taken between a node's deletion request and the removal of its finalizer -### `karpenter_deprovisioning_replacement_machine_initialized_seconds` -Amount of time required for a replacement machine to become initialized. +### `karpenter_nodes_terminated` +Number of nodes terminated in total by Karpenter. Labeled by owning nodepool. -### `karpenter_deprovisioning_replacement_machine_launch_failure_counter` -The number of times that Karpenter failed to launch a replacement node for deprovisioning. Labeled by deprovisioner. +### `karpenter_nodes_system_overhead` +Node system daemon overhead are the resources reserved for system overhead, the difference between the node's capacity and allocatable values are reported by the status. -## Interruption Metrics +### `karpenter_nodes_leases_deleted` +Number of deleted leaked leases. -### `karpenter_interruption_actions_performed` -Number of notification actions performed. Labeled by action +### `karpenter_nodes_created` +Number of nodes created in total by Karpenter. Labeled by owning nodepool. -### `karpenter_interruption_deleted_messages` -Count of messages deleted from the SQS queue. +### `karpenter_nodes_allocatable` +Node allocatable are the resources allocatable by nodes. -### `karpenter_interruption_message_latency_time_seconds` -Length of time between message creation in queue and an action taken on the message by the controller. +## Pods Metrics -### `karpenter_interruption_received_messages` -Count of messages received from the SQS queue. Broken down by message type and whether the message was actionable. +### `karpenter_pods_state` +Pod state is the current state of pods. This metric can be used several ways as it is labeled by the pod name, namespace, owner, node, nodepool name, zone, architecture, capacity type, instance type and pod phase. -## Machines Metrics +### `karpenter_pods_startup_time_seconds` +The time from pod creation until the pod is running. -### `karpenter_machines_created` -Number of machines created in total by Karpenter. Labeled by reason the machine was created and the owning provisioner. +## Provisioner Metrics -### `karpenter_machines_disrupted` -Number of machines disrupted in total by Karpenter. Labeled by disruption type of the machine and the owning provisioner. +### `karpenter_provisioner_scheduling_simulation_duration_seconds` +Duration of scheduling simulations used for deprovisioning and provisioning in seconds. -### `karpenter_machines_drifted` -Number of machine drifted reasons in total by Karpenter. Labeled by drift type of the machine and the owning provisioner.. +### `karpenter_provisioner_scheduling_duration_seconds` +Duration of scheduling process in seconds. -### `karpenter_machines_initialized` -Number of machines initialized in total by Karpenter. Labeled by the owning provisioner. +## Nodeclaims Metrics -### `karpenter_machines_launched` -Number of machines launched in total by Karpenter. Labeled by the owning provisioner. +### `karpenter_nodeclaims_terminated` +Number of nodeclaims terminated in total by Karpenter. Labeled by reason the nodeclaim was terminated and the owning nodepool. -### `karpenter_machines_registered` -Number of machines registered in total by Karpenter. Labeled by the owning provisioner. +### `karpenter_nodeclaims_registered` +Number of nodeclaims registered in total by Karpenter. Labeled by the owning nodepool. -### `karpenter_machines_terminated` -Number of machines terminated in total by Karpenter. Labeled by reason the machine was terminated and the owning provisioner. +### `karpenter_nodeclaims_launched` +Number of nodeclaims launched in total by Karpenter. Labeled by the owning nodepool. -## Provisioner Metrics +### `karpenter_nodeclaims_initialized` +Number of nodeclaims initialized in total by Karpenter. Labeled by the owning nodepool. -### `karpenter_provisioner_limit` -The Provisioner Limits are the limits specified on the provisioner that restrict the quantity of resources provisioned. Labeled by provisioner name and resource type. +### `karpenter_nodeclaims_drifted` +Number of nodeclaims drifted reasons in total by Karpenter. Labeled by drift type of the nodeclaim and the owning nodepool. -### `karpenter_provisioner_scheduling_duration_seconds` -Duration of scheduling process in seconds. +### `karpenter_nodeclaims_disrupted` +Number of nodeclaims disrupted in total by Karpenter. Labeled by disruption type of the nodeclaim and the owning nodepool. -### `karpenter_provisioner_scheduling_simulation_duration_seconds` -Duration of scheduling simulations used for deprovisioning and provisioning in seconds. +### `karpenter_nodeclaims_created` +Number of nodeclaims created in total by Karpenter. Labeled by reason the nodeclaim was created and the owning nodepool. -### `karpenter_provisioner_usage` -The Provisioner Usage is the amount of resources that have been provisioned by a particular provisioner. Labeled by provisioner name and resource type. +## Interruption Metrics -### `karpenter_provisioner_usage_pct` -The Provisioner Usage Percentage is the percentage of each resource used based on the resources provisioned and the limits that have been configured in the range [0,100]. Labeled by provisioner name and resource type. +### `karpenter_interruption_received_messages` +Count of messages received from the SQS queue. Broken down by message type and whether the message was actionable. -## Nodes Metrics +### `karpenter_interruption_message_latency_time_seconds` +Length of time between message creation in queue and an action taken on the message by the controller. -### `karpenter_nodes_allocatable` -Node allocatable are the resources allocatable by nodes. +### `karpenter_interruption_deleted_messages` +Count of messages deleted from the SQS queue. -### `karpenter_nodes_created` -Number of nodes created in total by Karpenter. Labeled by owning provisioner. +### `karpenter_interruption_actions_performed` +Number of notification actions performed. Labeled by action -### `karpenter_nodes_leases_deleted` -Number of deleted leaked leases. +## Disruption Metrics -### `karpenter_nodes_system_overhead` -Node system daemon overhead are the resources reserved for system overhead, the difference between the node's capacity and allocatable values are reported by the status. +### `karpenter_disruption_replacement_nodeclaim_initialized_seconds` +Amount of time required for a replacement nodeclaim to become initialized. -### `karpenter_nodes_terminated` -Number of nodes terminated in total by Karpenter. Labeled by owning provisioner. +### `karpenter_disruption_replacement_nodeclaim_failures_total` +The number of times that Karpenter failed to launch a replacement node for disruption. Labeled by disruption method. -### `karpenter_nodes_termination_time_seconds` -The time taken between a node's deletion request and the removal of its finalizer +### `karpenter_disruption_queue_depth` +The number of commands currently being waited on in the disruption orchestration queue. -### `karpenter_nodes_total_daemon_limits` -Node total daemon limits are the resources specified by DaemonSet pod limits. +### `karpenter_disruption_evaluation_duration_seconds` +Duration of the disruption evaluation process in seconds. -### `karpenter_nodes_total_daemon_requests` -Node total daemon requests are the resource requested by DaemonSet pods bound to nodes. +### `karpenter_disruption_eligible_nodes` +Number of nodes eligible for disruption by Karpenter. Labeled by disruption method. -### `karpenter_nodes_total_pod_limits` -Node total pod limits are the resources specified by non-DaemonSet pod limits. +### `karpenter_disruption_consolidation_timeouts_total` +Number of times the Consolidation algorithm has reached a timeout. Labeled by consolidation type. -### `karpenter_nodes_total_pod_requests` -Node total pod requests are the resources requested by non-DaemonSet pods bound to nodes. +### `karpenter_disruption_budgets_allowed_disruptions` +The number of nodes for a given NodePool that can be disrupted at a point in time. Labeled by NodePool. Note that allowed disruptions can change very rapidly, as new nodes may be created and others may be deleted at any point. -## Pods Metrics +### `karpenter_disruption_actions_performed_total` +Number of disruption actions performed. Labeled by disruption method. -### `karpenter_pods_startup_time_seconds` -The time from pod creation until the pod is running. +## Consistency Metrics -### `karpenter_pods_state` -Pod state is the current state of pods. This metric can be used several ways as it is labeled by the pod name, namespace, owner, node, provisioner name, zone, architecture, capacity type, instance type and pod phase. +### `karpenter_consistency_errors` +Number of consistency checks that have failed. ## Cloudprovider Metrics -### `karpenter_cloudprovider_duration_seconds` -Duration of cloud provider method calls. Labeled by the controller, method name and provider. +### `karpenter_cloudprovider_instance_type_price_estimate` +Estimated hourly price used when making informed decisions on node cost calculation. This is updated once on startup and then every 12 hours. -### `karpenter_cloudprovider_errors_total` -Total number of errors returned from CloudProvider calls. +### `karpenter_cloudprovider_instance_type_memory_bytes` +Memory, in bytes, for a given instance type. ### `karpenter_cloudprovider_instance_type_cpu_cores` VCPUs cores for a given instance type. -### `karpenter_cloudprovider_instance_type_memory_bytes` -Memory, in bytes, for a given instance type. +### `karpenter_cloudprovider_errors_total` +Total number of errors returned from CloudProvider calls. -### `karpenter_cloudprovider_instance_type_price_estimate` -Estimated hourly price used when making informed decisions on node cost calculation. This is updated once on startup and then every 12 hours. +### `karpenter_cloudprovider_duration_seconds` +Duration of cloud provider method calls. Labeled by the controller, method name and provider. ## Cloudprovider Batcher Metrics +### `karpenter_cloudprovider_batcher_batch_time_seconds` +Duration of the batching window per batcher + ### `karpenter_cloudprovider_batcher_batch_size` Size of the request batch per batcher -### `karpenter_cloudprovider_batcher_batch_time_seconds` -Duration of the batching window per batcher +## Controller Runtime Metrics + +### `controller_runtime_reconcile_total` +Total number of reconciliations per controller + +### `controller_runtime_reconcile_time_seconds` +Length of time per reconciliation per controller + +### `controller_runtime_reconcile_errors_total` +Total number of reconciliation errors per controller + +### `controller_runtime_max_concurrent_reconciles` +Maximum number of concurrent reconciles per controller + +### `controller_runtime_active_workers` +Number of currently used workers per controller diff --git a/website/content/en/v0.34/reference/settings.md b/website/content/en/v0.34/reference/settings.md new file mode 100644 index 000000000000..c7dbc250391f --- /dev/null +++ b/website/content/en/v0.34/reference/settings.md @@ -0,0 +1,68 @@ +--- +title: "Settings" +linkTitle: "Settings" +weight: 5 +description: > + Configure Karpenter +--- + +Karpenter surfaces environment variables and CLI parameters to allow you to configure certain global settings on the controllers. These settings are described below. + +[comment]: <> (the content below is generated from hack/docs/configuration_gen_docs.go) + +| Environment Variable | CLI Flag | Description | +|--|--|--| +| ASSUME_ROLE_ARN | \-\-assume-role-arn | Role to assume for calling AWS services.| +| ASSUME_ROLE_DURATION | \-\-assume-role-duration | Duration of assumed credentials in minutes. Default value is 15 minutes. Not used unless aws.assumeRole set. (default = 15m0s)| +| BATCH_IDLE_DURATION | \-\-batch-idle-duration | The maximum amount of time with no new pending pods that if exceeded ends the current batching window. If pods arrive faster than this time, the batching window will be extended up to the maxDuration. If they arrive slower, the pods will be batched separately. (default = 1s)| +| BATCH_MAX_DURATION | \-\-batch-max-duration | The maximum length of a batch window. The longer this is, the more pods we can consider for provisioning at one time which usually results in fewer but larger nodes. (default = 10s)| +| CLUSTER_CA_BUNDLE | \-\-cluster-ca-bundle | Cluster CA bundle for nodes to use for TLS connections with the API server. If not set, this is taken from the controller's TLS configuration.| +| CLUSTER_ENDPOINT | \-\-cluster-endpoint | The external kubernetes cluster endpoint for new nodes to connect with. If not specified, will discover the cluster endpoint using DescribeCluster API.| +| CLUSTER_NAME | \-\-cluster-name | [REQUIRED] The kubernetes cluster name for resource discovery.| +| DISABLE_WEBHOOK | \-\-disable-webhook | Disable the admission and validation webhooks| +| ENABLE_PROFILING | \-\-enable-profiling | Enable the profiling on the metric endpoint| +| FEATURE_GATES | \-\-feature-gates | Optional features can be enabled / disabled using feature gates. Current options are: Drift,SpotToSpotConsolidation (default = Drift=true,SpotToSpotConsolidation=false)| +| HEALTH_PROBE_PORT | \-\-health-probe-port | The port the health probe endpoint binds to for reporting controller health (default = 8081)| +| INTERRUPTION_QUEUE | \-\-interruption-queue | Interruption queue is disabled if not specified. Enabling interruption handling may require additional permissions on the controller service account. Additional permissions are outlined in the docs.| +| ISOLATED_VPC | \-\-isolated-vpc | If true, then assume we can't reach AWS services which don't have a VPC endpoint. This also has the effect of disabling look-ups to the AWS pricing endpoint.| +| KARPENTER_SERVICE | \-\-karpenter-service | The Karpenter Service name for the dynamic webhook certificate| +| KUBE_CLIENT_BURST | \-\-kube-client-burst | The maximum allowed burst of queries to the kube-apiserver (default = 300)| +| KUBE_CLIENT_QPS | \-\-kube-client-qps | The smoothed rate of qps to kube-apiserver (default = 200)| +| LEADER_ELECT | \-\-leader-elect | Start leader election client and gain leadership before executing the main loop. Enable this when running replicated components for high availability.| +| LOG_LEVEL | \-\-log-level | Log verbosity level. Can be one of 'debug', 'info', or 'error' (default = info)| +| MEMORY_LIMIT | \-\-memory-limit | Memory limit on the container running the controller. The GC soft memory limit is set to 90% of this value. (default = -1)| +| METRICS_PORT | \-\-metrics-port | The port the metric endpoint binds to for operating metrics about the controller itself (default = 8000)| +| RESERVED_ENIS | \-\-reserved-enis | Reserved ENIs are not included in the calculations for max-pods or kube-reserved. This is most often used in the VPC CNI custom networking setup https://docs.aws.amazon.com/eks/latest/userguide/cni-custom-network.html. (default = 0)| +| VM_MEMORY_OVERHEAD_PERCENT | \-\-vm-memory-overhead-percent | The VM memory overhead as a percent that will be subtracted from the total memory for all instance types. (default = 0.075)| +| WEBHOOK_METRICS_PORT | \-\-webhook-metrics-port | The port the webhook metric endpoing binds to for operating metrics about the webhook (default = 8001)| +| WEBHOOK_PORT | \-\-webhook-port | The port the webhook endpoint binds to for validation and mutation of resources (default = 8443)| + +[comment]: <> (end docs generated content from hack/docs/configuration_gen_docs.go) + +### Feature Gates + +Karpenter uses [feature gates](https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/#feature-gates-for-alpha-or-beta-features) You can enable the feature gates through the `--feature-gates` CLI environment variable or the `FEATURE_GATES` environment variable in the Karpenter deployment. For example, you can configure drift, spotToSpotConsolidation by setting the CLI argument: `--feature-gates Drift=true,SpotToSpotConsolidation=true`. + +| Feature | Default | Stage | Since | Until | +|-------------------------|---------|-------|---------|---------| +| Drift | false | Alpha | v0.21.x | v0.32.x | +| Drift | true | Beta | v0.33.x | | +| SpotToSpotConsolidation | false | Alpha | v0.34.x | | + +### Batching Parameters + +The batching parameters control how Karpenter batches an incoming stream of pending pods. Reducing these values may trade off a slightly faster time from pending pod to node launch, in exchange for launching smaller nodes. Increasing the values can do the inverse. Karpenter provides reasonable defaults for these values, but if you have specific knowledge about your workloads you can tweak these parameters to match the expected rate of incoming pods. + +For a standard deployment scale-up, the pods arrive at the QPS setting of the `kube-controller-manager`, and the default values are typically fine. These settings are intended for use cases where other systems may create large numbers of pods over a period of many seconds or minutes and there is a desire to batch them together. + +#### Batch Idle Duration + +The batch idle duration duration is the period of time that a new pending pod extends the current batching window. This can be increased to handle scenarios where pods arrive slower than one second part, but it would be preferable if they were batched together onto a single larger node. + +This value is expressed as a string value like `10s`, `1m` or `2h45m`. The valid time units are `ns`, `us` (or `µs`), `ms`, `s`, `m`, `h`. + +#### Batch Max Duration + +The batch max duration is the maximum period of time a batching window can be extended to. Increasing this value will allow the maximum batch window size to increase to collect more pending pods into a single batch at the expense of a longer delay from when the first pending pod was created. + +This value is expressed as a string value like `10s`, `1m` or `2h45m`. The valid time units are `ns`, `us` (or `µs`), `ms`, `s`, `m`, `h`. diff --git a/website/content/en/v0.34/reference/threat-model.md b/website/content/en/v0.34/reference/threat-model.md new file mode 100644 index 000000000000..3a0062aaddfe --- /dev/null +++ b/website/content/en/v0.34/reference/threat-model.md @@ -0,0 +1,120 @@ +--- +title: "Threat Model" +linkTitle: "Threat Model" +weight: 999 +--- + +Karpenter observes Kubernetes pods and launches nodes in response to those pods’ scheduling constraints. Karpenter does not perform the actual scheduling and instead waits for [kube-scheduler](https://kubernetes.io/docs/concepts/scheduling-eviction/kube-scheduler/) to schedule the pods. + +When running in AWS, Karpenter is typically installed onto EC2 instances that run in EKS Clusters. Karpenter relies on public facing AWS APIs and standard IAM Permissions. Karpenter uses AWS-SDK-Go v1, and AWS advises that credentials are provided using [IAM Roles for Service Accounts](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html). + + +## Architecture & Actors + +1. **Cluster Operator**: An identity that installs and configures Karpenter in a Kubernetes cluster, and configures Karpenter's cloud identity and permissions. +2. **Cluster Developer**: An identity that can create pods, typically through Deployments, DaemonSets, or other pod-controller types. +3. **Karpenter Controller:** The Karpenter application pod that operates inside a cluster. + +![threat-model](/threat-model.png) + +## Capabilities + +### Cluster Operator + +The Cluster Operator has full control to install and configure Karpenter including all [`NodePools`]({{}}) and [`EC2NodeClasses`]({{}}). The Cluster Operator has privileges to manage the cloud identities and permissions for Nodes, and the cloud identity and permissions for Karpenter. + +### Cluster Developer + +A Cluster Developer has the ability to create pods via `Deployments`, `ReplicaSets`, `StatefulSets`, `Jobs`, etc. This assumes that the Cluster Developer cannot modify the Karpenter pod or launch pods using Karpenter’s service account and gain access to Karpenter’s IAM role. + +### Karpenter Controller + +Karpenter has permissions to create and manage cloud instances. Karpenter has Kubernetes API permissions to create, update, and remove nodes, as well as evict pods. For a full list of the permissions, see the RBAC rules in the helm chart template. Karpenter also has AWS IAM permissions to create instances with IAM roles. + +* [aggregate-clusterrole.yaml](https://github.com/aws/karpenter/blob/v0.34.5/charts/karpenter/templates/aggregate-clusterrole.yaml) +* [clusterrole-core.yaml](https://github.com/aws/karpenter/blob/v0.34.5/charts/karpenter/templates/clusterrole-core.yaml) +* [clusterrole.yaml](https://github.com/aws/karpenter/blob/v0.34.5/charts/karpenter/templates/clusterrole.yaml) +* [rolebinding.yaml](https://github.com/aws/karpenter/blob/v0.34.5/charts/karpenter/templates/rolebinding.yaml) +* [role.yaml](https://github.com/aws/karpenter/blob/v0.34.5/charts/karpenter/templates/role.yaml) + +## Assumptions + +| Category | Assumption | Comment | +|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Generic | The Karpenter pod is operated on a node in the cluster, and uses a Service Account for authentication to the Kubernetes API | Cluster Operators may want to isolate the node running the Karpenter pod to a system-pool of nodes to mitigate the possibility of container breakout with Karpenter’s permissions. | +| Generic | Cluster Developer does not have any Kubernetes permissions to manage Karpenter running in the cluster (The deployment, pods, clusterrole, etc) | | +| Generic | Restrictions on the fields of pods a Cluster Developer can create are out of scope. | Cluster Operators can use policy frameworks to enforce restrictions on Pod capabilities | +| Generic | No sensitive data is included in non-Secret resources in the Kubernetes API. The Karpenter controller has the ability to list all pods, nodes, deployments, and many other pod-controller and storage resource types. | Karpenter does not have permission to list/watch cluster-wide ConfigMaps or Secrets | +| Generic | Karpenter has permissions to create, modify, and delete nodes from the cluster, and evict any pod. | Cluster Operators running applications with varying security profiles in the same cluster may want to configure dedicated nodes and scheduling rules for Karpenter to mitigate potential container escapes from other containers | +| AWS-Specific | The Karpenter IAM policy is encoded in the GitHub repo. Any additional permissions possibly granted to that role by the administrator are out of scope | | +| AWS-Specific | The Karpenter pod uses IRSA for AWS credentials | Setup of IRSA is out of scope for this document | + +## Generic Threats and Mitigations + +### Threat: Cluster Developer can influence creation of an arbitrary number of nodes + +**Background**: Karpenter creates new instances based on the count of pending pods. + +**Threat**: A Cluster Developer attempts to have Karpenter create more instances than intended by creating a large number of pods or by using anti-affinity to schedule one pod per node. + +**Mitigation**: In addition to [Kubernetes resource limits](https://kubernetes.io/docs/concepts/policy/resource-quotas/#object-count-quota), Cluster Operators can [configure limits on a NodePool]({{< ref "../concepts/nodepools#spec-limits" >}}) to limit the total amount of memory, CPU, or other resources provisioned across all nodes. + +## Threats + +### Threat: Using EC2 CreateTag/DeleteTag Permissions to Orchestrate Instance Creation/Deletion + +**Background**: As of v0.28.0, Karpenter creates a mapping between CloudProvider instances and CustomResources in the cluster for capacity tracking. To ensure this mapping is consistent, Karpenter utilizes the following tag keys: + +* `karpenter.sh/managed-by` +* `karpenter.sh/nodepool` +* `kubernetes.io/cluster/${CLUSTER_NAME}` +* `karpenter.sh/provisioner-name` (prior to `v0.32.0`) + +Any user that has the ability to Create/Delete tags on CloudProvider instances will have the ability to orchestrate Karpenter to Create/Delete CloudProvider instances as a side effect. + +In addition, as of v0.29.0, Karpenter will Drift on Security Groups and Subnets. If a user has the Create/Delete tags permission for either of resources, they can orchestrate Karpenter to Create/Delete CloudProvider instances as a side effect. + +**Threat:** A Cluster Operator attempts to create or delete a tag on a resource discovered by Karpenter. If it has the ability to create a tag it can effectively create or delete CloudProvider instances associated with the tagged resources. + +**Mitigation** Cluster Operators should [enforce tag-based IAM policies](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_tags.html) on these tags against any EC2 instance resource (`i-*`) for any users that might have [CreateTags](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateTags.html)/[DeleteTags](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DeleteTags.html) permissions but should not have [RunInstances](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_RunInstances.html)/[TerminateInstances](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_TerminateInstances.html) permissions. + +### Threat: Launching EC2 instances with IAM roles not intended for Karpenter nodes + +**Background**: Many IAM roles in an AWS account may trust the EC2 service principal. IAM administrators must grant the `iam:PassRole` permission to IAM principals to allow those principals in the account to launch instances with specific roles. + +**Threat:** A Cluster Operator attempts to create an `EC2NodeClass` with an IAM role not intended for Karpenter + +**Mitigation**: Cluster Operators must enumerate the roles in the resource section of the IAM policy granted to the Karpenter role for the `iam:PassRole` action. Karpenter will fail to generate an instance profile if role that is specified in the `spec.role` section of the `EC2NodeClass` is not enumerated in the `iam:PassRole` permission. + +### Threat: Karpenter can orchestrate the creation/deletion of IAM Instance Profiles it doesn't own + +**Background**: Karpenter has permission to create/update/delete instance profiles as part of its controller permissions to ensure that it can auto-generate instance profiles when EC2NodeClasses are created. + +**Threat**: An actor who obtains control of the Karpenter pod’s IAM role may delete instance profiles not owned by Karpenter, causing workload disruption to other instances using the profile in the account. + +**Mitigation**: Karpenter's controller permissions enforce that it creates instance profiles with tags which provide ownership information. These tags include: + +* `karpenter.sh/managed-by` +* `kubernetes.io/cluster/${CLUSTER_NAME}` +* `karpenter.k8s.aws/ec2nodeclass` +* `topology.kubernetes.io/region` + +These tags ensure that instance profiles created by Karpenter in the account are unique to that cluster. Karpenter's controller permissions _only_ allow it to act on instance profiles that contain these tags which match the cluster information. + +### Threat: Karpenter can be used to create or terminate EC2 instances outside the cluster + +**Background**: EC2 instances can exist in an AWS account outside of the Kubernetes cluster. + +**Threat**: An actor who obtains control of the Karpenter pod’s IAM role may be able to create or terminate EC2 instances not part of the Kubernetes cluster managed by Karpenter. + +**Mitigation**: Karpenter creates instances with tags, several of which are enforced in the IAM policy granted to the Karpenter IAM role that restrict the instances Karpenter can terminate. One tag requires that the instance was provisioned by a Karpenter controller (`karpenter.sh/nodepool`), another tag can include a cluster name to mitigate any termination between two clusters with Karpenter in the same account (`kubernetes.io/cluster/${CLUSTER_NAME}`. Cluster Operators also can restrict the region to prevent two clusters in the same account with the same name in different regions. + +Additionally, Karpenter does not allow tags to be modified on instances unowned by Karpenter after creation, except for the `Name` and `karpenter.sh/nodeclaim` tags. Though these tags can be changed after instance creation, `aws:ResourceTag` conditions enforce that the Karpenter controller is only able to change these tags on instances that it already owns, enforced through the `karpenter.sh/nodepool` and `kubernetes.io/cluster/${CLUSTER_NAME}` tags. + +### Threat: Karpenter launches an EC2 instance using an unintended AMI + +**Background**: Cluster Developers can create Node Templates that refer to an AMI by metadata, such as a name rather than an AMI resource ID. + +**Threat:** A threat actor creates a public AMI with the same name as a customer’s AMI in an attempt to get Karpenter to select the threat actor’s AMI instead of the intended AMI. + +**Mitigation**: When selecting AMIs by name or tags, Karpenter defaults to adding an ownership filter of `self,amazon` so AMI images external to the account are not used. diff --git a/website/content/en/v0.34/tasks/_index.md b/website/content/en/v0.34/tasks/_index.md new file mode 100644 index 000000000000..7d4ac8605f4e --- /dev/null +++ b/website/content/en/v0.34/tasks/_index.md @@ -0,0 +1,10 @@ +--- +title: "Tasks" +linkTitle: "Tasks" +weight: 25 +description: > + Tasks to run with Karpenter +cascade: + type: docs +--- + diff --git a/website/content/en/v0.34/tasks/managing-amis.md b/website/content/en/v0.34/tasks/managing-amis.md new file mode 100644 index 000000000000..1ef31f141e08 --- /dev/null +++ b/website/content/en/v0.34/tasks/managing-amis.md @@ -0,0 +1,153 @@ +--- +title: "Managing AMIs" +linkTitle: "Managing AMIs" +weight: 10 +description: > + Task for managing AMIs in Karpenter +--- + +Understanding how Karpenter assigns AMIs to nodes can help ensure that your workloads will run successfully on those nodes and continue to run if the nodes are upgraded to newer AMIs. +Below we describe how Karpenter assigns AMIs to nodes when they are first deployed and how newer AMIs are assigned later when nodes are spun up to replace old ones. +Later, it describes the options you have to assert control over how AMIs are used by Karpenter for your clusters. + +Features for managing AMIs described here should be considered as part of the larger upgrade policies that you have for your clusters. +See [How do I upgrade an EKS Cluster with Karpenter]({{< relref "../faq/#how-do-i-upgrade-an-eks-cluster-with-karpenter" >}}) for details on this process. + +## How Karpenter assigns AMIs to nodes + +Here is how Karpenter assigns AMIs nodes: + +* When you create an `EC2NodeClass`, you are required to set the family of AMIs to use. For example, for the AL2 family, you would set `amiFamily: AL2`. +* With that `amiFamily` set, any time Karpenter spins up a new node, it uses the latest [Amazon EKS optimized Amazon Linux 2 AMIs](https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami.html) release. +* Later, if an existing node needs to be replaced, Karpenter checks to see if a newer AMI in the AL2 family is available and automatically uses the new AMI instead to spin up the new node. In other words, you may automatically get an AMI that you have not tested with your workloads. + +You can manually delete a node managed by Karpenter, which will cause the default behavior just described to take effect. +However, there are situations that will cause node replacements with newer AMIs to happen automatically. +These include: Expiration (if node expiry is set, the node is marked for deletion at a certain time after the node is created), [**Consolidation**]({{< relref "../concepts/disruption/#consolidation" >}}) (if a node is empty of workloads, or deemed to be inefficiently running workloads, nodes can be deleted and more appropriately featured nodes are brought up to consolidate workloads), [Drift]({{< relref "../concepts/disruption/#drift" >}}) (nodes are set for deletion when they drift from the desired state of the `NodeClaims` and new nodes are brought up to replace them), and [Interruption]({{< relref "../concepts/disruption/#interruption" >}}) (nodes are sometimes involuntarily disrupted by things like Spot interruption, health changes, and instance events, requiring new nodes to be deployed). + +See [**Automated Methods**]({{< relref "../concepts/disruption/#automated-methods" >}}) for details on how Karpenter uses these automated actions to replace nodes. + +With these types of automated updates in place, there is some risk that the new AMI being used when replacing instances will introduce some regressions or bugs that cause your workloads to be degraded or fail altogether. +The options described below tell you how to take more control over the ways in which Karpenter selects AMIs for your nodes. + +{{% alert title="Important" color="warning" %}} +If you are new to Karpenter, you should know that the behavior described here is different than you get with Managed Node Groups (MNG). MNG will always use the assigned AMI when it creates a new node and will never automatically upgrade to a new AMI when a new node is required. See [Updating a Managed Node Group](https://docs.aws.amazon.com/eks/latest/userguide/update-managed-node-group.html) to see how you would manually update MNG to use new AMIs. +{{% /alert %}} + +## Choosing AMI options +One of Karpenter's greatest assets is its ability to provide the right node at the right time, with little intervention from the person managing the cluster. +Its default behavior of using a later AMI if one becomes available in the selected family means you automatically get the latest security fixes and features. +However, with this comes the risk that the new AMI could break or degrade your workloads. + +As the Karpenter team looks for new ways to manage AMIs, the options below offer some means of reducing these risks, based on your own security and ease-of-use requirements. +Here are the advantages and challenges of each of the options described below: + +* [Option 1]({{< relref "#option-1-manage-how-amis-are-tested-and-rolled-out" >}}) (Test AMIs): The safest way, and the one we recommend, for ensuring that a new AMI doesn't break your workloads is to test it before putting it into production. This takes the most effort on your part, but most effectively models how your workloads will run in production, allowing you to catch issues ahead of time. Note that you can sometimes get different results from your test environment when you roll a new AMI into production, since issues like scale and other factors can elevate problems you might not see in test. So combining this with other options, that do things like slow rollouts, can allow you to catch problems before they impact your whole cluster. +* [Option 2]({{< relref "#option-2-lock-down-which-amis-are-selected" >}}) (Lock down AMIs): If workloads require a particluar AMI, this option can make sure that it is the only AMI used by Karpenter. This can be used in combination with Option 1, where you lock down the AMI in production, but allow the newest AMIs in a test cluster while you test your workloads before upgrading production. Keep in mind that this makes upgrades a manual process for you. +* [Option 3]({{< relref "#option-3-control-the-pace-of-node-disruptions" >}}) ([Disruption budgets]({{< relref "../concepts/disruption/" >}})): This option can be used as a way of mitigating the scope of impact if a new AMI causes problems with your workloads. With Disruption budgets you can slow the pace of upgrades to nodes with new AMIs or make sure that upgrades only happen during selected dates and times (using `schedule`). This doesn't prevent a bad AMI from being deployed, but it allows you to control when nodes are upgraded, and gives you more time respond to rollout issues. + +## Options + +The following lays out the options you have to impact Karpenter’s behavior as it relates to how nodes are created and AMIs are consumed. + +### Option 1: Manage how AMIs are tested and rolled out + +Instead of just avoiding AMI upgrades, you can set up test clusters where you can try out new AMI releases before they are put into production. +For example, you could have: + +* **Test clusters**: On lower environment clusters, you can run the latest AMIs for your workloads in a safe environment. The `EC2NodeClass` for these clusters could be set with a chosen `amiFamily`, but no `amiSelectorTerms` set. For example, the `NodePool` and `EC2NodeClass` could begin with the following: + + ```yaml + apiVersion: karpenter.sh/v1beta1 + kind: NodePool + metadata: + name: default + spec: + template: + spec: + nodeClassRef: + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass + name: default + --- + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass + metadata: + name: default + spec: + # The latest AMI in this family will be used + amiFamily: AL2 + ``` +* **Production clusters**: After you've confirmed that the AMI works in your lower environments, you can pin the latest AMIs to be deployed in your production clusters to roll out the AMI. One way to do that is to use `amiSelectorTerms` to set the tested AMI to be used in your production cluster. Refer to Option 2 for how to choose a particular AMI by `name` or `id`. Remember that it is still best practice to gradually roll new AMIs into your cluster, even if they have been tested. So consider implementing that for your production clusters as described in Option 3. + +### Option 2: Lock down which AMIs are selected + +Instead of letting Karpenter always run the latest AMI, you can change Karpenter’s default behavior. +When you configure the [**EC2NodeClass**]({{< relref "../concepts/nodeclasses" >}}), you can set a specific AMI that you want Karpenter to always choose, using the `amiSelectorTerms` field. +This prevents a new and potentially untested AMI from replacing existing nodes when those nodes are terminated. + +With the `amiSelectorTerms` field in an `EC2NodeClass`, you can set a specific AMI for Karpenter to use, based on AMI name or id (only one is required). +These examples show two different ways to identify the same AMI: + +```yaml +amiSelectorTerms: +- tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" + environment: prod +- name: al2023-ami-2023.3.20240219.0-kernel-6.1-x86_64 +``` + +or + +```yaml +amiSelectorTerms: +- tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" + environment: prod +- id: ami-052c9ea013e6e3567 +``` + +See the [**spec.amiSelectorTerms**]({{< relref "../concepts/nodeclasses/#specamiselectorterms" >}}) section of the NodeClasses page for details. +Keep in mind, that this could prevent you from getting critical security patches when new AMIs are available, but it does give you control over exactly which AMI is running. + + +### Option 3: Control the pace of node disruptions + +To reduce the risk of entire workloads being immediately degraded when a new AMI is deployed, you can enable Karpenter [**Disruption Budgets**]({{< relref "../concepts/disruption/#disruption-budgets " >}}). +Disruption Budgets limit when and to what extent nodes can be disrupted. +You can prevent disruption based on nodes (a percentage or number of nodes that can be disrupted at a time) and schedule (excluding certain times from disrupting nodes). +You can set Disruption Budgets in a `NodePool` spec. +Here is an example: + +```yaml +disruption: + consolidationPolicy: WhenEmpty + expireAfter: 1440h + budgets: + - nodes: 15% + - nodes: "3" + - nodes: "0" + schedule: "0 7 * * sat-sun" + duration: 12h +``` + +The `disruption` settings define a few fields that indicate the state of a node that should be disrupted. +The `consolidationPolicy` field indicates that a node should be disrupted if the node is either underutilized (`WhenUnderutilized`) or not running any pods (`WhenEmpty`). +With `expireAfter` set to `1440` hours, the node expires after 60 days. +Extending those values causes longer times without disruption. + +Settings for budgets in the above example include the following: + +* **Percentage of nodes**: From the first `nodes` setting, only `15%` of the NodePool’s nodes can be disrupted at a time. +* **Number of nodes**: The second `nodes` setting limits the number of nodes that can be disrupted at a time to `3`. +* **Schedule**: The third `nodes` setting uses schedule to say that zero disruptions (`0`) are allowed starting at 7am on Saturday and Sunday and continues for 12 hours. +The format of the schedule follows the `crontab` format for identifying dates and times. +See the [crontab](https://man7.org/linux/man-pages/man5/crontab.5.html) page for information on the supported values for these fields. + +As with all disruption settings, keep in mind that avoiding updated AMIs for your nodes can result in not getting fixes for known security risks and bugs. +You need to balance that with your desire to not risk breaking the workloads on your cluster. + +## Follow-up + +The Karpenter project continues to add features to give you greater control over AMI upgrades on your clusters. +If you have opinions about features you would like to see to manage AMIs with Karpenter, feel free to enter a Karpenter [New Issue](https://github.com/aws/karpenter-provider-aws/issues/new/choose). diff --git a/website/content/en/v0.31/troubleshooting.md b/website/content/en/v0.34/troubleshooting.md similarity index 88% rename from website/content/en/v0.31/troubleshooting.md rename to website/content/en/v0.34/troubleshooting.md index eff5bd1ec6e6..ee912f7fe180 100644 --- a/website/content/en/v0.31/troubleshooting.md +++ b/website/content/en/v0.34/troubleshooting.md @@ -1,7 +1,7 @@ --- title: "Troubleshooting" linkTitle: "Troubleshooting" -weight: 90 +weight: 70 description: > Troubleshoot Karpenter problems --- @@ -53,6 +53,18 @@ This can be resolved by creating the [Service Linked Role](https://docs.aws.amaz aws iam create-service-linked-role --aws-service-name spot.amazonaws.com ``` +### Failed Resolving STS Credentials with I/O Timeout + +```bash +Checking EC2 API connectivity, WebIdentityErr: failed to retrieve credentials\ncaused by: RequestError: send request failed\ncaused by: Post \"https://sts.us-east-1.amazonaws.com/\": dial tcp: lookup sts.us-east-1.amazonaws.com: i/o timeout +``` + +If you see the error above when you attempt to install Karpenter, this indicates that Karpenter is unable to reach out to the STS endpoint due to failed DNS resolution. This can happen when Karpenter is running with `dnsPolicy: ClusterFirst` and your in-cluster DNS service is not yet running. + +You have two mitigations to resolve this error: +1. Let Karpenter manage your in-cluster DNS service - You can let Karpenter manage your DNS application pods' capacity by changing Karpenter's `dnsPolicy` to be `Default` (run `--set dnsPolicy=Default` with a Helm installation). This ensures that Karpenter reaches out to the [VPC DNS service](https://docs.aws.amazon.com/vpc/latest/userguide/vpc-dns.html) when running its controllers, allowing Karpenter to start-up without the DNS application pods running, enabling Karpenter to manage the capacity for these pods. +2. Let MNG/Fargate manage your in-cluster DNS service - If running a cluster with MNG, ensure that your group has enough capacity to support the DNS application pods and ensure that the application has the correct tolerations to schedule against the capacity. If running a cluster with Fargate, ensure that you have a [fargate profile](https://docs.aws.amazon.com/eks/latest/userguide/fargate-profile.html) that selects against your DNS application pods. + ### Karpenter Role names exceeding 64-character limit If you use a tool such as AWS CDK to generate your Kubernetes cluster name, when you add Karpenter to your cluster you could end up with a cluster name that is too long to incorporate into your KarpenterNodeRole name (which is limited to 64 characters). @@ -105,14 +117,14 @@ Karpenter v0.26.1+ introduced the `karpenter-crd` helm chart. When installing th - In the case of `invalid ownership metadata; label validation error: missing key "app.kubernetes.io/managed-by": must be set to "Helm"` run: ```shell -kubectl label crd awsnodetemplates.karpenter.k8s.aws provisioners.karpenter.sh machines.karpenter.sh app.kubernetes.io/managed-by=Helm --overwrite +kubectl label crd ec2nodeclasses.karpenter.k8s.aws nodepools.karpenter.sh nodeclaims.karpenter.sh app.kubernetes.io/managed-by=Helm --overwrite ``` - In the case of `annotation validation error: missing key "meta.helm.sh/release-namespace": must be set to "karpenter"` run: ```shell -kubectl annotate crd awsnodetemplates.karpenter.k8s.aws provisioners.karpenter.sh machines.karpenter.sh meta.helm.sh/release-name=karpenter-crd --overwrite -kubectl annotate crd awsnodetemplates.karpenter.k8s.aws provisioners.karpenter.sh machines.karpenter.sh meta.helm.sh/release-namespace=karpenter --overwrite +kubectl annotate crd ec2nodeclasses.karpenter.k8s.aws nodepools.karpenter.sh nodeclaims.karpenter.sh meta.helm.sh/release-name=karpenter-crd --overwrite +kubectl annotate crd ec2nodeclasses.karpenter.k8s.aws nodepools.karpenter.sh nodeclaims.karpenter.sh meta.helm.sh/release-namespace=karpenter --overwrite ``` ## Uninstallation @@ -213,7 +225,7 @@ See the Karpenter [Best Practices Guide](https://aws.github.io/aws-eks-best-prac ### Missing subnetSelector and securityGroupSelector tags causes provisioning failures -Starting with Karpenter v0.5.5, if you are using Karpenter-generated launch template, provisioners require that [subnetSelector]({{}}) and [securityGroupSelector]({{}}) tags be set to match your cluster. +Starting with Karpenter v0.5.5, if you are using Karpenter-generated launch template, provisioners require that [subnetSelector]({{}}) and [securityGroupSelector]({{}}) tags be set to match your cluster. The [Provisioner]({{}}) section in the Karpenter Getting Started Guide uses the following example: ```text @@ -266,7 +278,7 @@ spec: When attempting to schedule a large number of pods with PersistentVolumes, it's possible that these pods will co-locate on the same node. Pods will report the following errors in their events using a `kubectl describe pod` call -```console +```bash Warning FailedAttachVolume pod/example-pod AttachVolume.Attach failed for volume "***" : rpc error: code = Internal desc = Could not attach volume "***" to node "***": attachment of disk "***" failed, expected device to be attached but was attaching Warning FailedMount pod/example-pod Unable to attach or mount volumes: unmounted volumes=[***], unattached volumes=[***]: timed out waiting for the condition ``` @@ -277,14 +289,30 @@ In this case, Karpenter may fail to scale-up your nodes due to these pods due to Karpenter does not support [in-tree storage plugins](https://kubernetes.io/blog/2021/12/10/storage-in-tree-to-csi-migration-status-update/) to provision PersistentVolumes, since nearly all of the in-tree plugins have been deprecated in upstream Kubernetes. This means that, if you are using a statically-provisioned PersistentVolume that references a volume source like `AWSElasticBlockStore` or a dynamically-provisioned PersistentVolume that references a StorageClass with a in-tree storage plugin provisioner like `kubernetes.io/aws-ebs`, Karpenter will fail to discover the maxiumum volume attachments for the node. Instead, Karpenter may think the node still has more schedulable space due to memory and cpu constraints when there is really no more schedulable space on the node due to volume limits. When Karpenter sees you are using an in-tree storage plugin on your pod volumes, it will print the following error message into the logs. If you see this message, upgrade your StorageClasses and statically-provisioned PersistentVolumes to use the latest CSI drivers for your cloud provider. -```console +```bash 2023-04-05T23:56:53.363Z ERROR controller.node_state PersistentVolume source 'AWSElasticBlockStore' uses an in-tree storage plugin which is unsupported by Karpenter and is deprecated by Kubernetes. Scale-ups may fail because Karpenter will not discover driver limits. Use a PersistentVolume that references the 'CSI' volume source for Karpenter auto-scaling support. {"commit": "b2af562", "node": "ip-192-168-36-137.us-west-2.compute.internal", "pod": "inflate0-6c4bdb8b75-7qmfd", "volume": "mypd", "persistent-volume": "pvc-11db7489-3c6e-46f3-a958-91f9d5009d41"} 2023-04-05T23:56:53.464Z ERROR controller.node_state StorageClass .spec.provisioner uses an in-tree storage plugin which is unsupported by Karpenter and is deprecated by Kubernetes. Scale-ups may fail because Karpenter will not discover driver limits. Create a new StorageClass with a .spec.provisioner referencing the CSI driver plugin name 'ebs.csi.aws.com'. {"commit": "b2af562", "node": "ip-192-168-36-137.us-west-2.compute.internal", "pod": "inflate0-6c4bdb8b75-7qmfd", "volume": "mypd", "storage-class": "gp2", "provisioner": "kubernetes.io/aws-ebs"} ``` #### Pods were scheduled due to a race condition in Kubernetes -Due to [this race condition in Kubernetes](https://github.com/kubernetes/kubernetes/issues/95911), it's possible that the scheduler and the CSINode can race during node registration such that the scheduler assumes that a node can mount more volumes than the node attachments support. There is currently no solve for this problem other than enforcing `toplogySpreadConstraints` and `podAntiAffinity` on your workloads that use PVCs such that you attempt to reduce the number of PVCs that schedule to a given node. +Due to [this race condition in Kubernetes](https://github.com/kubernetes/kubernetes/issues/95911), it's possible that the scheduler and the CSINode can race during node registration such that the scheduler assumes that a node can mount more volumes than the node attachments support. There is currently no universal solve for this problem other than enforcing `toplogySpreadConstraints` and `podAntiAffinity` on your workloads that use PVCs such that you attempt to reduce the number of PVCs that schedule to a given node. + +The following is a list of known CSI drivers which support a startupTaint to eliminate this issue: +- [aws-ebs-csi-driver](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/master/docs/install.md#configure-node-startup-taint) +- [aws-efs-csi-driver](https://github.com/kubernetes-sigs/aws-efs-csi-driver/tree/master/docs#configure-node-startup-taint) + +These taints should be configured via `startupTaints` on your `NodePool`. For example, to enable this for EBS, add the following to your `NodePool`: +```yaml +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +spec: + template: + spec: + startupTaints: + - key: ebs.csi.aws.com/agent-not-ready + effect: NoExecute +``` ### CNI is unable to allocate IPs to pods @@ -300,7 +328,7 @@ time=2023-06-12T19:18:15Z type=Warning reason=FailedCreatePodSandBox from=kubele By default, the number of pods on a node is limited by both the number of networking interfaces (ENIs) that may be attached to an instance type and the number of IP addresses that can be assigned to each ENI. See [IP addresses per network interface per instance type](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-eni.html#AvailableIpPerENI) for a more detailed information on these instance types' limits. -If the max-pods (configured through your Provisioner [`kubeletConfiguration`]({{}})) is greater than the number of supported IPs for a given instance type, the CNI will fail to assign an IP to the pod and your pod will be left in a `ContainerCreating` state. +If the max-pods (configured through your Provisioner [`kubeletConfiguration`]({{}})) is greater than the number of supported IPs for a given instance type, the CNI will fail to assign an IP to the pod and your pod will be left in a `ContainerCreating` state. ##### Solutions @@ -308,13 +336,13 @@ To avoid this discrepancy between `maxPods` and the supported pod density of the 1. Enable [Prefix Delegation](https://www.eksworkshop.com/docs/networking/prefix/) to increase the number of allocatable IPs for the ENIs on each instance type 2. Reduce your `maxPods` value to be under the maximum pod density for the instance types assigned to your Provisioner -3. Remove the `maxPods` value from your [`kubeletConfiguration`]({{}}) if you no longer need it and instead rely on the defaulted values from Karpenter and EKS AMIs. +3. Remove the `maxPods` value from your [`kubeletConfiguration`]({{}}) if you no longer need it and instead rely on the defaulted values from Karpenter and EKS AMIs. -For more information on pod density, view the [Pod Density Conceptual Documentation]({{}}). +For more information on pod density, view the [Pod Density Section in the NodePools doc]({{}}). #### IP exhaustion in a subnet -When a node is launched by Karpenter, it is assigned to a subnet within your VPC based on the [`subnetSelector`]({{}}) value in your [`AWSNodeTemplate`]({{}})). When a subnet becomes IP address constrained, EC2 may think that it can successfully launch an instance in the subnet; however, when the CNI tries to assign IPs to the pods, there are none remaining. In this case, your pod will stay in a `ContainerCreating` state until an IP address is freed in the subnet and the CNI can assign one to the pod. +When a node is launched by Karpenter, it is assigned to a subnet within your VPC based on the [`subnetSelector`]({{}}) value in your [`AWSNodeTemplate`]({{}})). When a subnet becomes IP address constrained, EC2 may think that it can successfully launch an instance in the subnet; however, when the CNI tries to assign IPs to the pods, there are none remaining. In this case, your pod will stay in a `ContainerCreating` state until an IP address is freed in the subnet and the CNI can assign one to the pod. ##### Solutions @@ -579,7 +607,7 @@ This means that your CNI plugin is out of date. You can find instructions on how ### Node terminates before ready on failed encrypted EBS volume If you are using a custom launch template and an encrypted EBS volume, the IAM principal launching the node may not have sufficient permissions to use the KMS customer managed key (CMK) for the EC2 EBS root volume. -This issue also applies to [Block Device Mappings]({{}}) specified in the Provisioner. +This issue also applies to [Block Device Mappings]({{}}) specified in the Provisioner. In either case, this results in the node terminating almost immediately upon creation. Keep in mind that it is possible that EBS Encryption can be enabled without your knowledge. @@ -638,11 +666,11 @@ This typically occurs when the node has not been considered fully initialized fo This error indicates that the `vpc.amazonaws.com/pod-eni` resource was never reported on the node. If you've enabled Pod ENI for Karpenter nodes via the `aws.enablePodENI` setting, you will need to make the corresponding change to the VPC CNI to enable [security groups for pods](https://docs.aws.amazon.com/eks/latest/userguide/security-groups-for-pods.html) which will cause the resource to be registered. ### AWS Node Termination Handler (NTH) interactions -Karpenter [doesn't currently support draining and terminating on spot rebalance recommendations]({{< ref "concepts/deprovisioning#interruption" >}}). Users who want support for both drain and terminate on spot interruption as well as drain and termination on spot rebalance recommendations may install Node Termination Handler (NTH) on their clusters to support this behavior. +Karpenter [doesn't currently support draining and terminating on spot rebalance recommendations]({{< ref "concepts/disruption#interruption" >}}). Users who want support for both drain and terminate on spot interruption as well as drain and termination on spot rebalance recommendations may install Node Termination Handler (NTH) on their clusters to support this behavior. These two components do not share information between each other, meaning if you have drain and terminate functionality enabled on NTH, NTH may remove a node for a spot rebalance recommendation. Karpenter will replace the node to fulfill the pod capacity that was being fulfilled by the old node; however, Karpenter won't be aware of the reason that that node was terminated. This means that Karpenter may launch the same instance type that was just deprovisioned, causing a spot rebalance recommendation to be sent again. This can result in very short-lived instances where NTH continually removes nodes and Karpeneter re-launches the same instance type over and over again. -Karpenter doesn't recommend reacting to spot rebalance recommendations when running Karpenter with spot nodes; however, if you absolutely require this functionality, note that the above scenario is possible. +Karpenter doesn't recommend reacting to spot rebalance recommendations when running Karpenter with spot nodes; however, if you absolutely require this functionality, note that the above scenario is possible. Spot instances are time limited and, therefore, interruptible. When a signal is sent by AWS, it triggers actions from NTH and Karpenter, where the former signals a shutdown and the later provisions, creating a recursive situation. This can be mitigated by either completely removing NTH or by setting the following values: @@ -667,4 +695,4 @@ caused by: Post "https://api.pricing.us-east-1.amazonaws.com/": dial tcp 52.94.2 This network timeout occurs because there is no VPC endpoint available for the [Price List Query API.](https://docs.aws.amazon.com/awsaccountbilling/latest/aboutv2/using-pelong.html). To workaround this issue, Karpenter ships updated on-demand pricing data as part of the Karpenter binary; however, this means that pricing data will only be updated on Karpenter version upgrades. To disable pricing lookups and avoid the error messages, set the `AWS_ISOLATED_VPC` environment variable (or the `--aws-isolated-vpc` option) to true. -See [Environment Variables / CLI Flags]({{}}) for details. +See [Environment Variables / CLI Flags]({{}}) for details. diff --git a/website/content/en/v0.34/upgrading/_index.md b/website/content/en/v0.34/upgrading/_index.md new file mode 100644 index 000000000000..ef1368b22a75 --- /dev/null +++ b/website/content/en/v0.34/upgrading/_index.md @@ -0,0 +1,10 @@ +--- +title: "Upgrading" +linkTitle: "Upgrading" +weight: 30 +description: > + Upgrading Karpenter guide and reference +cascade: + type: docs +--- + diff --git a/website/content/en/v0.34/upgrading/compatibility.md b/website/content/en/v0.34/upgrading/compatibility.md new file mode 100644 index 000000000000..805e107b4f8f --- /dev/null +++ b/website/content/en/v0.34/upgrading/compatibility.md @@ -0,0 +1,101 @@ +--- +title: "Compatibility" +linkTitle: "Compatibility" +weight: 20 +description: > + Compatibility issues for Karpenter +--- + +# Compatibility + +To make upgrading easier we aim to minimize the introduction of breaking changes. +Before you begin upgrading Karpenter, consider Karpenter compatibility issues related to Kubernetes and the NodePool API (previously Provisioner). + +## Compatibility Matrix + +[comment]: <> (the content below is generated from hack/docs/compataiblitymetrix_gen_docs.go) + +| KUBERNETES | 1.23 | 1.24 | 1.25 | 1.26 | 1.27 | 1.28 | 1.29 | +|------------|----------|----------|----------|----------|----------|----------|------------| +| karpenter | \>= 0.21 | \>= 0.21 | \>= 0.25 | \>= 0.28 | \>= 0.28 | \>= 0.31 | \>= 0.34.0 | + +[comment]: <> (end docs generated content from hack/docs/compataiblitymetrix_gen_docs.go) + +{{% alert title="Note" color="warning" %}} +Karpenter currently does not support the following [new `topologySpreadConstraints` keys](https://kubernetes.io/blog/2023/04/17/fine-grained-pod-topology-spread-features-beta/), promoted to beta in Kubernetes 1.27: +- `matchLabelKeys` +- `nodeAffinityPolicy` +- `nodeTaintsPolicy` + +For more information on Karpenter's support for these keys, view [this tracking issue](https://github.com/aws/karpenter-core/issues/430). +{{% /alert %}} + +{{% alert title="Note" color="warning" %}} +Karpenter supports using [Kubernetes Common Expression Language](https://kubernetes.io/docs/reference/using-api/cel/) for validating its Custom Resource Definitions out-of-the-box; however, this feature is not supported on versions of Kubernetes < 1.25. If you are running an earlier version of Kubernetes, you will need to use the Karpenter admission webhooks for validation instead. You can enable these webhooks with `--set webhook.enabled=true` when applying the Karpenter helm chart. +{{% /alert %}} + +## Compatibility issues + +When we introduce a breaking change, we do so only as described in this document. + +Karpenter follows [Semantic Versioning 2.0.0](https://semver.org/) in its stable release versions, while in +major version zero (v0.y.z) [anything may change at any time](https://semver.org/#spec-item-4). +However, to further protect users during this phase we will only introduce breaking changes in minor releases (releases that increment y in x.y.z). +Note this does not mean every minor upgrade has a breaking change as we will also increment the +minor version when we release a new feature. + +Users should therefore check to see if there is a breaking change every time they are upgrading to a new minor version. + +### How Do We Break Incompatibility? + +When there is a breaking change we will: + +* Increment the minor version when in major version 0 +* Add a permanent separate section named `upgrading to vx.y.z+` under [release upgrade notes](#release-upgrade-notes) + clearly explaining the breaking change and what needs to be done on the user side to ensure a safe upgrade +* Add the sentence “This is a breaking change, please refer to the above link for upgrade instructions” to the top of the release notes and in all our announcements + +### How Do We Find Incompatibilities? + +Besides the peer review process for all changes to the code base we also do the followings in order to find +incompatibilities: +* (To be implemented) To check the compatibility of the application, we will automate tests for installing, uninstalling, upgrading from an older version, and downgrading to an older version +* (To be implemented) To check the compatibility of the documentation with the application, we will turn the commands in our documentation into scripts that we can automatically run + +### Security Patches + +While we are in major version 0 we will not release security patches to older versions. +Rather we provide the patches in the latest versions. +When at major version 1 we will have an EOL (end of life) policy where we provide security patches +for a subset of older versions and deprecate the others. + +## Release Types + +Karpenter offers three types of releases. This section explains the purpose of each release type and how the images for each release type are tagged in our [public image repository](https://gallery.ecr.aws/karpenter). + +### Stable Releases + +Stable releases are the most reliable releases that are released with weekly cadence. Stable releases are our only recommended versions for production environments. +Sometimes we skip a stable release because we find instability or problems that need to be fixed before having a stable release. +Stable releases are tagged with Semantic Versioning. For example `v0.13.0`. + +### Release Candidates + +We consider having release candidates for major and important minor versions. Our release candidates are tagged like `vx.y.z-rc.0`, `vx.y.z-rc.1`. The release candidate will then graduate to `vx.y.z` as a normal stable release. +By adopting this practice we allow our users who are early adopters to test out new releases before they are available to the wider community, thereby providing us with early feedback resulting in more stable releases. + +### Snapshot Releases + +We release a snapshot release for every commit that gets merged into [`aws/karpenter-provider-aws`](https://www.github.com/aws/karpenter-provider-aws). This enables users to immediately try a new feature or fix right after it's merged rather than waiting days or weeks for release. + +Snapshot releases are not made available in the same public ECR repository as other release types, they are instead published to a separate private ECR repository. +Helm charts are published to `oci://{{< param "snapshot_repo.account_id" >}}.dkr.ecr.{{< param "snapshot_repo.region" >}}.amazonaws.com/karpenter/snapshot/karpenter` and are tagged with the git commit hash prefixed by the Karpenter major version (e.g. `v0-fc17bfc89ebb30a3b102a86012b3e3992ec08adf`). +Anyone with an AWS account can pull from this repository, but must first authenticate: + +```bash +aws ecr get-login-password --region {{< param "snapshot_repo.region" >}} | docker login --username AWS --password-stdin {{< param "snapshot_repo.account_id" >}}.dkr.ecr.{{< param "snapshot_repo.region" >}}.amazonaws.com +``` + +{{% alert title="Note" color="warning" %}} +Snapshot releases are suitable for testing, and troubleshooting but they should not be used in production environments. Snapshot releases are ephemeral and will be removed 90 days after they were published. +{{% /alert %}} diff --git a/website/content/en/v0.31/upgrade-guide.md b/website/content/en/v0.34/upgrading/upgrade-guide.md similarity index 60% rename from website/content/en/v0.31/upgrade-guide.md rename to website/content/en/v0.34/upgrading/upgrade-guide.md index b1a418eda84c..fbe1e7ba16df 100644 --- a/website/content/en/v0.31/upgrade-guide.md +++ b/website/content/en/v0.34/upgrading/upgrade-guide.md @@ -8,61 +8,19 @@ description: > Karpenter is a controller that runs in your cluster, but it is not tied to a specific Kubernetes version, as the Cluster Autoscaler is. Use your existing upgrade mechanisms to upgrade your core add-ons in Kubernetes and keep Karpenter up to date on bug fixes and new features. +This guide contains information needed to upgrade to the latest release of Karpenter, along with compatibility issues you need to be aware of when upgrading from earlier Karpenter versions. -To make upgrading easier we aim to minimize introduction of breaking changes with the following: - -## Compatibility Matrix - -[comment]: <> (the content below is generated from hack/docs/compataiblitymetrix_gen_docs.go) - -| KUBERNETES | 1.23 | 1.24 | 1.25 | 1.26 | 1.27 | 1.28 | -|------------|---------|---------|---------|---------|---------|--------| -| karpenter | 0.21.x+ | 0.21.x+ | 0.25.x+ | 0.28.x+ | 0.28.x+ | 0.31.x | - -[comment]: <> (end docs generated content from hack/docs/compataiblitymetrix_gen_docs.go) - -{{% alert title="Note" color="warning" %}} -Karpenter currently does not support the following [new `topologySpreadConstraints` keys](https://kubernetes.io/blog/2023/04/17/fine-grained-pod-topology-spread-features-beta/), promoted to beta in Kubernetes 1.27: -- `matchLabelKeys` -- `nodeAffinityPolicy` -- `nodeTaintsPolicy` - -For more information on Karpenter's support for these keys, view [this tracking issue](https://github.com/aws/karpenter-core/issues/430). -{{% /alert %}} - -## Compatibility issues - -To make upgrading easier, we aim to minimize the introduction of breaking changes with the following components: - -* Provisioner API -* Helm Chart - -We try to maintain compatibility with: - -* The application itself -* The documentation of the application - -When we introduce a breaking change, we do so only as described in this document. - -Karpenter follows [Semantic Versioning 2.0.0](https://semver.org/) in its stable release versions, while in -major version zero (v0.y.z) [anything may change at any time](https://semver.org/#spec-item-4). -However, to further protect users during this phase we will only introduce breaking changes in minor releases (releases that increment y in x.y.z). -Note this does not mean every minor upgrade has a breaking change as we will also increment the -minor version when we release a new feature. - -Users should therefore check to see if there is a breaking change every time they are upgrading to a new minor version. - -### Custom Resource Definition (CRD) Upgrades +### CRD Upgrades Karpenter ships with a few Custom Resource Definitions (CRDs). These CRDs are published: -* As an independent helm chart [karpenter-crd](https://gallery.ecr.aws/karpenter/karpenter-crd) - [source](https://github.com/aws/karpenter/blob/main/charts/karpenter-crd) that can be used by Helm to manage the lifecycle of these CRDs. - * To upgrade or install `karpenter-crd` run: - ``` - helm upgrade --install karpenter-crd oci://public.ecr.aws/karpenter/karpenter-crd --version vx.y.z --namespace karpenter --create-namespace - ``` +* As an independent helm chart [karpenter-crd](https://gallery.ecr.aws/karpenter/karpenter-crd) - [source](https://github.com/aws/karpenter/blob/main/charts/karpenter-crd) that can be used by Helm to manage the lifecycle of these CRDs. To upgrade or install `karpenter-crd` run: + ```bash + KARPENTER_NAMESPACE=kube-system + helm upgrade --install karpenter-crd oci://public.ecr.aws/karpenter/karpenter-crd --version vx.y.z --namespace "${KARPENTER_NAMESPACE}" --create-namespace + ``` {{% alert title="Note" color="warning" %}} -If you get the error `invalid ownership metadata; label validation error:` while installing the `karpenter-crd` chart from an older version of Karpenter, follow the [Troubleshooting Guide]({{}}) for details on how to resolve these errors. +If you get the error `invalid ownership metadata; label validation error:` while installing the `karpenter-crd` chart from an older version of Karpenter, follow the [Troubleshooting Guide]({{}}) for details on how to resolve these errors. {{% /alert %}} * As part of the helm chart [karpenter](https://gallery.ecr.aws/karpenter/karpenter) - [source](https://github.com/aws/karpenter/blob/main/charts/karpenter/crds). Helm [does not manage the lifecycle of CRDs using this method](https://helm.sh/docs/chart_best_practices/custom_resource_definitions/), the tool will only install the CRD during the first installation of the helm chart. Subsequent chart upgrades will not add or remove CRDs, even if the CRDs have changed. When CRDs are changed, we will make a note in the version's upgrade guide. @@ -70,66 +28,67 @@ If you get the error `invalid ownership metadata; label validation error:` while In general, you can reapply the CRDs in the `crds` directory of the Karpenter helm chart: ```shell -kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.31.3/pkg/apis/crds/karpenter.sh_provisioners.yaml -kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.31.3/pkg/apis/crds/karpenter.sh_machines.yaml -kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.31.3/pkg/apis/crds/karpenter.k8s.aws_awsnodetemplates.yaml +kubectl apply -f https://raw.githubusercontent.com/aws/karpenter/v0.34.5/pkg/apis/crds/karpenter.sh_nodepools.yaml +kubectl apply -f https://raw.githubusercontent.com/aws/karpenter/v0.34.5/pkg/apis/crds/karpenter.sh_nodeclaims.yaml +kubectl apply -f https://raw.githubusercontent.com/aws/karpenter/v0.34.5/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml ``` -### How Do We Break Incompatibility? - -When there is a breaking change we will: +### Upgrading to v0.34.0+ -* Increment the minor version when in major version 0 -* Add a permanent separate section named `upgrading to vx.y.z+` under [released upgrade notes](#released-upgrade-notes) - clearly explaining the breaking change and what needs to be done on the user side to ensure a safe upgrade -* Add the sentence “This is a breaking change, please refer to the above link for upgrade instructions” to the top of the release notes and in all our announcements +[comment]: <> (WHEN CREATING A NEW SECTION OF THE UPGRADE GUIDANCE FOR NEWER VERSIONS, ENSURE THAT YOU COPY THE ALERT SECTION BELOW TO PROPERLY WARN USERS OF THE RISK OF UPGRADING WITHOUT GOING TO v0.32 FIRST) -### How Do We Find Incompatibilities? - -Besides the peer review process for all changes to the code base we also do the followings in order to find -incompatibilities: -* (To be implemented) To check the compatibility of the application, we will automate tests for installing, uninstalling, upgrading from an older version, and downgrading to an older version -* (To be implemented) To check the compatibility of the documentation with the application, we will turn the commands in our documentation into scripts that we can automatically run - -### Security Patches - -While we are in major version 0 we will not release security patches to older versions. -Rather we provide the patches in the latest versions. -When at major version 1 we will have an EOL (end of life) policy where we provide security patches -for a subset of older versions and deprecate the others. - -## Release Types +{{% alert title="Warning" color="warning" %}} +v0.33.0+ _only_ supports Karpenter v1beta1 APIs and will not work with existing Provisioner, AWSNodeTemplate or Machine alpha APIs. Do not upgrade to v0.33.0+ without first [upgrading to v0.32.x]({{}}). This version supports both the alpha and beta APIs, allowing you to migrate all of your existing APIs to beta APIs without experiencing downtime. +{{% /alert %}} -Karpenter offers three types of releases. This section explains the purpose of each release type and how the images for each release type are tagged in our [public image repository](https://gallery.ecr.aws/karpenter). +{{% alert title="Warning" color="warning" %}} +The Ubuntu EKS optimized AMI has moved from 20.04 to 22.04 for Kubernetes 1.29+. This new AMI version is __not currently__ supported for users relying on AMI auto-discovery with the Ubuntu AMI family. More details can be found in this [GitHub issue](https://github.com/aws/karpenter-provider-aws/issues/5572). Please review this issue before upgrading to Kubernetes 1.29 if you are using the Ubuntu AMI family. Upgrading to 1.29 without making any changes to your EC2NodeClass will result in Karpenter being unable to create new nodes. +{{% /alert %}} -### Stable Releases +* Karpenter now supports `nodepool.spec.disruption.budgets`, which allows users to control the speed of disruption in the cluster. Since this requires an update to the Custom Resource, before upgrading, you should re-apply the new updates to the CRDs. Check out [Disruption Budgets]({{}}) for more. +* With Disruption Budgets, Karpenter will disrupt multiple batches of nodes simultaneously, which can result in overall quicker scale-down of your cluster. Before v0.34, Karpenter had a hard-coded parallelism limit for each type of disruption. In v0.34, Karpenter will now disrupt at most 10% of nodes for a given NodePool. There is no setting that will be perfectly equivalent with the behavior prior to v0.34. When considering how to configure your budgets, please refer to the following limits for versions prior to v0.34: + * `Empty Expiration / Empty Drift / Empty Consolidation`: infinite parallelism + * `Non-Empty Expiration / Non-Empty Drift / Single-Node Consolidation`: one node at a time + * `Multi-Node Consolidation`: max 100 nodes +* To support Disruption Budgets, v0.34+ includes critical changes to Karpenter's core controllers, which allows Karpenter to consider multiple batches of disrupting nodes simultaneously. This increases Karpenter's performance with the potential downside of higher CPU and memory utilization from the Karpenter pod. While the magnitude of this difference varies on a case-by-case basis, when upgrading to Karpenter v0.34+, please note that you may need to increase the resources allocated to the Karpenter controller pods. +* Karpenter now adds a default `podSecurityContext` that configures the `fsgroup: 65536` of volumes in the pod. If you are using sidecar containers, you should review if this configuration is compatible for them. You can disable this default `podSecurityContext` through helm by performing `--set podSecurityContext=null` when installing/upgrading the chart. +* The `dnsPolicy` for the Karpenter controller pod has been changed back to the Kubernetes cluster default of `ClusterFirst`. Setting our `dnsPolicy` to `Default` (confusingly, this is not the Kubernetes cluster default) caused more confusion for any users running IPv6 clusters with dual-stack nodes or anyone running Karpenter with dependencies on cluster services (like clusters running service meshes). This change may be breaking for any users on Fargate or MNG who were allowing Karpenter to manage their in-cluster DNS service (`core-dns` on most clusters). If you still want the old behavior here, you can change the `dnsPolicy` to point to use `Default` by setting the helm value on install/upgrade with `--set dnsPolicy=Default`. More details on this issue can be found in the following Github issues: [#2186](https://github.com/aws/karpenter-provider-aws/issues/2186) and [#4947](https://github.com/aws/karpenter-provider-aws/issues/4947). +* Karpenter now disallows `nodepool.spec.template.spec.resources` to be set. The webhook validation never allowed `nodepool.spec.template.spec.resources`. We are now ensuring that CEL validation also disallows `nodepool.spec.template.spec.resources` to be set. If you were previously setting the resources field on your NodePool, ensure that you remove this field before upgrading to the newest version of Karpenter or else updates to the resource may fail on the new version. -Stable releases are the most reliable releases that are released with weekly cadence. Stable releases are our only recommended versions for production environments. -Sometimes we skip a stable release because we find instability or problems that need to be fixed before having a stable release. -Stable releases are tagged with Semantic Versioning. For example `v0.13.0`. +### Upgrading to v0.33.0+ -### Release Candidates +[comment]: <> (WHEN CREATING A NEW SECTION OF THE UPGRADE GUIDANCE FOR NEWER VERSIONS, ENSURE THAT YOU COPY THE ALERT SECTION BELOW TO PROPERLY WARN USERS OF THE RISK OF UPGRADING WITHOUT GOING TO v0.32 FIRST) -We consider having release candidates for major and important minor versions. Our release candidates are tagged like `vx.y.z-rc.0`, `vx.y.z-rc.1`. The release candidate will then graduate to `vx.y.z` as a normal stable release. -By adopting this practice we allow our users who are early adopters to test out new releases before they are available to the wider community, thereby providing us with early feedback resulting in more stable releases. +{{% alert title="Warning" color="warning" %}} +v0.33.0+ _only_ supports Karpenter v1beta1 APIs and will not work with existing Provisioner, AWSNodeTemplate or Machine alpha APIs. **Do not** upgrade to v0.33.0+ without first [upgrading to v0.32.x]({{}}). This version supports both the alpha and beta APIs, allowing you to migrate all of your existing APIs to beta APIs without experiencing downtime. +{{% /alert %}} -### Snapshot Releases +* Karpenter no longer supports using the `karpenter.sh/provisioner-name` label in NodePool labels and requirements or in application node selectors, affinities, or topologySpreadConstraints. If you were previously using this label to target applications to specific Provisioners, you should update your applications to use the `karpenter.sh/nodepool` label instead before upgrading. If you upgrade without changing these labels, you may begin to see pod scheduling failures for these applications. +* Karpenter now tags `spot-instances-request` with the same tags that it tags instances, volumes, and primary ENIs. This means that you will now need to add `ec2:CreateTags` permission for `spot-instances-request`. You can also further scope your controller policy for the `ec2:RunInstances` action to require that it launches the `spot-instances-request` with these specific tags. You can view an example of scoping these actions in the [Getting Started Guide's default CloudFormation controller policy](https://github.com/aws/karpenter/blob/v0.33.0/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml#L61). +* We now recommend that you set the installation namespace for your Karpenter controllers to `kube-system` to denote Karpenter as a critical cluster component. This ensures that requests from the Karpenter controllers are treated with higher priority by assigning them to a different [PriorityLevelConfiguration](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/#prioritylevelconfiguration) than generic requests from other namespaces. For more details on API Priority and Fairness, read the [Kubernetes API Priority and Fairness Conceptual Docs](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/). Note: Changing the namespace for your Karpenter release will cause the service account namespace to change. If you are using IRSA for authentication with AWS, you will need to change scoping set in the controller's trust policy from `karpenter:karpenter` to `kube-system:karpenter`. +* `v0.33.x` disables mutating and validating webhooks by default in favor of using [Common Expression Language for CRD validation](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#validation). The Common Expression Language Validation Feature [is enabled by default on EKS 1.25](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#validation-rules). If you are using Kubernetes version >= 1.25, no further action is required. If you are using a Kubernetes version below 1.25, you now need to set `DISABLE_WEBHOOK=false` in your container environment variables or `--set webhook.enabled=true` if using Helm. View the [Webhook Support Deprecated in Favor of CEL Section of the v1beta1 Migration Guide]({{}}). +* `v0.33.x` drops support for passing settings through the `karpenter-global-settings` ConfigMap. You should pass settings through the container environment variables in the Karpenter deployment manifest. View the [Global Settings Section of the v1beta1 Migration Guide]({{}}) for more details. +* `v0.33.x` enables `Drift=true` by default in the `FEATURE_GATES`. If you previously didn't enable the feature gate, Karpenter will now check if there is a difference between the desired state of your nodes declared in your NodePool and the actual state of your nodes. View the [Drift Section of Disruption Conceptual Docs]({{}}) for more details. +* `v0.33.x` drops looking up the `zap-logger-config` through ConfigMap discovery. Instead, Karpenter now expects the logging config to be mounted on the filesystem if you are using this to configure Zap logging. This is not enabled by default, but can be enabled through `--set logConfig.enabled=true` in the helm values. If you are setting any values in the `logConfig` from the `v0.32.x` upgrade, such as `logConfig.logEncoding`, note that you will have to explicitly set `logConfig.enabled=true` alongside it. Also, note that setting the Zap logging config is a deprecated feature in beta and is planned to be dropped at v1. View the [Logging Configuration Section of the v1beta1 Migration Guide]({{}}) for more details. +* `v0.33.x` change the default `LOG_LEVEL` from `debug` to `info` by default. If you are still enabling logging configuration through the `zap-logger-config`, no action is required. +* `v0.33.x` drops support for comma delimited lists on tags for `SubnetSelectorTerm`, `SecurityGroupsSelectorTerm`, and `AMISelectorTerm`. Karpenter now supports multiple terms for each of the selectors which means that we can specify a more explicit OR-based constraint through separate terms rather than a comma-delimited list of values. -We release a snapshot release for every commit that gets merged into [`aws/karpenter-provider-aws`](https://www.github.com/aws/karpenter-provider-aws). This enables users to immediately try a new feature or fix right after it's merged rather than waiting days or weeks for release. +### Upgrading to v0.32.0+ -Snapshot releases are not made available in the same public ECR repository as other release types, they are instead published to a separate private ECR repository. -Helm charts are published to `oci://{{< param "snapshot_repo.account_id" >}}.dkr.ecr.{{< param "snapshot_repo.region" >}}.amazonaws.com/karpenter/snapshot/karpenter` and are tagged with the git commit hash prefixed by the Karpenter major version (e.g. `v0-fc17bfc89ebb30a3b102a86012b3e3992ec08adf`). -Anyone with an AWS account can pull from this repository, but must first authenticate: +{{% alert title="Warning" color="warning" %}} +Karpenter v0.32.0 introduces v1beta1 APIs, including _significant_ changes to the API and installation procedures for the Karpenter controllers. **Do not** upgrade to v0.32.0+ without referencing the [v1beta1 Migration Upgrade Procedure]({{}}). -```bash -aws ecr get-login-password --region {{< param "snapshot_repo.region" >}} | docker login --username AWS --password-stdin {{< param "snapshot_repo.account_id" >}}.dkr.ecr.{{< param "snapshot_repo.region" >}}.amazonaws.com -``` +This version includes **dual support** for both alpha and beta APIs to ensure that you can slowly migrate your existing Provisioner, AWSNodeTemplate, and Machine alpha APIs to the newer NodePool, EC2NodeClass, and NodeClaim beta APIs. -{{% alert title="Note" color="warning" %}} -Snapshot releases are suitable for testing, and troubleshooting but they should not be used in production environments. Snapshot releases are ephemeral and will be removed 90 days after they were published. +Note that if you are rolling back after upgrading to v0.32.0, note that v0.31.4 is the only version that supports handling rollback after you have deployed the v1beta1 APIs to your cluster. {{% /alert %}} -## Released Upgrade Notes +* Karpenter now serves the webhook prometheus metrics server on port `8001`. If this port is already in-use on the pod or you are running in `hostNetworking` mode, you may need to change this port value. You can configure this port value through the `WEBHOOK_METRICS_PORT` environment variable or the `webhook.metrics.port` value if installing via Helm. +* Karpenter now exposes the ability to disable webhooks through the `webhook.enabled=false` value. This value will disable the webhook server and will prevent any permissions, mutating or validating webhook configurations from being deployed to the cluster. +* Karpenter now moves all logging configuration for the Zap logger into the `logConfig` values block. Configuring Karpenter logging with this mechanism _is_ deprecated and will be dropped at v1. Karpenter now only surfaces logLevel through the `logLevel` helm value. If you need more advanced configuration due to log parsing constraints, we recommend configuring your log parser to handle Karpenter's Zap JSON logging. +* The default log encoding changed from `console` to `json`. If you were previously not setting the type of log encoding, this default will change with the helm chart. If you were setting the value through `logEncoding`, this value will continue to work until v0.33.x but it is deprecated in favor of `logConfig.logEncoding` +* Karpenter now uses the `karpenter.sh/disruption:NoSchedule=disrupting` taint instead of the upstream `node.kubernetes.io/unschedulable` taint for nodes spawned with a NodePool to prevent pods from scheduling to nodes being disrupted. Pods that previously tolerated the `node.kubernetes.io/unschedulable` taint that previously weren't evicted during termination will now be evicted. This most notably affects DaemonSets, which have the `node.kubernetes.io/unschedulable` toleration by default, where Karpenter will now remove these pods during termination. If you want your specific pods to not be evicted when nodes are scaled down, you should add a toleration to the pods with the following: `Key=karpenter.sh/disruption, Effect=NoSchedule, Operator=Equals, Values=disrupting`. + * Note: Karpenter will continue to use the old `node.kubernetes.io/unschedulable` taint for nodes spawned with a Provisioner. ### Upgrading to v0.31.0+ @@ -137,7 +96,7 @@ Snapshot releases are suitable for testing, and troubleshooting but they should ### Upgrading to v0.30.0+ -* Karpenter will now [statically drift]({{}}) on both Provisioner and AWSNodeTemplate Fields. For Provisioner Static Drift, the `karpenter.sh/provisioner-hash` annotation must be present on both the Provisioner and Machine. For AWSNodeTemplate drift, the `karpenter.k8s.aws/nodetemplate-hash` annotation must be present on the AWSNodeTemplate and Machine. Karpenter will not add these annotations to pre-existing nodes, so each of these nodes will need to be recycled one time for the annotations to be added. +* Karpenter will now [statically drift]({{}}) on both Provisioner and AWSNodeTemplate Fields. For Provisioner Static Drift, the `karpenter.sh/provisioner-hash` annotation must be present on both the Provisioner and Machine. For AWSNodeTemplate drift, the `karpenter.k8s.aws/nodetemplate-hash` annotation must be present on the AWSNodeTemplate and Machine. Karpenter will not add these annotations to pre-existing nodes, so each of these nodes will need to be recycled one time for the annotations to be added. * Karpenter will now fail validation on AWSNodeTemplates and Provisioner `spec.provider` that have `amiSelectors`, `subnetSelectors`, or `securityGroupSelectors` set with a combination of id selectors (`aws-ids`, `aws::ids`) and other selectors. * Karpenter now statically sets the `securityContext` at both the pod and container-levels and doesn't allow override values to be passed through the helm chart. This change was made to adhere to [Restricted Pod Security Standard](https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted), which follows pod hardening best practices. @@ -154,7 +113,7 @@ Karpenter `v0.29.1` contains a [file descriptor and memory leak bug](https://git * Karpenter has changed the default metrics service port from 8080 to 8000 and the default webhook service port from 443 to 8443. In `v0.28.0`, the Karpenter pod port was changed to 8000, but referenced the service by name, allowing users to scrape the service at port 8080 for metrics. `v0.29.0` aligns the two ports so that service and pod metrics ports are the same. These ports are set by the `controller.metrics.port` and `webhook.port` helm chart values, so if you have previously set these to non-default values, you may need to update your Prometheus scraper to match these new values. * Karpenter will now reconcile nodes that are drifted due to their Security Groups or their Subnets. If your AWSNodeTemplate's Security Groups differ from the Security Groups used for an instance, Karpenter will consider it drifted. If the Subnet used by an instance is not contained in the allowed list of Subnets for an AWSNodeTemplate, Karpenter will also consider it drifted. - * Since Karpenter uses tags for discovery of Subnets and SecurityGroups, check the [Threat Model]({{}}) to see how to manage this IAM Permission. + * Since Karpenter uses tags for discovery of Subnets and SecurityGroups, check the [Threat Model]({{}}) to see how to manage this IAM Permission. ### Upgrading to v0.28.0+ @@ -163,8 +122,8 @@ Karpenter `v0.28.0` is incompatible with Kubernetes version 1.26+, which can res {{% /alert %}} * The `extraObjects` value is now removed from the Helm chart. Having this value in the chart proved to not work in the majority of Karpenter installs and often led to anti-patterns, where the Karpenter resources installed to manage Karpenter's capacity were directly tied to the install of the Karpenter controller deployments. The Karpenter team recommends that, if you want to install Karpenter manifests alongside the Karpenter helm chart, to do so by creating a separate chart for the manifests, creating a dependency on the controller chart. -* The `aws.nodeNameConvention` setting is now removed from the [`karpenter-global-settings`]({{}}) ConfigMap. Because Karpenter is now driving its orchestration of capacity through Machines, it no longer needs to know the node name, making this setting obsolete. Karpenter ignores configuration that it doesn't recognize in the [`karpenter-global-settings`]({{}}) ConfigMap, so leaving the `aws.nodeNameConvention` in the ConfigMap will simply cause this setting to be ignored. -* Karpenter now defines a set of "restricted tags" which can't be overridden with custom tagging in the AWSNodeTemplate or in the [`karpenter-global-settings`]({{}}) ConfigMap. If you are currently using any of these tag overrides when tagging your instances, webhook validation will now fail. These tags include: +* The `aws.nodeNameConvention` setting is now removed from the [`karpenter-global-settings`]({{}}) ConfigMap. Because Karpenter is now driving its orchestration of capacity through Machines, it no longer needs to know the node name, making this setting obsolete. Karpenter ignores configuration that it doesn't recognize in the [`karpenter-global-settings`]({{}}) ConfigMap, so leaving the `aws.nodeNameConvention` in the ConfigMap will simply cause this setting to be ignored. +* Karpenter now defines a set of "restricted tags" which can't be overridden with custom tagging in the AWSNodeTemplate or in the [`karpenter-global-settings`]({{}}) ConfigMap. If you are currently using any of these tag overrides when tagging your instances, webhook validation will now fail. These tags include: * `karpenter.sh/managed-by` * `karpenter.sh/provisioner-name` @@ -186,7 +145,7 @@ Karpenter creates a mapping between CloudProvider machines and CustomResources i * `karpenter.sh/provisioner-name` * `kubernetes.io/cluster/${CLUSTER_NAME}` -Because Karpenter takes this dependency, any user that has the ability to Create/Delete these tags on CloudProvider machines will have the ability to orchestrate Karpenter to Create/Delete CloudProvider machines as a side effect. Check the [Threat Model]({{}}) to see how this might affect you, and ways to mitigate this. +Because Karpenter takes this dependency, any user that has the ability to Create/Delete these tags on CloudProvider machines will have the ability to orchestrate Karpenter to Create/Delete CloudProvider machines as a side effect. Check the [Threat Model]({{}}) to see how this might affect you, and ways to mitigate this. {{% /alert %}} {{% alert title="Rolling Back" color="warning" %}} @@ -195,11 +154,10 @@ If, after upgrading to `v0.28.0+`, a rollback to an older version of Karpenter n Karpenter marks CloudProvider capacity as "managed by" a Machine using the `karpenter-sh/managed-by` tag on the CloudProvider machine. It uses this tag to ensure that the Machine CustomResources in the cluster match the CloudProvider capacity managed by Karpenter. If these states don't match, Karpenter will garbage collect the capacity. Because of this, if performing an upgrade, followed by a rollback, followed by another upgrade to `v0.28.0+`, ensure you remove the `karpenter.sh/managed-by` tags from existing capacity; otherwise, Karpenter will deprovision the capacity without a Machine CR counterpart. {{% /alert %}} - ### Upgrading to v0.27.3+ * The `defaulting.webhook.karpenter.sh` mutating webhook was removed in `v0.27.3`. If you are coming from an older version of Karpenter where this webhook existed and the webhook was not managed by Helm, you may need to delete the stale webhook. -```console +```bash kubectl delete mutatingwebhookconfigurations defaulting.webhook.karpenter.sh ``` @@ -221,7 +179,7 @@ kubectl delete mutatingwebhookconfigurations defaulting.webhook.karpenter.sh * The `karpenter_allocation_controller_scheduling_duration_seconds` metric name changed to `karpenter_provisioner_scheduling_duration_seconds` ### Upgrading to v0.26.0+ -* The `karpenter.sh/do-not-evict` annotation no longer blocks node termination when running `kubectl delete node`. This annotation on pods will only block automatic deprovisioning that is considered "voluntary," that is, disruptions that can be avoided. Disruptions that Karpenter deems as "involuntary" and will ignore the `karpenter.sh/do-not-evict` annotation include spot interruption and manual deletion of the node. See [Disabling Deprovisioning]({{}}) for more details. +* The `karpenter.sh/do-not-evict` annotation no longer blocks node termination when running `kubectl delete node`. This annotation on pods will only block automatic deprovisioning that is considered "voluntary," that is, disruptions that can be avoided. Disruptions that Karpenter deems as "involuntary" and will ignore the `karpenter.sh/do-not-evict` annotation include spot interruption and manual deletion of the node. See [Disabling Deprovisioning]({{}}) for more details. * Default resources `requests` and `limits` are removed from the Karpenter's controller deployment through the Helm chart. If you have not set custom resource `requests` or `limits` in your helm values and are using Karpenter's defaults, you will now need to set these values in your helm chart deployment. * The `controller.image` value in the helm chart has been broken out to a map consisting of `controller.image.repository`, `controller.image.tag`, and `controller.image.digest`. If manually overriding the `controller.image`, you will need to update your values to the new design. @@ -229,9 +187,9 @@ kubectl delete mutatingwebhookconfigurations defaulting.webhook.karpenter.sh * Cluster Endpoint can now be automatically discovered. If you are using Amazon Elastic Kubernetes Service (EKS), you can now omit the `clusterEndpoint` field in your configuration. In order to allow the resolving, you have to add the permission `eks:DescribeCluster` to the Karpenter Controller IAM role. ### Upgrading to v0.24.0+ -* Settings are no longer updated dynamically while Karpenter is running. If you manually make a change to the [`karpenter-global-settings`]({{}}) ConfigMap, you will need to reload the containers by restarting the deployment with `kubectl rollout restart -n karpenter deploy/karpenter` -* Karpenter no longer filters out instance types internally. Previously, `g2` (not supported by the NVIDIA device plugin) and FPGA instance types were filtered. The only way to filter instance types now is to set requirements on your provisioner or pods using well-known node labels described [here]({{}}). If you are currently using overly broad requirements that allows all of the `g` instance-category, you will want to tighten the requirement, or add an instance-generation requirement. -* `aws.tags` in [`karpenter-global-settings`]({{}}) ConfigMap is now a top-level field and expects the value associated with this key to be a JSON object of string to string. This is change from previous versions where keys were given implicitly by providing the key-value pair `aws.tags.: value` in the ConfigMap. +* Settings are no longer updated dynamically while Karpenter is running. If you manually make a change to the [`karpenter-global-settings`]({{}}) ConfigMap, you will need to reload the containers by restarting the deployment with `kubectl rollout restart -n karpenter deploy/karpenter` +* Karpenter no longer filters out instance types internally. Previously, `g2` (not supported by the NVIDIA device plugin) and FPGA instance types were filtered. The only way to filter instance types now is to set requirements on your provisioner or pods using well-known node labels described [here]({{}}). If you are currently using overly broad requirements that allows all of the `g` instance-category, you will want to tighten the requirement, or add an instance-generation requirement. +* `aws.tags` in [`karpenter-global-settings`]({{}}) ConfigMap is now a top-level field and expects the value associated with this key to be a JSON object of string to string. This is change from previous versions where keys were given implicitly by providing the key-value pair `aws.tags.: value` in the ConfigMap. ### Upgrading to v0.22.0+ * Do not upgrade to this version unless you are on Kubernetes >= v1.21. Karpenter no longer supports Kubernetes v1.20, but now supports Kubernetes v1.25. This change is due to the v1 PDB API, which was introduced in K8s v1.20 and subsequent removal of the v1beta1 API in K8s v1.25. @@ -241,11 +199,11 @@ kubectl delete mutatingwebhookconfigurations defaulting.webhook.karpenter.sh ### Upgrading to v0.19.0+ * The karpenter webhook and controller containers are combined into a single binary, which requires changes to the helm chart. If your Karpenter installation (helm or otherwise) currently customizes the karpenter webhook, your deployment tooling may require minor changes. -* Karpenter now supports native interruption handling. If you were previously using Node Termination Handler for spot interruption handling and health events, you will need to remove the component from your cluster before enabling `aws.interruptionQueueName`. For more details on Karpenter's interruption handling, see the [Interruption Handling Docs]({{< ref "./concepts/deprovisioning/#interruption" >}}). For common questions on the migration process, see the [FAQ]({{< ref "./faq/#interruption-handling" >}}) +* Karpenter now supports native interruption handling. If you were previously using Node Termination Handler for spot interruption handling and health events, you will need to remove the component from your cluster before enabling `aws.interruptionQueueName`. For more details on Karpenter's interruption handling, see the [Interruption Handling Docs]({{< ref "../concepts/disruption/#interruption" >}}). * Instance category defaults are now explicitly persisted in the Provisioner, rather than handled implicitly in memory. By default, Provisioners will limit instance category to c,m,r. If any instance type constraints are applied, it will override this default. If you have created Provisioners in the past with unconstrained instance type, family, or category, Karpenter will now more flexibly use instance types than before. If you would like to apply these constraints, they must be included in the Provisioner CRD. * Karpenter CRD raw YAML URLs have migrated from `https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.19.3/charts/karpenter/crds/...` to `https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.19.3/pkg/apis/crds/...`. If you reference static Karpenter CRDs or rely on `kubectl replace -f` to apply these CRDs from their remote location, you will need to migrate to the new location. * Pods without an ownerRef (also called "controllerless" or "naked" pods) will now be evicted by default during node termination and consolidation. Users can prevent controllerless pods from being voluntarily disrupted by applying the `karpenter.sh/do-not-evict: "true"` annotation to the pods in question. -* The following CLI options/environment variables are now removed and replaced in favor of pulling settings dynamically from the [`karpenter-global-settings`]({{}}) ConfigMap. See the [Settings docs]({{}}) for more details on configuring the new values in the ConfigMap. +* The following CLI options/environment variables are now removed and replaced in favor of pulling settings dynamically from the [`karpenter-global-settings`]({{}}) ConfigMap. See the [Settings docs]({{}}) for more details on configuring the new values in the ConfigMap. * `CLUSTER_NAME` -> `settings.aws.clusterName` * `CLUSTER_ENDPOINT` -> `settings.aws.clusterEndpoint` @@ -257,11 +215,11 @@ kubectl delete mutatingwebhookconfigurations defaulting.webhook.karpenter.sh * `VM_MEMORY_OVERHEAD` -> `settings.aws.vmMemoryOverheadPercent` ### Upgrading to v0.18.0+ -* v0.18.0 removes the `karpenter_consolidation_nodes_created` and `karpenter_consolidation_nodes_terminated` prometheus metrics in favor of the more generic `karpenter_nodes_created` and `karpenter_nodes_terminated` metrics. You can still see nodes created and terminated by consolidation by checking the `reason` label on the metrics. Check out all the metrics published by Karpenter [here]({{}}). +* v0.18.0 removes the `karpenter_consolidation_nodes_created` and `karpenter_consolidation_nodes_terminated` prometheus metrics in favor of the more generic `karpenter_nodes_created` and `karpenter_nodes_terminated` metrics. You can still see nodes created and terminated by consolidation by checking the `reason` label on the metrics. Check out all the metrics published by Karpenter [here]({{}}). ### Upgrading to v0.17.0+ Karpenter's Helm chart package is now stored in [Karpenter's OCI (Open Container Initiative) registry](https://gallery.ecr.aws/karpenter/karpenter). The Helm CLI supports the new format since [v3.8.0+](https://helm.sh/docs/topics/registries/). -With this change [charts.karpenter.sh](https://charts.karpenter.sh/) is no longer updated but preserved to allow using older Karpenter versions. For examples on working with the Karpenter helm charts look at [Install Karpenter Helm Chart]({{< ref "./getting-started/getting-started-with-karpenter/#install-karpenter-helm-chart" >}}). +With this change [charts.karpenter.sh](https://charts.karpenter.sh/) is no longer updated but preserved to allow using older Karpenter versions. For examples on working with the Karpenter helm charts look at [Install Karpenter Helm Chart]({{< ref "../getting-started/getting-started-with-karpenter/#install-karpenter-helm-chart" >}}). Users who have scripted the installation or upgrading of Karpenter need to adjust their scripts with the following changes: 1. There is no longer a need to add the Karpenter helm repo to helm @@ -311,13 +269,13 @@ aws ec2 delete-launch-template --launch-template-id operator: Exists ``` -* v0.14.0 introduces support for custom AMIs without the need for an entire launch template. You must add the `ec2:DescribeImages` permission to the Karpenter Controller Role for this feature to work. This permission is needed for Karpenter to discover custom images specified. Read the [Custom AMI documentation here]({{}}) to get started +* v0.14.0 introduces support for custom AMIs without the need for an entire launch template. You must add the `ec2:DescribeImages` permission to the Karpenter Controller Role for this feature to work. This permission is needed for Karpenter to discover custom images specified. Read the [Custom AMI documentation here]({{}}) to get started * v0.14.0 adds an an additional default toleration (CriticalAddonOnly=Exists) to the Karpenter helm chart. This may cause Karpenter to run on nodes with that use this Taint which previously would not have been schedulable. This can be overridden by using `--set tolerations[0]=null`. * v0.14.0 deprecates the `AWS_ENI_LIMITED_POD_DENSITY` environment variable in-favor of specifying `spec.kubeletConfiguration.maxPods` on the Provisioner. `AWS_ENI_LIMITED_POD_DENSITY` will continue to work when `maxPods` is not set on the Provisioner. If `maxPods` is set, it will override `AWS_ENI_LIMITED_POD_DENSITY` on that specific Provisioner. ### Upgrading to v0.13.0+ -* v0.13.0 introduces a new CRD named `AWSNodeTemplate` which can be used to specify AWS Cloud Provider parameters. Everything that was previously specified under `spec.provider` in the Provisioner resource, can now be specified in the spec of the new resource. The use of `spec.provider` is deprecated but will continue to function to maintain backwards compatibility for the current API version (v1alpha5) of the Provisioner resource. v0.13.0 also introduces support for custom user data that doesn't require the use of a custom launch template. The user data can be specified in-line in the AWSNodeTemplate resource. Read the [UserData documentation here](../aws/operating-systems) to get started. +* v0.13.0 introduces a new CRD named `AWSNodeTemplate` which can be used to specify AWS Cloud Provider parameters. Everything that was previously specified under `spec.provider` in the Provisioner resource, can now be specified in the spec of the new resource. The use of `spec.provider` is deprecated but will continue to function to maintain backwards compatibility for the current API version (v1alpha5) of the Provisioner resource. v0.13.0 also introduces support for custom user data that doesn't require the use of a custom launch template. The user data can be specified in-line in the AWSNodeTemplate resource. If you are upgrading from v0.10.1 - v0.11.1, a new CRD `awsnodetemplate` was added. In v0.12.0, this crd was renamed to `awsnodetemplates`. Since helm does not manage the lifecycle of CRDs, you will need to perform a few manual steps for this CRD upgrade: 1. Make sure any `awsnodetemplate` manifests are saved somewhere so that they can be reapplied to the cluster. @@ -325,8 +283,7 @@ aws ec2 delete-launch-template --launch-template-id 3. `kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.13.2/charts/karpenter/crds/karpenter.k8s.aws_awsnodetemplates.yaml` 4. Perform the Karpenter upgrade to v0.13.x, which will install the new `awsnodetemplates` CRD. 5. Reapply the `awsnodetemplate` manifests you saved from step 1, if applicable. -* v0.13.0 also adds EC2/spot price fetching to Karpenter to allow making more accurate decisions regarding node deployments. Our getting started guide documents this, but if you are upgrading Karpenter you will need to modify your Karpenter controller policy to add the `pricing:GetProducts` and `ec2:DescribeSpotPriceHistory` permissions. - +* v0.13.0 also adds EC2/spot price fetching to Karpenter to allow making more accurate decisions regarding node deployments. Our [getting started guide]({{< ref "../getting-started/getting-started-with-karpenter" >}}) documents this, but if you are upgrading Karpenter you will need to modify your Karpenter controller policy to add the `pricing:GetProducts` and `ec2:DescribeSpotPriceHistory` permissions. ### Upgrading to v0.12.0+ * v0.12.0 adds an OwnerReference to each Node created by a provisioner. Previously, deleting a provisioner would orphan nodes. Now, deleting a provisioner will cause Kubernetes [cascading delete](https://kubernetes.io/docs/concepts/architecture/garbage-collection/#cascading-deletion) logic to gracefully terminate the nodes using the Karpenter node finalizer. You may still orphan nodes by removing the owner reference. diff --git a/website/content/en/v0.31/_index.md b/website/content/en/v0.35/_index.md similarity index 81% rename from website/content/en/v0.31/_index.md rename to website/content/en/v0.35/_index.md index 93ddf9368a25..61ab45550ca3 100755 --- a/website/content/en/v0.31/_index.md +++ b/website/content/en/v0.35/_index.md @@ -1,4 +1,3 @@ - --- title: "Documentation" linkTitle: "Docs" @@ -8,23 +7,23 @@ cascade: tags: - preview --- -Karpenter is an open-source node provisioning project built for Kubernetes. +Karpenter is an open-source node lifecycle management project built for Kubernetes. Adding Karpenter to a Kubernetes cluster can dramatically improve the efficiency and cost of running workloads on that cluster. Karpenter works by: * **Watching** for pods that the Kubernetes scheduler has marked as unschedulable * **Evaluating** scheduling constraints (resource requests, nodeselectors, affinities, tolerations, and topology spread constraints) requested by the pods * **Provisioning** nodes that meet the requirements of the pods -* **Removing** the nodes when the nodes are no longer needed +* **Disrupting** the nodes when the nodes are no longer needed As someone using Karpenter, once your Kubernetes cluster and the Karpenter controller are up and running (see [Getting Started]({{}})), you can: -* **Set up provisioners**: By applying a provisioner to Karpenter, you can configure constraints on node provisioning and set timeout values for node expiry or Kubelet configuration values. -Provisioner-level constraints related to Kubernetes and your cloud provider (AWS, for example) include: +* **Set up NodePools**: By applying a NodePool to Karpenter, you can configure constraints on node provisioning and set values for node expiry, node consolidation, or Kubelet configuration values. + NodePool-level constraints related to Kubernetes and your cloud provider (AWS, for example) include: - Taints (`taints`): Identify taints to add to provisioned nodes. If a pod doesn't have a matching toleration for the taint, the effect set by the taint occurs (NoSchedule, PreferNoSchedule, or NoExecute). See Kubernetes [Taints and Tolerations](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/) for details. - Labels (`labels`): Apply arbitrary key-value pairs to nodes that can be matched by pods. - - Requirements (`requirements`): Set acceptable (`In`) and unacceptable (`Out`) Kubernetes and Karpenter values for node provisioning based on [Well-Known Labels](https://kubernetes.io/docs/reference/labels-annotations-taints/) and [cloud-specific settings]({{}}). These can include [instance types](https://kubernetes.io/docs/reference/labels-annotations-taints/#nodekubernetesioinstance-type), [zones](https://kubernetes.io/docs/reference/labels-annotations-taints/#topologykubernetesiozone), [computer architecture](https://kubernetes.io/docs/reference/labels-annotations-taints/#kubernetes-io-arch), and [capacity type]({{}}) (such as AWS spot or on-demand). + - Requirements (`requirements`): Set acceptable (`In`) and unacceptable (`Out`) Kubernetes and Karpenter values for node provisioning based on [Well-Known Labels](https://kubernetes.io/docs/reference/labels-annotations-taints/) and [cloud-specific settings]({{}}). These can include [instance types](https://kubernetes.io/docs/reference/labels-annotations-taints/#nodekubernetesioinstance-type), [zones](https://kubernetes.io/docs/reference/labels-annotations-taints/#topologykubernetesiozone), [computer architecture](https://kubernetes.io/docs/reference/labels-annotations-taints/#kubernetes-io-arch), and [capacity type]({{}}) (such as AWS spot or on-demand). - Limits (`limits`): Lets you set limits on the total CPU and Memory that can be used by the cluster, effectively stopping further node provisioning when those limits have been reached. * **Deploy workloads**: When deploying workloads, you can request that scheduling constraints be met to direct which nodes Karpenter provisions for those workloads. Use any of the following Pod spec constraints when you deploy pods: diff --git a/website/content/en/v0.35/concepts/_index.md b/website/content/en/v0.35/concepts/_index.md new file mode 100755 index 000000000000..67d573135369 --- /dev/null +++ b/website/content/en/v0.35/concepts/_index.md @@ -0,0 +1,113 @@ +--- +title: "Concepts" +linkTitle: "Concepts" +weight: 20 +description: > + Understand key concepts of Karpenter +--- + +Users fall under two basic roles: [Kubernetes cluster administrators]({{}}) and [application developers]({{}}). This document describes Karpenter concepts through the lens of those two types of users. + +## Cluster Administrator + +As a Kubernetes cluster administrator, you can engage with Karpenter to: + +* Install Karpenter +* Configure NodePools to set constraints and other features for managing nodes +* Disrupting nodes + +Concepts associated with this role are described below. + + +### Installing Karpenter + +Karpenter is designed to run on a node in your Kubernetes cluster. As part of the installation process, you need credentials from the underlying cloud provider to allow nodes to be started up and added to the cluster as they are needed. + +[Getting Started with Karpenter]({{}}) describes the process of installing Karpenter. Because requests to add and delete nodes and schedule pods are made through Kubernetes, AWS IAM Roles for Service Accounts (IRSA) are needed by your Kubernetes cluster to make privileged requests to AWS. For example, Karpenter uses AWS IRSA roles to grant the permissions needed to describe EC2 instance types and create EC2 instances. + +Once privileges are in place, Karpenter is deployed with a Helm chart. + +### Configuring NodePools + +Karpenter's job is to add nodes to handle unschedulable pods, schedule pods on those nodes, and remove the nodes when they are not needed. To configure Karpenter, you create [NodePools]({{}}) that define how Karpenter manages unschedulable pods and configures nodes. You will also define behaviors for your NodePools, capturing details like how Karpenter handles disruption of nodes and setting limits and weights for each NodePool + +Here are some things to know about Karpenter's NodePools: + +* **Unschedulable pods**: Karpenter only attempts to schedule pods that have a status condition `Unschedulable=True`, which the kube scheduler sets when it fails to schedule the pod to existing capacity. + +* [**Defining Constraints**]({{}}): Karpenter defines a Custom Resource called a NodePool to specify configuration. Each NodePool manages a distinct set of nodes, but pods can be scheduled to any NodePool that supports its scheduling constraints. A NodePool contains constraints that impact the nodes that can be provisioned and attributes of those nodes. See the [NodePools Documentation]({{}}) docs for a description of configuration and NodePool examples. + +* [**Defining Disruption**]({{}}): A NodePool can also include values to indicate when nodes should be disrupted. This includes configuration around concepts like [Consolidation]({{}}), [Drift]({{}}), and [Expiration]({{}}). + +* **Well-known labels**: The NodePool can use well-known Kubernetes labels to allow pods to request only certain instance types, architectures, operating systems, or other attributes when creating nodes. See [Well-Known Labels, Annotations and Taints](https://kubernetes.io/docs/reference/labels-annotations-taints/) for details. Keep in mind that only a subset of these labels are supported in Karpenter, as described later. + +* **Multiple NodePools**: Multiple NodePools can be configured on the same cluster. For example, you might want to configure different teams on the same cluster to run on completely separate capacity. One team could run on nodes using BottleRocket, while another uses EKSOptimizedAMI. + +Although most use cases are addressed with a single NodePool for multiple teams, multiple NodePools are useful to isolate nodes for billing, use different node constraints (such as no GPUs for a team), or use different disruption settings. + +### Disrupting nodes + +Karpenter deletes nodes when they are no longer needed. + +* [**Finalizer**]({{}}): Karpenter places a finalizer bit on each node it creates. +When a request comes in to delete one of those nodes (such as a TTL or a manual `kubectl delete node`), Karpenter will cordon the node, drain all the pods, terminate the EC2 instance, and delete the node object. +Karpenter handles all clean-up work needed to properly delete the node. +* [**Expiration**]({{}}): Karpenter will mark nodes as expired and disrupt them after they have lived a set number of seconds, based on the NodePool's `spec.disruption.expireAfter` value. You can use node expiry to periodically recycle nodes due to security concerns. +* [**Consolidation**]({{}}): Karpenter works to actively reduce cluster cost by identifying when: + * Nodes can be removed because the node is empty + * Nodes can be removed as their workloads will run on other nodes in the cluster. + * Nodes can be replaced with cheaper variants due to a change in the workloads. +* [**Drift**]({{}}): Karpenter will mark nodes as drifted and disrupt nodes that have drifted from their desired specification. See [Drift]({{}}) to see which fields are considered. +* [**Interruption**]({{}}): Karpenter will watch for upcoming interruption events that could affect your nodes (health events, spot interruption, etc.) and will cordon, drain, and terminate the node(s) ahead of the event to reduce workload disruption. + +For more details on how Karpenter deletes nodes, see the [Disruption Documentation]({{}}). + +### Scheduling + +Karpenter launches nodes in response to pods that the Kubernetes scheduler has marked unschedulable. After solving scheduling constraints and launching capacity, Karpenter launches a machine in your chosen cloud provider. + +Once Karpenter brings up a node, that node is available for the Kubernetes scheduler to schedule pods on it as well. + +#### Constraints + +The concept of layered constraints is key to using Karpenter. With no constraints defined in NodePools and none requested from pods being deployed, Karpenter chooses from the entire universe of features available to your cloud provider. Nodes can be created using any instance type and run in any zones. + +An application developer can tighten the constraints defined in a NodePool by the cluster administrator by defining additional scheduling constraints in their pod spec. Refer to the description of Karpenter constraints in the Application Developer section below for details. + +### Cloud Provider + +Karpenter makes requests to provision new nodes to the associated cloud provider. The first supported cloud provider is AWS, although Karpenter is designed to work with other cloud providers. Separating Kubernetes and AWS-specific settings allows Karpenter a clean path to integrating with other cloud providers. + +While using Kubernetes well-known labels, the NodePool can set some values that are specific to the cloud provider. For example, to include a certain instance type, you could use the Kubernetes label `node.kubernetes.io/instance-type`, but set its value to an AWS instance type (such as `m5.large` or `m5.2xlarge`). + +### Kubernetes Cluster Autoscaler + +Like Karpenter, [Kubernetes Cluster Autoscaler](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler) is designed to add nodes when requests come in to run pods that cannot be met by current capacity. Cluster autoscaler is part of the Kubernetes project, with implementations by most major Kubernetes cloud providers. By taking a fresh look at provisioning, Karpenter offers the following improvements: + +* **Designed to handle the full flexibility of the cloud**: Karpenter has the ability to efficiently address the full range of instance types available through AWS. Cluster autoscaler was not originally built with the flexibility to handle hundreds of instance types, zones, and purchase options. + +* **Quick node provisioning**: Karpenter manages each instance directly, without use of additional orchestration mechanisms like node groups. This enables it to retry in milliseconds instead of minutes when capacity is unavailable. It also allows Karpenter to leverage diverse instance types, availability zones, and purchase options without the creation of hundreds of node groups. + +## Application Developer + +As someone deploying pods that might be evaluated by Karpenter, you should know how to request the properties that your pods need of its compute resources. Karpenter's job is to efficiently assess and choose compute assets based on requests from pod deployments. These can include basic Kubernetes features or features that are specific to the cloud provider (such as AWS). + +Layered *constraints* are applied when a pod makes requests for compute resources that cannot be met by current capacity. A pod can specify `nodeAffinity` (to run in a particular zone or instance type) or a `topologySpreadConstraints` spread (to cause a set of pods to be balanced across multiple nodes). +The pod can specify a `nodeSelector` to run only on nodes with a particular label and `resource.requests` to ensure that the node has enough available memory. + +The Kubernetes scheduler tries to match those constraints with available nodes. If the pod is unschedulable, Karpenter creates compute resources that match its needs. When Karpenter tries to provision a node, it analyzes scheduling constraints before choosing the node to create. + +As long as the requests are not outside the NodePool's constraints, Karpenter will look to best match the request, comparing the same well-known labels defined by the pod's scheduling constraints. Note that if the constraints are such that a match is not possible, the pod will remain unscheduled. + +So, what constraints can you use as an application developer deploying pods that could be managed by Karpenter? + +Kubernetes features that Karpenter supports for scheduling pods include nodeAffinity and [nodeSelector](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector). +It also supports [PodDisruptionBudget](https://kubernetes.io/docs/tasks/run-application/configure-pdb/), [topologySpreadConstraints](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/), and [inter-pod affinity and anti-affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity). + +From the Kubernetes [Well-Known Labels, Annotations and Taints](https://kubernetes.io/docs/reference/labels-annotations-taints/) page, you can see a full list of Kubernetes labels, annotations and taints that determine scheduling. Those that are implemented in Karpenter include: + +* **kubernetes.io/arch**: For example, kubernetes.io/arch=amd64 +* **node.kubernetes.io/instance-type**: For example, node.kubernetes.io/instance-type=m3.medium +* **topology.kubernetes.io/zone**: For example, topology.kubernetes.io/zone=us-east-1c + +For more on how, as a developer, you can add constraints to your pod deployment, see [Scheduling](./scheduling/) for details. diff --git a/website/content/en/v0.35/concepts/disruption.md b/website/content/en/v0.35/concepts/disruption.md new file mode 100644 index 000000000000..48e0b479c5f3 --- /dev/null +++ b/website/content/en/v0.35/concepts/disruption.md @@ -0,0 +1,306 @@ +--- +title: "Disruption" +linkTitle: "Disruption" +weight: 4 +description: > + Understand different ways Karpenter disrupts nodes +--- + +## Control Flow + +Karpenter sets a Kubernetes [finalizer](https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/) on each node and node claim it provisions. +The finalizer blocks deletion of the node object while the Termination Controller taints and drains the node, before removing the underlying NodeClaim. Disruption is triggered by the Disruption Controller, by the user through manual disruption, or through an external system that sends a delete request to the node object. + +### Disruption Controller + +Karpenter automatically discovers disruptable nodes and spins up replacements when needed. Karpenter disrupts nodes by executing one [automated method](#automated-methods) at a time, in order of Expiration, Drift, and then Consolidation. Each method varies slightly, but they all follow the standard disruption process. Karpenter uses [disruption budgets]({{}}) to control the speed of disruption. +1. Identify a list of prioritized candidates for the disruption method. + * If there are [pods that cannot be evicted](#pod-eviction) on the node, Karpenter will ignore the node and try disrupting it later. + * If there are no disruptable nodes, continue to the next disruption method. +2. For each disruptable node: + 1. Check if disrupting it would violate its NodePool's disruption budget. + 2. Execute a scheduling simulation with the pods on the node to find if any replacement nodes are needed. +3. Add the `karpenter.sh/disruption:NoSchedule` taint to the node(s) to prevent pods from scheduling to it. +4. Pre-spin any replacement nodes needed as calculated in Step (2), and wait for them to become ready. + * If a replacement node fails to initialize, un-taint the node(s), and restart from Step (1), starting at the first disruption method again. +5. Delete the node(s) and wait for the Termination Controller to gracefully shutdown the node(s). +6. Once the Termination Controller terminates the node, go back to Step (1), starting at the first disruption method again. + +### Termination Controller + +When a Karpenter node is deleted, the Karpenter finalizer will block deletion and the APIServer will set the `DeletionTimestamp` on the node, allowing Karpenter to gracefully shutdown the node, modeled after [Kubernetes Graceful Node Shutdown](https://kubernetes.io/docs/concepts/architecture/nodes/#graceful-node-shutdown). Karpenter's graceful shutdown process will: +1. Add the `karpenter.sh/disruption=disrupting:NoSchedule` taint to the node to prevent pods from scheduling to it. +2. Begin evicting the pods on the node with the [Kubernetes Eviction API](https://kubernetes.io/docs/concepts/scheduling-eviction/api-eviction/) to respect PDBs, while ignoring all [static pods](https://kubernetes.io/docs/tasks/configure-pod-container/static-pod/), pods tolerating the `karpenter.sh/disruption=disrupting:NoSchedule` taint, and succeeded/failed pods. Wait for the node to be fully drained before proceeding to Step (3). + * While waiting, if the underlying NodeClaim for the node no longer exists, remove the finalizer to allow the APIServer to delete the node, completing termination. +3. Terminate the NodeClaim in the Cloud Provider. +4. Remove the finalizer from the node to allow the APIServer to delete the node, completing termination. + +## Manual Methods +* **Node Deletion**: You can use `kubectl` to manually remove a single Karpenter node or nodeclaim. Since each Karpenter node is owned by a NodeClaim, deleting either the node or the nodeclaim will cause cascade deletion of the other: + + ```bash + # Delete a specific nodeclaim + kubectl delete nodeclaim $NODECLAIM_NAME + + # Delete a specific node + kubectl delete node $NODE_NAME + + # Delete all nodeclaims + kubectl delete nodeclaims --all + + # Delete all nodes owned by any nodepool + kubectl delete nodes -l karpenter.sh/nodepool + + # Delete all nodeclaims owned by a specific nodepoolXS + kubectl delete nodeclaims -l karpenter.sh/nodepool=$NODEPOOL_NAME + ``` +* **NodePool Deletion**: NodeClaims are owned by the NodePool through an [owner reference](https://kubernetes.io/docs/concepts/overview/working-with-objects/owners-dependents/#owner-references-in-object-specifications) that launched them. Karpenter will gracefully terminate nodes through cascading deletion when the owning NodePool is deleted. + +{{% alert title="Note" color="primary" %}} +By adding the finalizer, Karpenter improves the default Kubernetes process of node deletion. +When you run `kubectl delete node` on a node without a finalizer, the node is deleted without triggering the finalization logic. The instance will continue running in EC2, even though there is no longer a node object for it. The kubelet isn’t watching for its own existence, so if a node is deleted, the kubelet doesn’t terminate itself. All the pod objects get deleted by a garbage collection process later, because the pods’ node is gone. +{{% /alert %}} + +## Automated Methods + +Automated methods can be rate limited through [NodePool Disruption Budgets]({{}}) + +* **Expiration**: Karpenter will mark nodes as expired and disrupt them after they have lived a set number of seconds, based on the NodePool's `spec.disruption.expireAfter` value. You can use node expiry to periodically recycle nodes due to security concerns. +* [**Consolidation**]({{}}): Karpenter works to actively reduce cluster cost by identifying when: + * Nodes can be removed because the node is empty + * Nodes can be removed as their workloads will run on other nodes in the cluster. + * Nodes can be replaced with lower priced variants due to a change in the workloads. +* [**Drift**]({{}}): Karpenter will mark nodes as drifted and disrupt nodes that have drifted from their desired specification. See [Drift]({{}}) to see which fields are considered. +* [**Interruption**]({{}}): Karpenter will watch for upcoming interruption events that could affect your nodes (health events, spot interruption, etc.) and will taint, drain, and terminate the node(s) ahead of the event to reduce workload disruption. + +{{% alert title="Defaults" color="secondary" %}} +Disruption is configured through the NodePool's disruption block by the `consolidationPolicy`, `expireAfter` and `consolidateAfter` fields. Karpenter will configure these fields with the following values by default if they are not set: + +```yaml +spec: + disruption: + consolidationPolicy: WhenUnderutilized + expireAfter: 720h +``` +{{% /alert %}} + +### Consolidation + +Karpenter has two mechanisms for cluster consolidation: +1. **Deletion** - A node is eligible for deletion if all of its pods can run on free capacity of other nodes in the cluster. +2. **Replace** - A node can be replaced if all of its pods can run on a combination of free capacity of other nodes in the cluster and a single lower price replacement node. + +Consolidation has three mechanisms that are performed in order to attempt to identify a consolidation action: +1. **Empty Node Consolidation** - Delete any entirely empty nodes in parallel +2. **Multi Node Consolidation** - Try to delete two or more nodes in parallel, possibly launching a single replacement whose price is lower than that of all nodes being removed +3. **Single Node Consolidation** - Try to delete any single node, possibly launching a single replacement whose price is lower than that of the node being removed + +It's impractical to examine all possible consolidation options for multi-node consolidation, so Karpenter uses a heuristic to identify a likely set of nodes that can be consolidated. For single-node consolidation we consider each node in the cluster individually. + +When there are multiple nodes that could be potentially deleted or replaced, Karpenter chooses to consolidate the node that overall disrupts your workloads the least by preferring to terminate: + +* Nodes running fewer pods +* Nodes that will expire soon +* Nodes with lower priority pods + +If consolidation is enabled, Karpenter periodically reports events against nodes that indicate why the node can't be consolidated. These events can be used to investigate nodes that you expect to have been consolidated, but still remain in your cluster. + +```bash +Events: + Type Reason Age From Message + ---- ------ ---- ---- ------- + Normal Unconsolidatable 66s karpenter pdb default/inflate-pdb prevents pod evictions + Normal Unconsolidatable 33s (x3 over 30m) karpenter can't replace with a lower-priced node +``` + +{{% alert title="Warning" color="warning" %}} +Using preferred anti-affinity and topology spreads can reduce the effectiveness of consolidation. At node launch, Karpenter attempts to satisfy affinity and topology spread preferences. In order to reduce node churn, consolidation must also attempt to satisfy these constraints to avoid immediately consolidating nodes after they launch. This means that consolidation may not disrupt nodes in order to avoid violating preferences, even if kube-scheduler can fit the host pods elsewhere. Karpenter reports these pods via logging to bring awareness to the possible issues they can cause (e.g. `pod default/inflate-anti-self-55894c5d8b-522jd has a preferred Anti-Affinity which can prevent consolidation`). +{{% /alert %}} + +#### Spot consolidation +For spot nodes, Karpenter has deletion consolidation enabled by default. If you would like to enable replacement with spot consolidation, you need to enable the feature through the [`SpotToSpotConsolidation` feature flag]({{}}). + +Lower priced spot instance types are selected with the [`price-capacity-optimized` strategy](https://aws.amazon.com/blogs/compute/introducing-price-capacity-optimized-allocation-strategy-for-ec2-spot-instances/). Sometimes, the lowest priced spot instance type is not launched due to the likelihood of interruption. As a result, Karpenter uses the number of available instance type options with a price lower than the currently launched spot instance as a heuristic for evaluating whether it should launch a replacement for the current spot node. + +We refer to the number of instances that Karpenter has within its launch decision as a launch's "instance type flexibility." When Karpenter is considering performing a spot-to-spot consolidation replacement, it will check whether replacing the instance type will lead to enough instance type flexibility in the subsequent launch request. As a result, we get the following properties when evaluating for consolidation: +1) We shouldn't continually consolidate down to the lowest priced spot instance which might have very high rates of interruption. +2) We launch with enough instance types that there’s high likelihood that our replacement instance has comparable availability to our current one. + +Karpenter requires a minimum instance type flexibility of 15 instance types when performing single node spot-to-spot consolidations (1 node to 1 node). It does not have the same instance type flexibility requirement for multi-node spot-to-spot consolidations (many nodes to 1 node) since doing so without requiring flexibility won't lead to "race to the bottom" scenarios. + + +### Drift +Drift handles changes to the NodePool/EC2NodeClass. For Drift, values in the NodePool/EC2NodeClass are reflected in the NodeClaimTemplateSpec/EC2NodeClassSpec in the same way that they’re set. A NodeClaim will be detected as drifted if the values in its owning NodePool/EC2NodeClass do not match the values in the NodeClaim. Similar to the upstream `deployment.spec.template` relationship to pods, Karpenter will annotate the owning NodePool and EC2NodeClass with a hash of the NodeClaimTemplateSpec to check for drift. Some special cases will be discovered either from Karpenter or through the CloudProvider interface, triggered by NodeClaim/Instance/NodePool/EC2NodeClass changes. + +#### Special Cases on Drift +In special cases, drift can correspond to multiple values and must be handled differently. Drift on resolved fields can create cases where drift occurs without changes to CRDs, or where CRD changes do not result in drift. For example, if a NodeClaim has `node.kubernetes.io/instance-type: m5.large`, and requirements change from `node.kubernetes.io/instance-type In [m5.large]` to `node.kubernetes.io/instance-type In [m5.large, m5.2xlarge]`, the NodeClaim will not be drifted because its value is still compatible with the new requirements. Conversely, if a NodeClaim is using a NodeClaim image `ami: ami-abc`, but a new image is published, Karpenter's `EC2NodeClass.spec.amiSelectorTerms` will discover that the new correct value is `ami: ami-xyz`, and detect the NodeClaim as drifted. + +##### NodePool +| Fields | +|----------------| +| spec.template.spec.requirements | + +##### EC2NodeClass +| Fields | +|-------------------------------| +| spec.subnetSelectorTerms | +| spec.securityGroupSelectorTerms | +| spec.amiSelectorTerms | + +#### Behavioral Fields +Behavioral Fields are treated as over-arching settings on the NodePool to dictate how Karpenter behaves. These fields don’t correspond to settings on the NodeClaim or instance. They’re set by the user to control Karpenter’s Provisioning and disruption logic. Since these don’t map to a desired state of NodeClaims, __behavioral fields are not considered for Drift__. + +##### NodePool +| Fields | +|---------------------| +| spec.weight | +| spec.limits | +| spec.disruption.* | + +Read the [Drift Design](https://github.com/aws/karpenter-core/blob/main/designs/drift.md) for more. + +To enable the drift feature flag, refer to the [Feature Gates]({{}}). + +Karpenter will add the `Drifted` status condition on NodeClaims if the NodeClaim is drifted from its owning NodePool. Karpenter will also remove the `Drifted` status condition if either: +1. The `Drift` feature gate is not enabled but the NodeClaim is drifted, Karpenter will remove the status condition. +2. The NodeClaim isn't drifted, but has the status condition, Karpenter will remove it. + +### Interruption + +If interruption-handling is enabled, Karpenter will watch for upcoming involuntary interruption events that would cause disruption to your workloads. These interruption events include: + +* Spot Interruption Warnings +* Scheduled Change Health Events (Maintenance Events) +* Instance Terminating Events +* Instance Stopping Events + +When Karpenter detects one of these events will occur to your nodes, it automatically taints, drains, and terminates the node(s) ahead of the interruption event to give the maximum amount of time for workload cleanup prior to compute disruption. This enables scenarios where the `terminationGracePeriod` for your workloads may be long or cleanup for your workloads is critical, and you want enough time to be able to gracefully clean-up your pods. + +For Spot interruptions, the NodePool will start a new node as soon as it sees the Spot interruption warning. Spot interruptions have a __2 minute notice__ before Amazon EC2 reclaims the instance. Karpenter's average node startup time means that, generally, there is sufficient time for the new node to become ready and to move the pods to the new node before the NodeClaim is reclaimed. + +{{% alert title="Note" color="primary" %}} +Karpenter publishes Kubernetes events to the node for all events listed above in addition to [__Spot Rebalance Recommendations__](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/rebalance-recommendations.html). Karpenter does not currently support taint, drain, and terminate logic for Spot Rebalance Recommendations. + +If you require handling for Spot Rebalance Recommendations, you can use the [AWS Node Termination Handler (NTH)](https://github.com/aws/aws-node-termination-handler) alongside Karpenter; however, note that the AWS Node Termination Handler cordons and drains nodes on rebalance recommendations, potentially causing more node churn in the cluster than with interruptions alone. Further information can be found in the [Troubleshooting Guide]({{< ref "../troubleshooting#aws-node-termination-handler-nth-interactions" >}}). +{{% /alert %}} + +Karpenter enables this feature by watching an SQS queue which receives critical events from AWS services which may affect your nodes. Karpenter requires that an SQS queue be provisioned and EventBridge rules and targets be added that forward interruption events from AWS services to the SQS queue. Karpenter provides details for provisioning this infrastructure in the [CloudFormation template in the Getting Started Guide](../../getting-started/getting-started-with-karpenter/#create-the-karpenter-infrastructure-and-iam-roles). + +To enable interruption handling, configure the `--interruption-queue-name` CLI argument with the name of the interruption queue provisioned to handle interruption events. + +## Controls + +### Disruption Budgets + +You can rate limit Karpenter's disruption through the NodePool's `spec.disruption.budgets`. If undefined, Karpenter will default to one budget with `nodes: 10%`. Budgets will consider nodes that are actively being deleted for any reason, and will only block Karpenter from disrupting nodes voluntarily through expiration, drift, emptiness, and consolidation. + +#### Nodes +When calculating if a budget will block nodes from disruption, Karpenter lists the total number of nodes owned by a NodePool, subtracting out the nodes owned by that NodePool that are currently being deleted and nodes that are NotReady. If the number of nodes being deleted by Karpenter or any other processes is greater than the number of allowed disruptions, disruption for this node will not proceed. + +If the budget is configured with a percentage value, such as `20%`, Karpenter will calculate the number of allowed disruptions as `allowed_disruptions = roundup(total * percentage) - total_deleting - total_notready`. If otherwise defined as a non-percentage value, Karpenter will simply subtract the number of nodes from the total `(total - non_percentage_value) - total_deleting - total_notready`. For multiple budgets in a NodePool, Karpenter will take the minimum value (most restrictive) of each of the budgets. + +For example, the following NodePool with three budgets defines the following requirements: +- The first budget will only allow 20% of nodes owned by that NodePool to be disrupted. For instance, if there were 19 nodes owned by the NodePool, 4 disruptions would be allowed, rounding up from `19 * .2 = 3.8`. +- The second budget acts as a ceiling to the previous budget, only allowing 5 disruptions when there are more than 25 nodes. +- The last budget only blocks disruptions during the first 10 minutes of the day, where 0 disruptions are allowed. + +```yaml +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: default +spec: + disruption: + consolidationPolicy: WhenUnderutilized + expireAfter: 720h # 30 * 24h = 720h + budgets: + - nodes: "20%" + - nodes: "5" + - nodes: "0" + schedule: "@daily" + duration: 10m +``` + +#### Schedule +Schedule is a cronjob schedule. Generally, the cron syntax is five space-delimited values with options below, with additional special macros like `@yearly`, `@monthly`, `@weekly`, `@daily`, `@hourly`. +Follow the [Kubernetes documentation](https://kubernetes.io/docs/concepts/workloads/controllers/cron-jobs/#writing-a-cronjob-spec) for more information on how to follow the cron syntax. + +```bash +# ┌───────────── minute (0 - 59) +# │ ┌───────────── hour (0 - 23) +# │ │ ┌───────────── day of the month (1 - 31) +# │ │ │ ┌───────────── month (1 - 12) +# │ │ │ │ ┌───────────── day of the week (0 - 6) (Sunday to Saturday; +# │ │ │ │ │ 7 is also Sunday on some systems) +# │ │ │ │ │ OR sun, mon, tue, wed, thu, fri, sat +# │ │ │ │ │ +# * * * * * +``` + +{{% alert title="Note" color="primary" %}} +Timezones are not supported. Most images default to UTC, but it is best practice to ensure this is the case when considering how to define your budgets. +{{% /alert %}} + +#### Duration +Duration allows compound durations with minutes and hours values such as `10h5m` or `30m` or `160h`. Since cron syntax does not accept denominations smaller than minutes, users can only define minutes or hours. + +{{% alert title="Note" color="primary" %}} +Duration and Schedule must be defined together. When omitted, the budget is always active. When defined, the schedule determines a starting point where the budget will begin being enforced, and the duration determines how long from that starting point the budget will be enforced. +{{% /alert %}} + +### Pod-Level Controls + +You can block Karpenter from voluntarily choosing to disrupt certain pods by setting the `karpenter.sh/do-not-disrupt: "true"` annotation on the pod. This is useful for pods that you want to run from start to finish without disruption. By opting pods out of this disruption, you are telling Karpenter that it should not voluntarily remove a node containing this pod. + +Examples of pods that you might want to opt-out of disruption include an interactive game that you don't want to interrupt or a long batch job (such as you might have with machine learning) that would need to start over if it were interrupted. + +```yaml +apiVersion: apps/v1 +kind: Deployment +spec: + template: + metadata: + annotations: + karpenter.sh/do-not-disrupt: "true" +``` + +{{% alert title="Note" color="primary" %}} +This annotation will be ignored for [terminating pods](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase) and [terminal pods](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase) (Failed/Succeeded). +{{% /alert %}} + +Examples of voluntary node removal that will be prevented by this annotation include: +- [Consolidation]({{}}) +- [Drift]({{}}) +- Expiration + +{{% alert title="Note" color="primary" %}} +Voluntary node removal does not include [Interruption]({{}}) or manual deletion initiated through `kubectl delete node`. Both of these are considered involuntary events, since node removal cannot be delayed. +{{% /alert %}} + +### Node-Level Controls + +You can block Karpenter from voluntarily choosing to disrupt certain nodes by setting the `karpenter.sh/do-not-disrupt: "true"` annotation on the node. This will prevent disruption actions on the node. + +```yaml +apiVersion: v1 +kind: Node +metadata: + annotations: + karpenter.sh/do-not-disrupt: "true" +``` + +#### Example: Disable Disruption on a NodePool + +NodePool `.spec.annotations` allow you to set annotations that will be applied to all nodes launched by this NodePool. By setting the annotation `karpenter.sh/do-not-disrupt: "true"` on the NodePool, you will selectively prevent all nodes launched by this NodePool from being considered in disruption actions. + +```yaml +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: default +spec: + template: + metadata: + annotations: # will be applied to all nodes + karpenter.sh/do-not-disrupt: "true" +``` diff --git a/website/content/en/v0.35/concepts/nodeclasses.md b/website/content/en/v0.35/concepts/nodeclasses.md new file mode 100644 index 000000000000..d426f579e7e6 --- /dev/null +++ b/website/content/en/v0.35/concepts/nodeclasses.md @@ -0,0 +1,1244 @@ + --- +title: "NodeClasses" +linkTitle: "NodeClasses" +weight: 2 +description: > + Configure AWS-specific settings with EC2NodeClasses +--- + +Node Classes enable configuration of AWS specific settings. +Each NodePool must reference an EC2NodeClass using `spec.template.spec.nodeClassRef`. +Multiple NodePools may point to the same EC2NodeClass. + +```yaml +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: default +spec: + template: + spec: + nodeClassRef: + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass + name: default +--- +apiVersion: karpenter.k8s.aws/v1beta1 +kind: EC2NodeClass +metadata: + name: default +spec: + # Required, resolves a default ami and userdata + amiFamily: AL2 + + # Required, discovers subnets to attach to instances + # Each term in the array of subnetSelectorTerms is ORed together + # Within a single term, all conditions are ANDed + subnetSelectorTerms: + # Select on any subnet that has the "karpenter.sh/discovery: ${CLUSTER_NAME}" + # AND the "environment: test" tag OR any subnet with ID "subnet-09fa4a0a8f233a921" + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" + environment: test + - id: subnet-09fa4a0a8f233a921 + + # Required, discovers security groups to attach to instances + # Each term in the array of securityGroupSelectorTerms is ORed together + # Within a single term, all conditions are ANDed + securityGroupSelectorTerms: + # Select on any security group that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any security group with the "my-security-group" name + # OR any security group with ID "sg-063d7acfb4b06c82c" + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" + environment: test + - name: my-security-group + - id: sg-063d7acfb4b06c82c + + # Optional, IAM role to use for the node identity. + # The "role" field is immutable after EC2NodeClass creation. This may change in the + # future, but this restriction is currently in place today to ensure that Karpenter + # avoids leaking managed instance profiles in your account. + # Must specify one of "role" or "instanceProfile" for Karpenter to launch nodes + role: "KarpenterNodeRole-${CLUSTER_NAME}" + + # Optional, IAM instance profile to use for the node identity. + # Must specify one of "role" or "instanceProfile" for Karpenter to launch nodes + instanceProfile: "KarpenterNodeInstanceProfile-${CLUSTER_NAME}" + + # Optional, discovers amis to override the amiFamily's default amis + # Each term in the array of amiSelectorTerms is ORed together + # Within a single term, all conditions are ANDed + amiSelectorTerms: + # Select on any AMI that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any AMI with the "my-ami" name + # OR any AMI with ID "ami-123" + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" + environment: test + - name: my-ami + - id: ami-123 + + # Optional, use instance-store volumes for node ephemeral-storage + instanceStorePolicy: RAID0 + + # Optional, overrides autogenerated userdata with a merge semantic + userData: | + echo "Hello world" + + # Optional, propagates tags to underlying EC2 resources + tags: + team: team-a + app: team-a-app + + # Optional, configures IMDS for the instance + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 2 + httpTokens: required + + # Optional, configures storage devices for the instance + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 100Gi + volumeType: gp3 + iops: 10000 + encrypted: true + kmsKeyID: "1234abcd-12ab-34cd-56ef-1234567890ab" + deleteOnTermination: true + throughput: 125 + snapshotID: snap-0123456789 + + # Optional, configures detailed monitoring for the instance + detailedMonitoring: true + + # Optional, configures if the instance should be launched with an associated public IP address. + # If not specified, the default value depends on the subnet's public IP auto-assign setting. + associatePublicIPAddress: true +status: + # Resolved subnets + subnets: + - id: subnet-0a462d98193ff9fac + zone: us-east-2b + - id: subnet-0322dfafd76a609b6 + zone: us-east-2c + - id: subnet-0727ef01daf4ac9fe + zone: us-east-2b + - id: subnet-00c99aeafe2a70304 + zone: us-east-2a + - id: subnet-023b232fd5eb0028e + zone: us-east-2c + - id: subnet-03941e7ad6afeaa72 + zone: us-east-2a + + # Resolved security groups + securityGroups: + - id: sg-041513b454818610b + name: ClusterSharedNodeSecurityGroup + - id: sg-0286715698b894bca + name: ControlPlaneSecurityGroup-1AQ073TSAAPW + + # Resolved AMIs + amis: + - id: ami-01234567890123456 + name: custom-ami-amd64 + requirements: + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - id: ami-01234567890123456 + name: custom-ami-arm64 + requirements: + - key: kubernetes.io/arch + operator: In + values: + - arm64 + + # Generated instance profile name from "role" + instanceProfile: "${CLUSTER_NAME}-0123456778901234567789" +``` +Refer to the [NodePool docs]({{}}) for settings applicable to all providers. To explore various `EC2NodeClass` configurations, refer to the examples provided [in the Karpenter Github repository](https://github.com/aws/karpenter/blob/main/examples/v1beta1/). + +## spec.amiFamily + +AMIFamily is a required field, dictating both the default bootstrapping logic for nodes provisioned through this `EC2NodeClass` but also selecting a group of recommended, latest AMIs by default. Currently, Karpenter supports `amiFamily` values `AL2`, `AL2023`, `Bottlerocket`, `Ubuntu`, `Windows2019`, `Windows2022` and `Custom`. GPUs are only supported by default with `AL2` and `Bottlerocket`. The `AL2` amiFamily does not support ARM64 GPU instance types unless you specify custom [`amiSelectorTerms`]({{}}). Default bootstrapping logic is shown below for each of the supported families. + +### AL2 + +```bash +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="//" + +--// +Content-Type: text/x-shellscript; charset="us-ascii" + +#!/bin/bash -xe +exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 +/etc/eks/bootstrap.sh 'test-cluster' --apiserver-endpoint 'https://test-cluster' --b64-cluster-ca 'ca-bundle' \ +--dns-cluster-ip '10.100.0.10' \ +--use-max-pods false \ +--kubelet-extra-args '--node-labels=karpenter.sh/capacity-type=on-demand,karpenter.sh/nodepool=test --max-pods=110' +--//-- +``` + +### AL2023 + +```text +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="//" + +--// +Content-Type: application/node.eks.aws + +# Karpenter Generated NodeConfig +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + cluster: + name: test-cluster + apiServerEndpoint: https://example.com + certificateAuthority: ca-bundle + cidr: 10.100.0.0/16 + kubelet: + config: + maxPods: 110 + flags: + - --node-labels=karpenter.sh/capacity-type=on-demand,karpenter.sh/nodepool=test + +--//-- + +``` + +### Bottlerocket + +```toml +[settings] +[settings.kubernetes] +api-server = 'https://test-cluster' +cluster-certificate = 'ca-bundle' +cluster-name = 'test-cluster' +cluster-dns-ip = '10.100.0.10' +max-pods = 110 + +[settings.kubernetes.node-labels] +'karpenter.sh/capacity-type' = 'on-demand' +'karpenter.sh/nodepool' = 'test' +``` + +### Ubuntu + +```bash +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="//" + +--// +Content-Type: text/x-shellscript; charset="us-ascii" + +#!/bin/bash -xe +exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 +/etc/eks/bootstrap.sh 'test-cluster' --apiserver-endpoint 'https://test-cluster' --b64-cluster-ca 'ca-bundle' \ +--dns-cluster-ip '10.100.0.10' \ +--use-max-pods false \ +--kubelet-extra-args '--node-labels="karpenter.sh/capacity-type=on-demand,karpenter.sh/nodepool=test" --max-pods=110' +--//-- +``` + +### Windows2019 + +```powershell + +[string]$EKSBootstrapScriptFile = "$env:ProgramFiles\Amazon\EKS\Start-EKSBootstrap.ps1" +& $EKSBootstrapScriptFile -EKSClusterName 'test-cluster' -APIServerEndpoint 'https://test-cluster' -Base64ClusterCA 'ca-bundle' -KubeletExtraArgs '--node-labels="karpenter.sh/capacity-type=on-demand,karpenter.sh/nodepool=test" --max-pods=110' -DNSClusterIP '10.100.0.10' + +``` + +### Windows2022 + +```powershell + +[string]$EKSBootstrapScriptFile = "$env:ProgramFiles\Amazon\EKS\Start-EKSBootstrap.ps1" +& $EKSBootstrapScriptFile -EKSClusterName 'test-cluster' -APIServerEndpoint 'https://test-cluster' -Base64ClusterCA 'ca-bundle' -KubeletExtraArgs '--node-labels="karpenter.sh/capacity-type=on-demand,karpenter.sh/nodepool=test" --max-pods=110' -DNSClusterIP '10.100.0.10' + +``` + +{{% alert title="Note" color="primary" %}} +Karpenter will automatically query for the appropriate [EKS optimized AMI](https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-amis.html) via AWS Systems Manager (SSM). In the case of the `Custom` AMIFamily, no default AMIs are defined. As a result, `amiSelectorTerms` must be specified to inform Karpenter on which custom AMIs are to be used. +{{% /alert %}} + +### Custom + +The `Custom` AMIFamily ships without any default userData to allow you to configure custom bootstrapping for control planes or images that don't support the default methods from the other families. + +## spec.subnetSelectorTerms + +Subnet Selector Terms allow you to specify selection logic for a set of subnet options that Karpenter can choose from when launching an instance from the `EC2NodeClass`. Karpenter discovers subnets through the `EC2NodeClass` using ids or [tags](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html). When launching nodes, a subnet is automatically chosen that matches the desired zone. If multiple subnets exist for a zone, the one with the most available IP addresses will be used. + +This selection logic is modeled as terms, where each term contains multiple conditions that must all be satisfied for the selector to match. Effectively, all requirements within a single term are ANDed together. It's possible that you may want to select on two different subnets that have unrelated requirements. In this case, you can specify multiple terms which will be ORed together to form your selection logic. The example below shows how this selection logic is fulfilled. + +```yaml +subnetSelectorTerms: + # Select on any subnet that has the "karpenter.sh/discovery: ${CLUSTER_NAME}" + # AND the "environment: test" tag OR any subnet with ID "subnet-09fa4a0a8f233a921" + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" + environment: test + - id: subnet-09fa4a0a8f233a921 +``` + +{{% alert title="Tip" color="secondary" %}} +Subnets may be specified by any tag, including `Name`. Selecting tag values using wildcards (`*`) is supported. +{{% /alert %}} + +#### Examples + +Select all with a specified tag key: +```yaml +spec: + subnetSelectorTerms: + - tags: + karpenter.sh/discovery/MyClusterName: '*' +``` + +Select by name and tag (all criteria must match): +```yaml +spec: + subnetSelectorTerms: + - tags: + Name: my-subnet + MyTag: '' # matches all resources with the tag +``` + +Select using multiple tag terms: +```yaml +spec: + subnetSelectorTerms: + - tags: + Name: "my-subnet-1" + - tags: + Name: "my-subnet-2" +``` + +Select using wildcards: +```yaml +spec: + subnetSelectorTerms: + - tags: + Name: "*Public*" + +``` + +Select using ids: +```yaml +spec: + subnetSelectorTerms: + - id: "subnet-09fa4a0a8f233a921" + - id: "subnet-0471ca205b8a129ae" +``` + + +## spec.securityGroupSelectorTerms + +Security Group Selector Terms allow you to specify selection logic for all security groups that will be attached to an instance launched from the `EC2NodeClass`. The security group of an instance is comparable to a set of firewall rules. +[EKS creates at least two security groups by default](https://docs.aws.amazon.com/eks/latest/userguide/sec-group-reqs.html). + +This selection logic is modeled as terms, where each term contains multiple conditions that must all be satisfied for the selector to match. Effectively, all requirements within a single term are ANDed together. It's possible that you may want to select on two different security groups that have unrelated requirements. In this case, you can specify multiple terms which will be ORed together to form your selection logic. The example below shows how this selection logic is fulfilled. + +```yaml +securityGroupSelectorTerms: + # Select on any security group that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any security group with the "my-security-group" name + # OR any security group with ID "sg-063d7acfb4b06c82c" + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" + environment: test + - name: my-security-group + - id: sg-063d7acfb4b06c82c +``` + +{{% alert title="Tip" color="secondary" %}} +Security groups may be specified by any tag, including "Name". Selecting tags using wildcards (`*`) is supported. +{{% /alert %}} + +{{% alert title="Note" color="primary" %}} +When launching nodes, Karpenter uses all the security groups that match the selector. If you choose to use the `kubernetes.io/cluster/$CLUSTER_NAME` tag for discovery, note that this may result in failures using the AWS Load Balancer controller. The Load Balancer controller only supports a single security group having that tag key. See [this issue](https://github.com/kubernetes-sigs/aws-load-balancer-controller/issues/2367) for more details. + +To verify if this restriction affects you, run the following commands. +```bash +CLUSTER_VPC_ID="$(aws eks describe-cluster --name $CLUSTER_NAME --query cluster.resourcesVpcConfig.vpcId --output text)" + +aws ec2 describe-security-groups --filters Name=vpc-id,Values=$CLUSTER_VPC_ID Name=tag-key,Values=kubernetes.io/cluster/$CLUSTER_NAME --query 'SecurityGroups[].[GroupName]' --output text +``` + +If multiple securityGroups are printed, you will need more specific securityGroupSelectorTerms. We generally recommend that you use the `karpenter.sh/discovery: $CLUSTER_NAME` tag selector instead. +{{% /alert %}} + +#### Examples + +Select all assigned to a cluster: +```yaml +spec: + securityGroupSelectorTerms: + - tags: + kubernetes.io/cluster/$CLUSTER_NAME: "owned" +``` + +Select all with a specified tag key: +```yaml +spec: + securityGroupSelectorTerms: + - tags: + MyTag: '*' +``` + +Select by name and tag (all criteria must match): +```yaml +spec: + securityGroupSelectorTerms: + - name: my-security-group + tags: + MyTag: '*' # matches all resources with the tag +``` + +Select using multiple tag terms: +```yaml +spec: + securityGroupSelectorTerms: + - tags: + Name: "my-security-group-1" + - tags: + Name: "my-security-group-2" +``` + +Select by name using a wildcard: +```yaml +spec: + securityGroupSelectorTerms: + - name: "*Public*" +``` + +Select using ids: +```yaml +spec: + securityGroupSelectorTerms: + - id: "sg-063d7acfb4b06c82c" + - id: "sg-06e0cf9c198874591" +``` + +## spec.amiSelectorTerms + +AMI Selector Terms are used to configure custom AMIs for Karpenter to use, where the AMIs are discovered through ids, owners, name, and [tags](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/Using_Tags.html). **When you specify `amiSelectorTerms`, you fully override the default AMIs that are selected on by your EC2NodeClass [`amiFamily`]({{< ref "#specamifamily" >}}).** + +This selection logic is modeled as terms, where each term contains multiple conditions that must all be satisfied for the selector to match. Effectively, all requirements within a single term are ANDed together. It's possible that you may want to select on two different AMIs that have unrelated requirements. In this case, you can specify multiple terms which will be ORed together to form your selection logic. The example below shows how this selection logic is fulfilled. + +```yaml +amiSelectorTerms: + # Select on any AMI that has both the "karpenter.sh/discovery: ${CLUSTER_NAME}" tag + # AND the "environment: test" tag OR any AMI with the "my-ami" name + # OR any AMI with ID "ami-123" + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" + environment: test + - name: my-ami + - id: ami-123 +``` + +This field is optional, and Karpenter will use the latest EKS-optimized AMIs for the AMIFamily if no amiSelectorTerms are specified. To select an AMI by name, use the `name` field in the selector term. To select an AMI by id, use the `id` field in the selector term. To ensure that AMIs are owned by the expected owner, use the `owner` field - you can use a combination of account aliases (e.g. `self` `amazon`, `your-aws-account-name`) and account IDs. + +If owner is not set for `name`, it defaults to `self,amazon`, preventing Karpenter from inadvertently selecting an AMI that is owned by a different account. Tags don't require an owner as tags can only be discovered by the user who created them. + +{{% alert title="Tip" color="secondary" %}} +AMIs may be specified by any AWS tag, including `Name`. Selecting by tag or by name using wildcards (`*`) is supported. +{{% /alert %}} + +{{% alert title="Note" color="primary" %}} +If `amiSelectorTerms` match more than one AMI, Karpenter will automatically determine which AMI best fits the workloads on the launched worker node under the following constraints: + +* When launching nodes, Karpenter automatically determines which architecture a custom AMI is compatible with and will use images that match an instanceType's requirements. + * Note that Karpenter **cannot** detect any requirement other than architecture. If you need to specify different AMIs for different kind of nodes (e.g. accelerated GPU AMIs), you should use a separate `EC2NodeClass`. +* If multiple AMIs are found that can be used, Karpenter will choose the latest one. +* If no AMIs are found that can be used, then no nodes will be provisioned. +{{% /alert %}} + +#### Examples + +Select all with a specified tag: +```yaml + amiSelectorTerms: + - tags: + karpenter.sh/discovery/MyClusterName: '*' +``` + +Select by name: +```yaml + amiSelectorTerms: + - name: my-ami +``` + +Select by `Name` tag: +```yaml + amiSelectorTerms: + - tags: + Name: my-ami +``` + +Select by name and owner: +```yaml + amiSelectorTerms: + - name: my-ami + owner: self + - name: my-ami + owner: 0123456789 +``` + +Select by name using a wildcard: +```yaml +spec: + amiSelectorTerms: + - name: "*EKS*" +``` + +Select by all under an owner: +```yaml +spec: + amiSelectorTerms: + - name: "*" + owner: self +``` + +Specify using ids: +```yaml + amiSelectorTerms: + - id: "ami-123" + - id: "ami-456" +``` + +## spec.role + +`Role` is an optional field and tells Karpenter which IAM identity nodes should assume. You must specify one of `role` or `instanceProfile` when creating a Karpenter `EC2NodeClass`. If using the [Karpenter Getting Started Guide]({{}}) to deploy Karpenter, you can use the `KarpenterNodeRole-$CLUSTER_NAME` role provisioned by that process. + +```yaml +spec: + role: "KarpenterNodeRole-$CLUSTER_NAME" +``` + +## spec.instanceProfile + +`InstanceProfile` is an optional field and tells Karpenter which IAM identity nodes should assume. You must specify one of `role` or `instanceProfile` when creating a Karpenter `EC2NodeClass`. If you use the `instanceProfile` field instead of `role`, Karpenter will not manage the InstanceProfile on your behalf; instead, it expects that you have pre-provisioned an IAM instance profile and assigned it a role. + +You can provision and assign a role to an IAM instance profile using [CloudFormation](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-iam-instanceprofile.html) or by using the [`aws iam create-instance-profile`](https://docs.aws.amazon.com/cli/latest/reference/iam/create-instance-profile.html) and [`aws iam add-role-to-instance-profile`](https://docs.aws.amazon.com/cli/latest/reference/iam/add-role-to-instance-profile.html) commands in the CLI. + +{{% alert title="Note" color="primary" %}} + +For [private clusters](https://docs.aws.amazon.com/eks/latest/userguide/private-clusters.html) that do not have access to the public internet, using `spec.instanceProfile` is required. `spec.role` cannot be used since Karpenter needs to access IAM endpoints to manage a generated instance profile. IAM [doesn't support private endpoints](https://docs.aws.amazon.com/vpc/latest/privatelink/aws-services-privatelink-support.html) to enable accessing the service without going to the public internet. + +{{% /alert %}} + +## spec.tags + +Karpenter adds tags to all resources it creates, including EC2 Instances, EBS volumes, and Launch Templates. The default set of tags are listed below. + +```yaml +Name: +karpenter.sh/nodeclaim: +karpenter.sh/nodepool: +karpenter.k8s.aws/ec2nodeclass: +kubernetes.io/cluster/: owned +``` + +Additional tags can be added in the tags section, which will be merged with the default tags specified above. +```yaml +spec: + tags: + InternalAccountingTag: 1234 + dev.corp.net/app: Calculator + dev.corp.net/team: MyTeam +``` + +{{% alert title="Note" color="primary" %}} +Karpenter allows overrides of the default "Name" tag but does not allow overrides to restricted domains (such as "karpenter.sh", "karpenter.k8s.aws", and "kubernetes.io/cluster"). This ensures that Karpenter is able to correctly auto-discover nodes that it owns. +{{% /alert %}} + +## spec.metadataOptions + +Control the exposure of [Instance Metadata Service](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) on EC2 Instances launched by this EC2NodeClass using a generated launch template. + +Refer to [recommended, security best practices](https://aws.github.io/aws-eks-best-practices/security/docs/iam/#restrict-access-to-the-instance-profile-assigned-to-the-worker-node) for limiting exposure of Instance Metadata and User Data to pods. + +If metadataOptions are omitted from this EC2NodeClass, the following default settings are applied: + +```yaml +spec: + metadataOptions: + httpEndpoint: enabled + httpProtocolIPv6: disabled + httpPutResponseHopLimit: 2 + httpTokens: required +``` + +## spec.blockDeviceMappings + +The `blockDeviceMappings` field in an `EC2NodeClass` can be used to control the [Elastic Block Storage (EBS) volumes](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/block-device-mapping-concepts.html#instance-block-device-mapping) that Karpenter attaches to provisioned nodes. Karpenter uses default block device mappings for the AMIFamily specified. For example, the `Bottlerocket` AMI Family defaults with two block device mappings, one for Bottlerocket's control volume and the other for container resources such as images and logs. + +```yaml +spec: + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 100Gi + volumeType: gp3 + iops: 10000 + encrypted: true + kmsKeyID: "1234abcd-12ab-34cd-56ef-1234567890ab" + deleteOnTermination: true + throughput: 125 + snapshotID: snap-0123456789 +``` + +The following blockDeviceMapping defaults are used for each `AMIFamily` if no `blockDeviceMapping` overrides are specified in the `EC2NodeClass` + +### AL2 +```yaml +spec: + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 20Gi + volumeType: gp3 + encrypted: true +``` + +### AL2023 +```yaml +spec: + blockDeviceMappings: + - deviceName: /dev/xvda + ebs: + volumeSize: 20Gi + volumeType: gp3 + encrypted: true +``` + +### Bottlerocket +```yaml +spec: + blockDeviceMappings: + # Root device + - deviceName: /dev/xvda + ebs: + volumeSize: 4Gi + volumeType: gp3 + encrypted: true + # Data device: Container resources such as images and logs + - deviceName: /dev/xvdb + ebs: + volumeSize: 20Gi + volumeType: gp3 + encrypted: true +``` + +### Ubuntu +```yaml +spec: + blockDeviceMappings: + - deviceName: /dev/sda1 + ebs: + volumeSize: 20Gi + volumeType: gp3 + encrypted: true +``` + +### Windows2019/Windows2022 +```yaml +spec: + blockDeviceMappings: + - deviceName: /dev/sda1 + ebs: + volumeSize: 50Gi + volumeType: gp3 + encrypted: true +``` + +### Custom + +The `Custom` AMIFamily ships without any default `blockDeviceMappings`. + +## spec.instanceStorePolicy + +The `instanceStorePolicy` field controls how [instance-store](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/InstanceStorage.html) volumes are handled. By default, Karpenter and Kubernetes will simply ignore them. + +### RAID0 + +If you intend to use these volumes for faster node ephemeral-storage, set `instanceStorePolicy` to `RAID0`: + +```yaml +spec: + instanceStorePolicy: RAID0 +``` + +This will set the allocatable ephemeral-storage of each node to the total size of the instance-store volume(s). + +The disks must be formatted & mounted in a RAID0 and be the underlying filesystem for the Kubelet & Containerd. Instructions for each AMI family are listed below: + +#### AL2 + +On AL2, Karpenter automatically configures the disks through an additional boostrap argument (`--local-disks raid0`). The device name is `/dev/md/0` and its mount point is `/mnt/k8s-disks/0`. You should ensure any additional disk setup does not interfere with these. + +#### AL2023 + +On AL2023, Karpenter automatically configures the disks via the generated `NodeConfig` object. Like AL2, the device name is `/dev/md/0` and its mount point is `/mnt/k8s-disks/0`. You should ensure any additional disk setup does not interfere with these. + +#### Others + +For all other AMI families, you must configure the disks yourself. Check out the [`setup-local-disks`](https://github.com/awslabs/amazon-eks-ami/blob/master/files/bin/setup-local-disks) script in [amazon-eks-ami](https://github.com/awslabs/amazon-eks-ami) to see how this is done for AL2. + +{{% alert title="Tip" color="secondary" %}} +Since the Kubelet & Containerd will be using the instance-store filesystem, you may consider using a more minimal root volume size. +{{% /alert %}} + +## spec.userData + +You can control the UserData that is applied to your worker nodes via this field. This allows you to run custom scripts or pass-through custom configuration to Karpenter instances on start-up. + +```yaml +apiVersion: karpenter.k8s.aws/v1beta1 +kind: EC2NodeClass +metadata: + name: bottlerocket-example +spec: + ... + amiFamily: Bottlerocket + userData: | + [settings.kubernetes] + "kube-api-qps" = 30 + "shutdown-grace-period" = "30s" + "shutdown-grace-period-for-critical-pods" = "30s" + [settings.kubernetes.eviction-hard] + "memory.available" = "20%" +``` + +This example adds SSH keys to allow remote login to the node (replace *my-authorized_keys* with your key file): + +{{% alert title="Note" color="primary" %}} +Instead of using SSH as set up in this example, you can use Session Manager (SSM) or EC2 Instance Connect to gain shell access to Karpenter nodes. +See [Node NotReady]({{< ref "../troubleshooting/#node-notready" >}}) troubleshooting for an example of starting an SSM session from the command line or [EC2 Instance Connect](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-connect-set-up.html) documentation to connect to nodes using SSH. +{{% /alert %}} + +```yaml +apiVersion: karpenter.k8s.aws/v1beta1 +kind: EC2NodeClass +metadata: + name: al2-example +spec: + ... + amiFamily: AL2 + userData: | + #!/bin/bash + mkdir -p ~ec2-user/.ssh/ + touch ~ec2-user/.ssh/authorized_keys + cat >> ~ec2-user/.ssh/authorized_keys <}}) section for more details on these defaults. View the sections below to understand the different merge strategies for each AMIFamily. + +### AL2/Ubuntu + +* Your UserData can be in the [MIME multi part archive](https://cloudinit.readthedocs.io/en/latest/topics/format.html#mime-multi-part-archive) format. +* Karpenter will transform your custom user-data as a MIME part, if necessary, and then merge a final MIME part to the end of your UserData parts which will bootstrap the worker node. Karpenter will have full control over all the parameters being passed to the bootstrap script. + * Karpenter will continue to set MaxPods, ClusterDNS and all other parameters defined in `spec.kubeletConfiguration` as before. + +Consider the following example to understand how your custom UserData will be merged - + +#### Passed-in UserData (bash) + +```bash +#!/bin/bash +echo "Running custom user data script (bash)" +``` + +#### Merged UserData (bash) + +```bash +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="//" + +--// +Content-Type: text/x-shellscript; charset="us-ascii" + +#!/bin/bash +echo "Running custom user data script (bash)" + +--// +Content-Type: text/x-shellscript; charset="us-ascii" + +#!/bin/bash -xe +exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 +/etc/eks/bootstrap.sh 'test-cluster' --apiserver-endpoint 'https://test-cluster' --b64-cluster-ca 'ca-bundle' \ +--use-max-pods false \ +--kubelet-extra-args '--node-labels=karpenter.sh/capacity-type=on-demand,karpenter.sh/nodepool=test --max-pods=110' +--//-- +``` + +#### Passed-in UserData (MIME) + +```bash +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="BOUNDARY" + +--BOUNDARY +Content-Type: text/x-shellscript; charset="us-ascii" + +#!/bin/bash +echo "Running custom user data script (mime)" + +--BOUNDARY-- +``` + +#### Merged UserData (MIME) + +```bash +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="//" + +--// +Content-Type: text/x-shellscript; charset="us-ascii" + +#!/bin/bash +echo "Running custom user data script (mime)" + +--// +Content-Type: text/x-shellscript; charset="us-ascii" + +#!/bin/bash -xe +exec > >(tee /var/log/user-data.log|logger -t user-data -s 2>/dev/console) 2>&1 +/etc/eks/bootstrap.sh 'test-cluster' --apiserver-endpoint 'https://test-cluster' --b64-cluster-ca 'ca-bundle' \ +--use-max-pods false \ +--kubelet-extra-args '--node-labels=karpenter.sh/capacity-type=on-demand,karpenter.sh/nodepool=test --max-pods=110' +--//-- +``` + +{{% alert title="Note" color="primary" %}} +You can also set kubelet-config properties by modifying the kubelet-config.json file before the EKS bootstrap script starts the kubelet: + +```yaml +apiVersion: karpenter.k8s.aws/v1beta1 +kind: EC2NodeClass +metadata: + name: kubelet-config-example +spec: + ... + amiFamily: AL2 + userData: | + #!/bin/bash + echo "$(jq '.kubeAPIQPS=50' /etc/kubernetes/kubelet/kubelet-config.json)" > /etc/kubernetes/kubelet/kubelet-config.json +``` +{{% /alert %}} + +### AL2023 + +* Your UserData may be in one of three formats: a [MIME multi part archive](https://cloudinit.readthedocs.io/en/latest/topics/format.html#mime-multi-part-archive), a NodeConfig YAML / JSON string, or a shell script. +* Karpenter will transform your custom UserData into a MIME part, if necessary, and then create a MIME multi-part archive. This archive will consist of a generated NodeConfig, containing Karpenter's default values, followed by the transformed custom UserData. For more information on the NodeConfig spec, refer to the [AL2023 EKS Optimized AMI docs](https://awslabs.github.io/amazon-eks-ami/nodeadm/doc/examples/). +* If a value is specified both in the Karpenter generated NodeConfig and the same value is specified in the custom user data, the value in the custom user data will take precedence. + +#### Passed-in UserData (NodeConfig) + +```yaml +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + kubelet: + config: + maxPods: 42 +``` + +#### Merged UserData (NodeConfig) + +```text +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="//" + +--// +# Karpenter Generated NodeConfig +Content-Type: application/node.eks.aws + +# Karpenter Generated NodeConfig +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + cluster: + apiServerEndpoint: https://test-cluster + certificateAuthority: cluster-ca + cidr: 10.100.0.0/16 + name: test-cluster + kubelet: + config: + clusterDNS: + - 10.100.0.10 + maxPods: 118 + flags: + - --node-labels="karpenter.sh/capacity-type=on-demand,karpenter.sh/nodepool=default" + +--// +Content-Type: application/node.eks.aws + +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + kubelet: + config: + maxPods: 42 +--//-- +``` + +#### Passed-in UserData (bash) + +```shell +#!/bin/bash +echo "Hello, AL2023!" +``` + +#### Merged UserData (bash) + +```text +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="//" + +--// +Content-Type: application/node.eks.aws + +# Karpenter Generated NodeConfig +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + cluster: + apiServerEndpoint: https://test-cluster + certificateAuthority: cluster-ca + cidr: 10.100.0.0/16 + name: test-cluster + kubelet: + config: + clusterDNS: + - 10.100.0.10 + maxPods: 118 + flags: + - --node-labels="karpenter.sh/capacity-type=on-demand,karpenter.sh/nodepool=default" + +--// +Content-Type: text/x-shellscript; charset="us-ascii" + +#!/bin/bash +echo "Hello, AL2023!" +--//-- +``` + +#### Passed-in UserData (MIME) + +```text +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="//" + +--// +Content-Type: application/node.eks.aws + +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + kubelet: + config: + maxPods: 42 +--// +Content-Type: text/x-shellscript; charset="us-ascii" + +#!/bin/bash +echo "Hello, AL2023!" +--// +``` + +#### Merged UserData (MIME) + +```text +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="//" + +--// +Content-Type: application/node.eks.aws + +# Karpenter Generated NodeConfig +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + cluster: + apiServerEndpoint: https://test-cluster + certificateAuthority: cluster-ca + cidr: 10.100.0.0/16 + name: test-cluster + kubelet: + config: + clusterDNS: + - 10.100.0.10 + maxPods: 118 + flags: + - --node-labels="karpenter.sh/capacity-type=on-demand,karpenter.sh/nodepool=default" + +--// +Content-Type: application/node.eks.aws + +apiVersion: node.eks.aws/v1alpha1 +kind: NodeConfig +spec: + kubelet: + config: + maxPods: 42 +--// +Content-Type: text/x-shellscript; charset="us-ascii" + +#!/bin/bash +echo "Hello, AL2023!" +--//-- +``` + +### Bottlerocket + +* Your UserData must be valid TOML. +* Karpenter will automatically merge settings to ensure successful bootstrap including `cluster-name`, `api-server` and `cluster-certificate`. Any labels and taints that need to be set based on pod requirements will also be specified in the final merged UserData. + * All Kubelet settings that Karpenter applies will override the corresponding settings in the provided UserData. For example, if you've specified `settings.kubernetes.cluster-name`, it will be overridden. + * If MaxPods is specified via the binary arg to Karpenter, the value will override anything specified in the UserData. + * If ClusterDNS is specified via `spec.kubeletConfiguration`, then that value will override anything specified in the UserData. +* Unknown TOML fields will be ignored when the final merged UserData is generated by Karpenter. + +Consider the following example to understand how your custom UserData settings will be merged in. + +#### Passed-in UserData + +```toml +[settings.kubernetes.eviction-hard] +"memory.available" = "12%" +[settings.kubernetes] +"unknown-setting" = "unknown" +[settings.kubernetes.node-labels] +'field.controlled.by/karpenter' = 'will-be-overridden' +``` + +#### Merged UserData + +```toml +[settings] +[settings.kubernetes] +api-server = 'https://cluster' +cluster-certificate = 'ca-bundle' +cluster-name = 'cluster' + +[settings.kubernetes.node-labels] +'karpenter.sh/capacity-type' = 'on-demand' +'karpenter.sh/nodepool' = 'default' + +[settings.kubernetes.node-taints] + +[settings.kubernetes.eviction-hard] +'memory.available' = '12%%' +``` + +### Windows2019/Windows2022 + +* Your UserData must be specified as PowerShell commands. +* The UserData specified will be prepended to a Karpenter managed section that will bootstrap the kubelet. +* Karpenter will continue to set ClusterDNS and all other parameters defined in spec.kubeletConfiguration as before. + +Consider the following example to understand how your custom UserData settings will be merged in. + +#### Passed-in UserData + +```powershell +Write-Host "Running custom user data script" +``` + +#### Merged UserData + +```powershell + +Write-Host "Running custom user data script" +[string]$EKSBootstrapScriptFile = "$env:ProgramFiles\Amazon\EKS\Start-EKSBootstrap.ps1" +& $EKSBootstrapScriptFile -EKSClusterName 'test-cluster' -APIServerEndpoint 'https://test-cluster' -Base64ClusterCA 'ca-bundle' -KubeletExtraArgs '--node-labels="karpenter.sh/capacity-type=spot,karpenter.sh/nodepool=windows2022" --max-pods=110' -DNSClusterIP '10.0.100.10' + +``` + +{{% alert title="Windows Support Notice" color="warning" %}} +Currently, Karpenter does not specify `-ServiceCIDR` to [EKS Windows AMI Bootstrap script](https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-windows-ami.html#bootstrap-script-configuration-parameters). +Windows worker nodes will use `172.20.0.0/16` or `10.100.0.0/16` for Kubernetes service IP address ranges based on the IP address of the primary interface. +The effective ServiceCIDR can be verified at `$env:ProgramData\Amazon\EKS\cni\config\vpc-bridge.conf` on the worker node. + +Support for the Windows ServiceCIDR argument can be tracked in a [Karpenter Github Issue](https://github.com/aws/karpenter/issues/4088). Currently, if the effective ServiceCIDR is incorrect for your windows worker nodes, you can add the following userData as a workaround. + +```yaml +spec: + userData: | + $global:EKSCluster = Get-EKSCluster -Name my-cluster +``` +{{% /alert %}} + +### Custom + +* No merging is performed, your UserData must perform all setup required of the node to allow it to join the cluster. + +## spec.detailedMonitoring + +Enabling detailed monitoring controls the [EC2 detailed monitoring](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-cloudwatch-new.html) feature. If you enable this option, the Amazon EC2 console displays monitoring graphs with a 1-minute period for the instances that Karpenter launches. + +```yaml +spec: + detailedMonitoring: true +``` + +## spec.associatePublicIPAddress + +A boolean field that controls whether instances created by Karpenter for this EC2NodeClass will have an associated public IP address. This overrides the `MapPublicIpOnLaunch` setting applied to the subnet the node is launched in. If this field is not set, the `MapPublicIpOnLaunch` field will be respected. + +{{% alert title="Note" color="warning" %}} +If a `NodeClaim` requests `vpc.amazonaws.com/efa` resources, `spec.associatePublicIPAddress` is respected. However, if this `NodeClaim` requests **multiple** EFA resources and the value for `spec.associatePublicIPAddress` is true, the instance will fail to launch. This is due to an EC2 restriction which +requires that the field is only set to true when configuring an instance with a single ENI at launch. When using this field, it is advised that users segregate their EFA workload to use a separate `NodePool` / `EC2NodeClass` pair. +{{% /alert %}} + +## status.subnets +[`status.subnets`]({{< ref "#statussubnets" >}}) contains the resolved `id` and `zone` of the subnets that were selected by the [`spec.subnetSelectorTerms`]({{< ref "#specsubnetselectorterms" >}}) for the node class. The subnets will be sorted by the available IP address count in decreasing order. + +#### Examples + +```yaml +spec: + subnetSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" +status: + subnets: + - id: subnet-0a462d98193ff9fac + zone: us-east-2b + - id: subnet-0322dfafd76a609b6 + zone: us-east-2c + - id: subnet-0727ef01daf4ac9fe + zone: us-east-2b + - id: subnet-00c99aeafe2a70304 + zone: us-east-2a + - id: subnet-023b232fd5eb0028e + zone: us-east-2c + - id: subnet-03941e7ad6afeaa72 + zone: us-east-2a +``` + +## status.securityGroups + +[`status.securityGroups`]({{< ref "#statussecuritygroups" >}}) contains the resolved `id` and `name` of the security groups that were selected by the [`spec.securityGroupSelectorTerms`]({{< ref "#specsecuritygroupselectorterms" >}}) for the node class. The subnets will be sorted by the available IP address count in decreasing order. + +#### Examples + +```yaml +spec: + securityGroupSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" +status: + securityGroups: + - id: sg-041513b454818610b + name: ClusterSharedNodeSecurityGroup + - id: sg-0286715698b894bca + name: ControlPlaneSecurityGroup-1AQ073TSAAPW +``` + +## status.amis + +[`status.amis`]({{< ref "#statusamis" >}}) contains the resolved `id`, `name`, and `requirements` of either the default AMIs for the [`spec.amiFamily`]({{< ref "#specamifamily" >}}) or the AMIs selected by the [`spec.amiSelectorTerms`]({{< ref "#specamiselectorterms" >}}) if this field is specified. + +#### Examples + +Default AMIs resolved from the AL2 AMIFamily: + +```yaml +spec: + amiFamily: AL2 +status: + amis: + - id: ami-03c3a3dcda64f5b75 + name: amazon-linux-2-gpu + requirements: + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: karpenter.k8s.aws/instance-gpu-count + operator: Exists + - id: ami-03c3a3dcda64f5b75 + name: amazon-linux-2-gpu + requirements: + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: karpenter.k8s.aws/instance-accelerator-count + operator: Exists + - id: ami-06afb2d101cc4b8bd + name: amazon-linux-2-arm64 + requirements: + - key: kubernetes.io/arch + operator: In + values: + - arm64 + - key: karpenter.k8s.aws/instance-gpu-count + operator: DoesNotExist + - key: karpenter.k8s.aws/instance-accelerator-count + operator: DoesNotExist + - id: ami-0e28b76d768af234e + name: amazon-linux-2 + requirements: + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - key: karpenter.k8s.aws/instance-gpu-count + operator: DoesNotExist + - key: karpenter.k8s.aws/instance-accelerator-count + operator: DoesNotExist +``` + +AMIs resolved from [`spec.amiSelectorTerms`]({{< ref "#specamiselectorterms" >}}): + +```yaml +spec: + amiFamily: AL2 + amiSelectorTerms: + - tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" +status: + amis: + - id: ami-01234567890123456 + name: custom-ami-amd64 + requirements: + - key: kubernetes.io/arch + operator: In + values: + - amd64 + - id: ami-01234567890123456 + name: custom-ami-arm64 + requirements: + - key: kubernetes.io/arch + operator: In + values: + - arm64 +``` + +## status.instanceProfile + +[`status.instanceProfile`]({{< ref "#statusinstanceprofile" >}}) contains the resolved instance profile generated by Karpenter from the [`spec.role`]({{< ref "#specrole" >}}) + +```yaml +spec: + role: "KarpenterNodeRole-${CLUSTER_NAME}" +status: + instanceProfile: "${CLUSTER_NAME}-0123456778901234567789" +``` diff --git a/website/content/en/v0.35/concepts/nodepools.md b/website/content/en/v0.35/concepts/nodepools.md new file mode 100644 index 000000000000..d679bb5f4373 --- /dev/null +++ b/website/content/en/v0.35/concepts/nodepools.md @@ -0,0 +1,573 @@ +--- +title: "NodePools" +linkTitle: "NodePools" +weight: 1 +description: > + Configure Karpenter with NodePools +--- + +When you first installed Karpenter, you set up a default NodePool. The NodePool sets constraints on the nodes that can be created by Karpenter and the pods that can run on those nodes. The NodePool can be set to do things like: + +* Define taints to limit the pods that can run on nodes Karpenter creates +* Define any startup taints to inform Karpenter that it should taint the node initially, but that the taint is temporary. +* Limit node creation to certain zones, instance types, and computer architectures +* Set defaults for node expiration + +You can change your NodePool or add other NodePools to Karpenter. +Here are things you should know about NodePools: + +* Karpenter won't do anything if there is not at least one NodePool configured. +* Each NodePool that is configured is looped through by Karpenter. +* If Karpenter encounters a taint in the NodePool that is not tolerated by a Pod, Karpenter won't use that NodePool to provision the pod. +* If Karpenter encounters a startup taint in the NodePool it will be applied to nodes that are provisioned, but pods do not need to tolerate the taint. Karpenter assumes that the taint is temporary and some other system will remove the taint. +* It is recommended to create NodePools that are mutually exclusive. So no Pod should match multiple NodePools. If multiple NodePools are matched, Karpenter will use the NodePool with the highest [weight](#specweight). + +For some example `NodePool` configurations, see the [examples in the Karpenter GitHub repository](https://github.com/aws/karpenter/blob/main/examples/v1beta1/). + +```yaml +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: default +spec: + # Template section that describes how to template out NodeClaim resources that Karpenter will provision + # Karpenter will consider this template to be the minimum requirements needed to provision a Node using this NodePool + # It will overlay this NodePool with Pods that need to schedule to further constrain the NodeClaims + # Karpenter will provision to launch new Nodes for the cluster + template: + metadata: + # Labels are arbitrary key-values that are applied to all nodes + labels: + billing-team: my-team + + # Annotations are arbitrary key-values that are applied to all nodes + annotations: + example.com/owner: "my-team" + spec: + # References the Cloud Provider's NodeClass resource, see your cloud provider specific documentation + nodeClassRef: + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass + name: default + + # Provisioned nodes will have these taints + # Taints may prevent pods from scheduling if they are not tolerated by the pod. + taints: + - key: example.com/special-taint + effect: NoSchedule + + # Provisioned nodes will have these taints, but pods do not need to tolerate these taints to be provisioned by this + # NodePool. These taints are expected to be temporary and some other entity (e.g. a DaemonSet) is responsible for + # removing the taint after it has finished initializing the node. + startupTaints: + - key: example.com/another-taint + effect: NoSchedule + + # Requirements that constrain the parameters of provisioned nodes. + # These requirements are combined with pod.spec.topologySpreadConstraints, pod.spec.affinity.nodeAffinity, pod.spec.affinity.podAffinity, and pod.spec.nodeSelector rules. + # Operators { In, NotIn, Exists, DoesNotExist, Gt, and Lt } are supported. + # https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#operators + requirements: + - key: "karpenter.k8s.aws/instance-category" + operator: In + values: ["c", "m", "r"] + # minValues here enforces the scheduler to consider at least that number of unique instance-category to schedule the pods. + # This field is ALPHA and can be dropped or replaced at any time + minValues: 2 + - key: "karpenter.k8s.aws/instance-family" + operator: In + values: ["m5","m5d","c5","c5d","c4","r4"] + minValues: 5 + - key: "karpenter.k8s.aws/instance-cpu" + operator: In + values: ["4", "8", "16", "32"] + - key: "karpenter.k8s.aws/instance-hypervisor" + operator: In + values: ["nitro"] + - key: "karpenter.k8s.aws/instance-generation" + operator: Gt + values: ["2"] + - key: "topology.kubernetes.io/zone" + operator: In + values: ["us-west-2a", "us-west-2b"] + - key: "kubernetes.io/arch" + operator: In + values: ["arm64", "amd64"] + - key: "karpenter.sh/capacity-type" + operator: In + values: ["spot", "on-demand"] + + # Karpenter provides the ability to specify a few additional Kubelet args. + # These are all optional and provide support for additional customization and use cases. + kubelet: + clusterDNS: ["10.0.1.100"] + systemReserved: + cpu: 100m + memory: 100Mi + ephemeral-storage: 1Gi + kubeReserved: + cpu: 200m + memory: 100Mi + ephemeral-storage: 3Gi + evictionHard: + memory.available: 5% + nodefs.available: 10% + nodefs.inodesFree: 10% + evictionSoft: + memory.available: 500Mi + nodefs.available: 15% + nodefs.inodesFree: 15% + evictionSoftGracePeriod: + memory.available: 1m + nodefs.available: 1m30s + nodefs.inodesFree: 2m + evictionMaxPodGracePeriod: 60 + imageGCHighThresholdPercent: 85 + imageGCLowThresholdPercent: 80 + cpuCFSQuota: true + podsPerCore: 2 + maxPods: 20 + + # Disruption section which describes the ways in which Karpenter can disrupt and replace Nodes + # Configuration in this section constrains how aggressive Karpenter can be with performing operations + # like rolling Nodes due to them hitting their maximum lifetime (expiry) or scaling down nodes to reduce cluster cost + disruption: + # Describes which types of Nodes Karpenter should consider for consolidation + # If using 'WhenUnderutilized', Karpenter will consider all nodes for consolidation and attempt to remove or replace Nodes when it discovers that the Node is underutilized and could be changed to reduce cost + # If using `WhenEmpty`, Karpenter will only consider nodes for consolidation that contain no workload pods + consolidationPolicy: WhenUnderutilized | WhenEmpty + + # The amount of time Karpenter should wait after discovering a consolidation decision + # This value can currently only be set when the consolidationPolicy is 'WhenEmpty' + # You can choose to disable consolidation entirely by setting the string value 'Never' here + consolidateAfter: 30s + + # The amount of time a Node can live on the cluster before being removed + # Avoiding long-running Nodes helps to reduce security vulnerabilities as well as to reduce the chance of issues that can plague Nodes with long uptimes such as file fragmentation or memory leaks from system processes + # You can choose to disable expiration entirely by setting the string value 'Never' here + expireAfter: 720h + + # Budgets control the speed Karpenter can scale down nodes. + # Karpenter will respect the minimum of the currently active budgets, and will round up + # when considering percentages. Duration and Schedule must be set together. + budgets: + - nodes: 10% + # On Weekdays during business hours, don't do any deprovisioning. + - schedule: "0 9 * * mon-fri" + duration: 8h + nodes: "0" + + # Resource limits constrain the total size of the cluster. + # Limits prevent Karpenter from creating new instances once the limit is exceeded. + limits: + cpu: "1000" + memory: 1000Gi + + # Priority given to the NodePool when the scheduler considers which NodePool + # to select. Higher weights indicate higher priority when comparing NodePools. + # Specifying no weight is equivalent to specifying a weight of 0. + weight: 10 +``` + +## spec.template.spec.requirements + +Kubernetes defines the following [Well-Known Labels](https://kubernetes.io/docs/reference/labels-annotations-taints/), and cloud providers (e.g., AWS) implement them. They are defined at the "spec.requirements" section of the NodePool API. + +In addition to the well-known labels from Kubernetes, Karpenter supports AWS-specific labels for more advanced scheduling. See the full list [here](../scheduling/#well-known-labels). + +These well-known labels may be specified at the NodePool level, or in a workload definition (e.g., nodeSelector on a pod.spec). Nodes are chosen using both the NodePool's and pod's requirements. If there is no overlap, nodes will not be launched. In other words, a pod's requirements must be within the NodePool's requirements. If a requirement is not defined for a well known label, any value available to the cloud provider may be chosen. + +For example, an instance type may be specified using a nodeSelector in a pod spec. If the instance type requested is not included in the NodePool list and the NodePool has instance type requirements, Karpenter will not create a node or schedule the pod. + +### Well-Known Labels + +#### Instance Types + +- key: `node.kubernetes.io/instance-type` +- key: `karpenter.k8s.aws/instance-family` +- key: `karpenter.k8s.aws/instance-category` +- key: `karpenter.k8s.aws/instance-generation` + +Generally, instance types should be a list and not a single value. Leaving these requirements undefined is recommended, as it maximizes choices for efficiently placing pods. + +Review [AWS instance types](../instance-types). Most instance types are supported with the exclusion of [non-HVM](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/virtualization_types.html). + +#### Availability Zones + +- key: `topology.kubernetes.io/zone` +- value example: `us-east-1c` +- value list: `aws ec2 describe-availability-zones --region ` + +Karpenter can be configured to create nodes in a particular zone. Note that the Availability Zone `us-east-1a` for your AWS account might not have the same location as `us-east-1a` for another AWS account. + +[Learn more about Availability Zone +IDs.](https://docs.aws.amazon.com/ram/latest/userguide/working-with-az-ids.html) + +#### Architecture + +- key: `kubernetes.io/arch` +- values + - `amd64` + - `arm64` + +Karpenter supports `amd64` nodes, and `arm64` nodes. + +#### Operating System + - key: `kubernetes.io/os` + - values + - `linux` + - `windows` + +Karpenter supports `linux` and `windows` operating systems. + +#### Capacity Type + +- key: `karpenter.sh/capacity-type` +- values + - `spot` + - `on-demand` + +Karpenter supports specifying capacity type, which is analogous to [EC2 purchase options](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-purchasing-options.html). + +Karpenter prioritizes Spot offerings if the NodePool allows Spot and on-demand instances. If the provider API (e.g. EC2 Fleet's API) indicates Spot capacity is unavailable, Karpenter caches that result across all attempts to provision EC2 capacity for that instance type and zone for the next 45 seconds. If there are no other possible offerings available for Spot, Karpenter will attempt to provision on-demand instances, generally within milliseconds. + +Karpenter also allows `karpenter.sh/capacity-type` to be used as a topology key for enforcing topology-spread. + +### Min Values + +Along with the combination of [key,operator,values] in the requirements, Karpenter also supports `minValues` in the NodePool requirements block, allowing the scheduler to be aware of user-specified flexibility minimums while scheduling pods to a cluster. If Karpenter cannot meet this minimum flexibility for each key when scheduling a pod, it will fail the scheduling loop for that NodePool, either falling back to another NodePool which meets the pod requirements or failing scheduling the pod altogether. + +For example, the below spec will use spot instance type for all provisioned instances and enforces `minValues` to various keys where it is defined +i.e at least 2 unique instance families from [c,m,r], 5 unique instance families [eg: "m5","m5d","r4","c5","c5d","c4" etc], 10 unique instance types [eg: "c5.2xlarge","c4.xlarge" etc] is required for scheduling the pods. + +```yaml +spec: + template: + spec: + requirements: + - key: kubernetes.io/arch + operator: In + values: ["amd64"] + - key: kubernetes.io/os + operator: In + values: ["linux"] + - key: karpenter.k8s.aws/instance-category + operator: In + values: ["c", "m", "r"] + minValues: 2 + - key: karpenter.k8s.aws/instance-family + operator: Exists + minValues: 5 + - key: node.kubernetes.io/instance-type + operator: Exists + minValues: 10 + - key: karpenter.k8s.aws/instance-generation + operator: Gt + values: ["2"] +``` + +Note that `minValues` can be used with multiple operators and multiple requirements. And if the `minValues` are defined with multiple operators for the same requirement key, scheduler considers the max of all the `minValues` for that requirement. For example, the below spec requires scheduler to consider at least 5 instance-family to schedule the pods. + +```yaml +spec: + template: + spec: + requirements: + - key: kubernetes.io/arch + operator: In + values: ["amd64"] + - key: kubernetes.io/os + operator: In + values: ["linux"] + - key: karpenter.k8s.aws/instance-category + operator: In + values: ["c", "m", "r"] + minValues: 2 + - key: karpenter.k8s.aws/instance-family + operator: Exists + minValues: 5 + - key: karpenter.k8s.aws/instance-family + operator: In + values: ["m5","m5d","c5","c5d","c4","r4"] + minValues: 3 + - key: node.kubernetes.io/instance-type + operator: Exists + minValues: 10 + - key: karpenter.k8s.aws/instance-generation + operator: Gt + values: ["2"] +``` + +{{% alert title="Recommended" color="primary" %}} +Karpenter allows you to be extremely flexible with your NodePools by only constraining your instance types in ways that are absolutely necessary for your cluster. By default, Karpenter will enforce that you specify the `spec.template.spec.requirements` field, but will not enforce that you specify any requirements within the field. If you choose to specify `requirements: []`, this means that you will completely flexible to _all_ instance types that your cloud provider supports. + +Though Karpenter doesn't enforce these defaults, for most use-cases, we recommend that you specify _some_ requirements to avoid odd behavior or exotic instance types. Below, is a high-level recommendation for requirements that should fit the majority of use-cases for generic workloads + +```yaml +spec: + template: + spec: + requirements: + - key: kubernetes.io/arch + operator: In + values: ["amd64"] + - key: kubernetes.io/os + operator: In + values: ["linux"] + - key: karpenter.sh/capacity-type + operator: In + values: ["on-demand"] + - key: karpenter.k8s.aws/instance-category + operator: In + values: ["c", "m", "r"] + - key: karpenter.k8s.aws/instance-generation + operator: Gt + values: ["2"] +``` + +{{% /alert %}} + +## spec.template.spec.nodeClassRef + +This field points to the Cloud Provider NodeClass resource. Learn more about [EC2NodeClasses]({{}}). + +## spec.template.spec.kubelet + +Karpenter provides the ability to specify a few additional Kubelet args. These are all optional and provide support for +additional customization and use cases. Adjust these only if you know you need to do so. For more details on kubelet configuration arguments, [see the KubeletConfiguration API specification docs](https://kubernetes.io/docs/reference/config-api/kubelet-config.v1beta1/). The implemented fields are a subset of the full list of upstream kubelet configuration arguments. Please cut an issue if you'd like to see another field implemented. + +```yaml +kubelet: + clusterDNS: ["10.0.1.100"] + systemReserved: + cpu: 100m + memory: 100Mi + ephemeral-storage: 1Gi + kubeReserved: + cpu: 200m + memory: 100Mi + ephemeral-storage: 3Gi + evictionHard: + memory.available: 5% + nodefs.available: 10% + nodefs.inodesFree: 10% + evictionSoft: + memory.available: 500Mi + nodefs.available: 15% + nodefs.inodesFree: 15% + evictionSoftGracePeriod: + memory.available: 1m + nodefs.available: 1m30s + nodefs.inodesFree: 2m + evictionMaxPodGracePeriod: 60 + imageGCHighThresholdPercent: 85 + imageGCLowThresholdPercent: 80 + cpuCFSQuota: true + podsPerCore: 2 + maxPods: 20 +``` + +### Reserved Resources + +Karpenter will automatically configure the system and kube reserved resource requests on the fly on your behalf. These requests are used to configure your node and to make scheduling decisions for your pods. If you have specific requirements or know that you will have additional capacity requirements, you can optionally override the `--system-reserved` configuration defaults with the `.spec.template.spec.kubelet.systemReserved` values and the `--kube-reserved` configuration defaults with the `.spec.template.spec.kubelet.kubeReserved` values. + +{{% alert title="Note" color="primary" %}} +Karpenter considers these reserved resources when computing the allocatable ephemeral storage on a given instance type. +If `kubeReserved` is not specified, Karpenter will compute the default reserved [CPU](https://github.com/awslabs/amazon-eks-ami/blob/db28da15d2b696bc08ac3aacc9675694f4a69933/files/bootstrap.sh#L251) and [memory](https://github.com/awslabs/amazon-eks-ami/blob/db28da15d2b696bc08ac3aacc9675694f4a69933/files/bootstrap.sh#L235) resources for the purpose of ephemeral storage computation. +These defaults are based on the defaults on Karpenter's supported AMI families, which are not the same as the kubelet defaults. +You should be aware of the CPU and memory default calculation when using Custom AMI Families. If they don't align, there may be a difference in Karpenter's computed allocatable ephemeral storage and the actually ephemeral storage available on the node. +{{% /alert %}} + +### Eviction Thresholds + +The kubelet supports eviction thresholds by default. When enough memory or file system pressure is exerted on the node, the kubelet will begin to evict pods to ensure that system daemons and other system processes can continue to run in a healthy manner. + +Kubelet has the notion of [hard evictions](https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#hard-eviction-thresholds) and [soft evictions](https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#soft-eviction-thresholds). In hard evictions, pods are evicted as soon as a threshold is met, with no grace period to terminate. Soft evictions, on the other hand, provide an opportunity for pods to be terminated gracefully. They do so by sending a termination signal to pods that are planning to be evicted and allowing those pods to terminate up to their grace period. + +Karpenter supports [hard evictions](https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#hard-eviction-thresholds) through the `.spec.template.spec.kubelet.evictionHard` field and [soft evictions](https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#soft-eviction-thresholds) through the `.spec.template.spec.kubelet.evictionSoft` field. `evictionHard` and `evictionSoft` are configured by listing [signal names](https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction/#eviction-signals) with either percentage values or resource values. + +```yaml +kubelet: + evictionHard: + memory.available: 500Mi + nodefs.available: 10% + nodefs.inodesFree: 10% + imagefs.available: 5% + imagefs.inodesFree: 5% + pid.available: 7% + evictionSoft: + memory.available: 1Gi + nodefs.available: 15% + nodefs.inodesFree: 15% + imagefs.available: 10% + imagefs.inodesFree: 10% + pid.available: 10% +``` + +#### Supported Eviction Signals + +| Eviction Signal | Description | +|--------------------|---------------------------------------------------------------------------------| +| memory.available | memory.available := node.status.capacity[memory] - node.stats.memory.workingSet | +| nodefs.available | nodefs.available := node.stats.fs.available | +| nodefs.inodesFree | nodefs.inodesFree := node.stats.fs.inodesFree | +| imagefs.available | imagefs.available := node.stats.runtime.imagefs.available | +| imagefs.inodesFree | imagefs.inodesFree := node.stats.runtime.imagefs.inodesFree | +| pid.available | pid.available := node.stats.rlimit.maxpid - node.stats.rlimit.curproc | + +For more information on eviction thresholds, view the [Node-pressure Eviction](https://kubernetes.io/docs/concepts/scheduling-eviction/node-pressure-eviction) section of the official Kubernetes docs. + +#### Soft Eviction Grace Periods + +Soft eviction pairs an eviction threshold with a specified grace period. With soft eviction thresholds, the kubelet will only begin evicting pods when the node exceeds its soft eviction threshold over the entire duration of its grace period. For example, if you specify `evictionSoft[memory.available]` of `500Mi` and a `evictionSoftGracePeriod[memory.available]` of `1m30`, the node must have less than `500Mi` of available memory over a minute and a half in order for the kubelet to begin evicting pods. + +Optionally, you can specify an `evictionMaxPodGracePeriod` which defines the administrator-specified maximum pod termination grace period to use during soft eviction. If a namespace-owner had specified a pod `terminationGracePeriodInSeconds` on pods in their namespace, the minimum of `evictionPodGracePeriod` and `terminationGracePeriodInSeconds` would be used. + +```yaml +kubelet: + evictionSoftGracePeriod: + memory.available: 1m + nodefs.available: 1m30s + nodefs.inodesFree: 2m + imagefs.available: 1m30s + imagefs.inodesFree: 2m + pid.available: 2m + evictionMaxPodGracePeriod: 60 +``` + +### Pod Density + +By default, the number of pods on a node is limited by both the number of networking interfaces (ENIs) that may be attached to an instance type and the number of IP addresses that can be assigned to each ENI. See [IP addresses per network interface per instance type](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-eni.html#AvailableIpPerENI) for a more detailed information on these instance types' limits. + +{{% alert title="Note" color="primary" %}} +By default, the VPC CNI allocates IPs for a node and pods from the same subnet. With [VPC CNI Custom Networking](https://aws.github.io/aws-eks-best-practices/networking/custom-networking), the pods will receive IP addresses from another subnet dedicated to pod IPs. This approach makes it easier to manage IP addresses and allows for separate Network Access Control Lists (NACLs) applied to your pods. VPC CNI Custom Networking reduces the pod density of a node since one of the ENI attachments will be used for the node and cannot share the allocated IPs on the interface to pods. Karpenter supports VPC CNI Custom Networking and similar CNI setups where the primary node interface is separated from the pods interfaces through a global [setting](./settings.md#configmap) within the karpenter-global-settings configmap: `aws.reservedENIs`. In the common case, `aws.reservedENIs` should be set to `"1"` if using Custom Networking. +{{% /alert %}} + +{{% alert title="Windows Support Notice" color="warning" %}} +It's currently not possible to specify custom networking with Windows nodes. +{{% /alert %}} + +#### Max Pods + +For small instances that require an increased pod density or large instances that require a reduced pod density, you can override this default value with `.spec.template.spec.kubelet.maxPods`. This value will be used during Karpenter pod scheduling and passed through to `--max-pods` on kubelet startup. + +{{% alert title="Note" color="primary" %}} +When using small instance types, it may be necessary to enable [prefix assignment mode](https://aws.amazon.com/blogs/containers/amazon-vpc-cni-increases-pods-per-node-limits/) in the AWS VPC CNI plugin to support a higher pod density per node. Prefix assignment mode was introduced in AWS VPC CNI v1.9 and allows ENIs to manage a broader set of IP addresses. Much higher pod densities are supported as a result. +{{% /alert %}} + +{{% alert title="Windows Support Notice" color="warning" %}} +Presently, Windows worker nodes do not support using more than one ENI. +As a consequence, the number of IP addresses, and subsequently, the number of pods that a Windows worker node can support is limited by the number of IPv4 addresses available on the primary ENI. +Currently, Karpenter will only consider individual secondary IP addresses when calculating the pod density limit. +{{% /alert %}} + +#### Pods Per Core + +An alternative way to dynamically set the maximum density of pods on a node is to use the `.spec.template.spec.kubelet.podsPerCore` value. Karpenter will calculate the pod density during scheduling by multiplying this value by the number of logical cores (vCPUs) on an instance type. This value will also be passed through to the `--pods-per-core` value on kubelet startup to configure the number of allocatable pods the kubelet can assign to the node instance. + +The value generated from `podsPerCore` cannot exceed `maxPods`, meaning, if both are set, the minimum of the `podsPerCore` dynamic pod density and the static `maxPods` value will be used for scheduling. + +{{% alert title="Note" color="primary" %}} +`maxPods` may not be set in the `kubelet` of a NodePool, but may still be restricted by the `ENI_LIMITED_POD_DENSITY` value. You may want to ensure that the `podsPerCore` value that will be used for instance families associated with the NodePool will not cause unexpected behavior by exceeding the `maxPods` value. +{{% /alert %}} + +{{% alert title="Pods Per Core on Bottlerocket" color="warning" %}} +Bottlerocket AMIFamily currently does not support `podsPerCore` configuration. If a NodePool contains a `provider` or `providerRef` to a node template that will launch a Bottlerocket instance, the `podsPerCore` value will be ignored for scheduling and for configuring the kubelet. +{{% /alert %}} + +## spec.disruption + +You can configure Karpenter to disrupt Nodes through your NodePool in multiple ways. You can use `spec.disruption.consolidationPolicy`, `spec.disruption.consolidateAfter` or `spec.disruption.expireAfter`. Read [Disruption]({{}}) for more. + +## spec.limits + +The NodePool spec includes a limits section (`spec.limits`), which constrains the maximum amount of resources that the NodePool will manage. + +Karpenter supports limits of any resource type reported by your cloudprovider. It limits instance types when scheduling to those that will not exceed the specified limits. If a limit has been exceeded, nodes provisioning is prevented until some nodes have been terminated. + +```yaml +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: default +spec: + template: + spec: + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["spot"] + limits: + cpu: 1000 + memory: 1000Gi + nvidia.com/gpu: 2 +``` + +{{% alert title="Note" color="primary" %}} +Karpenter provisioning is highly parallel. Because of this, limit checking is eventually consistent, which can result in overrun during rapid scale outs. +{{% /alert %}} + +CPU limits are described with a `DecimalSI` value. Note that the Kubernetes API will coerce this into a string, so we recommend against using integers to avoid GitOps skew. + +Memory limits are described with a [`BinarySI` value, such as 1000Gi.](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/#meaning-of-memory) + +You can view the current consumption of cpu and memory on your cluster by running: +``` +kubectl get nodepool -o=jsonpath='{.items[0].status}' +``` + +Review the [Kubernetes core API](https://github.com/kubernetes/api/blob/37748cca582229600a3599b40e9a82a951d8bbbf/core/v1/resource.go#L23) (`k8s.io/api/core/v1`) for more information on `resources`. + +## spec.weight + +Karpenter allows you to describe NodePool preferences through a `weight` mechanism similar to how weight is described with [pod and node affinities](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#affinity-and-anti-affinity). + +For more information on weighting NodePools, see the [Weighted NodePools section]({{}}) in the scheduling docs. + +## Examples + +### Isolating Expensive Hardware + +A NodePool can be set up to only provision nodes on particular processor types. +The following example sets a taint that only allows pods with tolerations for Nvidia GPUs to be scheduled: + +```yaml +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: gpu +spec: + disruption: + consolidationPolicy: WhenUnderutilized + template: + spec: + requirements: + - key: node.kubernetes.io/instance-type + operator: In + values: ["p3.8xlarge", "p3.16xlarge"] + taints: + - key: nvidia.com/gpu + value: "true" + effect: NoSchedule +``` +In order for a pod to run on a node defined in this NodePool, it must tolerate `nvidia.com/gpu` in its pod spec. + +### Cilium Startup Taint + +Per the Cilium [docs](https://docs.cilium.io/en/stable/installation/taints/#taint-effects), it's recommended to place a taint of `node.cilium.io/agent-not-ready=true:NoExecute` on nodes to allow Cilium to configure networking prior to other pods starting. This can be accomplished via the use of Karpenter `startupTaints`. These taints are placed on the node, but pods aren't required to tolerate these taints to be considered for provisioning. + +```yaml +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: cilium-startup +spec: + disruption: + consolidationPolicy: WhenUnderutilized + template: + spec: + startupTaints: + - key: node.cilium.io/agent-not-ready + value: "true" + effect: NoExecute +``` diff --git a/website/content/en/v0.30/concepts/scheduling.md b/website/content/en/v0.35/concepts/scheduling.md similarity index 79% rename from website/content/en/v0.30/concepts/scheduling.md rename to website/content/en/v0.35/concepts/scheduling.md index 0f70f1a0463f..5352f0706fb9 100755 --- a/website/content/en/v0.30/concepts/scheduling.md +++ b/website/content/en/v0.35/concepts/scheduling.md @@ -15,10 +15,10 @@ Reasons for constraining where your pods run could include: * Wanting to use techniques like topology spread to help ensure high availability Your Cloud Provider defines the first layer of constraints, including all instance types, architectures, zones, and purchase types available to its cloud. -The cluster administrator adds the next layer of constraints by creating one or more provisioners. +The cluster administrator adds the next layer of constraints by creating one or more NodePools. The final layer comes from you adding specifications to your Kubernetes pod deployments. -Pod scheduling constraints must fall within a provisioner's constraints or the pods will not deploy. -For example, if the provisioner sets limits that allow only a particular zone to be used, and a pod asks for a different zone, it will not be scheduled. +Pod scheduling constraints must fall within a NodePool's constraints or the pods will not deploy. +For example, if the NodePool sets limits that allow only a particular zone to be used, and a pod asks for a different zone, it will not be scheduled. Constraints you can request include: @@ -62,7 +62,7 @@ Its limits are set to 256MiB of memory and 1 CPU. Instance type selection math only uses `requests`, but `limits` may be configured to enable resource oversubscription. -See [Managing Resources for Containers](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) for details on resource types supported by Kubernetes, [Specify a memory request and a memory limit](https://kubernetes.io/docs/tasks/configure-pod-container/assign-memory-resource/#specify-a-memory-request-and-a-memory-limit) for examples of memory requests, and [Provisioner Configuration]({{}}) for a list of supported resources. +See [Managing Resources for Containers](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) for details on resource types supported by Kubernetes, [Specify a memory request and a memory limit](https://kubernetes.io/docs/tasks/configure-pod-container/assign-memory-resource/#specify-a-memory-request-and-a-memory-limit) for examples of memory requests, and [NodePools]({{}}) for a list of supported resources. ### Accelerators/GPU Resources @@ -129,7 +129,7 @@ This can include well-known labels or custom labels you create yourself. You can use `affinity` to define more complicated constraints, see [Node Affinity](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#node-affinity) for the complete specification. ### Labels -Well-known labels may be specified as provisioner requirements or pod scheduling constraints. You can also define your own custom labels by specifying `requirements` or `labels` on your Provisioner and select them using `nodeAffinity` or `nodeSelectors` on your Pods. +Well-known labels may be specified as NodePool requirements or pod scheduling constraints. You can also define your own custom labels by specifying `requirements` or `labels` on your NodePool and select them using `nodeAffinity` or `nodeSelectors` on your Pods. {{% alert title="Warning" color="warning" %}} Take care to ensure the label domains are correct. A well known label like `karpenter.k8s.aws/instance-family` will enforce node properties, but may be confused with `node.kubernetes.io/instance-family`, which is unknown to Karpenter, and treated as a custom label which will not enforce node properties. @@ -167,7 +167,7 @@ Karpenter translates the following deprecated labels to their stable equivalents #### User-Defined Labels -Karpenter is aware of several well-known labels, deriving them from instance type details. If you specify a `nodeSelector` or a required `nodeAffinity` using a label that is not well-known to Karpenter, it will not launch nodes with these labels and pods will remain pending. For Karpenter to become aware that it can schedule for these labels, you must specify the label in the Provisioner requirements with the `Exists` operator: +Karpenter is aware of several well-known labels, deriving them from instance type details. If you specify a `nodeSelector` or a required `nodeAffinity` using a label that is not well-known to Karpenter, it will not launch nodes with these labels and pods will remain pending. For Karpenter to become aware that it can schedule for these labels, you must specify the label in the NodePool requirements with the `Exists` operator: ```yaml requirements: @@ -186,7 +186,7 @@ nodeSelector: ``` This example features a well-known label (`topology.kubernetes.io/zone`) and a label that is well known to Karpenter (`karpenter.sh/capacity-type`). -If you want to create a custom label, you should do that at the provisioner level. +If you want to create a custom label, you should do that at the NodePool level. Then the pod can declare that custom label. @@ -204,8 +204,7 @@ When setting rules, the following Node affinity types define how hard or soft ea The `IgnoredDuringExecution` part of each tells the pod to keep running, even if conditions change on the node so the rules no longer matched. You can think of these concepts as `required` and `preferred`, since Kubernetes never implemented other variants of these rules. -All examples below assume that the provisioner doesn't have constraints to prevent those zones from being used. -The first constraint says you could use `us-west-2a` or `us-west-2b`, the second constraint makes it so only `us-west-2b` can be used. +All examples below assume that the NodePool doesn't have constraints to prevent those zones from being used. The first constraint says you could use `us-west-2a` or `us-west-2b`, the second constraint makes it so only `us-west-2b` can be used. ```yaml affinity: @@ -215,7 +214,7 @@ The first constraint says you could use `us-west-2a` or `us-west-2b`, the second - matchExpressions: - key: "topology.kubernetes.io/zone" operator: "In" - values: ["us-west-2a, us-west-2b"] + values: ["us-west-2a", "us-west-2b"] - key: "topology.kubernetes.io/zone" operator: "In" values: ["us-west-2b"] @@ -226,7 +225,7 @@ Changing the second operator to `NotIn` would allow the pod to run in `us-west-2 ```yaml - key: "topology.kubernetes.io/zone" operator: "In" - values: ["us-west-2a, us-west-2b"] + values: ["us-west-2a", "us-west-2b"] - key: "topology.kubernetes.io/zone" operator: "NotIn" values: ["us-west-2b"] @@ -244,7 +243,7 @@ Here, if `us-west-2a` is not available, the second term will cause the pod to ru - matchExpressions: # OR - key: "topology.kubernetes.io/zone" # AND operator: "In" - values: ["us-west-2a, us-west-2b"] + values: ["us-west-2a", "us-west-2b"] - key: "topology.kubernetes.io/zone" # AND operator: "NotIn" values: ["us-west-2b"] @@ -265,28 +264,28 @@ So if capacity becomes available, it will schedule the pod without user interven ## Taints and tolerations Taints are the opposite of affinity. -Setting a taint on a node tells the scheduler to not run a pod on it unless the pod has explicitly said it can tolerate that taint. -This example shows a Provisioner that was set up with a taint for only running pods that require a GPU, such as the following: - +Setting a taint on a node tells the scheduler to not run a pod on it unless the pod has explicitly said it can tolerate that taint. This example shows a NodePool that was set up with a taint for only running pods that require a GPU, such as the following: ```yaml -apiVersion: karpenter.sh/v1alpha5 -kind: Provisioner +apiVersion: karpenter.sh/v1beta1 +kind: NodePool metadata: name: gpu spec: - requirements: - - key: karpenter.k8s.aws/instance-family - operator: In - values: - - p3 - taints: - - key: nvidia.com/gpu - value: true - effect: "NoSchedule" + template: + spec: + requirements: + - key: karpenter.k8s.aws/instance-family + operator: In + values: + - p3 + taints: + - key: nvidia.com/gpu + value: true + effect: "NoSchedule" ``` -For a pod to request to run on a node that has provisioner, it could set a toleration as follows: +For a pod to request to run on a node that has this NodePool, it could set a toleration as follows: ```yaml apiVersion: v1 @@ -311,8 +310,7 @@ See [Taints and Tolerations](https://kubernetes.io/docs/concepts/scheduling-evic ## Topology Spread -By using the Kubernetes `topologySpreadConstraints` you can ask the provisioner to have pods push away from each other to limit the blast radius of an outage. -Think of it as the Kubernetes evolution for pod affinity: it lets you relate pods with respect to nodes while still allowing spread. +By using the Kubernetes `topologySpreadConstraints` you can ask the NodePool to have pods push away from each other to limit the blast radius of an outage. Think of it as the Kubernetes evolution for pod affinity: it lets you relate pods with respect to nodes while still allowing spread. For example: ```yaml @@ -351,12 +349,15 @@ The three supported `topologyKey` values that Karpenter supports are: - `kubernetes.io/hostname` - `karpenter.sh/capacity-type` - See [Pod Topology Spread Constraints](https://kubernetes.io/docs/concepts/workloads/pods/pod-topology-spread-constraints/) for details. +{{% alert title="Note" color="primary" %}} +NodePools do not attempt to balance or rebalance the availability zones for their nodes. Availability zone balancing may be achieved by defining zonal Topology Spread Constraints for Pods that require multi-zone durability, and NodePools will respect these constraints while optimizing for compute costs. +{{% /alert %}} + ## Pod affinity/anti-affinity -By using the `podAffinity` and `podAntiAffinity` configuration on a pod spec, you can inform the provisioner of your desire for pods to schedule together or apart with respect to different topology domains. For example: +By using the `podAffinity` and `podAntiAffinity` configuration on a pod spec, you can inform the Karpenter scheduler of your desire for pods to schedule together or apart with respect to different topology domains. For example: ```yaml spec: @@ -441,167 +442,174 @@ The EBS CSI driver uses `topology.ebs.csi.aws.com/zone` instead of the standard The topology key `topology.kubernetes.io/region` is not supported. Legacy in-tree CSI providers specify this label. Instead, install an out-of-tree CSI provider. [Learn more about moving to CSI providers.](https://kubernetes.io/blog/2021/12/10/storage-in-tree-to-csi-migration-status-update/#quick-recap-what-is-csi-migration-and-why-migrate) {{% /alert %}} -## Weighting Provisioners +## Weighted NodePools -Karpenter allows you to order your provisioners using the `.spec.weight` field so that the node scheduler will deterministically attempt to schedule with one provisioner before another. Below are a few example use-cases that are now supported with the provisioner weighting semantic. +Karpenter allows you to order your NodePools using the `.spec.weight` field so that the Karpenter scheduler will attempt to schedule one NodePool before another. ### Savings Plans and Reserved Instances If you have purchased a [Savings Plan](https://aws.amazon.com/savingsplans/) or [Reserved Instances](https://aws.amazon.com/ec2/pricing/reserved-instances/), you may want to tell Karpenter to prioritize this reserved capacity ahead of other instance types. -To enable this, you will need to tell the Karpenter controllers which instance types to prioritize and what is the maximum amount of capacity that should be provisioned using those instance types. We can set the `.spec.limits` on the provisioner to limit the capacity that can be launched by this provisioner. Combined with the `.spec.weight` value, we can tell Karpenter to pull from instance types in the reserved provisioner before defaulting to generic instance types. - -#### Reserved Instance Provisioner +To enable this, you will need to tell the Karpenter controllers which instance types to prioritize and what is the maximum amount of capacity that should be provisioned using those instance types. We can set the `.spec.limits` field on the NodePool to limit the capacity that can be launched by this NodePool. Combined with the `.spec.weight` value, we can tell Karpenter to pull from instance types in the reserved NodePool before defaulting to generic instance types. ```yaml -apiVersion: karpenter.sh/v1alpha5 -kind: Provisioner +apiVersion: karpenter.sh/v1beta1 +kind: NodePool metadata: name: reserved-instance spec: weight: 50 - requirements: - - key: "node.kubernetes.io/instance-type" - operator: In - values: ["c4.large"] limits: - resources: - cpu: 100 -``` - -#### Default Provisioner - -```yaml -apiVersion: karpenter.sh/v1alpha5 -kind: Provisioner + cpu: 100 + template: + spec: + requirements: + - key: "node.kubernetes.io/instance-type" + operator: In + values: ["c4.large"] +--- +apiVersion: karpenter.sh/v1beta1 +kind: NodePool metadata: name: default spec: - requirements: - - key: karpenter.sh/capacity-type - operator: In - values: ["spot", "on-demand"] - - key: kubernetes.io/arch - operator: In - values: ["amd64"] + template: + spec: + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["spot", "on-demand"] + - key: kubernetes.io/arch + operator: In + values: ["amd64"] ``` -### Default Node Configuration +### Fallback Pods that do not specify node selectors or affinities can potentially be assigned to any node with any configuration. There may be cases where you require these pods to schedule to a specific capacity type or architecture but assigning the relevant node selectors or affinities to all these workload pods may be too tedious or infeasible. Instead, we want to define a cluster-wide default configuration for nodes launched using Karpenter. -By assigning a higher `.spec.weight` value and restricting a provisioner to a specific capacity type or architecture, we can set default configuration for the nodes launched by pods that don't have node configuration restrictions. - -#### Default Provisioner +By assigning a higher `.spec.weight` value and restricting a NodePool to a specific capacity type or architecture, we can set default configuration for the nodes launched by pods that don't have node configuration restrictions. ```yaml -apiVersion: karpenter.sh/v1alpha5 -kind: Provisioner +apiVersion: karpenter.sh/v1beta1 +kind: NodePool metadata: name: default spec: weight: 50 - requirements: - - key: karpenter.sh/capacity-type - operator: In - values: ["spot", "on-demand"] - - key: kubernetes.io/arch - operator: In - values: ["amd64"] -``` - -#### ARM-64 Specific Provisioner - -```yaml -apiVersion: karpenter.sh/v1alpha5 -kind: Provisioner + template: + spec: + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["spot", "on-demand"] + - key: kubernetes.io/arch + operator: In + values: ["amd64"] +--- +apiVersion: karpenter.sh/v1beta1 +kind: NodePool metadata: name: arm64-specific spec: - requirements: - - key: karpenter.sh/capacity-type - operator: In - values: ["spot", "on-demand"] - - key: kubernetes.io/arch - operator: In - values: ["arm64"] - - key: node.kubernetes.io/instance-type - operator: In - values: ["a1.large", "a1.xlarge"] + template: + spec: + requirements: + - key: karpenter.sh/capacity-type + operator: In + values: ["spot", "on-demand"] + - key: kubernetes.io/arch + operator: In + values: ["arm64"] + - key: node.kubernetes.io/instance-type + operator: In + values: ["a1.large", "a1.xlarge"] ``` {{% alert title="Note" color="primary" %}} -Based on the way that Karpenter performs pod batching and bin packing, it is not guaranteed that Karpenter will always choose the highest priority provisioner given specific requirements. For example, if a pod can't be scheduled with the highest priority provisioner it will force creation of a node using a lower priority provisioner which may allow other pods from that batch to also schedule on that node. The behavior may also occur if existing capacity is available, as the kube-scheduler will schedule the pods instead of allowing Karpenter to provision a new node. +Based on the way that Karpenter performs pod batching and bin packing, it is not guaranteed that Karpenter will always choose the highest priority NodePool given specific requirements. For example, if a pod can't be scheduled with the highest priority NodePool, it will force creation of a node using a lower priority NodePool, allowing other pods from that batch to also schedule on that node. The behavior may also occur if existing capacity is available, as the kube-scheduler will schedule the pods instead of allowing Karpenter to provision a new node. {{% /alert %}} ## Advanced Scheduling Techniques ### `Exists` Operator -The `Exists` operator can be used on a provisioner to provide workload segregation across nodes. +The `Exists` operator can be used on a NodePool to provide workload segregation across nodes. ```yaml ... - requirements: - - key: company.com/team - operator: Exists +requirements: +- key: company.com/team + operator: Exists ... ``` -With the requirement on the provisioner in place, workloads can optionally specify a custom value as a required node affinity or node selector. Karpenter will then label the nodes it launches for these pods which prevents `kube-scheduler` from scheduling conflicting pods to those nodes. This provides a way to more dynamically isolate workloads without requiring a unique provisioner for each workload subset. +With the requirement on the NodePool, workloads can optionally specify a custom value as a required node affinity or node selector. Karpenter will then label the nodes it launches for these pods which prevents `kube-scheduler` from scheduling conflicting pods to those nodes. This provides a way to more dynamically isolate workloads without requiring a unique NodePool for each workload subset. ```yaml - nodeSelector: - company.com/team: team-a +nodeSelector: + company.com/team: team-a ``` {{% alert title="Note" color="primary" %}} -If a workload matches the provisioner but doesn't specify a label, Karpenter will generate a random label for the node. +If a workload matches the NodePool but doesn't specify a label, Karpenter will generate a random label for the node. {{% /alert %}} ### On-Demand/Spot Ratio Split Taking advantage of Karpenter's ability to assign labels to node and using a topology spread across those labels enables a crude method for splitting a workload across on-demand and spot instances in a desired ratio. -To do this, we create a provisioner each for spot and on-demand with disjoint values for a unique new label called `capacity-spread`. In the example below, we provide four unique values for the spot provisioner and one value for the on-demand provisioner. When we spread across our new label evenly, we'll end up with a ratio of 4:1 spot to on-demand nodes. +To do this, we create one NodePool each for spot and on-demand with disjoint values for a unique new label called `capacity-spread`. In the example below, we provide four unique values for the spot NodePool and one value for the on-demand NodePool. When we spread across our new label evenly, we'll end up with a ratio of 4:1 spot to on-demand nodes. {{% alert title="Warning" color="warning" %}} This is not identical to a topology spread with a specified ratio. We are constructing 'virtual domains' to spread evenly across and the ratio of those 'virtual domains' to spot and on-demand happen to coincide with the desired spot to on-demand ratio. As an example, if you launch pods using the provided example, Karpenter will launch nodes with `capacity-spread` labels of 1, 2, 3, 4, and 5. `kube-scheduler` will then schedule evenly across those nodes to give the desired ratio. {{% /alert %}} -#### Spot Provisioner -```yaml - requirements: - - key: "karpenter.sh/capacity-type" - operator: In - values: [ "spot"] - - key: capacity-spread - operator: In - values: - - "2" - - "3" - - "4" - - "5" -``` +#### NodePools -#### On-Demand Provisioner ```yaml - requirements: - - key: "karpenter.sh/capacity-type" - operator: In - values: [ "on-demand"] - - key: capacity-spread - operator: In - values: - - "1" +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: spot +spec: + template: + spec: + requirements: + - key: "karpenter.sh/capacity-type" + operator: In + values: ["spot"] + - key: capacity-spread + operator: In + values: + - "2" + - "3" + - "4" + - "5" +--- +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +metadata: + name: on-demand +spec: + template: + spec: + requirements: + - key: "karpenter.sh/capacity-type" + operator: In + values: ["on-demand"] + - key: capacity-spread + operator: In + values: + - "1" ``` #### Workload Topology Spread Constraint ```yaml topologySpreadConstraints: - - maxSkew: 1 - topologyKey: capacity-spread - whenUnsatisfiable: DoNotSchedule - labelSelector: - ... +- maxSkew: 1 + topologyKey: capacity-spread + whenUnsatisfiable: DoNotSchedule + labelSelector: + ... ``` diff --git a/website/content/en/v0.30/contributing/_index.md b/website/content/en/v0.35/contributing/_index.md similarity index 90% rename from website/content/en/v0.30/contributing/_index.md rename to website/content/en/v0.35/contributing/_index.md index 10bb749d39dc..6ec2c3df504e 100644 --- a/website/content/en/v0.30/contributing/_index.md +++ b/website/content/en/v0.35/contributing/_index.md @@ -1,7 +1,7 @@ --- title: "Contributing" linkTitle: "Contributing" -weight: 100 +weight: 40 description: > Learn about how to contribute to Karpenter --- diff --git a/website/content/en/v0.31/contributing/design-guide.md b/website/content/en/v0.35/contributing/design-guide.md similarity index 100% rename from website/content/en/v0.31/contributing/design-guide.md rename to website/content/en/v0.35/contributing/design-guide.md diff --git a/website/content/en/v0.31/contributing/development-guide.md b/website/content/en/v0.35/contributing/development-guide.md similarity index 94% rename from website/content/en/v0.31/contributing/development-guide.md rename to website/content/en/v0.35/contributing/development-guide.md index 8eab45dd2475..2afbb4d2ead7 100644 --- a/website/content/en/v0.31/contributing/development-guide.md +++ b/website/content/en/v0.35/contributing/development-guide.md @@ -73,9 +73,10 @@ make test # E2E correctness tests ### Change Log Level +By default, `make apply` will set the log level to debug. You can change the log level by setting the log level in your Helm values. + ```bash -kubectl patch configmap config-logging -n karpenter --patch '{"data":{"loglevel.controller":"debug"}}' # Debug Level -kubectl patch configmap config-logging -n karpenter --patch '{"data":{"loglevel.controller":"info"}}' # Info Level +--set logLevel=debug ``` ### Debugging Metrics @@ -105,8 +106,8 @@ stern -n karpenter -l app.kubernetes.io/name=karpenter ### AWS For local development on Karpenter you will need a Docker repo which can manage your images for Karpenter components. -You can use the following command to provision an ECR repository. We recommend using a single "dev" repository for -development across multiple projects, and to use specific image hashes instead of image tags. +You can use the following command to provision an ECR repository. We recommend using a single "dev" repository for +development across multiple projects, and to use specific image hashes instead of image tags. ```bash aws ecr create-repository \ diff --git a/website/content/en/v0.31/contributing/working-group.md b/website/content/en/v0.35/contributing/working-group.md similarity index 100% rename from website/content/en/v0.31/contributing/working-group.md rename to website/content/en/v0.35/contributing/working-group.md diff --git a/website/content/en/v0.30/faq.md b/website/content/en/v0.35/faq.md similarity index 50% rename from website/content/en/v0.30/faq.md rename to website/content/en/v0.35/faq.md index 18d83da2c091..a825e34056f6 100644 --- a/website/content/en/v0.30/faq.md +++ b/website/content/en/v0.35/faq.md @@ -1,101 +1,85 @@ --- title: "FAQs" linkTitle: "FAQs" -weight: 90 +weight: 60 description: > Review Karpenter Frequently Asked Questions --- ## General -### How does a provisioner decide to manage a particular node? -See [Configuring provisioners]({{< ref "./concepts/#configuring-provisioners" >}}) for information on how Karpenter provisions and manages nodes. +### How does a NodePool decide to manage a particular node? +See [Configuring NodePools]({{< ref "./concepts/#configuring-nodepools" >}}) for information on how Karpenter configures and manages nodes. ### What cloud providers are supported? AWS is the first cloud provider supported by Karpenter, although it is designed to be used with other cloud providers as well. ### Can I write my own cloud provider for Karpenter? -Yes, but there is no documentation yet for it. -Start with Karpenter's GitHub [cloudprovider](https://github.com/aws/karpenter-core/tree/v0.30.0/pkg/cloudprovider) documentation to see how the AWS provider is built, but there are other sections of the code that will require changes too. +Yes, but there is no documentation yet for it. Start with Karpenter's GitHub [cloudprovider](https://github.com/aws/karpenter-core/tree/v0.35.4/pkg/cloudprovider) documentation to see how the AWS provider is built, but there are other sections of the code that will require changes too. ### What operating system nodes does Karpenter deploy? -By default, Karpenter uses Amazon Linux 2 images. +Karpenter uses the OS defined by the [AMI Family in your EC2NodeClass]({{< ref "./concepts/nodeclasses#specamifamily" >}}). ### Can I provide my own custom operating system images? -Karpenter has multiple mechanisms for configuring the [operating system]({{< ref "./concepts/node-templates/#spec-amiselector" >}}) for your nodes. +Karpenter has multiple mechanisms for configuring the [operating system]({{< ref "./concepts/nodeclasses/#specamiselectorterms" >}}) for your nodes. ### Can Karpenter deal with workloads for mixed architecture cluster (arm vs. amd)? -Karpenter is flexible to multi architecture configurations using [well known labels]({{< ref "./concepts/scheduling/#supported-labels">}}). +Karpenter is flexible to multi-architecture configurations using [well known labels]({{< ref "./concepts/scheduling/#supported-labels">}}). ### What RBAC access is required? -All of the required RBAC rules can be found in the helm chart template. -See [clusterrolebinding.yaml](https://github.com/aws/karpenter/blob/v0.30.0/charts/karpenter/templates/clusterrolebinding.yaml), [clusterrole.yaml](https://github.com/aws/karpenter/blob/v0.30.0/charts/karpenter/templates/clusterrole.yaml), [rolebinding.yaml](https://github.com/aws/karpenter/blob/v0.30.0/charts/karpenter/templates/rolebinding.yaml), and [role.yaml](https://github.com/aws/karpenter/blob/v0.30.0/charts/karpenter/templates/role.yaml) files for details. +All the required RBAC rules can be found in the Helm chart template. See [clusterrole-core.yaml](https://github.com/aws/karpenter/blob/v0.35.4/charts/karpenter/templates/clusterrole-core.yaml), [clusterrole.yaml](https://github.com/aws/karpenter/blob/v0.35.4/charts/karpenter/templates/clusterrole.yaml), [rolebinding.yaml](https://github.com/aws/karpenter/blob/v0.35.4/charts/karpenter/templates/rolebinding.yaml), and [role.yaml](https://github.com/aws/karpenter/blob/v0.35.4/charts/karpenter/templates/role.yaml) files for details. ### Can I run Karpenter outside of a Kubernetes cluster? Yes, as long as the controller has network and IAM/RBAC access to the Kubernetes API and your provider API. +### What do I do if I encounter a security issue with Karpenter? +Refer to [Reporting Security Issues](https://github.com/aws/karpenter/security/policy) for information on how to report Karpenter security issues. Do not create a public GitHub issue. + ## Compatibility ### Which versions of Kubernetes does Karpenter support? -See the [Compatibility Matrix in the Upgrade Guide]({{< ref "./upgrade-guide#compatibility-matrix" >}}) to view the supported Kubernetes versions per Karpenter released version. +See the [Compatibility Matrix in the Upgrade Section]({{< ref "./upgrading/compatibility#compatibility-matrix" >}}) to view the supported Kubernetes versions per Karpenter released version. ### What Kubernetes distributions are supported? -Karpenter documents integration with a fresh or existing install of the latest AWS Elastic Kubernetes Service (EKS). -Other Kubernetes distributions (KOPs, etc.) can be used, but setting up cloud provider permissions for those distributions has not been documented. +Karpenter documents integration with a fresh or existing installation of the latest AWS Elastic Kubernetes Service (EKS). Other Kubernetes distributions (KOPs, etc.) can be used, but setting up cloud provider permissions for those distributions has not been documented. ### How does Karpenter interact with AWS node group features? -Provisioners are designed to work alongside static capacity management solutions like EKS Managed Node Groups and EC2 Auto Scaling Groups. -You can manage all capacity using provisioners, use a mixed model with dynamic and statically managed capacity, or use a fully static approach. -We expect most users will use a mixed approach in the near term and provisioner-managed in the long term. +NodePools are designed to work alongside static capacity management solutions like EKS Managed Node Groups and EC2 Auto Scaling Groups. You can manage all capacity using NodePools, use a mixed model with dynamic and statically managed capacity, or use a fully static approach. We expect most users will use a mixed approach in the near term and NodePool-managed in the long term. ### How does Karpenter interact with Kubernetes features? -* Kubernetes Cluster Autoscaler: Karpenter can work alongside cluster autoscaler. -See [Kubernetes cluster autoscaler]({{< ref "./concepts/#kubernetes-cluster-autoscaler" >}}) for details. -* Kubernetes Scheduler: Karpenter focuses on scheduling pods that the Kubernetes scheduler has marked as unschedulable. -See [Scheduling]({{< ref "./concepts/scheduling" >}}) for details on how Karpenter interacts with the Kubernetes scheduler. +* Kubernetes Cluster Autoscaler: Karpenter can work alongside Cluster Autoscaler. See [Kubernetes Cluster Autoscaler]({{< ref "./concepts/#kubernetes-cluster-autoscaler" >}}) for details. +* Kubernetes Scheduler: Karpenter focuses on scheduling pods that the Kubernetes scheduler has marked as unschedulable. See [Scheduling]({{< ref "./concepts/scheduling" >}}) for details on how Karpenter interacts with the Kubernetes scheduler. ## Provisioning -### What features does the Karpenter provisioner support? -See [Provisioner API]({{< ref "./concepts/provisioners" >}}) for provisioner examples and descriptions of features. +### What features does the Karpenter NodePool support? +See the [NodePool API docs]({{< ref "./concepts/nodepools" >}}) for NodePool examples and descriptions of features. -### Can I create multiple (team-based) provisioners on a cluster? -Yes, provisioners can identify multiple teams based on labels. -See [Provisioner API]({{< ref "./concepts/provisioners" >}}) for details. +### Can I create multiple (team-based) NodePools on a cluster? +Yes, NodePools can identify multiple teams based on labels. See the [NodePool API docs]({{< ref "./concepts/nodepools" >}}) for details. -### If multiple provisioners are defined, which will my pod use? +### If multiple NodePools are defined, which will my pod use? -Pending pods will be handled by any Provisioner that matches the requirements of the pod. -There is no ordering guarantee if multiple provisioners match pod requirements. -We recommend that Provisioners are setup to be mutually exclusive. -Read more about this recommendation in the [EKS Best Practices Guide for Karpenter](https://aws.github.io/aws-eks-best-practices/karpenter/#create-provisioners-that-are-mutually-exclusive). -To select a specific provisioner, use the node selector `karpenter.sh/provisioner-name: my-provisioner`. +Pending pods will be handled by any NodePools that matches the requirements of the pod. There is no ordering guarantee if multiple NodePools match pod requirements. We recommend that NodePools are set-up to be mutually exclusive. To select a specific NodePool, use the node selector `karpenter.sh/nodepool: my-nodepool`. ### How can I configure Karpenter to only provision pods for a particular namespace? -There is no native support for namespaced based provisioning. -Karpenter can be configured to provision a subset of pods based on a combination of taints/tolerations and node selectors. -This allows Karpenter to work in concert with the `kube-scheduler` in that the same mechanisms that `kube-scheduler` uses to determine if a pod can schedule to an existing node are also used for provisioning new nodes. -This avoids scenarios where pods are bound to nodes that were provisioned by Karpenter which Karpenter would not have bound itself. -If this were to occur, a node could remain non-empty and have its lifetime extended due to a pod that wouldn't have caused the node to be provisioned had the pod been unschedulable. +There is no native support for namespaced-based provisioning. Karpenter can be configured to provision a subset of pods based on a combination of taints/tolerations and node selectors. This allows Karpenter to work in concert with the `kube-scheduler` using the same mechanisms to determine if a pod can schedule to an existing node are also used for provisioning new nodes. This avoids scenarios where pods are bound to nodes that were provisioned by Karpenter which Karpenter would not have bound itself. If this were to occur, a node could remain non-empty and have its lifetime extended due to a pod that wouldn't have caused the node to be provisioned had the pod been unschedulable. -We recommend using Kubernetes native scheduling constraints to achieve namespace based scheduling segregation. Using native scheduling constraints ensures that Karpenter, `kube-scheduler` and any other scheduling or auto-provisioning mechanism all have an identical understanding of which pods can be scheduled on which nodes. This can be enforced via policy agents, an example of which can be seen [here](https://blog.mikesir87.io/2022/01/creating-tenant-node-pools-with-karpenter/). +We recommend using Kubernetes native scheduling constraints to achieve namespace-based scheduling segregation. Using native scheduling constraints ensures that Karpenter, `kube-scheduler` and any other scheduling or auto-provisioning mechanism all have an identical understanding of which pods can be scheduled on which nodes. This can be enforced via policy agents, an example of which can be seen [here](https://blog.mikesir87.io/2022/01/creating-tenant-node-pools-with-karpenter/). -### Can I add SSH keys to a provisioner? +### Can I add SSH keys to a NodePool? -Karpenter does not offer a way to add SSH keys via provisioners or secrets to the nodes it manages. -However, you can use Session Manager (SSM) or EC2 Instance Connect to gain shell access to Karpenter nodes. -See [Node NotReady]({{< ref "./troubleshooting/#node-notready" >}}) troubleshooting for an example of starting an SSM session from the command line or [EC2 Instance Connect](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-connect-set-up.html) documentation to connect to nodes using SSH. +Karpenter does not offer a way to add SSH keys via NodePools or secrets to the nodes it manages. +However, you can use Session Manager (SSM) or EC2 Instance Connect to gain shell access to Karpenter nodes. See [Node NotReady]({{< ref "./troubleshooting/#node-notready" >}}) troubleshooting for an example of starting an SSM session from the command line or [EC2 Instance Connect](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-connect-set-up.html) documentation to connect to nodes using SSH. -Though not recommended, if you need to access Karpenter-managed nodes without AWS credentials, you can add SSH keys using AWSNodeTemplate. -See [Custom User Data]({{< ref "./concepts/node-templates/#spec-userdata" >}}) for details. +Though not recommended, if you need to access Karpenter-managed nodes without AWS credentials, you can add SSH keys using EC2NodeClass User Data. See the [User Data section in the EC2NodeClass documentation]({{< ref "./concepts/nodeclasses/#specuserdata" >}}) for details. -### Can I set total limits of CPU and memory for a provisioner? -Yes, the setting is provider-specific. -See examples in [Accelerators, GPU]({{< ref "./concepts/scheduling/#accelerators-gpu-resources" >}}) Karpenter documentation. +### Can I set limits of CPU and memory for a NodePool? +Yes. View the [NodePool API docs]({{< ref "./concepts/nodepools#speclimits" >}}) for NodePool examples and descriptions of how to configure limits. ### Can I mix spot and on-demand EC2 run types? -Yes, see [Provisioning]({{< ref "./concepts/provisioners#examples" >}}) for an example. +Yes, see the [NodePool API docs]({{< ref "./concepts/nodepools#examples" >}}) for an example. ### Can I restrict EC2 instance types? @@ -104,53 +88,48 @@ Yes, see [Provisioning]({{< ref "./concepts/provisioners#examples" >}}) for an e ### Can I use Bare Metal instance types? -Yes, Karpenter supports provisioning metal instance types when a Provisioner's `node.kubernetes.io/instance-type` Requirements only include `metal` instance types. If other instance types fulfill pod requirements, then Karpenter will prioritize all non-metal instance types before metal ones are provisioned. +Yes, Karpenter supports provisioning metal instance types when a NodePool's `node.kubernetes.io/instance-type` Requirements only include `metal` instance types. If other instance types fulfill pod requirements, then Karpenter will prioritize all non-metal instance types before metal ones are provisioned. ### How does Karpenter dynamically select instance types? -Karpenter batches pending pods and then binpacks them based on CPU, memory, and GPUs required, taking into account node overhead, VPC CNI resources required, and daemonsets that will be packed when bringing up a new node. -By default Karpenter uses C, M, and R >= Gen 3 instance types, but it can be constrained in the provisioner spec with the [instance-type](https://kubernetes.io/docs/reference/labels-annotations-taints/#nodekubernetesioinstance-type) well-known label in the requirements section. +Karpenter batches pending pods and then binpacks them based on CPU, memory, and GPUs required, taking into account node overhead, VPC CNI resources required, and daemonsets that will be packed when bringing up a new node. Karpenter [recommends the use of C, M, and R >= Gen 3 instance types]({{< ref "./concepts/nodepools#spectemplatespecrequirements" >}}) for most generic workloads, but it can be constrained in the NodePool spec with the [instance-type](https://kubernetes.io/docs/reference/labels-annotations-taints/#nodekubernetesioinstance-type) well-known label in the requirements section. + After the pods are binpacked on the most efficient instance type (i.e. the smallest instance type that can fit the pod batch), Karpenter takes 59 other instance types that are larger than the most efficient packing, and passes all 60 instance type options to an API called Amazon EC2 Fleet. -The EC2 fleet API attempts to provision the instance type based on an allocation strategy. -If you are using the on-demand capacity type, then Karpenter uses the `lowest-price` allocation strategy. -So fleet will provision the lowest priced instance type it can get from the 60 instance types Karpenter passed to the EC2 fleet API. -If the instance type is unavailable for some reason, then fleet will move on to the next cheapest instance type. -If you are using the spot capacity type, Karpenter uses the price-capacity-optimized allocation strategy. This tells fleet to find the instance type that EC2 has the most capacity for while also considering price. This allocation strategy will balance cost and decrease the probability of a spot interruption happening in the near term. -See [Choose the appropriate allocation strategy](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-fleet-allocation-strategy.html#ec2-fleet-allocation-use-cases) for information on fleet optimization. + + +The EC2 fleet API attempts to provision the instance type based on the [Price Capacity Optimized allocation strategy](https://aws.amazon.com/blogs/compute/introducing-price-capacity-optimized-allocation-strategy-for-ec2-spot-instances/). For the on-demand capacity type, this is effectively equivalent to the `lowest-price` allocation strategy. For the spot capacity type, Fleet will determine an instance type that has both the lowest price combined with the lowest chance of being interrupted. Note that this may not give you the instance type with the strictly lowest price for spot. ### How does Karpenter calculate the resource usage of Daemonsets when simulating scheduling? -Karpenter currently calculates the applicable daemonsets at the provisioner level with label selectors/taints, etc. It does not look to see if there are requirements on the daemonsets that would exclude it from running on particular instances that the provisioner could or couldn't launch. -The recommendation for now is to use multiple provisioners with taints/tolerations or label selectors to limit daemonsets to only nodes launched from specific provisioners. +Karpenter currently calculates the applicable daemonsets at the NodePool level with label selectors/taints, etc. It does not look to see if there are requirements on the daemonsets that would exclude it from running on particular instances that the NodePool could or couldn't launch. +The recommendation for now is to use multiple NodePools with taints/tolerations or label selectors to limit daemonsets to only nodes launched from specific NodePoools. ### What if there is no Spot capacity? Will Karpenter use On-Demand? -The best defense against running out of Spot capacity is to allow Karpenter to provision as many different instance types as possible. -Even instance types that have higher specs, e.g. vCPU, memory, etc., than what you need can still be cheaper in the Spot market than using On-Demand instances. -When Spot capacity is constrained, On-Demand capacity can also be constrained since Spot is fundamentally spare On-Demand capacity. -Allowing Karpenter to provision nodes from a large, diverse set of instance types will help you to stay on Spot longer and lower your costs due to Spot’s discounted pricing. -Moreover, if Spot capacity becomes constrained, this diversity will also increase the chances that you’ll be able to continue to launch On-Demand capacity for your workloads. +The best defense against running out of Spot capacity is to allow Karpenter to provision as many distinct instance types as possible. Even instance types that have higher specs (e.g. vCPU, memory, etc.) than what you need can still be cheaper in the Spot market than using On-Demand instances. When Spot capacity is constrained, On-Demand capacity can also be constrained since Spot is fundamentally spare On-Demand capacity. -If your Karpenter Provisioner specifies flexibility to both Spot and On-Demand capacity, Karpenter will attempt to provision On-Demand capacity if there is no Spot capacity available. -However, it’s strongly recommended that you specify at least 20 instance types in your Provisioner (or none and allow Karpenter to pick the best instance types) as our research indicates that this additional diversity increases the chances that your workloads will not need to launch On-Demand capacity at all. -Today, Karpenter will warn you if the number of instances in your Provisioner isn’t sufficiently diverse. +Allowing Karpenter to provision nodes from a large, diverse set of instance types will help you to stay on Spot longer and lower your costs due to Spot’s discounted pricing. Moreover, if Spot capacity becomes constrained, this diversity will also increase the chances that you’ll be able to continue to launch On-Demand capacity for your workloads. -Technically, Karpenter has a concept of an “offering” for each instance type, which is a combination of zone and capacity type (equivalent in the AWS cloud provider to an EC2 purchase option – Spot or On-Demand). -Whenever the Fleet API returns an insufficient capacity error for Spot instances, those particular offerings are temporarily removed from consideration (across the entire provisioner) so that Karpenter can make forward progress with different options. +If your Karpenter NodePool specifies allows both Spot and On-Demand capacity, Karpenter will fallback to provision On-Demand capacity if there is no Spot capacity available. However, it’s strongly recommended that you allow at least 20 instance types in your NodePool since this additional diversity increases the chances that your workloads will not need to launch On-Demand capacity at all. + +Karpenter has a concept of an “offering” for each instance type, which is a combination of zone and capacity type. Whenever the Fleet API returns an insufficient capacity error for Spot instances, those particular offerings are temporarily removed from consideration (across the entire NodePool) so that Karpenter can make forward progress with different options. ### Does Karpenter support IPv6? -Yes! Karpenter dynamically discovers if you are running in an IPv6 cluster by checking the kube-dns service's cluster-ip. When using an AMI Family such as `AL2`, Karpenter will automatically configure the EKS Bootstrap script for IPv6. Some EC2 instance types do not support IPv6 and the Amazon VPC CNI only supports instance types that run on the Nitro hypervisor. It's best to add a requirement to your Provisioner to only allow Nitro instance types: +Yes! Karpenter dynamically discovers if you are running in an IPv6 cluster by checking the kube-dns service's cluster-ip. When using an AMI Family such as `AL2`, Karpenter will automatically configure the EKS Bootstrap script for IPv6. Some EC2 instance types do not support IPv6 and the Amazon VPC CNI only supports instance types that run on the Nitro hypervisor. It's best to add a requirement to your NodePool to only allow Nitro instance types: ``` -kind: Provisioner +apiVersion: karpenter.sh/v1beta1 +kind: NodePool ... spec: - requirements: - - key: karpenter.k8s.aws/instance-hypervisor - operator: In - values: - - nitro + template: + spec: + requirements: + - key: karpenter.k8s.aws/instance-hypervisor + operator: In + values: + - nitro ``` For more documentation on enabling IPv6 with the Amazon VPC CNI, see the [docs](https://docs.aws.amazon.com/eks/latest/userguide/cni-ipv6.html). @@ -165,7 +144,7 @@ Windows nodes do not support IPv6. `kube-scheduler` is responsible for the scheduling of pods, while Karpenter launches the capacity. When using any sort of preferred scheduling constraint, `kube-scheduler` will schedule pods to nodes anytime it is possible. -As an example, suppose you scale up a deployment with a preferred zonal topology spread and none of the newly created pods can run on your existing cluster. Karpenter will then launch multiple nodes to satisfy that preference. If a) one of the nodes becomes ready slightly faster than other nodes and b) has enough capacity for multiple pods, `kube-scheduler` will schedule as many pods as possible to the single ready node so they won't remain unschedulable. It doesn't consider the in-flight capacity that will be ready in a few seconds. If all of the pods fit on the single node, the remaining nodes that Karpenter has launched aren't needed when they become ready and consolidation will delete them. +As an example, suppose you scale up a deployment with a preferred zonal topology spread and none of the newly created pods can run on your existing cluster. Karpenter will then launch multiple nodes to satisfy that preference. If a) one of the nodes becomes ready slightly faster than other nodes and b) has enough capacity for multiple pods, `kube-scheduler` will schedule as many pods as possible to the single ready node, so they won't remain unschedulable. It doesn't consider the in-flight capacity that will be ready in a few seconds. If all the pods fit on the single node, the remaining nodes that Karpenter has launched aren't needed when they become ready and consolidation will delete them. ### When deploying an additional DaemonSet to my cluster, why does Karpenter not scale-up my nodes to support the extra DaemonSet? @@ -194,21 +173,26 @@ See [Application developer]({{< ref "./concepts/#application-developer" >}}) for Yes. See [Persistent Volume Topology]({{< ref "./concepts/scheduling#persistent-volume-topology" >}}) for details. ### Can I set `--max-pods` on my nodes? -Yes, see the [KubeletConfiguration Section in the Provisioners Documentation]({{}}) to learn more. +Yes, see the [KubeletConfiguration Section in the NodePool docs]({{}}) to learn more. ### Why do the Windows2019 and Windows2022 AMI families only support Windows Server Core? The difference between the Core and Full variants is that Core is a minimal OS with less components and no graphic user interface (GUI) or desktop experience. `Windows2019` and `Windows2022` AMI families use the Windows Server Core option for simplicity, but if required, you can specify a custom AMI to run Windows Server Full. -You can specify the [Amazon EKS optimized AMI](https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-windows-ami.html) with Windows Server 2022 Full for Kubernetes 1.27 by configuring an `amiSelector` that references the AMI name. -``` -amiSelector: - aws::name: Windows_Server-2022-English-Full-EKS_Optimized-1.27* +You can specify the [Amazon EKS optimized AMI](https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-windows-ami.html) with Windows Server 2022 Full for Kubernetes 1.29 by configuring an `amiSelector` that references the AMI name. +```yaml +amiSelectorTerms: + - name: Windows_Server-2022-English-Full-EKS_Optimized-1.29* ``` +### Can I use Karpenter to scale my workload's pods? +Karpenter is a node autoscaler which will create new nodes in response to unschedulable pods. Scaling the pods themselves is outside of its scope. +This is the realm of pod autoscalers such as the [Vertical Pod Autoscaler](https://github.com/kubernetes/autoscaler/tree/master/vertical-pod-autoscaler) (for scaling an individual pod's resources) or the [Horizontal Pod Autoscaler](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale/) (for scaling replicas). +We also recommend taking a look at [Keda](https://keda.sh/) if you're looking for more advanced autoscaling capabilities for pods. + ## Deprovisioning ### How does Karpenter deprovision nodes? -See [Deprovisioning nodes]({{< ref "./concepts/deprovisioning" >}}) for information on how Karpenter deprovisions nodes. +See [Deprovisioning nodes]({{< ref "./concepts/disruption" >}}) for information on how Karpenter deprovisions nodes. ## Upgrading Karpenter @@ -218,26 +202,24 @@ Use your existing upgrade mechanisms to upgrade your core add-ons in Kubernetes Karpenter requires proper permissions in the `KarpenterNode IAM Role` and the `KarpenterController IAM Role`. To upgrade Karpenter to version `$VERSION`, make sure that the `KarpenterNode IAM Role` and the `KarpenterController IAM Role` have the right permission described in `https://karpenter.sh/$VERSION/getting-started/getting-started-with-karpenter/cloudformation.yaml`. -Next, locate `KarpenterController IAM Role` ARN (i.e., ARN of the resource created in [Create the KarpenterController IAM Role](../getting-started/getting-started-with-karpenter/#create-the-karpentercontroller-iam-role)) and pass them to the helm upgrade command. +Next, locate `KarpenterController IAM Role` ARN (i.e., ARN of the resource created in [Create the KarpenterController IAM Role](../getting-started/getting-started-with-karpenter/#create-the-karpentercontroller-iam-role)) and pass them to the Helm upgrade command. {{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step08-apply-helm-chart.sh" language="bash"%}} -For information on upgrading Karpenter, see the [Upgrade Guide]({{< ref "./upgrade-guide/" >}}). - -### Why do I get an `unknown field "startupTaints"` error when creating a provisioner with startupTaints? - -```bash -error: error validating "provisioner.yaml": error validating data: ValidationError(Provisioner.spec): unknown field "startupTaints" in sh.karpenter.v1alpha5.Provisioner.spec; if you choose to ignore these errors, turn validation off with --validate=false -``` - -The `startupTaints` parameter was added in v0.10.0. Helm upgrades do not upgrade the CRD describing the provisioner, so it must be done manually. For specific details, see the [Upgrade Guide]({{< ref "./upgrade-guide/#upgrading-to-v0100" >}}) +For information on upgrading Karpenter, see the [Upgrade Guide]({{< ref "./upgrading/upgrade-guide/" >}}). ## Upgrading Kubernetes Cluster ### How do I upgrade an EKS Cluster with Karpenter? -When upgrading an Amazon EKS cluster, [Karpenter's Drift feature]({{}}) can automatically upgrade the Karpenter-provisioned nodes to stay in-sync with the EKS control plane. Karpenter Drift currently needs to be enabled using a [feature gate]({{}}). Karpenter's default [AWSNodeTemplate `amiFamily` configuration]({{}}) uses the latest EKS Optimized AL2 AMI for the same major and minor version as the EKS cluster's control plane. Karpenter's AWSNodeTemplate can be configured to not use the EKS optimized AL2 AMI in favor of a custom AMI by configuring the [`amiSelector`]({{}}). If using a custom AMI, you will need to trigger the rollout of this new worker node image through the publication of a new AMI with tags matching the [`amiSelector`]({{}}), or a change to the [`amiSelector`]({{}}) field. +When upgrading an Amazon EKS cluster, [Karpenter's Drift feature]({{}}) can automatically upgrade the Karpenter-provisioned nodes to stay in-sync with the EKS control plane. Karpenter Drift is enabled by default starting `0.33.0`. + +{{% alert title="Note" color="primary" %}} +Karpenter's default [EC2NodeClass `amiFamily` configuration]({{}}) uses the latest EKS Optimized AL2 AMI for the same major and minor version as the EKS cluster's control plane, meaning that an upgrade of the control plane will cause Karpenter to auto-discover the new AMIs for that version. + +If using a custom AMI, you will need to trigger the rollout of this new worker node image through the publication of a new AMI with tags matching the [`amiSelector`]({{}}), or a change to the [`amiSelector`]({{}}) field. +{{% /alert %}} -Start by [upgrading the EKS Cluster control plane](https://docs.aws.amazon.com/eks/latest/userguide/update-cluster.html). After the EKS Cluster upgrade completes, Karpenter's Drift feature will detect that the Karpenter-provisioned nodes are using EKS Optimized AMIs for the previous cluster version, and [automatically cordon, drain, and replace those nodes]({{}}). To support pods moving to new nodes, follow Kubernetes best practices by setting appropriate pod [Resource Quotas](https://kubernetes.io/docs/concepts/policy/resource-quotas/), and using [Pod Disruption Budgets](https://kubernetes.io/docs/concepts/workloads/pods/disruptions/) (PDB). Karpenter's Drift feature will spin up replacement nodes based on the pod resource requests, and will respect the PDBs when deprovisioning nodes. +Start by [upgrading the EKS Cluster control plane](https://docs.aws.amazon.com/eks/latest/userguide/update-cluster.html). After the EKS Cluster upgrade completes, Karpenter's Drift feature will detect that the Karpenter-provisioned nodes are using EKS Optimized AMIs for the previous cluster version, and [automatically cordon, drain, and replace those nodes]({{}}). To support pods moving to new nodes, follow Kubernetes best practices by setting appropriate pod [Resource Quotas](https://kubernetes.io/docs/concepts/policy/resource-quotas/), and using [Pod Disruption Budgets](https://kubernetes.io/docs/concepts/workloads/pods/disruptions/) (PDB). Karpenter's Drift feature will spin up replacement nodes based on the pod resource requests, and will respect the PDBs when deprovisioning nodes. ## Interruption Handling @@ -249,10 +231,10 @@ Karpenter's native interruption handling offers two main benefits over the stand 1. You don't have to manage and maintain a separate component to exclusively handle interruption events. 2. Karpenter's native interruption handling coordinates with other deprovisioning so that consolidation, expiration, etc. can be aware of interruption events and vice-versa. -### Why am I receiving QueueNotFound errors when I set `aws.interruptionQueueName`? +### Why am I receiving QueueNotFound errors when I set `--interruption-queue-name`? Karpenter requires a queue to exist that receives event messages from EC2 and health services in order to handle interruption messages properly for nodes. -Details on the types of events that Karpenter handles can be found in the [Interruption Handling Docs]({{< ref "./concepts/deprovisioning/#interruption" >}}). +Details on the types of events that Karpenter handles can be found in the [Interruption Handling Docs]({{< ref "./concepts/disruption/#interruption" >}}). Details on provisioning the SQS queue and EventBridge rules can be found in the [Getting Started Guide]({{< ref "./getting-started/getting-started-with-karpenter/#create-the-karpenter-infrastructure-and-iam-roles" >}}). @@ -268,4 +250,4 @@ Consolidation packs pods tightly onto nodes which can leave little free allocata Karpenter uses [uber-go/zap](https://github.com/uber-go/zap) for logging. You can customize or configure the log messages by editing the [configmap-logging.yaml](https://github.com/aws/karpenter/blob/main/charts/karpenter/templates/configmap-logging.yaml) `ConfigMap`'s [data.zap-logger-config](https://github.com/aws/karpenter/blob/main/charts/karpenter/templates/configmap-logging.yaml#L26) field. -The available configuration options are specified in the [zap.Config godocs](https://pkg.go.dev/go.uber.org/zap#Config). \ No newline at end of file +The available configuration options are specified in the [zap.Config godocs](https://pkg.go.dev/go.uber.org/zap#Config). diff --git a/website/content/en/v0.30/getting-started/_index.md b/website/content/en/v0.35/getting-started/_index.md similarity index 75% rename from website/content/en/v0.30/getting-started/_index.md rename to website/content/en/v0.35/getting-started/_index.md index c159f68f5e48..cebdf66a99fd 100644 --- a/website/content/en/v0.30/getting-started/_index.md +++ b/website/content/en/v0.35/getting-started/_index.md @@ -1,18 +1,16 @@ --- title: "Getting Started" linkTitle: "Getting Started" -weight: 1 +weight: 10 description: > Choose from different methods to get started with Karpenter -cascade: - type: docs --- To get started with Karpenter, the [Getting Started with Karpenter]({{< relref "getting-started-with-karpenter" >}}) guide provides an end-to-end procedure for creating a cluster (with `eksctl`) and adding Karpenter. If you prefer, the following instructions use Terraform to create a cluster and add Karpenter: -* [Amazon EKS Blueprints for Terraform](https://aws-ia.github.io/terraform-aws-eks-blueprints): Follow a basic [Getting Started](https://aws-ia.github.io/terraform-aws-eks-blueprints/v4.18.0/getting-started/) guide and also add modules and add-ons. This includes a [Karpenter](https://aws-ia.github.io/terraform-aws-eks-blueprints/v4.18.0/add-ons/karpenter/) add-on that lets you bypass the instructions in this guide for setting up Karpenter. +* [Amazon EKS Blueprints for Terraform](https://aws-ia.github.io/terraform-aws-eks-blueprints): Follow a basic [Getting Started](https://aws-ia.github.io/terraform-aws-eks-blueprints/getting-started/) guide and also add modules and add-ons. This includes a [Karpenter](https://aws-ia.github.io/terraform-aws-eks-blueprints/patterns/karpenter/) add-on that lets you bypass the instructions in this guide for setting up Karpenter. Although not supported, you could also try Karpenter on other Kubernetes distributions running on AWS. For example: @@ -23,6 +21,6 @@ Learn more about Karpenter and how to get started below. * [Karpenter EKS Best Practices](https://aws.github.io/aws-eks-best-practices/karpenter/) guide * [EC2 Spot Workshop for Karpenter](https://ec2spotworkshops.com/karpenter.html) * [EKS Karpenter Workshop](https://www.eksworkshop.com/docs/autoscaling/compute/karpenter/) -* [Advanced EKS Immersion Karpenter Workshop](https://catalog.workshops.aws/eks-advanced/karpenter/) +* [Advanced EKS Immersion Karpenter Workshop](https://catalog.us-east-1.prod.workshops.aws/workshops/76a5dd80-3249-4101-8726-9be3eeee09b2/en-US/autoscaling/karpenter) * [Karpenter Blueprints](https://github.com/aws-samples/karpenter-blueprints) * [Tutorial: Run Kubernetes Clusters for Less with Amazon EC2 Spot and Karpenter](https://community.aws/tutorials/run-kubernetes-clusters-for-less-with-amazon-ec2-spot-and-karpenter#step-6-optional-simulate-spot-interruption) diff --git a/website/content/en/v0.35/getting-started/getting-started-with-karpenter/_index.md b/website/content/en/v0.35/getting-started/getting-started-with-karpenter/_index.md new file mode 100644 index 000000000000..fa34d319bd24 --- /dev/null +++ b/website/content/en/v0.35/getting-started/getting-started-with-karpenter/_index.md @@ -0,0 +1,247 @@ + +--- +title: "Getting Started with Karpenter" +linkTitle: "Getting Started with Karpenter" +weight: 10 +description: > + Set up a cluster and add Karpenter +--- + +Karpenter automatically provisions new nodes in response to unschedulable pods. Karpenter does this by observing events within the Kubernetes cluster, and then sending commands to the underlying cloud provider. + +This guide shows how to get started with Karpenter by creating a Kubernetes cluster and installing Karpenter. +To use Karpenter, you must be running a supported Kubernetes cluster on a supported cloud provider. +Currently, only EKS on AWS is supported. + +## Create a cluster and add Karpenter + +This guide uses `eksctl` to create the cluster. +It should take less than 1 hour to complete, and cost less than $0.25. +Follow the clean-up instructions to reduce any charges. + +### 1. Install utilities + +Karpenter is installed in clusters with a Helm chart. + +Karpenter requires cloud provider permissions to provision nodes, for AWS IAM +Roles for Service Accounts (IRSA) should be used. IRSA permits Karpenter +(within the cluster) to make privileged requests to AWS (as the cloud provider) +via a ServiceAccount. + +Install these tools before proceeding: + +1. [AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/install-cliv2-linux.html) +2. `kubectl` - [the Kubernetes CLI](https://kubernetes.io/docs/tasks/tools/install-kubectl-linux/) +3. `eksctl` (>= v0.169.0) - [the CLI for AWS EKS](https://docs.aws.amazon.com/eks/latest/userguide/eksctl.html) +4. `helm` - [the package manager for Kubernetes](https://helm.sh/docs/intro/install/) + +[Configure the AWS CLI](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-quickstart.html) +with a user that has sufficient privileges to create an EKS cluster. Verify that the CLI can +authenticate properly by running `aws sts get-caller-identity`. + +### 2. Set environment variables + +After setting up the tools, set the Karpenter and Kubernetes version: + +```bash +export KARPENTER_NAMESPACE="kube-system" +export KARPENTER_VERSION="0.35.4" +export K8S_VERSION="1.29" +``` + +Then set the following environment variable: + +{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step01-config.sh" language="bash"%}} + +{{% alert title="Warning" color="warning" %}} +If you open a new shell to run steps in this procedure, you need to set some or all of the environment variables again. +To remind yourself of these values, type: + +```bash +echo "${KARPENTER_NAMESPACE}" "${KARPENTER_VERSION}" "${K8S_VERSION}" "${CLUSTER_NAME}" "${AWS_DEFAULT_REGION}" "${AWS_ACCOUNT_ID}" "${TEMPOUT}" "${ARM_AMI_ID}" "${AMD_AMI_ID}" "${GPU_AMI_ID}" +``` + +{{% /alert %}} + + +### 3. Create a Cluster + +Create a basic cluster with `eksctl`. +The following cluster configuration will: + +* Use CloudFormation to set up the infrastructure needed by the EKS cluster. See [CloudFormation]({{< relref "../../reference/cloudformation/" >}}) for a complete description of what `cloudformation.yaml` does for Karpenter. +* Create a Kubernetes service account and AWS IAM Role, and associate them using IRSA to let Karpenter launch instances. +* Add the Karpenter node role to the aws-auth configmap to allow nodes to connect. +* Use [AWS EKS managed node groups](https://docs.aws.amazon.com/eks/latest/userguide/managed-node-groups.html) for the kube-system and karpenter namespaces. Uncomment fargateProfiles settings (and comment out managedNodeGroups settings) to use Fargate for both namespaces instead. +* Set KARPENTER_IAM_ROLE_ARN variables. +* Create a role to allow spot instances. +* Run Helm to install Karpenter + +{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step02-create-cluster.sh" language="bash"%}} + +{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step06-add-spot-role.sh" language="bash"%}} + +{{% alert title="Windows Support Notice" color="warning" %}} +In order to run Windows workloads, Windows support should be enabled in your EKS Cluster. +See [Enabling Windows support](https://docs.aws.amazon.com/eks/latest/userguide/windows-support.html#enable-windows-support) to learn more. +{{% /alert %}} + +### 4. Install Karpenter + +{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step08-apply-helm-chart.sh" language="bash"%}} + +{{% alert title="DNS Policy Notice" color="warning" %}} +Karpenter uses the `ClusterFirst` pod DNS policy by default. This is the Kubernetes cluster default and this ensures that Karpetner can reach-out to internal Kubernetes services during its lifetime. There may be cases where you do not have the DNS service that you are using on your cluster up-and-running before Karpenter starts up. The most common case of this is you want Karpenter to manage the node capacity where your DNS service pods are running. + +If you need Karpenter to manage the DNS service pods' capacity, this means that DNS won't be running when Karpenter starts-up. In this case, you will need to set the pod DNS policy to `Default` with `--set dnsPolicy=Default`. This will tell Karpenter to use the host's DNS resolution instead of the internal DNS resolution, ensuring that you don't have a dependency on the DNS service pods to run. More details on this issue can be found in the following Github issues: [#2186](https://github.com/aws/karpenter-provider-aws/issues/2186) and [#4947](https://github.com/aws/karpenter-provider-aws/issues/4947). +{{% /alert %}} + +{{% alert title="Common Expression Language/Webhooks Notice" color="warning" %}} +Karpenter supports using [Kubernetes Common Expression Language](https://kubernetes.io/docs/reference/using-api/cel/) for validating its Custom Resource Definitions out-of-the-box; however, this feature is not supported on versions of Kubernetes < 1.25. If you are running an earlier version of Kubernetes, you will need to use the Karpenter admission webhooks for validation instead. You can enable these webhooks with `--set webhook.enabled=true` when applying the Karpenter Helm chart. +{{% /alert %}} + +{{% alert title="Pod Identity Supports Notice" color="warning" %}} +Karpenter now supports using [Pod Identity](https://docs.aws.amazon.com/eks/latest/userguide/pod-identities.html) to authenticate AWS SDK to make API requests to AWS services using AWS Identity and Access Management (IAM) permissions. This feature not supported on versions of Kubernetes < 1.24. If you are running an earlier version of Kubernetes, you will need to use the [IAM Roles for Service Accounts(IRSA)](https://docs.aws.amazon.com/emr/latest/EMR-on-EKS-DevelopmentGuide/setting-up-enable-IAM.html) for pod authentication instead. You can enable these IRSA with `--set "serviceAccount.annotations.eks\.amazonaws\.com/role-arn=${KARPENTER_IAM_ROLE_ARN}"` when applying the Karpenter Helm chart. +{{% /alert %}} + +{{% alert title="Warning" color="warning" %}} +Karpenter creates a mapping between CloudProvider machines and CustomResources in the cluster for capacity tracking. To ensure this mapping is consistent, Karpenter utilizes the following tag keys: + +* `karpenter.sh/managed-by` +* `karpenter.sh/nodepool` +* `kubernetes.io/cluster/${CLUSTER_NAME}` + +Because Karpenter takes this dependency, any user that has the ability to Create/Delete these tags on CloudProvider machines will have the ability to orchestrate Karpenter to Create/Delete CloudProvider machines as a side effect. We recommend that you [enforce tag-based IAM policies](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_tags.html) on these tags against any EC2 instance resource (`i-*`) for any users that might have [CreateTags](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateTags.html)/[DeleteTags](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DeleteTags.html) permissions but should not have [RunInstances](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_RunInstances.html)/[TerminateInstances](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_TerminateInstances.html) permissions. +{{% /alert %}} + +### 5. Create NodePool + +A single Karpenter NodePool is capable of handling many different pod shapes. Karpenter makes scheduling and provisioning decisions based on pod attributes such as labels and affinity. In other words, Karpenter eliminates the need to manage many different node groups. + +Create a default NodePool using the command below. This NodePool uses `securityGroupSelectorTerms` and `subnetSelectorTerms` to discover resources used to launch nodes. We applied the tag `karpenter.sh/discovery` in the `eksctl` command above. Depending on how these resources are shared between clusters, you may need to use different tagging schemes. + +The `consolidationPolicy` set to `WhenUnderutilized` in the `disruption` block configures Karpenter to reduce cluster cost by removing and replacing nodes. As a result, consolidation will terminate any empty nodes on the cluster. This behavior can be disabled by setting `consolidateAfter` to `Never`, telling Karpenter that it should never consolidate nodes. Review the [NodePool API docs]({{}}) for more information. + +Note: This NodePool will create capacity as long as the sum of all created capacity is less than the specified limit. + +{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step12-add-nodepool.sh" language="bash"%}} + +Karpenter is now active and ready to begin provisioning nodes. + +### 6. Scale up deployment + +This deployment uses the [pause image](https://www.ianlewis.org/en/almighty-pause-container) and starts with zero replicas. + +{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step13-automatic-node-provisioning.sh" language="bash"%}} + +### 7. Scale down deployment + +Now, delete the deployment. After a short amount of time, Karpenter should terminate the empty nodes due to consolidation. + +{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step14-deprovisioning.sh" language="bash"%}} + +### 8. Delete Karpenter nodes manually + +If you delete a node with kubectl, Karpenter will gracefully cordon, drain, +and shutdown the corresponding instance. Under the hood, Karpenter adds a +finalizer to the node object, which blocks deletion until all pods are +drained and the instance is terminated. Keep in mind, this only works for +nodes provisioned by Karpenter. + +{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step16-delete-node.sh" language="bash"%}} + +### 9. Delete the cluster +To avoid additional charges, remove the demo infrastructure from your AWS account. + +{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step17-cleanup.sh" language="bash"%}} + +## Monitoring with Grafana (optional) + +This section describes optional ways to configure Karpenter to enhance its capabilities. +In particular, the following commands deploy a Prometheus and Grafana stack that is suitable for this guide but does not include persistent storage or other configurations that would be necessary for monitoring a production deployment of Karpenter. +This deployment includes two Karpenter dashboards that are automatically onboarded to Grafana. They provide a variety of visualization examples on Karpenter metrics. + +{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step09-add-prometheus-grafana.sh" language="bash"%}} + +The Grafana instance may be accessed using port forwarding. + +{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step10-add-grafana-port-forward.sh" language="bash"%}} + +The new stack has only one user, `admin`, and the password is stored in a secret. The following command will retrieve the password. + +{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step11-grafana-get-password.sh" language="bash"%}} + +## Advanced Installation + +The section below covers advanced installation techniques for installing Karpenter. This includes things such as running Karpenter on a cluster without public internet access or ensuring that Karpenter avoids getting throttled by other components in your cluster. + +### Private Clusters + +You can optionally install Karpenter on a [private cluster](https://docs.aws.amazon.com/eks/latest/userguide/private-clusters.html#private-cluster-requirements) using the `eksctl` installation by setting `privateCluster.enabled` to true in your [ClusterConfig](https://eksctl.io/usage/eks-private-cluster/#eks-fully-private-cluster) and by setting `--set settings.isolatedVPC=true` when installing the `karpenter` Helm chart. + +```bash +privateCluster: + enabled: true +``` + +Private clusters have no outbound access to the internet. This means that in order for Karpenter to reach out to the services that it needs to access, you need to enable specific VPC private endpoints. Below shows the endpoints that you need to enable to successfully run Karpenter in a private cluster: + +```text +com.amazonaws..ec2 +com.amazonaws..ecr.api +com.amazonaws..ecr.dkr +com.amazonaws..s3 – For pulling container images +com.amazonaws..sts – For IAM roles for service accounts +com.amazonaws..ssm - For resolving default AMIs +com.amazonaws..sqs - For accessing SQS if using interruption handling +com.amazonaws..eks - For Karpenter to discover the cluster endpoint +``` + +If you do not currently have these endpoints surfaced in your VPC, you can add the endpoints by running + +```bash +aws ec2 create-vpc-endpoint --vpc-id ${VPC_ID} --service-name ${SERVICE_NAME} --vpc-endpoint-type Interface --subnet-ids ${SUBNET_IDS} --security-group-ids ${SECURITY_GROUP_IDS} +``` + +{{% alert title="Note" color="primary" %}} + +Karpenter (controller and webhook deployment) container images must be in or copied to Amazon ECR private or to another private registry accessible from inside the VPC. If these are not available from within the VPC, or from networks peered with the VPC, you will get Image pull errors when Kubernetes tries to pull these images from ECR public. + +{{% /alert %}} + +{{% alert title="Note" color="primary" %}} + +There is currently no VPC private endpoint for the [IAM API](https://docs.aws.amazon.com/IAM/latest/APIReference/welcome.html). As a result, you cannot use the default `spec.role` field in your `EC2NodeClass`. Instead, you need to provision and manage an instance profile manually and then specify Karpenter to use this instance profile through the `spec.instanceProfile` field. + +You can provision an instance profile manually and assign a Node role to it by calling the following command + +```bash +aws iam create-instance-profile --instance-profile-name "KarpenterNodeInstanceProfile-${CLUSTER_NAME}" +aws iam add-role-to-instance-profile --instance-profile-name "KarpenterNodeInstanceProfile-${CLUSTER_NAME}" --role-name "KarpenterNodeRole-${CLUSTER_NAME}" +``` + +{{% /alert %}} + +{{% alert title="Note" color="primary" %}} + +There is currently no VPC private endpoint for the [Price List Query API](https://docs.aws.amazon.com/awsaccountbilling/latest/aboutv2/using-price-list-query-api.html). As a result, pricing data can go stale over time. By default, Karpenter ships a static price list that is updated when each binary is released. + +Failed requests for pricing data will result in the following error messages + +```bash +ERROR controller.aws.pricing updating on-demand pricing, RequestError: send request failed +caused by: Post "https://api.pricing.us-east-1.amazonaws.com/": dial tcp 52.94.231.236:443: i/o timeout; RequestError: send request failed +caused by: Post "https://api.pricing.us-east-1.amazonaws.com/": dial tcp 52.94.231.236:443: i/o timeout, using existing pricing data from 2022-08-17T00:19:52Z {"commit": "4b5f953"} +``` + +{{% /alert %}} + +### Preventing APIServer Request Throttling + +Kubernetes uses [FlowSchemas](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/#flowschema) and [PriorityLevelConfigurations](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/#prioritylevelconfiguration) to map calls to the API server into buckets which determine each user agent's throttling limits. + +By default, Karpenter is installed into the `kube-system` namespace, which leverages the `system-leader-election` and `kube-system-service-accounts` [FlowSchemas](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/#flowschema) to map calls from the `kube-system` namespace to the `leader-election` and `workload-high` PriorityLevelConfigurations respectively. By putting Karpenter in these PriorityLevelConfigurations, we ensure that Karpenter and other critical cluster components are able to run even if other components on the cluster are throttled in other PriorityLevelConfigurations. + +If you install Karpenter in a different namespace than the default `kube-system` namespace, Karpenter will not be put into these higher-priority FlowSchemas by default. Instead, you will need to create custom FlowSchemas for the namespace and service account where Karpenter is installed to ensure that requests are put into this higher PriorityLevelConfiguration. + +{{% script file="./content/en/{VERSION}/getting-started/getting-started-with-karpenter/scripts/step15-apply-flowschemas.sh" language="bash"%}} diff --git a/website/content/en/v0.30/getting-started/getting-started-with-karpenter/cloudformation.yaml b/website/content/en/v0.35/getting-started/getting-started-with-karpenter/cloudformation.yaml similarity index 71% rename from website/content/en/v0.30/getting-started/getting-started-with-karpenter/cloudformation.yaml rename to website/content/en/v0.35/getting-started/getting-started-with-karpenter/cloudformation.yaml index b8c8d64614e7..207761007d2d 100644 --- a/website/content/en/v0.30/getting-started/getting-started-with-karpenter/cloudformation.yaml +++ b/website/content/en/v0.35/getting-started/getting-started-with-karpenter/cloudformation.yaml @@ -5,13 +5,6 @@ Parameters: Type: String Description: "EKS cluster name" Resources: - KarpenterNodeInstanceProfile: - Type: "AWS::IAM::InstanceProfile" - Properties: - InstanceProfileName: !Sub "KarpenterNodeInstanceProfile-${ClusterName}" - Path: "/" - Roles: - - !Ref "KarpenterNodeRole" KarpenterNodeRole: Type: "AWS::IAM::Role" Properties: @@ -42,21 +35,36 @@ Resources: "Version": "2012-10-17", "Statement": [ { - "Sid": "AllowScopedEC2InstanceActions", + "Sid": "AllowScopedEC2InstanceAccessActions", "Effect": "Allow", "Resource": [ "arn:${AWS::Partition}:ec2:${AWS::Region}::image/*", "arn:${AWS::Partition}:ec2:${AWS::Region}::snapshot/*", - "arn:${AWS::Partition}:ec2:${AWS::Region}:*:spot-instances-request/*", "arn:${AWS::Partition}:ec2:${AWS::Region}:*:security-group/*", - "arn:${AWS::Partition}:ec2:${AWS::Region}:*:subnet/*", - "arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*" + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:subnet/*" ], "Action": [ "ec2:RunInstances", "ec2:CreateFleet" ] }, + { + "Sid": "AllowScopedEC2LaunchTemplateAccessActions", + "Effect": "Allow", + "Resource": "arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*", + "Action": [ + "ec2:RunInstances", + "ec2:CreateFleet" + ], + "Condition": { + "StringEquals": { + "aws:ResourceTag/kubernetes.io/cluster/${ClusterName}": "owned" + }, + "StringLike": { + "aws:ResourceTag/karpenter.sh/nodepool": "*" + } + } + }, { "Sid": "AllowScopedEC2InstanceActionsWithTags", "Effect": "Allow", @@ -65,7 +73,8 @@ Resources: "arn:${AWS::Partition}:ec2:${AWS::Region}:*:instance/*", "arn:${AWS::Partition}:ec2:${AWS::Region}:*:volume/*", "arn:${AWS::Partition}:ec2:${AWS::Region}:*:network-interface/*", - "arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*" + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*", + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:spot-instances-request/*" ], "Action": [ "ec2:RunInstances", @@ -77,7 +86,7 @@ Resources: "aws:RequestTag/kubernetes.io/cluster/${ClusterName}": "owned" }, "StringLike": { - "aws:RequestTag/karpenter.sh/provisioner-name": "*" + "aws:RequestTag/karpenter.sh/nodepool": "*" } } }, @@ -89,7 +98,8 @@ Resources: "arn:${AWS::Partition}:ec2:${AWS::Region}:*:instance/*", "arn:${AWS::Partition}:ec2:${AWS::Region}:*:volume/*", "arn:${AWS::Partition}:ec2:${AWS::Region}:*:network-interface/*", - "arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*" + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*", + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:spot-instances-request/*" ], "Action": "ec2:CreateTags", "Condition": { @@ -102,27 +112,26 @@ Resources: ] }, "StringLike": { - "aws:RequestTag/karpenter.sh/provisioner-name": "*" + "aws:RequestTag/karpenter.sh/nodepool": "*" } } }, { - "Sid": "AllowMachineMigrationTagging", + "Sid": "AllowScopedResourceTagging", "Effect": "Allow", "Resource": "arn:${AWS::Partition}:ec2:${AWS::Region}:*:instance/*", "Action": "ec2:CreateTags", "Condition": { "StringEquals": { - "aws:ResourceTag/kubernetes.io/cluster/${ClusterName}": "owned", - "aws:RequestTag/karpenter.sh/managed-by": "${ClusterName}" + "aws:ResourceTag/kubernetes.io/cluster/${ClusterName}": "owned" }, "StringLike": { - "aws:RequestTag/karpenter.sh/provisioner-name": "*" + "aws:ResourceTag/karpenter.sh/nodepool": "*" }, "ForAllValues:StringEquals": { "aws:TagKeys": [ - "karpenter.sh/provisioner-name", - "karpenter.sh/managed-by" + "karpenter.sh/nodeclaim", + "Name" ] } } @@ -143,7 +152,7 @@ Resources: "aws:ResourceTag/kubernetes.io/cluster/${ClusterName}": "owned" }, "StringLike": { - "aws:ResourceTag/karpenter.sh/provisioner-name": "*" + "aws:ResourceTag/karpenter.sh/nodepool": "*" } } }, @@ -186,7 +195,6 @@ Resources: "Resource": "${KarpenterInterruptionQueue.Arn}", "Action": [ "sqs:DeleteMessage", - "sqs:GetQueueAttributes", "sqs:GetQueueUrl", "sqs:ReceiveMessage" ] @@ -202,6 +210,68 @@ Resources: } } }, + { + "Sid": "AllowScopedInstanceProfileCreationActions", + "Effect": "Allow", + "Resource": "*", + "Action": [ + "iam:CreateInstanceProfile" + ], + "Condition": { + "StringEquals": { + "aws:RequestTag/kubernetes.io/cluster/${ClusterName}": "owned", + "aws:RequestTag/topology.kubernetes.io/region": "${AWS::Region}" + }, + "StringLike": { + "aws:RequestTag/karpenter.k8s.aws/ec2nodeclass": "*" + } + } + }, + { + "Sid": "AllowScopedInstanceProfileTagActions", + "Effect": "Allow", + "Resource": "*", + "Action": [ + "iam:TagInstanceProfile" + ], + "Condition": { + "StringEquals": { + "aws:ResourceTag/kubernetes.io/cluster/${ClusterName}": "owned", + "aws:ResourceTag/topology.kubernetes.io/region": "${AWS::Region}", + "aws:RequestTag/kubernetes.io/cluster/${ClusterName}": "owned", + "aws:RequestTag/topology.kubernetes.io/region": "${AWS::Region}" + }, + "StringLike": { + "aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass": "*", + "aws:RequestTag/karpenter.k8s.aws/ec2nodeclass": "*" + } + } + }, + { + "Sid": "AllowScopedInstanceProfileActions", + "Effect": "Allow", + "Resource": "*", + "Action": [ + "iam:AddRoleToInstanceProfile", + "iam:RemoveRoleFromInstanceProfile", + "iam:DeleteInstanceProfile" + ], + "Condition": { + "StringEquals": { + "aws:ResourceTag/kubernetes.io/cluster/${ClusterName}": "owned", + "aws:ResourceTag/topology.kubernetes.io/region": "${AWS::Region}" + }, + "StringLike": { + "aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass": "*" + } + } + }, + { + "Sid": "AllowInstanceProfileReadActions", + "Effect": "Allow", + "Resource": "*", + "Action": "iam:GetInstanceProfile" + }, { "Sid": "AllowAPIServerEndpointDiscovery", "Effect": "Allow", diff --git a/website/content/en/v0.31/getting-started/getting-started-with-karpenter/grafana-values.yaml b/website/content/en/v0.35/getting-started/getting-started-with-karpenter/grafana-values.yaml similarity index 83% rename from website/content/en/v0.31/getting-started/getting-started-with-karpenter/grafana-values.yaml rename to website/content/en/v0.35/getting-started/getting-started-with-karpenter/grafana-values.yaml index 9294e9af9e52..67d28f71217c 100644 --- a/website/content/en/v0.31/getting-started/getting-started-with-karpenter/grafana-values.yaml +++ b/website/content/en/v0.35/getting-started/getting-started-with-karpenter/grafana-values.yaml @@ -22,6 +22,6 @@ dashboardProviders: dashboards: default: capacity-dashboard: - url: https://karpenter.sh/v0.31/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json + url: https://karpenter.sh/v0.35/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json performance-dashboard: - url: https://karpenter.sh/v0.31/getting-started/getting-started-with-karpenter/karpenter-performance-dashboard.json + url: https://karpenter.sh/v0.35/getting-started/getting-started-with-karpenter/karpenter-performance-dashboard.json diff --git a/website/content/en/v0.31/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json b/website/content/en/v0.35/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json similarity index 94% rename from website/content/en/v0.31/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json rename to website/content/en/v0.35/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json index 9d9cb6ad14a8..d474d01f4e16 100644 --- a/website/content/en/v0.31/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json +++ b/website/content/en/v0.35/getting-started/getting-started-with-karpenter/karpenter-capacity-dashboard.json @@ -484,7 +484,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "sum by ($distribution_filter)(\n karpenter_pods_state{arch=~\"$arch\", capacity_type=~\"$capacity_type\", instance_type=~\"$instance_type\", provisioner=~\"$provisioner\"}\n)", + "expr": "sum by ($distribution_filter)(\n karpenter_pods_state{arch=~\"$arch\", capacity_type=~\"$capacity_type\", instance_type=~\"$instance_type\", nodepool=~\"$nodepool\"}\n)", "legendFormat": "{{label_name}}", "range": true, "refId": "A" @@ -588,7 +588,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "karpenter_provisioner_usage{resource_type=\"cpu\"} / karpenter_provisioner_limit{resource_type=\"cpu\"}", + "expr": "karpenter_nodepool_usage{resource_type=\"cpu\"} / karpenter_nodepool_limit{resource_type=\"cpu\"}", "format": "table", "instant": true, "legendFormat": "CPU Limit Utilization", @@ -602,7 +602,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "count by (provisioner)(karpenter_nodes_allocatable{provisioner!=\"N/A\",resource_type=\"cpu\"}) # Selects a single resource type to get node count", + "expr": "count by (nodepool)(karpenter_nodes_allocatable{nodepool!=\"N/A\",resource_type=\"cpu\"}) # Selects a single resource type to get node count", "format": "table", "hide": false, "instant": true, @@ -616,7 +616,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "karpenter_provisioner_usage{resource_type=\"memory\"} / karpenter_provisioner_limit{resource_type=\"memory\"}", + "expr": "karpenter_nodepool_usage{resource_type=\"memory\"} / karpenter_nodepool_limit{resource_type=\"memory\"}", "format": "table", "hide": false, "instant": true, @@ -631,7 +631,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum by (provisioner)(karpenter_nodes_allocatable{provisioner!=\"N/A\",resource_type=\"cpu\"})", + "expr": "sum by (nodepool)(karpenter_nodes_allocatable{nodepool!=\"N/A\",resource_type=\"cpu\"})", "format": "table", "hide": false, "instant": true, @@ -645,7 +645,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "sum by (provisioner)(karpenter_nodes_allocatable{provisioner!=\"N/A\",resource_type=\"memory\"})", + "expr": "sum by (nodepool)(karpenter_nodes_allocatable{nodepool!=\"N/A\",resource_type=\"memory\"})", "format": "table", "hide": false, "instant": true, @@ -653,12 +653,12 @@ "refId": "Memory Capacity" } ], - "title": "Provisioner Summary", + "title": "Nodepool Summary", "transformations": [ { "id": "seriesToColumns", "options": { - "byField": "provisioner" + "byField": "nodepool" } }, { @@ -697,7 +697,7 @@ "instance 2": 12, "job 1": 9, "job 2": 13, - "provisioner": 0, + "nodepool": 0, "resource_type 1": 10, "resource_type 2": 14 }, @@ -714,7 +714,7 @@ "instance": "", "instance 1": "", "job": "", - "provisioner": "Provisioner" + "nodepool": "Nodepool" } } } @@ -804,7 +804,7 @@ "uid": "${datasource}" }, "editorMode": "code", - "expr": "(count(karpenter_nodes_allocatable{arch=~\"$arch\",capacity_type=\"spot\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"}) or vector(0)) / count(karpenter_nodes_allocatable{arch=~\"$arch\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"})", + "expr": "(count(karpenter_nodes_allocatable{arch=~\"$arch\",capacity_type=\"spot\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}) or vector(0)) / count(karpenter_nodes_allocatable{arch=~\"$arch\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"})", "legendFormat": "Percentage", "range": true, "refId": "A" @@ -946,7 +946,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "((karpenter_nodes_total_daemon_requests{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0) + \n(karpenter_nodes_total_pod_requests{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0)) / \nkarpenter_nodes_allocatable{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"}", + "expr": "((karpenter_nodes_total_daemon_requests{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0) + \n(karpenter_nodes_total_pod_requests{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0)) / \nkarpenter_nodes_allocatable{resource_type=\"cpu\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}", "format": "table", "hide": false, "instant": true, @@ -961,7 +961,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "((karpenter_nodes_total_daemon_requests{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0) + \n(karpenter_nodes_total_pod_requests{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0)) / \nkarpenter_nodes_allocatable{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"}", + "expr": "((karpenter_nodes_total_daemon_requests{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0) + \n(karpenter_nodes_total_pod_requests{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} or karpenter_nodes_allocatable*0)) / \nkarpenter_nodes_allocatable{resource_type=\"memory\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}", "format": "table", "hide": false, "instant": true, @@ -976,7 +976,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "karpenter_nodes_total_daemon_requests{resource_type=\"pods\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"} + \nkarpenter_nodes_total_pod_requests{resource_type=\"pods\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",provisioner=~\"$provisioner\",zone=~\"$zone\"}", + "expr": "karpenter_nodes_total_daemon_requests{resource_type=\"pods\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"} + \nkarpenter_nodes_total_pod_requests{resource_type=\"pods\",arch=~\"$arch\",capacity_type=~\"$capacity_type\",instance_type=~\"$instance_type\",nodepool=~\"$nodepool\",zone=~\"$zone\"}", "format": "table", "hide": false, "instant": true, @@ -1091,9 +1091,9 @@ "os 1": true, "os 2": true, "os 3": true, - "provisioner 1": false, - "provisioner 2": true, - "provisioner 3": true, + "nodepool 1": false, + "nodepool 2": true, + "nodepool 3": true, "resource_type": true, "resource_type 1": true, "resource_type 2": true, @@ -1161,9 +1161,9 @@ "os 1": 23, "os 2": 41, "os 3": 61, - "provisioner 1": 2, - "provisioner 2": 42, - "provisioner 3": 62, + "nodepool 1": 2, + "nodepool 2": 42, + "nodepool 3": 62, "resource_type 1": 24, "resource_type 2": 43, "resource_type 3": 63, @@ -1190,7 +1190,7 @@ "instance_type": "Instance Type", "instance_type 1": "Instance Type", "node_name": "Node Name", - "provisioner 1": "Provisioner", + "nodepool 1": "Nodepool", "zone 1": "Zone" } } @@ -1237,14 +1237,14 @@ "type": "prometheus", "uid": "${datasource}" }, - "definition": "label_values(karpenter_nodes_allocatable, provisioner)", + "definition": "label_values(karpenter_nodes_allocatable, nodepool)", "hide": 0, "includeAll": true, "multi": true, - "name": "provisioner", + "name": "nodepool", "options": [], "query": { - "query": "label_values(karpenter_nodes_allocatable, provisioner)", + "query": "label_values(karpenter_nodes_allocatable, nodepool)", "refId": "StandardVariableQuery" }, "refresh": 2, @@ -1376,8 +1376,8 @@ { "current": { "selected": true, - "text": "provisioner", - "value": "provisioner" + "text": "nodepool", + "value": "nodepool" }, "hide": 0, "includeAll": false, @@ -1411,8 +1411,8 @@ }, { "selected": true, - "text": "provisioner", - "value": "provisioner" + "text": "nodepool", + "value": "nodepool" }, { "selected": false, @@ -1420,7 +1420,7 @@ "value": "zone" } ], - "query": "arch,capacity_type,instance_type,namespace,node,provisioner,zone", + "query": "arch,capacity_type,instance_type,namespace,node,nodepool,zone", "queryValue": "", "skipUrlSync": false, "type": "custom" diff --git a/website/content/en/v0.31/getting-started/getting-started-with-karpenter/karpenter-controllers-allocation.json b/website/content/en/v0.35/getting-started/getting-started-with-karpenter/karpenter-controllers-allocation.json similarity index 100% rename from website/content/en/v0.31/getting-started/getting-started-with-karpenter/karpenter-controllers-allocation.json rename to website/content/en/v0.35/getting-started/getting-started-with-karpenter/karpenter-controllers-allocation.json diff --git a/website/content/en/v0.31/getting-started/getting-started-with-karpenter/karpenter-controllers.json b/website/content/en/v0.35/getting-started/getting-started-with-karpenter/karpenter-controllers.json similarity index 100% rename from website/content/en/v0.31/getting-started/getting-started-with-karpenter/karpenter-controllers.json rename to website/content/en/v0.35/getting-started/getting-started-with-karpenter/karpenter-controllers.json diff --git a/website/content/en/v0.31/getting-started/getting-started-with-karpenter/karpenter-performance-dashboard.json b/website/content/en/v0.35/getting-started/getting-started-with-karpenter/karpenter-performance-dashboard.json similarity index 100% rename from website/content/en/v0.31/getting-started/getting-started-with-karpenter/karpenter-performance-dashboard.json rename to website/content/en/v0.35/getting-started/getting-started-with-karpenter/karpenter-performance-dashboard.json diff --git a/website/content/en/v0.31/getting-started/getting-started-with-karpenter/prometheus-values.yaml b/website/content/en/v0.35/getting-started/getting-started-with-karpenter/prometheus-values.yaml similarity index 63% rename from website/content/en/v0.31/getting-started/getting-started-with-karpenter/prometheus-values.yaml rename to website/content/en/v0.35/getting-started/getting-started-with-karpenter/prometheus-values.yaml index 4cd78495351b..588762a8c9dc 100644 --- a/website/content/en/v0.31/getting-started/getting-started-with-karpenter/prometheus-values.yaml +++ b/website/content/en/v0.35/getting-started/getting-started-with-karpenter/prometheus-values.yaml @@ -13,8 +13,10 @@ extraScrapeConfigs: | - role: endpoints namespaces: names: - - karpenter + - $KARPENTER_NAMESPACE relabel_configs: - - source_labels: [__meta_kubernetes_endpoint_port_name] - regex: http-metrics + - source_labels: + - __meta_kubernetes_endpoints_name + - __meta_kubernetes_endpoint_port_name action: keep + regex: karpenter;http-metrics diff --git a/website/content/en/v0.35/getting-started/getting-started-with-karpenter/scripts/step01-config.sh b/website/content/en/v0.35/getting-started/getting-started-with-karpenter/scripts/step01-config.sh new file mode 100755 index 000000000000..b8209b641640 --- /dev/null +++ b/website/content/en/v0.35/getting-started/getting-started-with-karpenter/scripts/step01-config.sh @@ -0,0 +1,8 @@ +export AWS_PARTITION="aws" # if you are not using standard partitions, you may need to configure to aws-cn / aws-us-gov +export CLUSTER_NAME="${USER}-karpenter-demo" +export AWS_DEFAULT_REGION="us-west-2" +export AWS_ACCOUNT_ID="$(aws sts get-caller-identity --query Account --output text)" +export TEMPOUT="$(mktemp)" +export ARM_AMI_ID="$(aws ssm get-parameter --name /aws/service/eks/optimized-ami/${K8S_VERSION}/amazon-linux-2-arm64/recommended/image_id --query Parameter.Value --output text)" +export AMD_AMI_ID="$(aws ssm get-parameter --name /aws/service/eks/optimized-ami/${K8S_VERSION}/amazon-linux-2/recommended/image_id --query Parameter.Value --output text)" +export GPU_AMI_ID="$(aws ssm get-parameter --name /aws/service/eks/optimized-ami/${K8S_VERSION}/amazon-linux-2-gpu/recommended/image_id --query Parameter.Value --output text)" diff --git a/website/content/en/v0.31/getting-started/getting-started-with-karpenter/scripts/step02-create-cluster-fargate.sh b/website/content/en/v0.35/getting-started/getting-started-with-karpenter/scripts/step02-create-cluster-fargate.sh similarity index 80% rename from website/content/en/v0.31/getting-started/getting-started-with-karpenter/scripts/step02-create-cluster-fargate.sh rename to website/content/en/v0.35/getting-started/getting-started-with-karpenter/scripts/step02-create-cluster-fargate.sh index d5ca27944cf1..e957e364acd7 100755 --- a/website/content/en/v0.31/getting-started/getting-started-with-karpenter/scripts/step02-create-cluster-fargate.sh +++ b/website/content/en/v0.35/getting-started/getting-started-with-karpenter/scripts/step02-create-cluster-fargate.sh @@ -5,13 +5,13 @@ kind: ClusterConfig metadata: name: ${CLUSTER_NAME} region: ${AWS_DEFAULT_REGION} - version: "1.24" + version: "${K8S_VERSION}" tags: karpenter.sh/discovery: ${CLUSTER_NAME} fargateProfiles: - name: karpenter selectors: - - namespace: karpenter + - namespace: "${KARPENTER_NAMESPACE}" iam: withOIDC: true EOF diff --git a/website/content/en/v0.30/getting-started/getting-started-with-karpenter/scripts/step02-create-cluster.sh b/website/content/en/v0.35/getting-started/getting-started-with-karpenter/scripts/step02-create-cluster.sh similarity index 57% rename from website/content/en/v0.30/getting-started/getting-started-with-karpenter/scripts/step02-create-cluster.sh rename to website/content/en/v0.35/getting-started/getting-started-with-karpenter/scripts/step02-create-cluster.sh index 7e7d70ce77b0..cdad96eeca23 100755 --- a/website/content/en/v0.30/getting-started/getting-started-with-karpenter/scripts/step02-create-cluster.sh +++ b/website/content/en/v0.35/getting-started/getting-started-with-karpenter/scripts/step02-create-cluster.sh @@ -1,4 +1,4 @@ -curl -fsSL https://raw.githubusercontent.com/aws/karpenter-provider-aws/"${KARPENTER_VERSION}"/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml > $TEMPOUT \ +curl -fsSL https://raw.githubusercontent.com/aws/karpenter-provider-aws/v"${KARPENTER_VERSION}"/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml > "${TEMPOUT}" \ && aws cloudformation deploy \ --stack-name "Karpenter-${CLUSTER_NAME}" \ --template-file "${TEMPOUT}" \ @@ -12,20 +12,32 @@ kind: ClusterConfig metadata: name: ${CLUSTER_NAME} region: ${AWS_DEFAULT_REGION} - version: "1.27" + version: "${K8S_VERSION}" tags: karpenter.sh/discovery: ${CLUSTER_NAME} iam: withOIDC: true - serviceAccounts: - - metadata: - name: karpenter - namespace: karpenter + podIdentityAssociations: + - namespace: "${KARPENTER_NAMESPACE}" + serviceAccountName: karpenter roleName: ${CLUSTER_NAME}-karpenter - attachPolicyARNs: + permissionPolicyARNs: - arn:${AWS_PARTITION}:iam::${AWS_ACCOUNT_ID}:policy/KarpenterControllerPolicy-${CLUSTER_NAME} - roleOnly: true + +## Optionally run on fargate or on k8s 1.23 +# Pod Identity is not available on fargate +# https://docs.aws.amazon.com/eks/latest/userguide/pod-identities.html +# iam: +# withOIDC: true +# serviceAccounts: +# - metadata: +# name: karpenter +# namespace: "${KARPENTER_NAMESPACE}" +# roleName: ${CLUSTER_NAME}-karpenter +# attachPolicyARNs: +# - arn:${AWS_PARTITION}:iam::${AWS_ACCOUNT_ID}:policy/KarpenterControllerPolicy-${CLUSTER_NAME} +# roleOnly: true iamIdentityMappings: - arn: "arn:${AWS_PARTITION}:iam::${AWS_ACCOUNT_ID}:role/KarpenterNodeRole-${CLUSTER_NAME}" @@ -45,14 +57,17 @@ managedNodeGroups: minSize: 1 maxSize: 10 +addons: +- name: eks-pod-identity-agent + ## Optionally run on fargate # fargateProfiles: # - name: karpenter # selectors: -# - namespace: karpenter +# - namespace: "${KARPENTER_NAMESPACE}" EOF -export CLUSTER_ENDPOINT="$(aws eks describe-cluster --name ${CLUSTER_NAME} --query "cluster.endpoint" --output text)" +export CLUSTER_ENDPOINT="$(aws eks describe-cluster --name "${CLUSTER_NAME}" --query "cluster.endpoint" --output text)" export KARPENTER_IAM_ROLE_ARN="arn:${AWS_PARTITION}:iam::${AWS_ACCOUNT_ID}:role/${CLUSTER_NAME}-karpenter" -echo $CLUSTER_ENDPOINT $KARPENTER_IAM_ROLE_ARN +echo "${CLUSTER_ENDPOINT} ${KARPENTER_IAM_ROLE_ARN}" diff --git a/website/content/en/v0.31/getting-started/getting-started-with-karpenter/scripts/step03-iam-cloud-formation.sh b/website/content/en/v0.35/getting-started/getting-started-with-karpenter/scripts/step03-iam-cloud-formation.sh similarity index 62% rename from website/content/en/v0.31/getting-started/getting-started-with-karpenter/scripts/step03-iam-cloud-formation.sh rename to website/content/en/v0.35/getting-started/getting-started-with-karpenter/scripts/step03-iam-cloud-formation.sh index b8e610f7bee6..54e826db269b 100755 --- a/website/content/en/v0.31/getting-started/getting-started-with-karpenter/scripts/step03-iam-cloud-formation.sh +++ b/website/content/en/v0.35/getting-started/getting-started-with-karpenter/scripts/step03-iam-cloud-formation.sh @@ -1,6 +1,6 @@ -TEMPOUT=$(mktemp) +TEMPOUT="$(mktemp)" -curl -fsSL https://raw.githubusercontent.com/aws/karpenter-provider-aws/"${KARPENTER_VERSION}"/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml > $TEMPOUT \ +curl -fsSL https://raw.githubusercontent.com/aws/karpenter-provider-aws/v"${KARPENTER_VERSION}"/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml > "${TEMPOUT}" \ && aws cloudformation deploy \ --stack-name "Karpenter-${CLUSTER_NAME}" \ --template-file "${TEMPOUT}" \ diff --git a/website/content/en/v0.31/getting-started/getting-started-with-karpenter/scripts/step04-grant-access.sh b/website/content/en/v0.35/getting-started/getting-started-with-karpenter/scripts/step04-grant-access.sh similarity index 100% rename from website/content/en/v0.31/getting-started/getting-started-with-karpenter/scripts/step04-grant-access.sh rename to website/content/en/v0.35/getting-started/getting-started-with-karpenter/scripts/step04-grant-access.sh diff --git a/website/content/en/v0.31/getting-started/getting-started-with-karpenter/scripts/step05-controller-iam.sh b/website/content/en/v0.35/getting-started/getting-started-with-karpenter/scripts/step05-controller-iam.sh similarity index 79% rename from website/content/en/v0.31/getting-started/getting-started-with-karpenter/scripts/step05-controller-iam.sh rename to website/content/en/v0.35/getting-started/getting-started-with-karpenter/scripts/step05-controller-iam.sh index 923317b0b93f..6975d31ea9e4 100755 --- a/website/content/en/v0.31/getting-started/getting-started-with-karpenter/scripts/step05-controller-iam.sh +++ b/website/content/en/v0.35/getting-started/getting-started-with-karpenter/scripts/step05-controller-iam.sh @@ -1,5 +1,5 @@ eksctl create iamserviceaccount \ - --cluster "${CLUSTER_NAME}" --name karpenter --namespace karpenter \ + --cluster "${CLUSTER_NAME}" --name karpenter --namespace "${KARPENTER_NAMESPACE}" \ --role-name "${CLUSTER_NAME}-karpenter" \ --attach-policy-arn "arn:${AWS_PARTITION}:iam::${AWS_ACCOUNT_ID}:policy/KarpenterControllerPolicy-${CLUSTER_NAME}" \ --role-only \ diff --git a/website/content/en/v0.31/getting-started/getting-started-with-karpenter/scripts/step06-add-spot-role.sh b/website/content/en/v0.35/getting-started/getting-started-with-karpenter/scripts/step06-add-spot-role.sh similarity index 100% rename from website/content/en/v0.31/getting-started/getting-started-with-karpenter/scripts/step06-add-spot-role.sh rename to website/content/en/v0.35/getting-started/getting-started-with-karpenter/scripts/step06-add-spot-role.sh diff --git a/website/content/en/v0.31/getting-started/getting-started-with-karpenter/scripts/step08-apply-helm-chart.sh b/website/content/en/v0.35/getting-started/getting-started-with-karpenter/scripts/step08-apply-helm-chart.sh similarity index 50% rename from website/content/en/v0.31/getting-started/getting-started-with-karpenter/scripts/step08-apply-helm-chart.sh rename to website/content/en/v0.35/getting-started/getting-started-with-karpenter/scripts/step08-apply-helm-chart.sh index 99fa927b12b6..aca3c191d684 100755 --- a/website/content/en/v0.31/getting-started/getting-started-with-karpenter/scripts/step08-apply-helm-chart.sh +++ b/website/content/en/v0.35/getting-started/getting-started-with-karpenter/scripts/step08-apply-helm-chart.sh @@ -1,11 +1,9 @@ # Logout of helm registry to perform an unauthenticated pull against the public ECR helm registry logout public.ecr.aws -helm upgrade --install karpenter oci://public.ecr.aws/karpenter/karpenter --version ${KARPENTER_VERSION} --namespace karpenter --create-namespace \ - --set serviceAccount.annotations."eks\.amazonaws\.com/role-arn"=${KARPENTER_IAM_ROLE_ARN} \ - --set settings.aws.clusterName=${CLUSTER_NAME} \ - --set settings.aws.defaultInstanceProfile=KarpenterNodeInstanceProfile-${CLUSTER_NAME} \ - --set settings.aws.interruptionQueueName=${CLUSTER_NAME} \ +helm upgrade --install karpenter oci://public.ecr.aws/karpenter/karpenter --version "${KARPENTER_VERSION}" --namespace "${KARPENTER_NAMESPACE}" --create-namespace \ + --set "settings.clusterName=${CLUSTER_NAME}" \ + --set "settings.interruptionQueue=${CLUSTER_NAME}" \ --set controller.resources.requests.cpu=1 \ --set controller.resources.requests.memory=1Gi \ --set controller.resources.limits.cpu=1 \ diff --git a/website/content/en/v0.31/getting-started/getting-started-with-karpenter/scripts/step09-add-prometheus-grafana.sh b/website/content/en/v0.35/getting-started/getting-started-with-karpenter/scripts/step09-add-prometheus-grafana.sh similarity index 62% rename from website/content/en/v0.31/getting-started/getting-started-with-karpenter/scripts/step09-add-prometheus-grafana.sh rename to website/content/en/v0.35/getting-started/getting-started-with-karpenter/scripts/step09-add-prometheus-grafana.sh index a539f7491e10..1107c2ec1d24 100755 --- a/website/content/en/v0.31/getting-started/getting-started-with-karpenter/scripts/step09-add-prometheus-grafana.sh +++ b/website/content/en/v0.35/getting-started/getting-started-with-karpenter/scripts/step09-add-prometheus-grafana.sh @@ -4,8 +4,8 @@ helm repo update kubectl create namespace monitoring -curl -fsSL https://raw.githubusercontent.com/aws/karpenter-provider-aws/"${KARPENTER_VERSION}"/website/content/en/preview/getting-started/getting-started-with-karpenter/prometheus-values.yaml | tee prometheus-values.yaml +curl -fsSL https://raw.githubusercontent.com/aws/karpenter-provider-aws/v"${KARPENTER_VERSION}"/website/content/en/preview/getting-started/getting-started-with-karpenter/prometheus-values.yaml | envsubst | tee prometheus-values.yaml helm install --namespace monitoring prometheus prometheus-community/prometheus --values prometheus-values.yaml -curl -fsSL https://raw.githubusercontent.com/aws/karpenter-provider-aws/"${KARPENTER_VERSION}"/website/content/en/preview/getting-started/getting-started-with-karpenter/grafana-values.yaml | tee grafana-values.yaml +curl -fsSL https://raw.githubusercontent.com/aws/karpenter-provider-aws/v"${KARPENTER_VERSION}"/website/content/en/preview/getting-started/getting-started-with-karpenter/grafana-values.yaml | tee grafana-values.yaml helm install --namespace monitoring grafana grafana-charts/grafana --values grafana-values.yaml diff --git a/website/content/en/v0.31/getting-started/getting-started-with-karpenter/scripts/step10-add-grafana-port-forward.sh b/website/content/en/v0.35/getting-started/getting-started-with-karpenter/scripts/step10-add-grafana-port-forward.sh similarity index 100% rename from website/content/en/v0.31/getting-started/getting-started-with-karpenter/scripts/step10-add-grafana-port-forward.sh rename to website/content/en/v0.35/getting-started/getting-started-with-karpenter/scripts/step10-add-grafana-port-forward.sh diff --git a/website/content/en/v0.31/getting-started/getting-started-with-karpenter/scripts/step11-grafana-get-password.sh b/website/content/en/v0.35/getting-started/getting-started-with-karpenter/scripts/step11-grafana-get-password.sh similarity index 100% rename from website/content/en/v0.31/getting-started/getting-started-with-karpenter/scripts/step11-grafana-get-password.sh rename to website/content/en/v0.35/getting-started/getting-started-with-karpenter/scripts/step11-grafana-get-password.sh diff --git a/website/content/en/v0.35/getting-started/getting-started-with-karpenter/scripts/step12-add-nodepool.sh b/website/content/en/v0.35/getting-started/getting-started-with-karpenter/scripts/step12-add-nodepool.sh new file mode 100755 index 000000000000..33f1cb553b1b --- /dev/null +++ b/website/content/en/v0.35/getting-started/getting-started-with-karpenter/scripts/step12-add-nodepool.sh @@ -0,0 +1,53 @@ +cat < ``` @@ -43,35 +44,31 @@ Now attach the required policies to the role {{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step03-node-policies.sh" language="bash" %}} -Attach the IAM role to an EC2 instance profile. - -{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step04-instance-profile.sh" language="bash" %}} - Now we need to create an IAM role that the Karpenter controller will use to provision new instances. The controller will be using [IAM Roles for Service Accounts (IRSA)](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html) which requires an OIDC endpoint. If you have another option for using IAM credentials with workloads (e.g. [kube2iam](https://github.com/jtblin/kube2iam)) your steps will be different. -{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step05-controller-iam.sh" language="bash" %}} +{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step04-controller-iam.sh" language="bash" %}} ## Add tags to subnets and security groups We need to add tags to our nodegroup subnets so Karpenter will know which subnets to use. -{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step06-tag-subnets.sh" language="bash" %}} +{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step05-tag-subnets.sh" language="bash" %}} Add tags to our security groups. This command only tags the security groups for the first nodegroup in the cluster. If you have multiple nodegroups or multiple security groups you will need to decide which one Karpenter should use. -{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step07-tag-security-groups.sh" language="bash" %}} +{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step06-tag-security-groups.sh" language="bash" %}} ## Update aws-auth ConfigMap We need to allow nodes that are using the node IAM role we just created to join the cluster. To do that we have to modify the `aws-auth` ConfigMap in the cluster. -{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step08-edit-aws-auth.sh" language="bash" %}} +{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step07-edit-aws-auth.sh" language="bash" %}} You will need to add a section to the mapRoles that looks something like this. Replace the `${AWS_PARTITION}` variable with the account partition, `${AWS_ACCOUNT_ID}` variable with your account ID, and `${CLUSTER_NAME}` variable with the cluster name, but do not replace the `{{EC2PrivateDNSName}}`. @@ -95,12 +92,12 @@ One for your Karpenter node role and one for your existing node group. First set the Karpenter release you want to deploy. ```bash -export KARPENTER_VERSION=v0.31.3 +export KARPENTER_VERSION="0.35.4" ``` -We can now generate a full Karpenter deployment yaml from the helm chart. +We can now generate a full Karpenter deployment yaml from the Helm chart. -{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step09-generate-chart.sh" language="bash" %}} +{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step08-generate-chart.sh" language="bash" %}} Modify the following lines in the karpenter.yaml file. @@ -118,7 +115,7 @@ affinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - - key: karpenter.sh/provisioner-name + - key: karpenter.sh/nodepool operator: DoesNotExist - matchExpressions: - key: eks.amazonaws.com/nodegroup @@ -130,16 +127,15 @@ affinity: - topologyKey: "kubernetes.io/hostname" ``` -Now that our deployment is ready we can create the karpenter namespace, create the provisioner CRD, and then deploy the rest of the karpenter resources. +Now that our deployment is ready we can create the karpenter namespace, create the NodePool CRD, and then deploy the rest of the karpenter resources. -{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step10-deploy.sh" language="bash" %}} +{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step09-deploy.sh" language="bash" %}} -## Create default provisioner +## Create default NodePool -We need to create a default provisioner so Karpenter knows what types of nodes we want for unscheduled workloads. -You can refer to some of the [example provisioners](https://github.com/aws/karpenter/tree/v0.31.3/examples/provisioner) for specific needs. +We need to create a default NodePool so Karpenter knows what types of nodes we want for unscheduled workloads. You can refer to some of the [example NodePool](https://github.com/aws/karpenter/tree/v0.35.4/examples/v1beta1) for specific needs. -{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step11-create-provisioner.sh" language="bash" %}} +{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step10-create-nodepool.sh" language="bash" %}} ## Set nodeAffinity for critical workloads (optional) @@ -170,7 +166,7 @@ affinity: Now that karpenter is running we can disable the cluster autoscaler. To do that we will scale the number of replicas to zero. -{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step12-scale-cas.sh" language="bash" %}} +{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step11-scale-cas.sh" language="bash" %}} To get rid of the instances that were added from the node group we can scale our nodegroup down to a minimum size to support Karpenter and other critical services. @@ -178,11 +174,11 @@ To get rid of the instances that were added from the node group we can scale our If you have a single multi-AZ node group, we suggest a minimum of 2 instances. -{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step13-scale-single-ng.sh" language="bash" %}} +{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step12-scale-single-ng.sh" language="bash" %}} Or, if you have multiple single-AZ node groups, we suggest a minimum of 1 instance each. -{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step13-scale-multiple-ng.sh" language="bash" %}} +{{% script file="./content/en/{VERSION}/getting-started/migrating-from-cas/scripts/step12-scale-multiple-ng.sh" language="bash" %}} {{% alert title="Note" color="warning" %}} If you have a lot of nodes or workloads you may want to slowly scale down your node groups by a few instances at a time. It is recommended to watch the transition carefully for workloads that may not have enough replicas running or disruption budgets configured. diff --git a/website/content/en/v0.35/getting-started/migrating-from-cas/scripts/step01-env.sh b/website/content/en/v0.35/getting-started/migrating-from-cas/scripts/step01-env.sh new file mode 100644 index 000000000000..b3ee0fa3d0ef --- /dev/null +++ b/website/content/en/v0.35/getting-started/migrating-from-cas/scripts/step01-env.sh @@ -0,0 +1,10 @@ +AWS_PARTITION="aws" # if you are not using standard partitions, you may need to configure to aws-cn / aws-us-gov +AWS_REGION="$(aws configure list | grep region | tr -s " " | cut -d" " -f3)" +OIDC_ENDPOINT="$(aws eks describe-cluster --name "${CLUSTER_NAME}" \ + --query "cluster.identity.oidc.issuer" --output text)" +AWS_ACCOUNT_ID=$(aws sts get-caller-identity --query 'Account' \ + --output text) +K8S_VERSION=1.28 +ARM_AMI_ID="$(aws ssm get-parameter --name /aws/service/eks/optimized-ami/${K8S_VERSION}/amazon-linux-2-arm64/recommended/image_id --query Parameter.Value --output text)" +AMD_AMI_ID="$(aws ssm get-parameter --name /aws/service/eks/optimized-ami/${K8S_VERSION}/amazon-linux-2/recommended/image_id --query Parameter.Value --output text)" +GPU_AMI_ID="$(aws ssm get-parameter --name /aws/service/eks/optimized-ami/${K8S_VERSION}/amazon-linux-2-gpu/recommended/image_id --query Parameter.Value --output text)" diff --git a/website/content/en/v0.31/getting-started/migrating-from-cas/scripts/step02-node-iam.sh b/website/content/en/v0.35/getting-started/migrating-from-cas/scripts/step02-node-iam.sh similarity index 100% rename from website/content/en/v0.31/getting-started/migrating-from-cas/scripts/step02-node-iam.sh rename to website/content/en/v0.35/getting-started/migrating-from-cas/scripts/step02-node-iam.sh diff --git a/website/content/en/v0.35/getting-started/migrating-from-cas/scripts/step03-node-policies.sh b/website/content/en/v0.35/getting-started/migrating-from-cas/scripts/step03-node-policies.sh new file mode 100644 index 000000000000..fbc5455e541b --- /dev/null +++ b/website/content/en/v0.35/getting-started/migrating-from-cas/scripts/step03-node-policies.sh @@ -0,0 +1,11 @@ +aws iam attach-role-policy --role-name "KarpenterNodeRole-${CLUSTER_NAME}" \ + --policy-arn "arn:${AWS_PARTITION}:iam::aws:policy/AmazonEKSWorkerNodePolicy" + +aws iam attach-role-policy --role-name "KarpenterNodeRole-${CLUSTER_NAME}" \ + --policy-arn "arn:${AWS_PARTITION}:iam::aws:policy/AmazonEKS_CNI_Policy" + +aws iam attach-role-policy --role-name "KarpenterNodeRole-${CLUSTER_NAME}" \ + --policy-arn "arn:${AWS_PARTITION}:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" + +aws iam attach-role-policy --role-name "KarpenterNodeRole-${CLUSTER_NAME}" \ + --policy-arn "arn:${AWS_PARTITION}:iam::aws:policy/AmazonSSMManagedInstanceCore" diff --git a/website/content/en/v0.35/getting-started/migrating-from-cas/scripts/step04-controller-iam.sh b/website/content/en/v0.35/getting-started/migrating-from-cas/scripts/step04-controller-iam.sh new file mode 100644 index 000000000000..cc3d7f929986 --- /dev/null +++ b/website/content/en/v0.35/getting-started/migrating-from-cas/scripts/step04-controller-iam.sh @@ -0,0 +1,143 @@ +cat << EOF > controller-trust-policy.json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Federated": "arn:${AWS_PARTITION}:iam::${AWS_ACCOUNT_ID}:oidc-provider/${OIDC_ENDPOINT#*//}" + }, + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { + "${OIDC_ENDPOINT#*//}:aud": "sts.amazonaws.com", + "${OIDC_ENDPOINT#*//}:sub": "system:serviceaccount:${KARPENTER_NAMESPACE}:karpenter" + } + } + } + ] +} +EOF + +aws iam create-role --role-name "KarpenterControllerRole-${CLUSTER_NAME}" \ + --assume-role-policy-document file://controller-trust-policy.json + +cat << EOF > controller-policy.json +{ + "Statement": [ + { + "Action": [ + "ssm:GetParameter", + "ec2:DescribeImages", + "ec2:RunInstances", + "ec2:DescribeSubnets", + "ec2:DescribeSecurityGroups", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeInstances", + "ec2:DescribeInstanceTypes", + "ec2:DescribeInstanceTypeOfferings", + "ec2:DescribeAvailabilityZones", + "ec2:DeleteLaunchTemplate", + "ec2:CreateTags", + "ec2:CreateLaunchTemplate", + "ec2:CreateFleet", + "ec2:DescribeSpotPriceHistory", + "pricing:GetProducts" + ], + "Effect": "Allow", + "Resource": "*", + "Sid": "Karpenter" + }, + { + "Action": "ec2:TerminateInstances", + "Condition": { + "StringLike": { + "ec2:ResourceTag/karpenter.sh/nodepool": "*" + } + }, + "Effect": "Allow", + "Resource": "*", + "Sid": "ConditionalEC2Termination" + }, + { + "Effect": "Allow", + "Action": "iam:PassRole", + "Resource": "arn:${AWS_PARTITION}:iam::${AWS_ACCOUNT_ID}:role/KarpenterNodeRole-${CLUSTER_NAME}", + "Sid": "PassNodeIAMRole" + }, + { + "Effect": "Allow", + "Action": "eks:DescribeCluster", + "Resource": "arn:${AWS_PARTITION}:eks:${AWS_REGION}:${AWS_ACCOUNT_ID}:cluster/${CLUSTER_NAME}", + "Sid": "EKSClusterEndpointLookup" + }, + { + "Sid": "AllowScopedInstanceProfileCreationActions", + "Effect": "Allow", + "Resource": "*", + "Action": [ + "iam:CreateInstanceProfile" + ], + "Condition": { + "StringEquals": { + "aws:RequestTag/kubernetes.io/cluster/${CLUSTER_NAME}": "owned", + "aws:RequestTag/topology.kubernetes.io/region": "${AWS_REGION}" + }, + "StringLike": { + "aws:RequestTag/karpenter.k8s.aws/ec2nodeclass": "*" + } + } + }, + { + "Sid": "AllowScopedInstanceProfileTagActions", + "Effect": "Allow", + "Resource": "*", + "Action": [ + "iam:TagInstanceProfile" + ], + "Condition": { + "StringEquals": { + "aws:ResourceTag/kubernetes.io/cluster/${CLUSTER_NAME}": "owned", + "aws:ResourceTag/topology.kubernetes.io/region": "${AWS_REGION}", + "aws:RequestTag/kubernetes.io/cluster/${CLUSTER_NAME}": "owned", + "aws:RequestTag/topology.kubernetes.io/region": "${AWS_REGION}" + }, + "StringLike": { + "aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass": "*", + "aws:RequestTag/karpenter.k8s.aws/ec2nodeclass": "*" + } + } + }, + { + "Sid": "AllowScopedInstanceProfileActions", + "Effect": "Allow", + "Resource": "*", + "Action": [ + "iam:AddRoleToInstanceProfile", + "iam:RemoveRoleFromInstanceProfile", + "iam:DeleteInstanceProfile" + ], + "Condition": { + "StringEquals": { + "aws:ResourceTag/kubernetes.io/cluster/${CLUSTER_NAME}": "owned", + "aws:ResourceTag/topology.kubernetes.io/region": "${AWS_REGION}" + }, + "StringLike": { + "aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass": "*" + } + } + }, + { + "Sid": "AllowInstanceProfileReadActions", + "Effect": "Allow", + "Resource": "*", + "Action": "iam:GetInstanceProfile" + } + ], + "Version": "2012-10-17" +} +EOF + +aws iam put-role-policy --role-name "KarpenterControllerRole-${CLUSTER_NAME}" \ + --policy-name "KarpenterControllerPolicy-${CLUSTER_NAME}" \ + --policy-document file://controller-policy.json diff --git a/website/content/en/v0.35/getting-started/migrating-from-cas/scripts/step05-tag-subnets.sh b/website/content/en/v0.35/getting-started/migrating-from-cas/scripts/step05-tag-subnets.sh new file mode 100644 index 000000000000..47df188dc87d --- /dev/null +++ b/website/content/en/v0.35/getting-started/migrating-from-cas/scripts/step05-tag-subnets.sh @@ -0,0 +1,6 @@ +for NODEGROUP in $(aws eks list-nodegroups --cluster-name "${CLUSTER_NAME}" --query 'nodegroups' --output text); do + aws ec2 create-tags \ + --tags "Key=karpenter.sh/discovery,Value=${CLUSTER_NAME}" \ + --resources "$(aws eks describe-nodegroup --cluster-name "${CLUSTER_NAME}" \ + --nodegroup-name "${NODEGROUP}" --query 'nodegroup.subnets' --output text )" +done diff --git a/website/content/en/v0.35/getting-started/migrating-from-cas/scripts/step06-tag-security-groups.sh b/website/content/en/v0.35/getting-started/migrating-from-cas/scripts/step06-tag-security-groups.sh new file mode 100644 index 000000000000..c63bde3b78dc --- /dev/null +++ b/website/content/en/v0.35/getting-started/migrating-from-cas/scripts/step06-tag-security-groups.sh @@ -0,0 +1,22 @@ +NODEGROUP=$(aws eks list-nodegroups --cluster-name "${CLUSTER_NAME}" \ + --query 'nodegroups[0]' --output text) + +LAUNCH_TEMPLATE=$(aws eks describe-nodegroup --cluster-name "${CLUSTER_NAME}" \ + --nodegroup-name "${NODEGROUP}" --query 'nodegroup.launchTemplate.{id:id,version:version}' \ + --output text | tr -s "\t" ",") + +# If your EKS setup is configured to use only Cluster security group, then please execute - + +SECURITY_GROUPS=$(aws eks describe-cluster \ + --name "${CLUSTER_NAME}" --query "cluster.resourcesVpcConfig.clusterSecurityGroupId" --output text) + +# If your setup uses the security groups in the Launch template of a managed node group, then : + +SECURITY_GROUPS="$(aws ec2 describe-launch-template-versions \ + --launch-template-id "${LAUNCH_TEMPLATE%,*}" --versions "${LAUNCH_TEMPLATE#*,}" \ + --query 'LaunchTemplateVersions[0].LaunchTemplateData.[NetworkInterfaces[0].Groups||SecurityGroupIds]' \ + --output text)" + +aws ec2 create-tags \ + --tags "Key=karpenter.sh/discovery,Value=${CLUSTER_NAME}" \ + --resources "${SECURITY_GROUPS}" diff --git a/website/content/en/v0.31/getting-started/migrating-from-cas/scripts/step08-edit-aws-auth.sh b/website/content/en/v0.35/getting-started/migrating-from-cas/scripts/step07-edit-aws-auth.sh similarity index 100% rename from website/content/en/v0.31/getting-started/migrating-from-cas/scripts/step08-edit-aws-auth.sh rename to website/content/en/v0.35/getting-started/migrating-from-cas/scripts/step07-edit-aws-auth.sh diff --git a/website/content/en/v0.35/getting-started/migrating-from-cas/scripts/step08-generate-chart.sh b/website/content/en/v0.35/getting-started/migrating-from-cas/scripts/step08-generate-chart.sh new file mode 100644 index 000000000000..f81ac0b90cc8 --- /dev/null +++ b/website/content/en/v0.35/getting-started/migrating-from-cas/scripts/step08-generate-chart.sh @@ -0,0 +1,7 @@ +helm template karpenter oci://public.ecr.aws/karpenter/karpenter --version "${KARPENTER_VERSION}" --namespace "${KARPENTER_NAMESPACE}" \ + --set "settings.clusterName=${CLUSTER_NAME}" \ + --set "serviceAccount.annotations.eks\.amazonaws\.com/role-arn=arn:${AWS_PARTITION}:iam::${AWS_ACCOUNT_ID}:role/KarpenterControllerRole-${CLUSTER_NAME}" \ + --set controller.resources.requests.cpu=1 \ + --set controller.resources.requests.memory=1Gi \ + --set controller.resources.limits.cpu=1 \ + --set controller.resources.limits.memory=1Gi > karpenter.yaml diff --git a/website/content/en/v0.35/getting-started/migrating-from-cas/scripts/step09-deploy.sh b/website/content/en/v0.35/getting-started/migrating-from-cas/scripts/step09-deploy.sh new file mode 100644 index 000000000000..e46742fd22ea --- /dev/null +++ b/website/content/en/v0.35/getting-started/migrating-from-cas/scripts/step09-deploy.sh @@ -0,0 +1,8 @@ +kubectl create namespace "${KARPENTER_NAMESPACE}" || true +kubectl create -f \ + "https://raw.githubusercontent.com/aws/karpenter-provider-aws/v${KARPENTER_VERSION}/pkg/apis/crds/karpenter.sh_nodepools.yaml" +kubectl create -f \ + "https://raw.githubusercontent.com/aws/karpenter-provider-aws/v${KARPENTER_VERSION}/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml" +kubectl create -f \ + "https://raw.githubusercontent.com/aws/karpenter-provider-aws/v${KARPENTER_VERSION}/pkg/apis/crds/karpenter.sh_nodeclaims.yaml" +kubectl apply -f karpenter.yaml diff --git a/website/content/en/v0.35/getting-started/migrating-from-cas/scripts/step10-create-nodepool.sh b/website/content/en/v0.35/getting-started/migrating-from-cas/scripts/step10-create-nodepool.sh new file mode 100644 index 000000000000..33f1cb553b1b --- /dev/null +++ b/website/content/en/v0.35/getting-started/migrating-from-cas/scripts/step10-create-nodepool.sh @@ -0,0 +1,53 @@ +cat < + Reference documentation for Karpenter +--- \ No newline at end of file diff --git a/website/content/en/v0.35/reference/cloudformation.md b/website/content/en/v0.35/reference/cloudformation.md new file mode 100644 index 000000000000..e764749ecbd1 --- /dev/null +++ b/website/content/en/v0.35/reference/cloudformation.md @@ -0,0 +1,614 @@ +--- +title: "CloudFormation" +linkTitle: "CloudFormation" +weight: 5 +description: > + A description of the Getting Started CloudFormation file and permissions +--- +The [Getting Started with Karpenter]({{< relref "../getting-started/getting-started-with-karpenter" >}}) guide uses CloudFormation to bootstrap the cluster to enable Karpenter to create and manage nodes, as well as to allow Karpenter to respond to interruption events. +This document describes the `cloudformation.yaml` file used in that guide. +These descriptions should allow you to understand: + +* What Karpenter is authorized to do with your EKS cluster and AWS resources when using the `cloudformation.yaml` file +* What permissions you need to set up if you are adding Karpenter to an existing cluster + +## Overview + +To download a particular version of `cloudformation.yaml`, set the version and use `curl` to pull the file to your local system: + +```bash +export KARPENTER_VERSION="0.35.4" +curl https://raw.githubusercontent.com/aws/karpenter-provider-aws/v"${KARPENTER_VERSION}"/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml > cloudformation.yaml +``` + +Following some header information, the rest of the `cloudformation.yaml` file describes the resources that CloudFormation deploys. +The sections of that file can be grouped together under the following general headings: + +* [**Node Authorization**]({{< relref "#node-authorization" >}}): Creates a NodeInstanceProfile, attaches a NodeRole to it, and connects it to an IAM Identity Mapping used to authorize nodes to the cluster. This defines the permissions each node managed by Karpenter has to access EC2 and other AWS resources. This doesn't actually create the IAM Identity Mapping. That part is orchestrated by `eksctl` in the Getting Started guide. +* [**Controller Authorization**]({{< relref "#controller-authorization" >}}): Creates the `KarpenterControllerPolicy` that is attached to the service account. +Again, the actual service account creation (`karpenter`), that is combined with the `KarpenterControllerPolicy`, is orchestrated by `eksctl` in the Getting Started guide. +* [**Interruption Handling**]({{< relref "#interruption-handling" >}}): Allows the Karpenter controller to see and respond to interruptions that occur with the nodes that Karpenter is managing. See the [Interruption]({{< relref "../concepts/disruption#interruption" >}}) section of the Disruption page for details. + +A lot of the object naming that is done by `cloudformation.yaml` is based on the following: + +* Cluster name: With a username of `bob` the Getting Started Guide would name your cluster `bob-karpenter-demo` +That name would then be appended to any name below where `${ClusterName}` is included. + +* Partition: Any time an ARN is used, it includes the [partition name](https://docs.aws.amazon.com/whitepapers/latest/aws-fault-isolation-boundaries/partitions.html) to identify where the object is found. In most cases, that partition name is `aws`. However, it could also be `aws-cn` (for China Regions) or `aws-us-gov` (for AWS GovCloud US Regions). + +## Node Authorization + +The following sections of the `cloudformation.yaml` file set up IAM permissions for Kubernetes nodes created by Karpenter. +In particular, this involves setting up a node role that can be attached and passed to instance profiles that Karpenter generates at runtime: + +* KarpenterNodeRole + +### KarpenterNodeRole + +This section of the template defines the IAM role attached to generated instance profiles. +Given a cluster name of `bob-karpenter-demo`, this role would end up being named `"KarpenterNodeRole-bob-karpenter-demo`. + +```yaml +KarpenterNodeRole: + Type: "AWS::IAM::Role" + Properties: + RoleName: !Sub "KarpenterNodeRole-${ClusterName}" + Path: / + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + Service: + !Sub "ec2.${AWS::URLSuffix}" + Action: + - "sts:AssumeRole" + ManagedPolicyArns: + - !Sub "arn:${AWS::Partition}:iam::aws:policy/AmazonEKS_CNI_Policy" + - !Sub "arn:${AWS::Partition}:iam::aws:policy/AmazonEKSWorkerNodePolicy" + - !Sub "arn:${AWS::Partition}:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly" + - !Sub "arn:${AWS::Partition}:iam::aws:policy/AmazonSSMManagedInstanceCore" +``` + +The role created here includes several AWS managed policies, which are designed to provide permissions for specific uses needed by the nodes to work with EC2 and other AWS resources. These include: + +* [AmazonEKS_CNI_Policy](https://docs.aws.amazon.com/aws-managed-policy/latest/reference/AmazonEKS_CNI_Policy.html): Provides the permissions that the Amazon VPC CNI Plugin needs to configure EKS worker nodes. +* [AmazonEKSWorkerNodePolicy](https://docs.aws.amazon.com/aws-managed-policy/latest/reference/AmazonEKSWorkerNodePolicy.html): Lets Amazon EKS worker nodes connect to EKS Clusters. +* [AmazonEC2ContainerRegistryReadOnly](https://docs.aws.amazon.com/aws-managed-policy/latest/reference/AmazonEC2ContainerRegistryReadOnly.html): Allows read-only access to repositories in the Amazon EC2 Container Registry. +* [AmazonSSMManagedInstanceCore](https://docs.aws.amazon.com/aws-managed-policy/latest/reference/AmazonSSMManagedInstanceCore.html): Adds AWS Systems Manager service core functions for Amazon EC2. + +If you were to use a node role from an existing cluster, you could skip this provisioning step and pass this node role to any EC2NodeClasses that you create. Additionally, you would ensure that the [Controller Policy]({{< relref "#controllerpolicy" >}}) has `iam:PassRole` permission to the role attached to the generated instance profiles. + +## Controller Authorization + +This section sets the AWS permissions for the Karpenter Controller. When used in the Getting Started guide, `eksctl` uses these permissions to create a service account (karpenter) that is combined with the KarpenterControllerPolicy. + +The resources defined in this section are associated with: + +* KarpenterControllerPolicy + +Because the scope of the KarpenterControllerPolicy is an AWS region, the cluster's AWS region is included in the `AllowScopedEC2InstanceAccessActions`. + +### KarpenterControllerPolicy + +A `KarpenterControllerPolicy` object sets the name of the policy, then defines a set of resources and actions allowed for those resources. +For our example, the KarpenterControllerPolicy would be named: `KarpenterControllerPolicy-bob-karpenter-demo` + +```yaml +KarpenterControllerPolicy: + Type: AWS::IAM::ManagedPolicy + Properties: + ManagedPolicyName: !Sub "KarpenterControllerPolicy-${ClusterName}" + # The PolicyDocument must be in JSON string format because we use a StringEquals condition that uses an interpolated + # value in one of its key parameters which isn't natively supported by CloudFormation + PolicyDocument: !Sub | + { + "Version": "2012-10-17", + "Statement": [ +``` + +Someone wanting to add Karpenter to an existing cluster, instead of using `cloudformation.yaml`, would need to create the IAM policy directly and assign that policy to the role leveraged by the service account using IRSA. + +#### AllowScopedEC2InstanceAccessActions + +The AllowScopedEC2InstanceAccessActions statement ID (Sid) identifies a set of EC2 resources that are allowed to be accessed with +[RunInstances](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_RunInstances.html) and [CreateFleet](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html) actions. +For `RunInstances` and `CreateFleet` actions, the Karpenter controller can read (but not create) `image`, `snapshot`, `security-group`, `subnet` and `launch-template` EC2 resources, scoped for the particular AWS partition and region. + +```json +{ + "Sid": "AllowScopedEC2InstanceAccessActions", + "Effect": "Allow", + "Resource": [ + "arn:${AWS::Partition}:ec2:${AWS::Region}::image/*", + "arn:${AWS::Partition}:ec2:${AWS::Region}::snapshot/*", + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:security-group/*", + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:subnet/*" + ], + "Action": [ + "ec2:RunInstances", + "ec2:CreateFleet" + ] +} +``` + +#### AllowScopedEC2LaunchTemplateAccessActions + +The AllowScopedEC2InstanceAccessActions statement ID (Sid) identifies launch templates that are allowed to be accessed with +[RunInstances](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_RunInstances.html) and [CreateFleet](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html) actions. +For `RunInstances` and `CreateFleet` actions, the Karpenter controller can read (but not create) `launch-template` EC2 resources that have the `kubernetes.io/cluster/${ClusterName}` tag be set to `owned` and a `karpenter.sh/nodepool` tag, scoped for the particular AWS partition and region. This ensures that an instance launch can't access launch templates that weren't provisioned by Karpenter. + +```json +{ + "Sid": "AllowScopedEC2LaunchTemplateAccessActions", + "Effect": "Allow", + "Resource": "arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*", + "Action": [ + "ec2:RunInstances", + "ec2:CreateFleet" + ], + "Condition": { + "StringEquals": { + "aws:ResourceTag/kubernetes.io/cluster/${ClusterName}": "owned" + }, + "StringLike": { + "aws:ResourceTag/karpenter.sh/nodepool": "*" + } + } +} +``` + +#### AllowScopedEC2InstanceActionsWithTags + +The AllowScopedEC2InstanceActionsWithTags Sid allows the +[RunInstances](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_RunInstances.html), [CreateFleet](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateFleet.html), and [CreateLaunchTemplate](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateLaunchTemplate.html) +actions requested by the Karpenter controller to create all `fleet`, `instance`, `volume`, `network-interface`, `launch-template` or `spot-instances-request` EC2 resources (for the partition and region), and requires that the `kubernetes.io/cluster/${ClusterName}` tag be set to `owned` and a `karpenter.sh/nodepool` tag be set to any value. This ensures that Karpenter is only allowed to create instances for a single EKS cluster. + +```json +{ + "Sid": "AllowScopedEC2InstanceActionsWithTags", + "Effect": "Allow", + "Resource": [ + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:fleet/*", + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:instance/*", + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:volume/*", + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:network-interface/*", + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*", + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:spot-instances-request/*" + ], + "Action": [ + "ec2:RunInstances", + "ec2:CreateFleet", + "ec2:CreateLaunchTemplate" + ], + "Condition": { + "StringEquals": { + "aws:RequestTag/kubernetes.io/cluster/${ClusterName}": "owned" + }, + "StringLike": { + "aws:RequestTag/karpenter.sh/nodepool": "*" + } + } +} +``` + +#### AllowScopedResourceCreationTagging + +The AllowScopedResourceCreationTagging Sid allows EC2 [CreateTags](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateTags.html) +actions on `fleet`, `instance`, `volume`, `network-interface`, `launch-template` and `spot-instances-request` resources, While making `RunInstance`, `CreateFleet`, or `CreateLaunchTemplate` calls. Additionally, this ensures that resources can't be tagged arbitrarily by Karpenter after they are created. + +```json +{ + "Sid": "AllowScopedResourceCreationTagging", + "Effect": "Allow", + "Resource": [ + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:fleet/*", + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:instance/*", + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:volume/*", + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:network-interface/*", + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*", + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:spot-instances-request/*" + ], + "Action": "ec2:CreateTags", + "Condition": { + "StringEquals": { + "aws:RequestTag/kubernetes.io/cluster/${ClusterName}": "owned", + "ec2:CreateAction": [ + "RunInstances", + "CreateFleet", + "CreateLaunchTemplate" + ] + }, + "StringLike": { + "aws:RequestTag/karpenter.sh/nodepool": "*" + } + } +} +``` + +#### AllowScopedResourceTagging + +The AllowScopedResourceTagging Sid allows EC2 [CreateTags](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateTags.html) actions on all instances created by Karpenter after their creation. It enforces that Karpenter is only able to update the tags on cluster instances it is operating on through the `kubernetes.io/cluster/${ClusterName}`" and `karpenter.sh/nodepool` tags. +```json +{ + "Sid": "AllowScopedResourceTagging", + "Effect": "Allow", + "Resource": "arn:${AWS::Partition}:ec2:${AWS::Region}:*:instance/*", + "Action": "ec2:CreateTags", + "Condition": { + "StringEquals": { + "aws:ResourceTag/kubernetes.io/cluster/${ClusterName}": "owned" + }, + "StringLike": { + "aws:ResourceTag/karpenter.sh/nodepool": "*" + }, + "ForAllValues:StringEquals": { + "aws:TagKeys": [ + "karpenter.sh/nodeclaim", + "Name" + ] + } + } +} +``` + +#### AllowScopedDeletion + +The AllowScopedDeletion Sid allows [TerminateInstances](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_TerminateInstances.html) and [DeleteLaunchTemplate](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DeleteLaunchTemplate.html) actions to delete instance and launch-template resources, provided that `karpenter.sh/nodepool` and `kubernetes.io/cluster/${ClusterName}` tags are set. These tags must be present on all resources that Karpenter is going to delete. This ensures that Karpenter can only delete instances and launch templates that are associated with it. + +```json +{ + "Sid": "AllowScopedDeletion", + "Effect": "Allow", + "Resource": [ + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:instance/*", + "arn:${AWS::Partition}:ec2:${AWS::Region}:*:launch-template/*" + ], + "Action": [ + "ec2:TerminateInstances", + "ec2:DeleteLaunchTemplate" + ], + "Condition": { + "StringEquals": { + "aws:ResourceTag/kubernetes.io/cluster/${ClusterName}": "owned" + }, + "StringLike": { + "aws:ResourceTag/karpenter.sh/nodepool": "*" + } + } +} +``` + +#### AllowRegionalReadActions + +The AllowRegionalReadActions Sid allows [DescribeAvailabilityZones](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeAvailabilityZones.html), [DescribeImages](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeImages.html), [DescribeInstances](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeInstances.html), [DescribeInstanceTypeOfferings](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeInstanceTypeOfferings.html), [DescribeInstanceTypes](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeInstanceTypes.html), [DescribeLaunchTemplates](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeLaunchTemplates.html), [DescribeSecurityGroups](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeSecurityGroups.html), [DescribeSpotPriceHistory](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeSpotPriceHistory.html), and [DescribeSubnets](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DescribeSubnets.html) actions for the current AWS region. +This allows the Karpenter controller to do any of those read-only actions across all related resources for that AWS region. + +```json +{ + "Sid": "AllowRegionalReadActions", + "Effect": "Allow", + "Resource": "*", + "Action": [ + "ec2:DescribeAvailabilityZones", + "ec2:DescribeImages", + "ec2:DescribeInstances", + "ec2:DescribeInstanceTypeOfferings", + "ec2:DescribeInstanceTypes", + "ec2:DescribeLaunchTemplates", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSpotPriceHistory", + "ec2:DescribeSubnets" + ], + "Condition": { + "StringEquals": { + "aws:RequestedRegion": "${AWS::Region}" + } + } +} +``` + +#### AllowSSMReadActions + +The AllowSSMReadActions Sid allows the Karpenter controller to read SSM parameters (`ssm:GetParameter`) from the current region for SSM parameters generated by ASW services. + +**NOTE**: If potentially sensitive information is stored in SSM parameters, you could consider restricting access to these messages further. +```json +{ + "Sid": "AllowSSMReadActions", + "Effect": "Allow", + "Resource": "arn:${AWS::Partition}:ssm:${AWS::Region}::parameter/aws/service/*", + "Action": "ssm:GetParameter" +} +``` + +#### AllowPricingReadActions + +Because pricing information does not exist in every region at the moment, the AllowPricingReadActions Sid allows the Karpenter controller to get product pricing information (`pricing:GetProducts`) for all related resources across all regions. + +```json +{ + "Sid": "AllowPricingReadActions", + "Effect": "Allow", + "Resource": "*", + "Action": "pricing:GetProducts" +} +``` + +#### AllowInterruptionQueueActions + +Karpenter supports interruption queues, that you can create as described in the [Interruption]({{< relref "../concepts/disruption#interruption" >}}) section of the Disruption page. +This section of the cloudformation.yaml template can give Karpenter permission to access those queues by specifying the resource ARN. +For the interruption queue you created (`${KarpenterInterruptionQueue.Arn}`), the AllowInterruptionQueueActions Sid lets the Karpenter controller have permission to delete messages ([DeleteMessage](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_DeleteMessage.html)), get queue URL ([GetQueueUrl](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_GetQueueUrl.html)), and receive messages ([ReceiveMessage](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_ReceiveMessage.html)). + +```json +{ + "Sid": "AllowInterruptionQueueActions", + "Effect": "Allow", + "Resource": "${KarpenterInterruptionQueue.Arn}", + "Action": [ + "sqs:DeleteMessage", + "sqs:GetQueueUrl", + "sqs:ReceiveMessage" + ] +} +``` + +#### AllowPassingInstanceRole + +The AllowPassingInstanceRole Sid gives the Karpenter controller permission to pass (`iam:PassRole`) the node role (`KarpenterNodeRole-${ClusterName}`) to generated instance profiles. +This gives EC2 permission explicit permission to use the `KarpenterNodeRole-${ClusterName}` when assigning permissions to generated instance profiles while launching nodes. + +```json +{ + "Sid": "AllowPassingInstanceRole", + "Effect": "Allow", + "Resource": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/KarpenterNodeRole-${ClusterName}", + "Action": "iam:PassRole", + "Condition": { + "StringEquals": { + "iam:PassedToService": "ec2.amazonaws.com" + } + } +} +``` + +#### AllowScopedInstanceProfileCreationActions + +The AllowScopedInstanceProfileCreationActions Sid gives the Karpenter controller permission to create a new instance profile with [`iam:CreateInstanceProfile`](https://docs.aws.amazon.com/IAM/latest/APIReference/API_CreateInstanceProfile.html), +provided that the request is made to a cluster with `kubernetes.io/cluster/${ClusterName` set to owned and is made in the current region. +Also, `karpenter.k8s.aws/ec2nodeclass` must be set to some value. This ensures that Karpenter can generate instance profiles on your behalf based on roles specified in your `EC2NodeClasses` that you use to configure Karpenter. + +```json +{ + "Sid": "AllowScopedInstanceProfileCreationActions", + "Effect": "Allow", + "Resource": "*", + "Action": [ + "iam:CreateInstanceProfile" + ], + "Condition": { + "StringEquals": { + "aws:RequestTag/kubernetes.io/cluster/${ClusterName}": "owned", + "aws:RequestTag/topology.kubernetes.io/region": "${AWS::Region}" + }, + "StringLike": { + "aws:RequestTag/karpenter.k8s.aws/ec2nodeclass": "*" + } + } +} +``` + +#### AllowScopedInstanceProfileTagActions + +The AllowScopedInstanceProfileTagActions Sid gives the Karpenter controller permission to tag an instance profile with [`iam:TagInstanceProfile`](https://docs.aws.amazon.com/IAM/latest/APIReference/API_TagInstanceProfile.html), based on the values shown below, +Also, `karpenter.k8s.aws/ec2nodeclass` must be set to some value. This ensures that Karpenter is only able to act on instance profiles that it provisions for this cluster. + +```json +{ + "Sid": "AllowScopedInstanceProfileTagActions", + "Effect": "Allow", + "Resource": "*", + "Action": [ + "iam:TagInstanceProfile" + ], + "Condition": { + "StringEquals": { + "aws:ResourceTag/kubernetes.io/cluster/${ClusterName}": "owned", + "aws:ResourceTag/topology.kubernetes.io/region": "${AWS::Region}", + "aws:RequestTag/kubernetes.io/cluster/${ClusterName}": "owned", + "aws:RequestTag/topology.kubernetes.io/region": "${AWS::Region}" + }, + "StringLike": { + "aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass": "*", + "aws:RequestTag/karpenter.k8s.aws/ec2nodeclass": "*" + } + } +} +``` + + +#### AllowScopedInstanceProfileActions + +The AllowScopedInstanceProfileActions Sid gives the Karpenter controller permission to perform [`iam:AddRoleToInstanceProfile`](https://docs.aws.amazon.com/IAM/latest/APIReference/API_AddRoleToInstanceProfile.html), [`iam:RemoveRoleFromInstanceProfile`](https://docs.aws.amazon.com/IAM/latest/APIReference/API_RemoveRoleFromInstanceProfile.html), and [`iam:DeleteInstanceProfile`](https://docs.aws.amazon.com/IAM/latest/APIReference/API_DeleteInstanceProfile.html) actions, +provided that the request is made to a cluster with `kubernetes.io/cluster/${ClusterName` set to owned and is made in the current region. +Also, `karpenter.k8s.aws/ec2nodeclass` must be set to some value. This permission is further enforced by the `iam:PassRole` permission. If Karpenter attempts to add a role to an instance profile that it doesn't have `iam:PassRole` permission on, that call will fail. Therefore, if you configure Karpenter to use a new role through the `EC2NodeClass`, ensure that you also specify that role within your `iam:PassRole` permission. + +```json +{ + "Sid": "AllowScopedInstanceProfileActions", + "Effect": "Allow", + "Resource": "*", + "Action": [ + "iam:AddRoleToInstanceProfile", + "iam:RemoveRoleFromInstanceProfile", + "iam:DeleteInstanceProfile" + ], + "Condition": { + "StringEquals": { + "aws:ResourceTag/kubernetes.io/cluster/${ClusterName}": "owned", + "aws:ResourceTag/topology.kubernetes.io/region": "${AWS::Region}" + }, + "StringLike": { + "aws:ResourceTag/karpenter.k8s.aws/ec2nodeclass": "*" + } + } +} +``` + +#### AllowInstanceProfileActions + +The AllowInstanceProfileActions Sid gives the Karpenter controller permission to perform [`iam:GetInstanceProfile`](https://docs.aws.amazon.com/IAM/latest/APIReference/API_GetInstanceProfile.html) actions to retrieve information about a specified instance profile, including understanding if an instance profile has been provisioned for an `EC2NodeClass` or needs to be re-provisioned. + +```json +{ + "Sid": "AllowInstanceProfileReadActions", + "Effect": "Allow", + "Resource": "*", + "Action": "iam:GetInstanceProfile" +} +``` + +#### AllowAPIServerEndpointDiscovery + +You can optionally allow the Karpenter controller to discover the Kubernetes cluster's external API endpoint to enable EC2 nodes to successfully join the EKS cluster. + +> **Note**: If you are not using an EKS control plane, you will have to specify this endpoint explicitly. See the description of the `aws.clusterEndpoint` setting in the [ConfigMap](.settings/#configmap) documentation for details. + +The AllowAPIServerEndpointDiscovery Sid allows the Karpenter controller to get that information (`eks:DescribeCluster`) for the cluster (`cluster/${ClusterName}`). +```json +{ + "Sid": "AllowAPIServerEndpointDiscovery", + "Effect": "Allow", + "Resource": "arn:${AWS::Partition}:eks:${AWS::Region}:${AWS::AccountId}:cluster/${ClusterName}", + "Action": "eks:DescribeCluster" +} +``` + +## Interruption Handling + +Settings in this section allow the Karpenter controller to stand-up an interruption queue to receive notification messages from other AWS services about the health and status of instances. For example, this interruption queue allows Karpenter to be aware of spot instance interruptions that are sent 2 minutes before spot instances are reclaimed by EC2. Adding this queue allows Karpenter to be proactive in migrating workloads to new nodes. +See the [Interruption]({{< relref "../concepts/disruption#interruption" >}}) section of the Disruption page for details. + +Defining the `KarpenterInterruptionQueuePolicy` allows Karpenter to see and respond to the following: + +* AWS health events +* Spot interruptions +* Spot rebalance recommendations +* Instance state changes + +The resources defined in this section include: + +* KarpenterInterruptionQueue +* KarpenterInterruptionQueuePolicy +* ScheduledChangeRule +* SpotInterruptionRule +* RebalanceRule +* InstanceStateChangeRule + +### KarpenterInterruptionQueue + +The [AWS::SQS::Queue](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-sqs-queue.html) resource is used to create an Amazon SQS standard queue. +Properties of that resource set the `QueueName` to the name of your cluster, the time for which SQS retains each message (`MessageRetentionPeriod`) to 300 seconds, and enabling serverside-side encryption using SQS owned encryption keys (`SqsManagedSseEnabled`) to `true`. +See [SetQueueAttributes](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_SetQueueAttributes.html) for descriptions of some of these attributes. + +```yaml +KarpenterInterruptionQueue: + Type: AWS::SQS::Queue + Properties: + QueueName: !Sub "${ClusterName}" + MessageRetentionPeriod: 300 + SqsManagedSseEnabled: true +``` + +### KarpenterInterruptionQueuePolicy + +The Karpenter interruption queue policy is created to allow AWS services that we want to receive instance notifications from to push notification messages to the queue. +The [AWS::SQS::QueuePolicy](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-sqs-queuepolicy.html) resource here applies `EC2InterruptionPolicy` to the `KarpenterInterruptionQueue`. The policy allows [sqs:SendMessage](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/APIReference/API_SendMessage.html) actions to `events.amazonaws.com` and `sqs.amazonaws.com` services. It also allows the `GetAtt` function to get attributes from `KarpenterInterruptionQueue.Arn`. + +```yaml +KarpenterInterruptionQueuePolicy: + Type: AWS::SQS::QueuePolicy + Properties: + Queues: + - !Ref KarpenterInterruptionQueue + PolicyDocument: + Id: EC2InterruptionPolicy + Statement: + - Effect: Allow + Principal: + Service: + - events.amazonaws.com + - sqs.amazonaws.com + Action: sqs:SendMessage + Resource: !GetAtt KarpenterInterruptionQueue.Arn +``` + +### Rules + +This section allows Karpenter to gather [AWS Health Events](https://docs.aws.amazon.com/health/latest/ug/cloudwatch-events-health.html#about-public-events) and direct them to a queue where they can be consumed by Karpenter. +These rules include: + +* ScheduledChangeRule: The [AWS::Events::Rule](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-events-rule.html) creates a rule where the [EventPattern](https://docs.aws.amazon.com/eventbridge/latest/userguide/eb-event-patterns.html) is set to send events from the `aws.health` source to `KarpenterInterruptionQueue`. + + ```yaml + ScheduledChangeRule: + Type: 'AWS::Events::Rule' + Properties: + EventPattern: + source: + - aws.health + detail-type: + - AWS Health Event + Targets: + - Id: KarpenterInterruptionQueueTarget + Arn: !GetAtt KarpenterInterruptionQueue.Arn + ``` + +* SpotInterruptionRule: An EC2 Spot Instance Interruption warning tells you that AWS is about to reclaim a Spot instance you are using. This rule allows Karpenter to gather [EC2 Spot Instance Interruption Warning](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-interruptions.html) events and direct them to a queue where they can be consumed by Karpenter. In particular, the [AWS::Events::Rule](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-events-rule.html) here creates a rule where the [EventPattern](https://docs.aws.amazon.com/eventbridge/latest/userguide/eb-event-patterns.html) is set to send events from the `aws.ec2` source to `KarpenterInterruptionQueue`. + + ```yaml + SpotInterruptionRule: + Type: 'AWS::Events::Rule' + Properties: + EventPattern: + source: + - aws.ec2 + detail-type: + - EC2 Spot Instance Interruption Warning + Targets: + - Id: KarpenterInterruptionQueueTarget + Arn: !GetAtt KarpenterInterruptionQueue.Arn + ``` + +* RebalanceRule: An EC2 Instance Rebalance Recommendation signal tells you that a Spot instance is at a heightened risk of being interrupted, allowing Karpenter to get new instances or simply rebalance workloads. This rule allows Karpenter to gather [EC2 Instance Rebalance Recommendation](https://docs.aws.amazon.com/AWSEC2/latest/WindowsGuide/rebalance-recommendations.html) signals and direct them to a queue where they can be consumed by Karpenter. In particular, the [AWS::Events::Rule](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-events-rule.html) here creates a rule where the [EventPattern](https://docs.aws.amazon.com/eventbridge/latest/userguide/eb-event-patterns.html) is set to send events from the `aws.ec2` source to `KarpenterInterruptionQueue`. + + ```yaml + RebalanceRule: + Type: 'AWS::Events::Rule' + Properties: + EventPattern: + source: + - aws.ec2 + detail-type: + - EC2 Instance Rebalance Recommendation + Targets: + - Id: KarpenterInterruptionQueueTarget + Arn: !GetAtt KarpenterInterruptionQueue.Arn + ``` + +* InstanceStateChangeRule: An EC2 Instance State-change Notification signal tells you that the state of an instance has changed to one of the following states: pending, running, stopping, stopped, shutting-down, or terminated. This rule allows Karpenter to gather [EC2 Instance State-change](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/monitoring-instance-state-changes.html) signals and direct them to a queue where they can be consumed by Karpenter. In particular, the [AWS::Events::Rule](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-events-rule.html) here creates a rule where the [EventPattern](https://docs.aws.amazon.com/eventbridge/latest/userguide/eb-event-patterns.html) is set to send events from the `aws.ec2` source to `KarpenterInterruptionQueue`. + + ```yaml + InstanceStateChangeRule: + Type: 'AWS::Events::Rule' + Properties: + EventPattern: + source: + - aws.ec2 + detail-type: + - EC2 Instance State-change Notification + Targets: + - Id: KarpenterInterruptionQueueTarget + Arn: !GetAtt KarpenterInterruptionQueue.Arn + ``` diff --git a/website/content/en/v0.30/concepts/instance-types.md b/website/content/en/v0.35/reference/instance-types.md similarity index 91% rename from website/content/en/v0.30/concepts/instance-types.md rename to website/content/en/v0.35/reference/instance-types.md index 6394e03e9475..6978c20ef258 100644 --- a/website/content/en/v0.30/concepts/instance-types.md +++ b/website/content/en/v0.35/reference/instance-types.md @@ -25,7 +25,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|2048| |karpenter.k8s.aws/instance-network-bandwidth|500| - |karpenter.k8s.aws/instance-pods|8| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -50,7 +49,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|4096| |karpenter.k8s.aws/instance-network-bandwidth|750| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -75,7 +73,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -100,7 +97,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -125,7 +121,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -150,7 +145,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -175,7 +169,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|1| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|1740| - |karpenter.k8s.aws/instance-pods|12| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -198,7 +191,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|1| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|7168| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -222,7 +214,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|3| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|3840| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -245,7 +236,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|3| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|7680| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -268,7 +258,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|3| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|15360| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -291,7 +280,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|3| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|30720| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -315,7 +303,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|61440| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -339,7 +326,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|4| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|3840| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -362,7 +348,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|4| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|7680| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -385,7 +370,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|4| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|15360| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -408,7 +392,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|4| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|30720| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -432,7 +415,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|61440| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -457,7 +439,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|4096| |karpenter.k8s.aws/instance-network-bandwidth|750| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -482,7 +463,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -507,7 +487,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -532,7 +511,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -557,7 +535,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|73728| |karpenter.k8s.aws/instance-network-bandwidth|12000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|9xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -582,7 +559,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|98304| |karpenter.k8s.aws/instance-network-bandwidth|12000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -607,7 +583,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|147456| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|18xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -632,7 +607,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -657,7 +631,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -683,7 +656,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|4096| |karpenter.k8s.aws/instance-network-bandwidth|750| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -708,7 +680,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -733,7 +704,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -758,7 +728,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -783,7 +752,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -808,7 +776,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|98304| |karpenter.k8s.aws/instance-network-bandwidth|12000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -833,7 +800,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|20000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -858,7 +824,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|20000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -885,7 +850,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|75| |karpenter.k8s.aws/instance-memory|4096| |karpenter.k8s.aws/instance-network-bandwidth|750| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -911,7 +875,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|150| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -937,7 +900,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|300| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -963,7 +925,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|600| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -989,7 +950,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1200| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1015,7 +975,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1800| |karpenter.k8s.aws/instance-memory|98304| |karpenter.k8s.aws/instance-network-bandwidth|12000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1041,7 +1000,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|2400| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|20000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1067,7 +1025,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|20000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1094,7 +1051,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|50| |karpenter.k8s.aws/instance-memory|4096| |karpenter.k8s.aws/instance-network-bandwidth|750| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1120,7 +1076,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|100| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1146,7 +1101,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|200| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1172,7 +1126,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|400| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1198,7 +1151,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|900| |karpenter.k8s.aws/instance-memory|73728| |karpenter.k8s.aws/instance-network-bandwidth|12000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|9xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1224,7 +1176,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1800| |karpenter.k8s.aws/instance-memory|98304| |karpenter.k8s.aws/instance-network-bandwidth|12000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1250,7 +1201,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1800| |karpenter.k8s.aws/instance-memory|147456| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|18xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1276,7 +1226,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3600| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1302,7 +1251,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3600| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1328,7 +1276,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|5376| |karpenter.k8s.aws/instance-network-bandwidth|3000| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1353,7 +1300,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|10752| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1378,7 +1324,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|21504| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1403,7 +1348,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|43008| |karpenter.k8s.aws/instance-network-bandwidth|15000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1428,7 +1372,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|98304| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|9xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1440,6 +1383,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|88002Mi| |pods|234| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|54| ### `c5n.18xlarge` #### Labels @@ -1453,7 +1397,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|18xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1465,6 +1408,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|173400Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ### `c5n.metal` #### Labels @@ -1478,7 +1422,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1490,6 +1433,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|173400Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## c6a Family ### `c6a.large` @@ -1504,7 +1448,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|4096| |karpenter.k8s.aws/instance-network-bandwidth|781| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1529,7 +1472,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|1562| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1554,7 +1496,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|3125| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1579,7 +1520,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|6250| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1604,7 +1544,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1629,7 +1568,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|98304| |karpenter.k8s.aws/instance-network-bandwidth|18750| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1654,7 +1592,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1679,7 +1616,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|37500| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1704,7 +1640,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|32xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1729,7 +1664,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|48xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1741,6 +1675,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|355262Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ### `c6a.metal` #### Labels @@ -1754,7 +1689,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -1766,6 +1700,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|355262Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## c6g Family ### `c6g.medium` @@ -1780,7 +1715,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|2048| |karpenter.k8s.aws/instance-network-bandwidth|500| - |karpenter.k8s.aws/instance-pods|8| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -1805,7 +1739,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|4096| |karpenter.k8s.aws/instance-network-bandwidth|750| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -1830,7 +1763,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -1855,7 +1787,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -1880,7 +1811,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -1905,7 +1835,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|12000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -1930,7 +1859,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|98304| |karpenter.k8s.aws/instance-network-bandwidth|20000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -1955,7 +1883,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -1980,7 +1907,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -2007,7 +1933,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|59| |karpenter.k8s.aws/instance-memory|2048| |karpenter.k8s.aws/instance-network-bandwidth|500| - |karpenter.k8s.aws/instance-pods|8| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -2033,7 +1958,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|118| |karpenter.k8s.aws/instance-memory|4096| |karpenter.k8s.aws/instance-network-bandwidth|750| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -2059,7 +1983,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|237| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -2085,7 +2008,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|474| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -2111,7 +2033,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|950| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -2137,7 +2058,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1900| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|12000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -2163,7 +2083,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|2850| |karpenter.k8s.aws/instance-memory|98304| |karpenter.k8s.aws/instance-network-bandwidth|20000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -2189,7 +2108,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -2215,7 +2133,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -2241,7 +2158,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|2048| |karpenter.k8s.aws/instance-network-bandwidth|1600| - |karpenter.k8s.aws/instance-pods|8| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -2266,7 +2182,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|4096| |karpenter.k8s.aws/instance-network-bandwidth|3000| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -2291,7 +2206,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|6300| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -2316,7 +2230,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -2340,8 +2253,7 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|6| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| - |karpenter.k8s.aws/instance-network-bandwidth|15000| - |karpenter.k8s.aws/instance-pods|234| + |karpenter.k8s.aws/instance-network-bandwidth|25000| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -2366,7 +2278,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -2391,7 +2302,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|98304| |karpenter.k8s.aws/instance-network-bandwidth|75000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -2416,7 +2326,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -2428,6 +2337,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|112720Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## c6i Family ### `c6i.large` @@ -2442,7 +2352,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|4096| |karpenter.k8s.aws/instance-network-bandwidth|781| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -2467,7 +2376,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|1562| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -2492,7 +2400,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|3125| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -2517,7 +2424,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|6250| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -2542,7 +2448,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -2567,7 +2472,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|98304| |karpenter.k8s.aws/instance-network-bandwidth|18750| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -2592,7 +2496,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -2617,7 +2520,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|37500| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -2642,7 +2544,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|32xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -2654,6 +2555,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|234021Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ### `c6i.metal` #### Labels @@ -2667,7 +2569,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -2679,6 +2580,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|234021Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## c6id Family ### `c6id.large` @@ -2694,7 +2596,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|118| |karpenter.k8s.aws/instance-memory|4096| |karpenter.k8s.aws/instance-network-bandwidth|781| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -2720,7 +2621,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|237| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|1562| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -2746,7 +2646,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|474| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|3125| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -2772,7 +2671,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|950| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|6250| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -2798,7 +2696,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1900| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -2824,7 +2721,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|2850| |karpenter.k8s.aws/instance-memory|98304| |karpenter.k8s.aws/instance-network-bandwidth|18750| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -2850,7 +2746,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -2876,7 +2771,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|5700| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|37500| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -2902,7 +2796,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|7600| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|32xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -2914,6 +2807,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|234021Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ### `c6id.metal` #### Labels @@ -2928,7 +2822,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|7600| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -2940,6 +2833,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|234021Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## c6in Family ### `c6in.large` @@ -2954,7 +2848,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|4096| |karpenter.k8s.aws/instance-network-bandwidth|3125| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -2979,7 +2872,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|6250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3004,7 +2896,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3029,7 +2920,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3054,7 +2944,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3079,7 +2968,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|98304| |karpenter.k8s.aws/instance-network-bandwidth|75000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3104,7 +2992,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3129,7 +3016,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|150000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3154,7 +3040,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|200000| - |karpenter.k8s.aws/instance-pods|345| |karpenter.k8s.aws/instance-size|32xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3166,6 +3051,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|238333Mi| |pods|345| + |vpc.amazonaws.com/efa|2| |vpc.amazonaws.com/pod-eni|108| ### `c6in.metal` #### Labels @@ -3179,7 +3065,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|200000| - |karpenter.k8s.aws/instance-pods|345| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3191,7 +3076,299 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|238333Mi| |pods|345| + |vpc.amazonaws.com/efa|2| |vpc.amazonaws.com/pod-eni|108| +## c7a Family +### `c7a.medium` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|1| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7a| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|2048| + |karpenter.k8s.aws/instance-network-bandwidth|390| + |karpenter.k8s.aws/instance-size|medium| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7a.medium| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|940m| + |ephemeral-storage|17Gi| + |memory|1451Mi| + |pods|8| + |vpc.amazonaws.com/pod-eni|4| +### `c7a.large` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7a| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|4096| + |karpenter.k8s.aws/instance-network-bandwidth|781| + |karpenter.k8s.aws/instance-size|large| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7a.large| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|1930m| + |ephemeral-storage|17Gi| + |memory|3114Mi| + |pods|29| + |vpc.amazonaws.com/pod-eni|9| +### `c7a.xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7a| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|8192| + |karpenter.k8s.aws/instance-network-bandwidth|1562| + |karpenter.k8s.aws/instance-size|xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7a.xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|3920m| + |ephemeral-storage|17Gi| + |memory|6584Mi| + |pods|58| + |vpc.amazonaws.com/pod-eni|18| +### `c7a.2xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7a| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|16384| + |karpenter.k8s.aws/instance-network-bandwidth|3125| + |karpenter.k8s.aws/instance-size|2xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7a.2xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|7910m| + |ephemeral-storage|17Gi| + |memory|14162Mi| + |pods|58| + |vpc.amazonaws.com/pod-eni|38| +### `c7a.4xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7a| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|32768| + |karpenter.k8s.aws/instance-network-bandwidth|6250| + |karpenter.k8s.aws/instance-size|4xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7a.4xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|15890m| + |ephemeral-storage|17Gi| + |memory|27381Mi| + |pods|234| + |vpc.amazonaws.com/pod-eni|54| +### `c7a.8xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7a| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|65536| + |karpenter.k8s.aws/instance-network-bandwidth|12500| + |karpenter.k8s.aws/instance-size|8xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7a.8xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|31850m| + |ephemeral-storage|17Gi| + |memory|57691Mi| + |pods|234| + |vpc.amazonaws.com/pod-eni|84| +### `c7a.12xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7a| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|98304| + |karpenter.k8s.aws/instance-network-bandwidth|18750| + |karpenter.k8s.aws/instance-size|12xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7a.12xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|47810m| + |ephemeral-storage|17Gi| + |memory|88002Mi| + |pods|234| + |vpc.amazonaws.com/pod-eni|114| +### `c7a.16xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7a| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|131072| + |karpenter.k8s.aws/instance-network-bandwidth|25000| + |karpenter.k8s.aws/instance-size|16xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7a.16xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|63770m| + |ephemeral-storage|17Gi| + |memory|112779Mi| + |pods|737| + |vpc.amazonaws.com/pod-eni|107| +### `c7a.24xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7a| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|196608| + |karpenter.k8s.aws/instance-network-bandwidth|37500| + |karpenter.k8s.aws/instance-size|24xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7a.24xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|95690m| + |ephemeral-storage|17Gi| + |memory|173400Mi| + |pods|737| + |vpc.amazonaws.com/pod-eni|107| +### `c7a.32xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7a| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|262144| + |karpenter.k8s.aws/instance-network-bandwidth|50000| + |karpenter.k8s.aws/instance-size|32xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7a.32xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|127610m| + |ephemeral-storage|17Gi| + |memory|234021Mi| + |pods|737| + |vpc.amazonaws.com/pod-eni|107| +### `c7a.48xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|192| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7a| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|393216| + |karpenter.k8s.aws/instance-network-bandwidth|50000| + |karpenter.k8s.aws/instance-size|48xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7a.48xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|191450m| + |ephemeral-storage|17Gi| + |memory|355262Mi| + |pods|737| + |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| +### `c7a.metal-48xl` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|192| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7a| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|| + |karpenter.k8s.aws/instance-memory|393216| + |karpenter.k8s.aws/instance-network-bandwidth|50000| + |karpenter.k8s.aws/instance-size|metal-48xl| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7a.metal-48xl| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|191450m| + |ephemeral-storage|17Gi| + |memory|355262Mi| + |pods|737| + |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| ## c7g Family ### `c7g.medium` #### Labels @@ -3205,7 +3382,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|2048| |karpenter.k8s.aws/instance-network-bandwidth|520| - |karpenter.k8s.aws/instance-pods|8| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3230,7 +3406,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|4096| |karpenter.k8s.aws/instance-network-bandwidth|937| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3255,7 +3430,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|1876| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3280,7 +3454,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|3750| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3305,7 +3478,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|7500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3330,7 +3502,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|15000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3355,7 +3526,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|98304| |karpenter.k8s.aws/instance-network-bandwidth|22500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3380,7 +3550,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|30000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3392,6 +3561,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|112720Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ### `c7g.metal` #### Labels @@ -3405,7 +3575,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|30000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3417,6 +3586,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|112720Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## c7gd Family ### `c7gd.medium` @@ -3432,7 +3602,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|59| |karpenter.k8s.aws/instance-memory|2048| |karpenter.k8s.aws/instance-network-bandwidth|520| - |karpenter.k8s.aws/instance-pods|8| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3458,7 +3627,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|118| |karpenter.k8s.aws/instance-memory|4096| |karpenter.k8s.aws/instance-network-bandwidth|937| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3484,7 +3652,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|237| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|1876| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3510,7 +3677,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|474| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|3750| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3536,7 +3702,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|950| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|7500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3562,7 +3727,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1900| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|15000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3588,7 +3752,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|2850| |karpenter.k8s.aws/instance-memory|98304| |karpenter.k8s.aws/instance-network-bandwidth|22500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3614,7 +3777,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|30000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3626,6 +3788,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|112720Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## c7gn Family ### `c7gn.medium` @@ -3640,7 +3803,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|2048| |karpenter.k8s.aws/instance-network-bandwidth|3125| - |karpenter.k8s.aws/instance-pods|8| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3665,7 +3827,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|4096| |karpenter.k8s.aws/instance-network-bandwidth|6250| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3690,7 +3851,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3715,7 +3875,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3740,7 +3899,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3765,7 +3923,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3790,7 +3947,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|98304| |karpenter.k8s.aws/instance-network-bandwidth|150000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3815,7 +3971,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|200000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -3827,9 +3982,277 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|112720Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| -## d2 Family -### `d2.xlarge` +## c7i Family +### `c7i.large` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7i| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|4096| + |karpenter.k8s.aws/instance-network-bandwidth|781| + |karpenter.k8s.aws/instance-size|large| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7i.large| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|1930m| + |ephemeral-storage|17Gi| + |memory|3114Mi| + |pods|29| + |vpc.amazonaws.com/pod-eni|9| +### `c7i.xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7i| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|8192| + |karpenter.k8s.aws/instance-network-bandwidth|1562| + |karpenter.k8s.aws/instance-size|xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7i.xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|3920m| + |ephemeral-storage|17Gi| + |memory|6584Mi| + |pods|58| + |vpc.amazonaws.com/pod-eni|18| +### `c7i.2xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7i| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|16384| + |karpenter.k8s.aws/instance-network-bandwidth|3125| + |karpenter.k8s.aws/instance-size|2xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7i.2xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|7910m| + |ephemeral-storage|17Gi| + |memory|14162Mi| + |pods|58| + |vpc.amazonaws.com/pod-eni|38| +### `c7i.4xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7i| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|32768| + |karpenter.k8s.aws/instance-network-bandwidth|6250| + |karpenter.k8s.aws/instance-size|4xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7i.4xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|15890m| + |ephemeral-storage|17Gi| + |memory|27381Mi| + |pods|234| + |vpc.amazonaws.com/pod-eni|54| +### `c7i.8xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7i| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|65536| + |karpenter.k8s.aws/instance-network-bandwidth|12500| + |karpenter.k8s.aws/instance-size|8xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7i.8xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|31850m| + |ephemeral-storage|17Gi| + |memory|57691Mi| + |pods|234| + |vpc.amazonaws.com/pod-eni|84| +### `c7i.12xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7i| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|98304| + |karpenter.k8s.aws/instance-network-bandwidth|18750| + |karpenter.k8s.aws/instance-size|12xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7i.12xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|47810m| + |ephemeral-storage|17Gi| + |memory|88002Mi| + |pods|234| + |vpc.amazonaws.com/pod-eni|114| +### `c7i.16xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7i| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|131072| + |karpenter.k8s.aws/instance-network-bandwidth|25000| + |karpenter.k8s.aws/instance-size|16xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7i.16xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|63770m| + |ephemeral-storage|17Gi| + |memory|112779Mi| + |pods|737| + |vpc.amazonaws.com/pod-eni|107| +### `c7i.24xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7i| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|196608| + |karpenter.k8s.aws/instance-network-bandwidth|37500| + |karpenter.k8s.aws/instance-size|24xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7i.24xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|95690m| + |ephemeral-storage|17Gi| + |memory|173400Mi| + |pods|737| + |vpc.amazonaws.com/pod-eni|107| +### `c7i.metal-24xl` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7i| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|| + |karpenter.k8s.aws/instance-memory|196608| + |karpenter.k8s.aws/instance-network-bandwidth|37500| + |karpenter.k8s.aws/instance-size|metal-24xl| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7i.metal-24xl| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|95690m| + |ephemeral-storage|17Gi| + |memory|173400Mi| + |pods|737| + |vpc.amazonaws.com/pod-eni|107| +### `c7i.48xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|192| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7i| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|393216| + |karpenter.k8s.aws/instance-network-bandwidth|50000| + |karpenter.k8s.aws/instance-size|48xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7i.48xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|191450m| + |ephemeral-storage|17Gi| + |memory|355262Mi| + |pods|737| + |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| +### `c7i.metal-48xl` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|c| + |karpenter.k8s.aws/instance-cpu|192| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|c7i| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|| + |karpenter.k8s.aws/instance-memory|393216| + |karpenter.k8s.aws/instance-network-bandwidth|50000| + |karpenter.k8s.aws/instance-size|metal-48xl| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|c7i.metal-48xl| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|191450m| + |ephemeral-storage|17Gi| + |memory|355262Mi| + |pods|737| + |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| +## d2 Family +### `d2.xlarge` #### Labels | Label | Value | |--|--| @@ -3840,7 +4263,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|2| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|31232| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3863,7 +4285,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|2| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|62464| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3886,7 +4307,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|2| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|124928| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3910,7 +4330,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|249856| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3936,7 +4355,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|5940| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|3000| - |karpenter.k8s.aws/instance-pods|10| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3962,7 +4380,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|11880| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|6000| - |karpenter.k8s.aws/instance-pods|18| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -3988,7 +4405,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|23760| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|38| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4014,7 +4430,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|47520| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|59| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4041,7 +4456,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|27960| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|6000| - |karpenter.k8s.aws/instance-pods|10| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4067,7 +4481,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|55920| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|18| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4093,7 +4506,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|111840| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|38| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4119,7 +4531,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|167760| |karpenter.k8s.aws/instance-memory|98304| |karpenter.k8s.aws/instance-network-bandwidth|40000| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|6xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4145,7 +4556,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|223680| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|78| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4171,7 +4581,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|335520| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|75000| - |karpenter.k8s.aws/instance-pods|89| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4202,7 +4611,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|4000| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|400000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4215,6 +4623,7 @@ below are the resources available with some assumptions and after the instance o |habana.ai/gaudi|8| |memory|718987Mi| |pods|737| + |vpc.amazonaws.com/efa|4| |vpc.amazonaws.com/pod-eni|62| ## f1 Family ### `f1.2xlarge` @@ -4230,7 +4639,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|470| |karpenter.k8s.aws/instance-memory|124928| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4255,7 +4663,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|940| |karpenter.k8s.aws/instance-memory|249856| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4280,7 +4687,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3760| |karpenter.k8s.aws/instance-memory|999424| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|394| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4292,94 +4698,36 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|919778Mi| |pods|394| -## g2 Family -### `g2.2xlarge` +## g3 Family +### `g3.4xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|g| - |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| - |karpenter.k8s.aws/instance-family|g2| - |karpenter.k8s.aws/instance-generation|2| + |karpenter.k8s.aws/instance-family|g3| + |karpenter.k8s.aws/instance-generation|3| |karpenter.k8s.aws/instance-gpu-count|1| |karpenter.k8s.aws/instance-gpu-manufacturer|nvidia| - |karpenter.k8s.aws/instance-gpu-memory|4096| - |karpenter.k8s.aws/instance-gpu-name|k520| + |karpenter.k8s.aws/instance-gpu-memory|8192| + |karpenter.k8s.aws/instance-gpu-name|m60| |karpenter.k8s.aws/instance-hypervisor|xen| - |karpenter.k8s.aws/instance-memory|15360| - |karpenter.k8s.aws/instance-pods|58| - |karpenter.k8s.aws/instance-size|2xlarge| + |karpenter.k8s.aws/instance-memory|124928| + |karpenter.k8s.aws/instance-network-bandwidth|5000| + |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|g2.2xlarge| + |node.kubernetes.io/instance-type|g3.4xlarge| #### Resources | Resource | Quantity | |--|--| - |cpu|7910m| + |cpu|15890m| |ephemeral-storage|17Gi| - |memory|13215Mi| + |memory|112629Mi| |nvidia.com/gpu|1| - |pods|58| -### `g2.8xlarge` -#### Labels - | Label | Value | - |--|--| - |karpenter.k8s.aws/instance-category|g| - |karpenter.k8s.aws/instance-cpu|32| - |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| - |karpenter.k8s.aws/instance-family|g2| - |karpenter.k8s.aws/instance-generation|2| - |karpenter.k8s.aws/instance-gpu-count|4| - |karpenter.k8s.aws/instance-gpu-manufacturer|nvidia| - |karpenter.k8s.aws/instance-gpu-memory|4096| - |karpenter.k8s.aws/instance-gpu-name|k520| - |karpenter.k8s.aws/instance-hypervisor|xen| - |karpenter.k8s.aws/instance-memory|61440| - |karpenter.k8s.aws/instance-pods|234| - |karpenter.k8s.aws/instance-size|8xlarge| - |kubernetes.io/arch|amd64| - |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|g2.8xlarge| -#### Resources - | Resource | Quantity | - |--|--| - |cpu|31850m| - |ephemeral-storage|17Gi| - |memory|53903Mi| - |nvidia.com/gpu|4| - |pods|234| -## g3 Family -### `g3.4xlarge` -#### Labels - | Label | Value | - |--|--| - |karpenter.k8s.aws/instance-category|g| - |karpenter.k8s.aws/instance-cpu|16| - |karpenter.k8s.aws/instance-encryption-in-transit-supported|false| - |karpenter.k8s.aws/instance-family|g3| - |karpenter.k8s.aws/instance-generation|3| - |karpenter.k8s.aws/instance-gpu-count|1| - |karpenter.k8s.aws/instance-gpu-manufacturer|nvidia| - |karpenter.k8s.aws/instance-gpu-memory|8192| - |karpenter.k8s.aws/instance-gpu-name|m60| - |karpenter.k8s.aws/instance-hypervisor|xen| - |karpenter.k8s.aws/instance-memory|124928| - |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| - |karpenter.k8s.aws/instance-size|4xlarge| - |kubernetes.io/arch|amd64| - |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|g3.4xlarge| -#### Resources - | Resource | Quantity | - |--|--| - |cpu|15890m| - |ephemeral-storage|17Gi| - |memory|112629Mi| - |nvidia.com/gpu|1| - |pods|234| -### `g3.8xlarge` + |pods|234| +### `g3.8xlarge` #### Labels | Label | Value | |--|--| @@ -4395,7 +4743,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|249856| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4424,7 +4771,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|499712| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4453,7 +4799,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-gpu-name|m60| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|31232| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4484,7 +4829,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|150| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|2000| - |karpenter.k8s.aws/instance-pods|8| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4515,7 +4859,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|300| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|4167| - |karpenter.k8s.aws/instance-pods|8| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4546,7 +4889,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|600| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|8333| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4577,7 +4919,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1200| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|15000| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4608,7 +4949,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|2400| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4640,7 +4980,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|125| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4671,7 +5010,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|225| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4702,7 +5040,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|225| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|20000| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4733,7 +5070,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|900| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4746,6 +5082,7 @@ below are the resources available with some assumptions and after the instance o |memory|120248Mi| |nvidia.com/gpu|1| |pods|58| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|58| ### `g4dn.12xlarge` #### Labels @@ -4764,7 +5101,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|900| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4777,6 +5113,7 @@ below are the resources available with some assumptions and after the instance o |memory|178933Mi| |nvidia.com/gpu|4| |pods|234| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|54| ### `g4dn.16xlarge` #### Labels @@ -4795,7 +5132,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|900| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4808,6 +5144,7 @@ below are the resources available with some assumptions and after the instance o |memory|241490Mi| |nvidia.com/gpu|1| |pods|58| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|118| ### `g4dn.metal` #### Labels @@ -4826,7 +5163,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1800| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4839,6 +5175,7 @@ below are the resources available with some assumptions and after the instance o |memory|355262Mi| |nvidia.com/gpu|8| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## g5 Family ### `g5.xlarge` @@ -4858,7 +5195,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|250| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4889,7 +5225,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|450| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4920,7 +5255,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|600| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4951,7 +5285,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|900| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4964,6 +5297,7 @@ below are the resources available with some assumptions and after the instance o |memory|118312Mi| |nvidia.com/gpu|1| |pods|234| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|84| ### `g5.12xlarge` #### Labels @@ -4982,7 +5316,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|40000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -4995,6 +5328,7 @@ below are the resources available with some assumptions and after the instance o |memory|173400Mi| |nvidia.com/gpu|4| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ### `g5.16xlarge` #### Labels @@ -5013,7 +5347,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1900| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5026,6 +5359,7 @@ below are the resources available with some assumptions and after the instance o |memory|239554Mi| |nvidia.com/gpu|1| |pods|234| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|114| ### `g5.24xlarge` #### Labels @@ -5044,7 +5378,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5057,6 +5390,7 @@ below are the resources available with some assumptions and after the instance o |memory|355262Mi| |nvidia.com/gpu|4| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ### `g5.48xlarge` #### Labels @@ -5075,7 +5409,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|7600| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|345| |karpenter.k8s.aws/instance-size|48xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5088,6 +5421,7 @@ below are the resources available with some assumptions and after the instance o |memory|723299Mi| |nvidia.com/gpu|8| |pods|345| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|115| ## g5g Family ### `g5g.xlarge` @@ -5106,7 +5440,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -5136,7 +5469,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -5166,7 +5498,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -5196,7 +5527,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|12000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -5226,7 +5556,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -5256,7 +5585,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -5283,7 +5611,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5307,7 +5634,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5331,7 +5657,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5355,7 +5680,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5365,8 +5689,8 @@ below are the resources available with some assumptions and after the instance o |--|--| |cpu|63770m| |ephemeral-storage|17Gi| - |memory|234021Mi| - |pods|737| + |memory|237794Mi| + |pods|394| ## hpc7g Family ### `hpc7g.4xlarge` #### Labels @@ -5380,7 +5704,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|200000| - |karpenter.k8s.aws/instance-pods|198| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -5392,6 +5715,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|118649Mi| |pods|198| + |vpc.amazonaws.com/efa|1| ### `hpc7g.8xlarge` #### Labels | Label | Value | @@ -5404,7 +5728,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|200000| - |karpenter.k8s.aws/instance-pods|198| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -5416,6 +5739,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|118649Mi| |pods|198| + |vpc.amazonaws.com/efa|1| ### `hpc7g.16xlarge` #### Labels | Label | Value | @@ -5428,7 +5752,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|200000| - |karpenter.k8s.aws/instance-pods|198| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -5440,6 +5763,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|118649Mi| |pods|198| + |vpc.amazonaws.com/efa|1| ## i2 Family ### `i2.xlarge` #### Labels @@ -5452,7 +5776,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|2| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|31232| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5475,7 +5798,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|2| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|62464| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5498,7 +5820,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|2| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|124928| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5522,7 +5843,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|249856| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5548,7 +5868,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|475| |karpenter.k8s.aws/instance-memory|15616| |karpenter.k8s.aws/instance-network-bandwidth|750| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5573,7 +5892,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|950| |karpenter.k8s.aws/instance-memory|31232| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5598,7 +5916,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1900| |karpenter.k8s.aws/instance-memory|62464| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5623,7 +5940,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|124928| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5648,7 +5964,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|7600| |karpenter.k8s.aws/instance-memory|249856| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5673,7 +5988,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|15200| |karpenter.k8s.aws/instance-memory|499712| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5698,7 +6012,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|15200| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5725,7 +6038,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1250| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|2100| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5751,7 +6063,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|2500| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|4200| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5777,7 +6088,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|5000| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|8400| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5803,7 +6113,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|7500| |karpenter.k8s.aws/instance-memory|98304| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|3xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5829,7 +6138,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|15000| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|6xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5855,7 +6163,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|30000| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5867,6 +6174,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|360795Mi| |pods|234| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|54| ### `i3en.24xlarge` #### Labels @@ -5881,7 +6189,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|60000| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5893,6 +6200,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|718987Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ### `i3en.metal` #### Labels @@ -5907,7 +6215,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|60000| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -5919,6 +6226,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|718987Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## i4g Family ### `i4g.large` @@ -5934,7 +6242,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|468| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|781| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -5960,7 +6267,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|937| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|1875| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -5986,7 +6292,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1875| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|4687| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -6012,7 +6317,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3750| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|9375| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -6038,7 +6342,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|7500| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|18750| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -6064,7 +6367,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|15000| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|37500| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -6076,6 +6378,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|476445Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## i4i Family ### `i4i.large` @@ -6091,7 +6394,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|468| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|781| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6116,7 +6418,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|937| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|1875| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6142,7 +6443,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1875| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|4687| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6168,7 +6468,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3750| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|9375| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6194,7 +6493,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|7500| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|18750| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6207,6 +6505,31 @@ below are the resources available with some assumptions and after the instance o |memory|239554Mi| |pods|234| |vpc.amazonaws.com/pod-eni|112| +### `i4i.12xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|i| + |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|i4i| + |karpenter.k8s.aws/instance-generation|4| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-local-nvme|11250| + |karpenter.k8s.aws/instance-memory|393216| + |karpenter.k8s.aws/instance-network-bandwidth|28120| + |karpenter.k8s.aws/instance-size|12xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|i4i.12xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|47810m| + |ephemeral-storage|17Gi| + |memory|360795Mi| + |pods|234| + |vpc.amazonaws.com/pod-eni|112| ### `i4i.16xlarge` #### Labels | Label | Value | @@ -6220,7 +6543,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|15000| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|37500| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6233,6 +6555,31 @@ below are the resources available with some assumptions and after the instance o |memory|476504Mi| |pods|737| |vpc.amazonaws.com/pod-eni|120| +### `i4i.24xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|i| + |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|i4i| + |karpenter.k8s.aws/instance-generation|4| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-local-nvme|22500| + |karpenter.k8s.aws/instance-memory|786432| + |karpenter.k8s.aws/instance-network-bandwidth|56250| + |karpenter.k8s.aws/instance-size|24xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|i4i.24xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|95690m| + |ephemeral-storage|17Gi| + |memory|722287Mi| + |pods|437| + |vpc.amazonaws.com/pod-eni|105| ### `i4i.32xlarge` #### Labels | Label | Value | @@ -6246,7 +6593,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|30000| |karpenter.k8s.aws/instance-memory|1048576| |karpenter.k8s.aws/instance-network-bandwidth|75000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|32xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6258,6 +6604,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|961470Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|120| ### `i4i.metal` #### Labels @@ -6272,7 +6619,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|30000| |karpenter.k8s.aws/instance-memory|1048576| |karpenter.k8s.aws/instance-network-bandwidth|75000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6284,6 +6630,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|961470Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|120| ## im4gn Family ### `im4gn.large` @@ -6299,7 +6646,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|937| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|3125| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -6325,7 +6671,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1875| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|6250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -6351,7 +6696,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3750| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -6377,7 +6721,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|7500| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -6403,7 +6746,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|15000| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -6429,7 +6771,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|30000| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -6441,6 +6782,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|233962Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## inf1 Family ### `inf1.xlarge` @@ -6458,7 +6800,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|38| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6487,7 +6828,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|38| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6516,7 +6856,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|49152| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|6xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6545,7 +6884,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|321| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6558,6 +6896,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|177976Mi| |pods|321| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|111| ## inf2 Family ### `inf2.xlarge` @@ -6575,7 +6914,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|2083| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6588,6 +6926,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|14162Mi| |pods|58| + |vpc.amazonaws.com/pod-eni|18| ### `inf2.8xlarge` #### Labels | Label | Value | @@ -6603,7 +6942,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|16667| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6616,6 +6954,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|118312Mi| |pods|234| + |vpc.amazonaws.com/pod-eni|84| ### `inf2.24xlarge` #### Labels | Label | Value | @@ -6631,7 +6970,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6644,6 +6982,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|355262Mi| |pods|737| + |vpc.amazonaws.com/pod-eni|107| ### `inf2.48xlarge` #### Labels | Label | Value | @@ -6659,7 +6998,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|48xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6672,6 +7010,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|718987Mi| |pods|737| + |vpc.amazonaws.com/pod-eni|107| ## is4gen Family ### `is4gen.medium` #### Labels @@ -6686,7 +7025,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|937| |karpenter.k8s.aws/instance-memory|6144| |karpenter.k8s.aws/instance-network-bandwidth|1562| - |karpenter.k8s.aws/instance-pods|8| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -6712,7 +7050,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1875| |karpenter.k8s.aws/instance-memory|12288| |karpenter.k8s.aws/instance-network-bandwidth|3125| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -6738,7 +7075,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3750| |karpenter.k8s.aws/instance-memory|24576| |karpenter.k8s.aws/instance-network-bandwidth|6250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -6764,7 +7100,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|7500| |karpenter.k8s.aws/instance-memory|49152| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -6790,7 +7125,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|15000| |karpenter.k8s.aws/instance-memory|98304| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -6816,7 +7150,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|30000| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -6841,7 +7174,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|1| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|1740| - |karpenter.k8s.aws/instance-pods|8| |karpenter.k8s.aws/instance-size|small| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6864,7 +7196,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|1| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|3788| - |karpenter.k8s.aws/instance-pods|12| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6887,7 +7218,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|1| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|7680| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6910,7 +7240,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|1| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|15360| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6934,7 +7263,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|2| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|17510| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6957,7 +7285,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|2| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|35020| - |karpenter.k8s.aws/instance-pods|118| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -6980,7 +7307,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|2| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|70041| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7004,7 +7330,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|3| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|3840| - |karpenter.k8s.aws/instance-pods|12| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7027,7 +7352,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|3| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|7680| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7050,7 +7374,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|3| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|15360| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7073,7 +7396,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|3| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|30720| - |karpenter.k8s.aws/instance-pods|118| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7097,7 +7419,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|4| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|8192| - |karpenter.k8s.aws/instance-pods|20| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7120,7 +7441,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|4| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|16384| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7143,7 +7463,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|4| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|32768| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7166,7 +7485,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|4| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|65536| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7190,7 +7508,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|163840| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|10xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7214,7 +7531,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7239,7 +7555,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|750| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7264,7 +7579,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7289,7 +7603,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7314,7 +7627,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7339,7 +7651,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7364,7 +7675,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|12000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7389,7 +7699,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|20000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7414,7 +7723,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7439,7 +7747,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7465,7 +7772,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|750| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7490,7 +7796,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7515,7 +7820,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7540,7 +7844,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7565,7 +7868,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|7500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7590,7 +7892,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7615,7 +7916,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|12000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7640,7 +7940,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|20000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7667,7 +7966,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|75| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|750| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7693,7 +7991,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|150| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7719,7 +8016,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|300| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7745,7 +8041,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|600| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7771,7 +8066,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1200| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|7500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7797,7 +8091,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1800| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7823,7 +8116,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|2400| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|12000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7849,7 +8141,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3600| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|20000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7876,7 +8167,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|75| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|750| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7902,7 +8192,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|150| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7928,7 +8217,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|300| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7954,7 +8242,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|600| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -7980,7 +8267,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1200| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8006,7 +8292,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1800| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|12000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8032,7 +8317,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|2400| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|20000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8058,7 +8342,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3600| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8084,7 +8367,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3600| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8111,7 +8393,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|75| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|2100| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8137,7 +8418,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|150| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|4100| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8163,7 +8443,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|300| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|8125| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8189,7 +8468,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|600| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|16250| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8215,7 +8493,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1200| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8241,7 +8518,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1800| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8267,7 +8543,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|2400| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|75000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8293,7 +8568,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3600| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8305,6 +8579,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|355262Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ### `m5dn.metal` #### Labels @@ -8319,7 +8594,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3600| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8331,6 +8605,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|355262Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## m5n Family ### `m5n.large` @@ -8345,7 +8620,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|2100| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8370,7 +8644,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|4100| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8395,7 +8668,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|8125| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8420,7 +8692,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|16250| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8445,7 +8716,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8470,7 +8740,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8495,7 +8764,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|75000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8520,7 +8788,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8532,6 +8799,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|355262Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ### `m5n.metal` #### Labels @@ -8545,7 +8813,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8557,6 +8824,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|355262Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## m5zn Family ### `m5zn.large` @@ -8571,7 +8839,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|3000| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8596,7 +8863,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8621,7 +8887,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8646,7 +8911,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|49152| |karpenter.k8s.aws/instance-network-bandwidth|15000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|3xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8671,7 +8935,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|98304| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|6xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8696,7 +8959,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8708,6 +8970,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|173400Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ### `m5zn.metal` #### Labels @@ -8721,7 +8984,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8733,6 +8995,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|173400Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## m6a Family ### `m6a.large` @@ -8747,7 +9010,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|781| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8772,7 +9034,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|1562| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8797,7 +9058,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|3125| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8822,7 +9082,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|6250| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8847,7 +9106,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8872,7 +9130,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|18750| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8897,7 +9154,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8922,7 +9178,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|37500| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8947,7 +9202,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|32xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8972,7 +9226,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|48xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -8984,6 +9237,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|718987Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ### `m6a.metal` #### Labels @@ -8997,7 +9251,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -9009,6 +9262,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|718987Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## m6g Family ### `m6g.medium` @@ -9023,7 +9277,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|4096| |karpenter.k8s.aws/instance-network-bandwidth|500| - |karpenter.k8s.aws/instance-pods|8| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -9048,7 +9301,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|750| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -9073,7 +9325,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -9098,7 +9349,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -9123,7 +9373,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -9148,7 +9397,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|12000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -9173,7 +9421,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|20000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -9198,7 +9445,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -9223,7 +9469,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -9250,7 +9495,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|59| |karpenter.k8s.aws/instance-memory|4096| |karpenter.k8s.aws/instance-network-bandwidth|500| - |karpenter.k8s.aws/instance-pods|8| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -9276,7 +9520,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|118| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|750| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -9302,7 +9545,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|237| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -9328,7 +9570,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|474| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -9354,7 +9595,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|950| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -9380,7 +9620,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1900| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|12000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -9406,7 +9645,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|2850| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|20000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -9432,7 +9670,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -9458,7 +9695,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -9484,7 +9720,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|781| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -9509,7 +9744,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|1562| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -9534,7 +9768,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|3125| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -9559,7 +9792,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|6250| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -9584,7 +9816,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -9609,7 +9840,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|18750| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -9634,7 +9864,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -9659,7 +9888,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|37500| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -9684,7 +9912,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|32xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -9696,6 +9923,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|476504Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ### `m6i.metal` #### Labels @@ -9709,7 +9937,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -9721,6 +9948,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|476504Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## m6id Family ### `m6id.large` @@ -9736,7 +9964,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|118| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|781| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -9762,7 +9989,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|237| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|1562| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -9788,7 +10014,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|474| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|3125| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -9814,7 +10039,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|950| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|6250| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -9840,7 +10064,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1900| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -9866,7 +10089,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|2850| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|18750| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -9892,7 +10114,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -9918,7 +10139,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|5700| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|37500| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -9944,7 +10164,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|7600| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|32xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -9956,6 +10175,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|476504Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ### `m6id.metal` #### Labels @@ -9970,7 +10190,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|7600| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -9982,6 +10201,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|476504Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## m6idn Family ### `m6idn.large` @@ -9997,7 +10217,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|118| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|3125| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10023,7 +10242,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|237| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|6250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10049,7 +10267,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|474| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10075,7 +10292,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|950| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10101,7 +10317,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1900| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10127,7 +10342,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|2850| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|75000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10153,7 +10367,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10179,7 +10392,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|5700| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|150000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10205,7 +10417,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|7600| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|200000| - |karpenter.k8s.aws/instance-pods|345| |karpenter.k8s.aws/instance-size|32xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10217,6 +10428,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|480816Mi| |pods|345| + |vpc.amazonaws.com/efa|2| |vpc.amazonaws.com/pod-eni|108| ### `m6idn.metal` #### Labels @@ -10231,7 +10443,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|7600| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|200000| - |karpenter.k8s.aws/instance-pods|345| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10243,6 +10454,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|480816Mi| |pods|345| + |vpc.amazonaws.com/efa|2| |vpc.amazonaws.com/pod-eni|108| ## m6in Family ### `m6in.large` @@ -10257,7 +10469,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|3125| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10282,7 +10493,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|6250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10307,7 +10517,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10332,7 +10541,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10357,7 +10565,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10382,7 +10589,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|75000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10407,7 +10613,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10432,7 +10637,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|150000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10457,7 +10661,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|200000| - |karpenter.k8s.aws/instance-pods|345| |karpenter.k8s.aws/instance-size|32xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10469,6 +10672,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|480816Mi| |pods|345| + |vpc.amazonaws.com/efa|2| |vpc.amazonaws.com/pod-eni|108| ### `m6in.metal` #### Labels @@ -10482,7 +10686,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|200000| - |karpenter.k8s.aws/instance-pods|345| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10494,6 +10697,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|480816Mi| |pods|345| + |vpc.amazonaws.com/efa|2| |vpc.amazonaws.com/pod-eni|108| ## m7a Family ### `m7a.medium` @@ -10508,7 +10712,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|4096| |karpenter.k8s.aws/instance-network-bandwidth|390| - |karpenter.k8s.aws/instance-pods|8| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10520,6 +10723,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|3345Mi| |pods|8| + |vpc.amazonaws.com/pod-eni|4| ### `m7a.large` #### Labels | Label | Value | @@ -10532,7 +10736,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|781| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10544,6 +10747,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|6903Mi| |pods|29| + |vpc.amazonaws.com/pod-eni|9| ### `m7a.xlarge` #### Labels | Label | Value | @@ -10556,7 +10760,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|1562| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10568,6 +10771,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|14162Mi| |pods|58| + |vpc.amazonaws.com/pod-eni|18| ### `m7a.2xlarge` #### Labels | Label | Value | @@ -10580,7 +10784,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|3125| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10592,6 +10795,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|29317Mi| |pods|58| + |vpc.amazonaws.com/pod-eni|38| ### `m7a.4xlarge` #### Labels | Label | Value | @@ -10604,7 +10808,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|6250| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10616,6 +10819,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|57691Mi| |pods|234| + |vpc.amazonaws.com/pod-eni|54| ### `m7a.8xlarge` #### Labels | Label | Value | @@ -10628,7 +10832,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10640,6 +10843,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|118312Mi| |pods|234| + |vpc.amazonaws.com/pod-eni|84| ### `m7a.12xlarge` #### Labels | Label | Value | @@ -10652,7 +10856,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|18750| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10664,6 +10867,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|178933Mi| |pods|234| + |vpc.amazonaws.com/pod-eni|114| ### `m7a.16xlarge` #### Labels | Label | Value | @@ -10676,7 +10880,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10688,6 +10891,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|234021Mi| |pods|737| + |vpc.amazonaws.com/pod-eni|107| ### `m7a.24xlarge` #### Labels | Label | Value | @@ -10700,7 +10904,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|37500| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10712,6 +10915,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|355262Mi| |pods|737| + |vpc.amazonaws.com/pod-eni|107| ### `m7a.32xlarge` #### Labels | Label | Value | @@ -10724,7 +10928,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|32xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10736,6 +10939,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|476504Mi| |pods|737| + |vpc.amazonaws.com/pod-eni|107| ### `m7a.48xlarge` #### Labels | Label | Value | @@ -10748,7 +10952,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|48xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10760,6 +10963,8 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|718987Mi| |pods|737| + |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| ### `m7a.metal-48xl` #### Labels | Label | Value | @@ -10772,7 +10977,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal-48xl| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -10784,6 +10988,8 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|718987Mi| |pods|737| + |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| ## m7g Family ### `m7g.medium` #### Labels @@ -10797,7 +11003,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|4096| |karpenter.k8s.aws/instance-network-bandwidth|520| - |karpenter.k8s.aws/instance-pods|8| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -10822,7 +11027,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|937| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -10847,7 +11051,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|1876| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -10872,7 +11075,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|3750| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -10897,7 +11099,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|7500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -10922,7 +11123,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|15000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -10947,7 +11147,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|22500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -10972,7 +11171,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|30000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -10984,6 +11182,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|233962Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ### `m7g.metal` #### Labels @@ -10997,7 +11196,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|30000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -11009,6 +11207,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|233962Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## m7gd Family ### `m7gd.medium` @@ -11024,7 +11223,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|59| |karpenter.k8s.aws/instance-memory|4096| |karpenter.k8s.aws/instance-network-bandwidth|520| - |karpenter.k8s.aws/instance-pods|8| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -11050,7 +11248,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|118| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|937| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -11076,7 +11273,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|237| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|1876| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -11102,7 +11298,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|474| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|3750| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -11128,7 +11323,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|950| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|7500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -11154,7 +11348,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1900| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|15000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -11180,7 +11373,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|2850| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|22500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -11206,7 +11398,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|30000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -11218,7 +11409,32 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|233962Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| +### `m7gd.metal` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|m| + |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|m7gd| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|| + |karpenter.k8s.aws/instance-local-nvme|3800| + |karpenter.k8s.aws/instance-memory|262144| + |karpenter.k8s.aws/instance-size|metal| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|m7gd.metal| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|63770m| + |ephemeral-storage|17Gi| + |memory|233962Mi| + |pods|737| + |vpc.amazonaws.com/efa|1| ## m7i Family ### `m7i.large` #### Labels @@ -11232,7 +11448,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|781| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11244,6 +11459,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|6903Mi| |pods|29| + |vpc.amazonaws.com/pod-eni|9| ### `m7i.xlarge` #### Labels | Label | Value | @@ -11256,7 +11472,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|1562| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11268,6 +11483,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|14162Mi| |pods|58| + |vpc.amazonaws.com/pod-eni|18| ### `m7i.2xlarge` #### Labels | Label | Value | @@ -11280,7 +11496,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|3125| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11292,6 +11507,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|29317Mi| |pods|58| + |vpc.amazonaws.com/pod-eni|38| ### `m7i.4xlarge` #### Labels | Label | Value | @@ -11304,7 +11520,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|6250| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11316,6 +11531,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|57691Mi| |pods|234| + |vpc.amazonaws.com/pod-eni|54| ### `m7i.8xlarge` #### Labels | Label | Value | @@ -11328,7 +11544,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11340,6 +11555,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|118312Mi| |pods|234| + |vpc.amazonaws.com/pod-eni|84| ### `m7i.12xlarge` #### Labels | Label | Value | @@ -11352,7 +11568,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|18750| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11364,6 +11579,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|178933Mi| |pods|234| + |vpc.amazonaws.com/pod-eni|114| ### `m7i.16xlarge` #### Labels | Label | Value | @@ -11376,7 +11592,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11388,6 +11603,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|234021Mi| |pods|737| + |vpc.amazonaws.com/pod-eni|107| ### `m7i.24xlarge` #### Labels | Label | Value | @@ -11400,7 +11616,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|37500| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11412,6 +11627,31 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|355262Mi| |pods|737| + |vpc.amazonaws.com/pod-eni|107| +### `m7i.metal-24xl` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|m| + |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|m7i| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|| + |karpenter.k8s.aws/instance-memory|393216| + |karpenter.k8s.aws/instance-network-bandwidth|37500| + |karpenter.k8s.aws/instance-size|metal-24xl| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|m7i.metal-24xl| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|95690m| + |ephemeral-storage|17Gi| + |memory|355262Mi| + |pods|737| + |vpc.amazonaws.com/pod-eni|107| ### `m7i.48xlarge` #### Labels | Label | Value | @@ -11424,7 +11664,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|48xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11436,6 +11675,33 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|718987Mi| |pods|737| + |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| +### `m7i.metal-48xl` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|m| + |karpenter.k8s.aws/instance-cpu|192| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|m7i| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|| + |karpenter.k8s.aws/instance-memory|786432| + |karpenter.k8s.aws/instance-network-bandwidth|50000| + |karpenter.k8s.aws/instance-size|metal-48xl| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|m7i.metal-48xl| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|191450m| + |ephemeral-storage|17Gi| + |memory|718987Mi| + |pods|737| + |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| ## m7i-flex Family ### `m7i-flex.large` #### Labels @@ -11449,7 +11715,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|390| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11461,6 +11726,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|6903Mi| |pods|29| + |vpc.amazonaws.com/pod-eni|3| ### `m7i-flex.xlarge` #### Labels | Label | Value | @@ -11473,7 +11739,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|781| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11485,6 +11750,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|14162Mi| |pods|58| + |vpc.amazonaws.com/pod-eni|8| ### `m7i-flex.2xlarge` #### Labels | Label | Value | @@ -11497,7 +11763,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|1562| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11509,6 +11774,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|29317Mi| |pods|58| + |vpc.amazonaws.com/pod-eni|18| ### `m7i-flex.4xlarge` #### Labels | Label | Value | @@ -11521,7 +11787,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|3125| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11533,6 +11798,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|57691Mi| |pods|234| + |vpc.amazonaws.com/pod-eni|34| ### `m7i-flex.8xlarge` #### Labels | Label | Value | @@ -11545,7 +11811,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|6250| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11557,6 +11822,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|118312Mi| |pods|234| + |vpc.amazonaws.com/pod-eni|54| ## p2 Family ### `p2.xlarge` #### Labels @@ -11573,7 +11839,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-gpu-name|k80| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|62464| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11602,7 +11867,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|499712| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11631,7 +11895,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|749568| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11660,7 +11923,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-gpu-name|v100| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|62464| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11673,7 +11935,6 @@ below are the resources available with some assumptions and after the instance o |memory|56786Mi| |nvidia.com/gpu|1| |pods|58| - |vpc.amazonaws.com/pod-eni|38| ### `p3.8xlarge` #### Labels | Label | Value | @@ -11690,7 +11951,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|249856| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11703,7 +11963,6 @@ below are the resources available with some assumptions and after the instance o |memory|228187Mi| |nvidia.com/gpu|4| |pods|234| - |vpc.amazonaws.com/pod-eni|54| ### `p3.16xlarge` #### Labels | Label | Value | @@ -11720,7 +11979,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|499712| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11733,7 +11991,6 @@ below are the resources available with some assumptions and after the instance o |memory|459304Mi| |nvidia.com/gpu|8| |pods|234| - |vpc.amazonaws.com/pod-eni|114| ## p3dn Family ### `p3dn.24xlarge` #### Labels @@ -11752,7 +12009,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1800| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11765,6 +12021,7 @@ below are the resources available with some assumptions and after the instance o |memory|718987Mi| |nvidia.com/gpu|8| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## p4d Family ### `p4d.24xlarge` @@ -11784,7 +12041,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|8000| |karpenter.k8s.aws/instance-memory|1179648| |karpenter.k8s.aws/instance-network-bandwidth|400000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11797,6 +12053,7 @@ below are the resources available with some assumptions and after the instance o |memory|1082712Mi| |nvidia.com/gpu|8| |pods|737| + |vpc.amazonaws.com/efa|4| |vpc.amazonaws.com/pod-eni|62| ## p5 Family ### `p5.48xlarge` @@ -11816,7 +12073,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|30400| |karpenter.k8s.aws/instance-memory|2097152| |karpenter.k8s.aws/instance-network-bandwidth|3200000| - |karpenter.k8s.aws/instance-pods|100| |karpenter.k8s.aws/instance-size|48xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11829,6 +12085,7 @@ below are the resources available with some assumptions and after the instance o |memory|1938410Mi| |nvidia.com/gpu|8| |pods|100| + |vpc.amazonaws.com/efa|32| |vpc.amazonaws.com/pod-eni|120| ## r3 Family ### `r3.large` @@ -11842,7 +12099,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|3| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|15360| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11865,7 +12121,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|3| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|31232| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11888,7 +12143,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|3| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|62464| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11911,7 +12165,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|3| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|124928| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11935,7 +12188,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|249856| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11960,7 +12212,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|15616| |karpenter.k8s.aws/instance-network-bandwidth|750| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -11984,7 +12235,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|31232| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12008,7 +12258,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|62464| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12032,7 +12281,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|124928| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12056,7 +12304,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|249856| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12080,7 +12327,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|499712| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12105,7 +12351,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|750| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12130,7 +12375,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12155,7 +12399,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12180,7 +12423,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12205,7 +12447,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12230,7 +12471,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|12000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12255,7 +12495,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|20000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12280,7 +12519,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12305,7 +12543,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12331,7 +12568,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|750| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12356,7 +12592,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12381,7 +12616,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12406,7 +12640,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12431,7 +12664,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|7500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12456,7 +12688,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12481,7 +12712,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|12000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12506,7 +12736,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|20000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12533,7 +12762,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|75| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|750| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12559,7 +12787,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|150| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12585,7 +12812,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|300| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12611,7 +12837,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|600| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12637,7 +12862,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1200| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|7500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12663,7 +12887,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1800| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12689,7 +12912,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|2400| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|12000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12715,7 +12937,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3600| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|20000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12741,7 +12962,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|750| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12766,7 +12986,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12791,7 +13010,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12816,7 +13034,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12841,7 +13058,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12866,7 +13082,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|12000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12891,7 +13106,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|20000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12916,7 +13130,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12941,7 +13154,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12968,7 +13180,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|75| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|750| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -12994,7 +13205,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|150| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13020,7 +13230,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|300| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13046,7 +13255,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|600| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13072,7 +13280,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1200| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13098,7 +13305,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1800| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|12000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13124,7 +13330,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|2400| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|20000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13150,7 +13355,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3600| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13176,7 +13380,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3600| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13203,7 +13406,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|75| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|2100| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13229,7 +13431,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|150| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|4100| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13255,7 +13456,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|300| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|8125| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13281,7 +13481,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|600| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|16250| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13307,7 +13506,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1200| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13333,7 +13531,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1800| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13359,7 +13556,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|2400| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|75000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13385,7 +13581,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3600| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13397,6 +13592,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|718987Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ### `r5dn.metal` #### Labels @@ -13411,7 +13607,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3600| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13423,6 +13618,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|718987Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## r5n Family ### `r5n.large` @@ -13437,7 +13633,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|2100| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13462,7 +13657,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|4100| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13487,7 +13681,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|8125| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13512,7 +13705,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|16250| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13537,7 +13729,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13562,7 +13753,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13587,7 +13777,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|75000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13612,7 +13801,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13624,6 +13812,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|718987Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ### `r5n.metal` #### Labels @@ -13637,7 +13826,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13649,6 +13837,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|718987Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## r6a Family ### `r6a.large` @@ -13663,7 +13852,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|781| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13688,7 +13876,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|1562| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13713,7 +13900,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|3125| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13738,7 +13924,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|6250| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13763,7 +13948,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13788,7 +13972,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|18750| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13813,7 +13996,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13838,7 +14020,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|37500| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13863,7 +14044,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|1048576| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|32xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13888,7 +14068,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|1572864| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|48xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13900,6 +14079,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|1446437Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ### `r6a.metal` #### Labels @@ -13913,7 +14093,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|1572864| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -13925,6 +14104,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|1446437Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## r6g Family ### `r6g.medium` @@ -13939,7 +14119,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|500| - |karpenter.k8s.aws/instance-pods|8| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -13964,7 +14143,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|750| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -13989,7 +14167,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -14014,7 +14191,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -14039,7 +14215,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -14064,7 +14239,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|12000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -14089,7 +14263,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|20000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -14114,7 +14287,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -14139,7 +14311,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -14166,7 +14337,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|59| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|500| - |karpenter.k8s.aws/instance-pods|8| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -14192,7 +14362,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|118| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|750| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -14218,7 +14387,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|237| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -14244,7 +14412,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|474| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -14270,7 +14437,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|950| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -14296,7 +14462,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1900| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|12000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -14322,7 +14487,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|2850| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|20000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -14348,7 +14512,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -14374,7 +14537,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -14400,7 +14562,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|781| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -14425,7 +14586,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|1562| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -14450,7 +14610,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|3125| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -14475,7 +14634,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|6250| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -14500,7 +14658,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -14525,7 +14682,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|18750| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -14550,7 +14706,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -14575,7 +14730,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|37500| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -14600,7 +14754,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|1048576| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|32xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -14612,6 +14765,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|961470Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ### `r6i.metal` #### Labels @@ -14625,7 +14779,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|1048576| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -14637,6 +14790,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|961470Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## r6id Family ### `r6id.large` @@ -14652,7 +14806,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|118| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|781| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -14678,7 +14831,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|237| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|1562| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -14704,7 +14856,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|474| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|3125| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -14730,7 +14881,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|950| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|6250| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -14756,7 +14906,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1900| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -14782,7 +14931,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|2850| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|18750| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -14808,7 +14956,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -14834,7 +14981,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|5700| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|37500| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -14860,7 +15006,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|7600| |karpenter.k8s.aws/instance-memory|1048576| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|32xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -14872,6 +15017,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|961470Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ### `r6id.metal` #### Labels @@ -14886,7 +15032,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|7600| |karpenter.k8s.aws/instance-memory|1048576| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -14898,6 +15043,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|961470Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## r6idn Family ### `r6idn.large` @@ -14913,7 +15059,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|118| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|3125| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -14939,7 +15084,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|237| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|6250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -14965,7 +15109,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|474| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -14991,7 +15134,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|950| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15017,7 +15159,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1900| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15043,7 +15184,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|2850| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|75000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15069,7 +15209,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15095,7 +15234,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|5700| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|150000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15121,7 +15259,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|7600| |karpenter.k8s.aws/instance-memory|1048576| |karpenter.k8s.aws/instance-network-bandwidth|200000| - |karpenter.k8s.aws/instance-pods|345| |karpenter.k8s.aws/instance-size|32xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15133,6 +15270,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|965782Mi| |pods|345| + |vpc.amazonaws.com/efa|2| |vpc.amazonaws.com/pod-eni|108| ### `r6idn.metal` #### Labels @@ -15147,7 +15285,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|7600| |karpenter.k8s.aws/instance-memory|1048576| |karpenter.k8s.aws/instance-network-bandwidth|200000| - |karpenter.k8s.aws/instance-pods|345| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15159,6 +15296,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|965782Mi| |pods|345| + |vpc.amazonaws.com/efa|2| |vpc.amazonaws.com/pod-eni|108| ## r6in Family ### `r6in.large` @@ -15173,7 +15311,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|3125| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15198,7 +15335,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|6250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15223,7 +15359,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15248,7 +15383,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15273,7 +15407,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15298,7 +15431,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|75000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15323,7 +15455,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15348,7 +15479,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|150000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15373,7 +15503,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|1048576| |karpenter.k8s.aws/instance-network-bandwidth|200000| - |karpenter.k8s.aws/instance-pods|345| |karpenter.k8s.aws/instance-size|32xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15385,6 +15514,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|965782Mi| |pods|345| + |vpc.amazonaws.com/efa|2| |vpc.amazonaws.com/pod-eni|108| ### `r6in.metal` #### Labels @@ -15398,7 +15528,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|1048576| |karpenter.k8s.aws/instance-network-bandwidth|200000| - |karpenter.k8s.aws/instance-pods|345| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15410,252 +15539,316 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|965782Mi| |pods|345| + |vpc.amazonaws.com/efa|2| |vpc.amazonaws.com/pod-eni|108| -## r7g Family -### `r7g.medium` +## r7a Family +### `r7a.medium` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|1| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| - |karpenter.k8s.aws/instance-family|r7g| + |karpenter.k8s.aws/instance-family|r7a| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| - |karpenter.k8s.aws/instance-network-bandwidth|520| - |karpenter.k8s.aws/instance-pods|8| + |karpenter.k8s.aws/instance-network-bandwidth|390| |karpenter.k8s.aws/instance-size|medium| - |kubernetes.io/arch|arm64| + |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|r7g.medium| + |node.kubernetes.io/instance-type|r7a.medium| #### Resources | Resource | Quantity | |--|--| |cpu|940m| |ephemeral-storage|17Gi| - |memory|7075Mi| + |memory|7134Mi| |pods|8| |vpc.amazonaws.com/pod-eni|4| -### `r7g.large` +### `r7a.large` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| - |karpenter.k8s.aws/instance-family|r7g| + |karpenter.k8s.aws/instance-family|r7a| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| - |karpenter.k8s.aws/instance-network-bandwidth|937| - |karpenter.k8s.aws/instance-pods|29| + |karpenter.k8s.aws/instance-network-bandwidth|781| |karpenter.k8s.aws/instance-size|large| - |kubernetes.io/arch|arm64| + |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|r7g.large| + |node.kubernetes.io/instance-type|r7a.large| #### Resources | Resource | Quantity | |--|--| |cpu|1930m| |ephemeral-storage|17Gi| - |memory|14422Mi| + |memory|14481Mi| |pods|29| |vpc.amazonaws.com/pod-eni|9| -### `r7g.xlarge` +### `r7a.xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| - |karpenter.k8s.aws/instance-family|r7g| + |karpenter.k8s.aws/instance-family|r7a| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| - |karpenter.k8s.aws/instance-network-bandwidth|1876| - |karpenter.k8s.aws/instance-pods|58| + |karpenter.k8s.aws/instance-network-bandwidth|1562| |karpenter.k8s.aws/instance-size|xlarge| - |kubernetes.io/arch|arm64| + |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|r7g.xlarge| + |node.kubernetes.io/instance-type|r7a.xlarge| #### Resources | Resource | Quantity | |--|--| |cpu|3920m| |ephemeral-storage|17Gi| - |memory|29258Mi| + |memory|29317Mi| |pods|58| |vpc.amazonaws.com/pod-eni|18| -### `r7g.2xlarge` +### `r7a.2xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| - |karpenter.k8s.aws/instance-family|r7g| + |karpenter.k8s.aws/instance-family|r7a| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|65536| - |karpenter.k8s.aws/instance-network-bandwidth|3750| - |karpenter.k8s.aws/instance-pods|58| + |karpenter.k8s.aws/instance-network-bandwidth|3125| |karpenter.k8s.aws/instance-size|2xlarge| - |kubernetes.io/arch|arm64| + |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|r7g.2xlarge| + |node.kubernetes.io/instance-type|r7a.2xlarge| #### Resources | Resource | Quantity | |--|--| |cpu|7910m| |ephemeral-storage|17Gi| - |memory|59568Mi| + |memory|59627Mi| |pods|58| |vpc.amazonaws.com/pod-eni|38| -### `r7g.4xlarge` +### `r7a.4xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| - |karpenter.k8s.aws/instance-family|r7g| + |karpenter.k8s.aws/instance-family|r7a| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|131072| - |karpenter.k8s.aws/instance-network-bandwidth|7500| - |karpenter.k8s.aws/instance-pods|234| + |karpenter.k8s.aws/instance-network-bandwidth|6250| |karpenter.k8s.aws/instance-size|4xlarge| - |kubernetes.io/arch|arm64| + |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|r7g.4xlarge| + |node.kubernetes.io/instance-type|r7a.4xlarge| #### Resources | Resource | Quantity | |--|--| |cpu|15890m| |ephemeral-storage|17Gi| - |memory|118253Mi| + |memory|118312Mi| |pods|234| |vpc.amazonaws.com/pod-eni|54| -### `r7g.8xlarge` +### `r7a.8xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| - |karpenter.k8s.aws/instance-family|r7g| + |karpenter.k8s.aws/instance-family|r7a| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| - |karpenter.k8s.aws/instance-network-bandwidth|15000| - |karpenter.k8s.aws/instance-pods|234| + |karpenter.k8s.aws/instance-network-bandwidth|12500| |karpenter.k8s.aws/instance-size|8xlarge| - |kubernetes.io/arch|arm64| + |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|r7g.8xlarge| + |node.kubernetes.io/instance-type|r7a.8xlarge| #### Resources | Resource | Quantity | |--|--| |cpu|31850m| |ephemeral-storage|17Gi| - |memory|239495Mi| + |memory|239554Mi| |pods|234| - |vpc.amazonaws.com/pod-eni|54| -### `r7g.12xlarge` + |vpc.amazonaws.com/pod-eni|84| +### `r7a.12xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| - |karpenter.k8s.aws/instance-family|r7g| + |karpenter.k8s.aws/instance-family|r7a| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|393216| - |karpenter.k8s.aws/instance-network-bandwidth|22500| - |karpenter.k8s.aws/instance-pods|234| + |karpenter.k8s.aws/instance-network-bandwidth|18750| |karpenter.k8s.aws/instance-size|12xlarge| - |kubernetes.io/arch|arm64| + |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|r7g.12xlarge| + |node.kubernetes.io/instance-type|r7a.12xlarge| #### Resources | Resource | Quantity | |--|--| |cpu|47810m| |ephemeral-storage|17Gi| - |memory|360736Mi| + |memory|360795Mi| |pods|234| - |vpc.amazonaws.com/pod-eni|54| -### `r7g.16xlarge` + |vpc.amazonaws.com/pod-eni|114| +### `r7a.16xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| - |karpenter.k8s.aws/instance-family|r7g| + |karpenter.k8s.aws/instance-family|r7a| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|524288| - |karpenter.k8s.aws/instance-network-bandwidth|30000| - |karpenter.k8s.aws/instance-pods|737| + |karpenter.k8s.aws/instance-network-bandwidth|25000| |karpenter.k8s.aws/instance-size|16xlarge| - |kubernetes.io/arch|arm64| + |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|r7g.16xlarge| + |node.kubernetes.io/instance-type|r7a.16xlarge| #### Resources | Resource | Quantity | |--|--| |cpu|63770m| |ephemeral-storage|17Gi| - |memory|476445Mi| + |memory|476504Mi| |pods|737| |vpc.amazonaws.com/pod-eni|107| -### `r7g.metal` +### `r7a.24xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| - |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-cpu|96| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| - |karpenter.k8s.aws/instance-family|r7g| + |karpenter.k8s.aws/instance-family|r7a| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|786432| + |karpenter.k8s.aws/instance-network-bandwidth|37500| + |karpenter.k8s.aws/instance-size|24xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7a.24xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|95690m| + |ephemeral-storage|17Gi| + |memory|718987Mi| + |pods|737| + |vpc.amazonaws.com/pod-eni|107| +### `r7a.32xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7a| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|1048576| + |karpenter.k8s.aws/instance-network-bandwidth|50000| + |karpenter.k8s.aws/instance-size|32xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7a.32xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|127610m| + |ephemeral-storage|17Gi| + |memory|961470Mi| + |pods|737| + |vpc.amazonaws.com/pod-eni|107| +### `r7a.48xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|192| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7a| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|1572864| + |karpenter.k8s.aws/instance-network-bandwidth|50000| + |karpenter.k8s.aws/instance-size|48xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7a.48xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|191450m| + |ephemeral-storage|17Gi| + |memory|1446437Mi| + |pods|737| + |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| +### `r7a.metal-48xl` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|192| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7a| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|| - |karpenter.k8s.aws/instance-memory|524288| - |karpenter.k8s.aws/instance-network-bandwidth|30000| - |karpenter.k8s.aws/instance-pods|737| - |karpenter.k8s.aws/instance-size|metal| - |kubernetes.io/arch|arm64| + |karpenter.k8s.aws/instance-memory|1572864| + |karpenter.k8s.aws/instance-network-bandwidth|50000| + |karpenter.k8s.aws/instance-size|metal-48xl| + |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|r7g.metal| + |node.kubernetes.io/instance-type|r7a.metal-48xl| #### Resources | Resource | Quantity | |--|--| - |cpu|63770m| + |cpu|191450m| |ephemeral-storage|17Gi| - |memory|476445Mi| + |memory|1446437Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| -## r7gd Family -### `r7gd.medium` +## r7g Family +### `r7g.medium` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|1| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| - |karpenter.k8s.aws/instance-family|r7gd| + |karpenter.k8s.aws/instance-family|r7g| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| - |karpenter.k8s.aws/instance-local-nvme|59| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|520| - |karpenter.k8s.aws/instance-pods|8| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|r7gd.medium| + |node.kubernetes.io/instance-type|r7g.medium| #### Resources | Resource | Quantity | |--|--| @@ -15664,24 +15857,22 @@ below are the resources available with some assumptions and after the instance o |memory|7075Mi| |pods|8| |vpc.amazonaws.com/pod-eni|4| -### `r7gd.large` +### `r7g.large` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|2| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| - |karpenter.k8s.aws/instance-family|r7gd| + |karpenter.k8s.aws/instance-family|r7g| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| - |karpenter.k8s.aws/instance-local-nvme|118| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|937| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|r7gd.large| + |node.kubernetes.io/instance-type|r7g.large| #### Resources | Resource | Quantity | |--|--| @@ -15690,24 +15881,22 @@ below are the resources available with some assumptions and after the instance o |memory|14422Mi| |pods|29| |vpc.amazonaws.com/pod-eni|9| -### `r7gd.xlarge` +### `r7g.xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|4| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| - |karpenter.k8s.aws/instance-family|r7gd| + |karpenter.k8s.aws/instance-family|r7g| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| - |karpenter.k8s.aws/instance-local-nvme|237| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|1876| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|r7gd.xlarge| + |node.kubernetes.io/instance-type|r7g.xlarge| #### Resources | Resource | Quantity | |--|--| @@ -15716,24 +15905,22 @@ below are the resources available with some assumptions and after the instance o |memory|29258Mi| |pods|58| |vpc.amazonaws.com/pod-eni|18| -### `r7gd.2xlarge` +### `r7g.2xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|8| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| - |karpenter.k8s.aws/instance-family|r7gd| + |karpenter.k8s.aws/instance-family|r7g| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| - |karpenter.k8s.aws/instance-local-nvme|474| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|3750| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|r7gd.2xlarge| + |node.kubernetes.io/instance-type|r7g.2xlarge| #### Resources | Resource | Quantity | |--|--| @@ -15742,24 +15929,22 @@ below are the resources available with some assumptions and after the instance o |memory|59568Mi| |pods|58| |vpc.amazonaws.com/pod-eni|38| -### `r7gd.4xlarge` +### `r7g.4xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|16| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| - |karpenter.k8s.aws/instance-family|r7gd| + |karpenter.k8s.aws/instance-family|r7g| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| - |karpenter.k8s.aws/instance-local-nvme|950| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|7500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|r7gd.4xlarge| + |node.kubernetes.io/instance-type|r7g.4xlarge| #### Resources | Resource | Quantity | |--|--| @@ -15768,76 +15953,95 @@ below are the resources available with some assumptions and after the instance o |memory|118253Mi| |pods|234| |vpc.amazonaws.com/pod-eni|54| -### `r7gd.8xlarge` +### `r7g.8xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|32| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| - |karpenter.k8s.aws/instance-family|r7gd| + |karpenter.k8s.aws/instance-family|r7g| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| - |karpenter.k8s.aws/instance-local-nvme|1900| - |karpenter.k8s.aws/instance-memory|196608| + |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|15000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|r7gd.8xlarge| + |node.kubernetes.io/instance-type|r7g.8xlarge| #### Resources | Resource | Quantity | |--|--| |cpu|31850m| |ephemeral-storage|17Gi| - |memory|178874Mi| + |memory|239495Mi| |pods|234| |vpc.amazonaws.com/pod-eni|54| -### `r7gd.12xlarge` +### `r7g.12xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|48| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| - |karpenter.k8s.aws/instance-family|r7gd| + |karpenter.k8s.aws/instance-family|r7g| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| - |karpenter.k8s.aws/instance-local-nvme|2850| - |karpenter.k8s.aws/instance-memory|262144| + |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|22500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|r7gd.12xlarge| + |node.kubernetes.io/instance-type|r7g.12xlarge| #### Resources | Resource | Quantity | |--|--| |cpu|47810m| |ephemeral-storage|17Gi| - |memory|239495Mi| + |memory|360736Mi| |pods|234| |vpc.amazonaws.com/pod-eni|54| -### `r7gd.16xlarge` +### `r7g.16xlarge` #### Labels | Label | Value | |--|--| |karpenter.k8s.aws/instance-category|r| |karpenter.k8s.aws/instance-cpu|64| |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| - |karpenter.k8s.aws/instance-family|r7gd| + |karpenter.k8s.aws/instance-family|r7g| |karpenter.k8s.aws/instance-generation|7| |karpenter.k8s.aws/instance-hypervisor|nitro| - |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|30000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| - |node.kubernetes.io/instance-type|r7gd.16xlarge| + |node.kubernetes.io/instance-type|r7g.16xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|63770m| + |ephemeral-storage|17Gi| + |memory|476445Mi| + |pods|737| + |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| +### `r7g.metal` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7g| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|| + |karpenter.k8s.aws/instance-memory|524288| + |karpenter.k8s.aws/instance-network-bandwidth|30000| + |karpenter.k8s.aws/instance-size|metal| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7g.metal| #### Resources | Resource | Quantity | |--|--| @@ -15845,6 +16049,719 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|476445Mi| |pods|737| + |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| +## r7gd Family +### `r7gd.medium` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|1| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7gd| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-local-nvme|59| + |karpenter.k8s.aws/instance-memory|8192| + |karpenter.k8s.aws/instance-network-bandwidth|520| + |karpenter.k8s.aws/instance-size|medium| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7gd.medium| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|940m| + |ephemeral-storage|17Gi| + |memory|7075Mi| + |pods|8| + |vpc.amazonaws.com/pod-eni|4| +### `r7gd.large` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7gd| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-local-nvme|118| + |karpenter.k8s.aws/instance-memory|16384| + |karpenter.k8s.aws/instance-network-bandwidth|937| + |karpenter.k8s.aws/instance-size|large| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7gd.large| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|1930m| + |ephemeral-storage|17Gi| + |memory|14422Mi| + |pods|29| + |vpc.amazonaws.com/pod-eni|9| +### `r7gd.xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7gd| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-local-nvme|237| + |karpenter.k8s.aws/instance-memory|32768| + |karpenter.k8s.aws/instance-network-bandwidth|1876| + |karpenter.k8s.aws/instance-size|xlarge| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7gd.xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|3920m| + |ephemeral-storage|17Gi| + |memory|29258Mi| + |pods|58| + |vpc.amazonaws.com/pod-eni|18| +### `r7gd.2xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7gd| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-local-nvme|474| + |karpenter.k8s.aws/instance-memory|65536| + |karpenter.k8s.aws/instance-network-bandwidth|3750| + |karpenter.k8s.aws/instance-size|2xlarge| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7gd.2xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|7910m| + |ephemeral-storage|17Gi| + |memory|59568Mi| + |pods|58| + |vpc.amazonaws.com/pod-eni|38| +### `r7gd.4xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7gd| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-local-nvme|950| + |karpenter.k8s.aws/instance-memory|131072| + |karpenter.k8s.aws/instance-network-bandwidth|7500| + |karpenter.k8s.aws/instance-size|4xlarge| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7gd.4xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|15890m| + |ephemeral-storage|17Gi| + |memory|118253Mi| + |pods|234| + |vpc.amazonaws.com/pod-eni|54| +### `r7gd.8xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7gd| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-local-nvme|1900| + |karpenter.k8s.aws/instance-memory|262144| + |karpenter.k8s.aws/instance-network-bandwidth|15000| + |karpenter.k8s.aws/instance-size|8xlarge| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7gd.8xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|31850m| + |ephemeral-storage|17Gi| + |memory|239495Mi| + |pods|234| + |vpc.amazonaws.com/pod-eni|54| +### `r7gd.12xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7gd| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-local-nvme|2850| + |karpenter.k8s.aws/instance-memory|393216| + |karpenter.k8s.aws/instance-network-bandwidth|22500| + |karpenter.k8s.aws/instance-size|12xlarge| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7gd.12xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|47810m| + |ephemeral-storage|17Gi| + |memory|360736Mi| + |pods|234| + |vpc.amazonaws.com/pod-eni|54| +### `r7gd.16xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7gd| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-local-nvme|3800| + |karpenter.k8s.aws/instance-memory|524288| + |karpenter.k8s.aws/instance-network-bandwidth|30000| + |karpenter.k8s.aws/instance-size|16xlarge| + |kubernetes.io/arch|arm64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7gd.16xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|63770m| + |ephemeral-storage|17Gi| + |memory|476445Mi| + |pods|737| + |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| +## r7i Family +### `r7i.large` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7i| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|16384| + |karpenter.k8s.aws/instance-network-bandwidth|781| + |karpenter.k8s.aws/instance-size|large| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7i.large| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|1930m| + |ephemeral-storage|17Gi| + |memory|14481Mi| + |pods|29| + |vpc.amazonaws.com/pod-eni|9| +### `r7i.xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7i| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|32768| + |karpenter.k8s.aws/instance-network-bandwidth|1562| + |karpenter.k8s.aws/instance-size|xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7i.xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|3920m| + |ephemeral-storage|17Gi| + |memory|29317Mi| + |pods|58| + |vpc.amazonaws.com/pod-eni|18| +### `r7i.2xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7i| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|65536| + |karpenter.k8s.aws/instance-network-bandwidth|3125| + |karpenter.k8s.aws/instance-size|2xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7i.2xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|7910m| + |ephemeral-storage|17Gi| + |memory|59627Mi| + |pods|58| + |vpc.amazonaws.com/pod-eni|38| +### `r7i.4xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7i| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|131072| + |karpenter.k8s.aws/instance-network-bandwidth|6250| + |karpenter.k8s.aws/instance-size|4xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7i.4xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|15890m| + |ephemeral-storage|17Gi| + |memory|118312Mi| + |pods|234| + |vpc.amazonaws.com/pod-eni|54| +### `r7i.8xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7i| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|262144| + |karpenter.k8s.aws/instance-network-bandwidth|12500| + |karpenter.k8s.aws/instance-size|8xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7i.8xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|31850m| + |ephemeral-storage|17Gi| + |memory|239554Mi| + |pods|234| + |vpc.amazonaws.com/pod-eni|84| +### `r7i.12xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7i| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|393216| + |karpenter.k8s.aws/instance-network-bandwidth|18750| + |karpenter.k8s.aws/instance-size|12xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7i.12xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|47810m| + |ephemeral-storage|17Gi| + |memory|360795Mi| + |pods|234| + |vpc.amazonaws.com/pod-eni|114| +### `r7i.16xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7i| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|524288| + |karpenter.k8s.aws/instance-network-bandwidth|25000| + |karpenter.k8s.aws/instance-size|16xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7i.16xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|63770m| + |ephemeral-storage|17Gi| + |memory|476504Mi| + |pods|737| + |vpc.amazonaws.com/pod-eni|107| +### `r7i.24xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7i| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|786432| + |karpenter.k8s.aws/instance-network-bandwidth|37500| + |karpenter.k8s.aws/instance-size|24xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7i.24xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|95690m| + |ephemeral-storage|17Gi| + |memory|718987Mi| + |pods|737| + |vpc.amazonaws.com/pod-eni|107| +### `r7i.metal-24xl` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|96| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7i| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|| + |karpenter.k8s.aws/instance-memory|786432| + |karpenter.k8s.aws/instance-network-bandwidth|37500| + |karpenter.k8s.aws/instance-size|metal-24xl| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7i.metal-24xl| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|95690m| + |ephemeral-storage|17Gi| + |memory|718987Mi| + |pods|737| + |vpc.amazonaws.com/pod-eni|107| +### `r7i.48xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|192| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7i| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|1572864| + |karpenter.k8s.aws/instance-network-bandwidth|50000| + |karpenter.k8s.aws/instance-size|48xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7i.48xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|191450m| + |ephemeral-storage|17Gi| + |memory|1446437Mi| + |pods|737| + |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| +### `r7i.metal-48xl` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|192| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7i| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|| + |karpenter.k8s.aws/instance-memory|1572864| + |karpenter.k8s.aws/instance-network-bandwidth|50000| + |karpenter.k8s.aws/instance-size|metal-48xl| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7i.metal-48xl| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|191450m| + |ephemeral-storage|17Gi| + |memory|1446437Mi| + |pods|737| + |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| +## r7iz Family +### `r7iz.large` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|2| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7iz| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|16384| + |karpenter.k8s.aws/instance-network-bandwidth|781| + |karpenter.k8s.aws/instance-size|large| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7iz.large| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|1930m| + |ephemeral-storage|17Gi| + |memory|14481Mi| + |pods|29| + |vpc.amazonaws.com/pod-eni|9| +### `r7iz.xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|4| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7iz| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|32768| + |karpenter.k8s.aws/instance-network-bandwidth|1562| + |karpenter.k8s.aws/instance-size|xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7iz.xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|3920m| + |ephemeral-storage|17Gi| + |memory|29317Mi| + |pods|58| + |vpc.amazonaws.com/pod-eni|18| +### `r7iz.2xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|8| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7iz| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|65536| + |karpenter.k8s.aws/instance-network-bandwidth|3125| + |karpenter.k8s.aws/instance-size|2xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7iz.2xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|7910m| + |ephemeral-storage|17Gi| + |memory|59627Mi| + |pods|58| + |vpc.amazonaws.com/pod-eni|38| +### `r7iz.4xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|16| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7iz| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|131072| + |karpenter.k8s.aws/instance-network-bandwidth|6250| + |karpenter.k8s.aws/instance-size|4xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7iz.4xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|15890m| + |ephemeral-storage|17Gi| + |memory|118312Mi| + |pods|234| + |vpc.amazonaws.com/pod-eni|54| +### `r7iz.8xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|32| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7iz| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|262144| + |karpenter.k8s.aws/instance-network-bandwidth|12500| + |karpenter.k8s.aws/instance-size|8xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7iz.8xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|31850m| + |ephemeral-storage|17Gi| + |memory|239554Mi| + |pods|234| + |vpc.amazonaws.com/pod-eni|84| +### `r7iz.12xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|48| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7iz| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|393216| + |karpenter.k8s.aws/instance-network-bandwidth|25000| + |karpenter.k8s.aws/instance-size|12xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7iz.12xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|47810m| + |ephemeral-storage|17Gi| + |memory|360795Mi| + |pods|234| + |vpc.amazonaws.com/pod-eni|114| +### `r7iz.16xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7iz| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|524288| + |karpenter.k8s.aws/instance-network-bandwidth|25000| + |karpenter.k8s.aws/instance-size|16xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7iz.16xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|63770m| + |ephemeral-storage|17Gi| + |memory|476504Mi| + |pods|737| + |vpc.amazonaws.com/pod-eni|107| +### `r7iz.metal-16xl` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|64| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7iz| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|| + |karpenter.k8s.aws/instance-memory|524288| + |karpenter.k8s.aws/instance-network-bandwidth|25000| + |karpenter.k8s.aws/instance-size|metal-16xl| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7iz.metal-16xl| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|63770m| + |ephemeral-storage|17Gi| + |memory|476504Mi| + |pods|737| + |vpc.amazonaws.com/pod-eni|107| +### `r7iz.32xlarge` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7iz| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|nitro| + |karpenter.k8s.aws/instance-memory|1048576| + |karpenter.k8s.aws/instance-network-bandwidth|50000| + |karpenter.k8s.aws/instance-size|32xlarge| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7iz.32xlarge| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|127610m| + |ephemeral-storage|17Gi| + |memory|961470Mi| + |pods|737| + |vpc.amazonaws.com/efa|1| + |vpc.amazonaws.com/pod-eni|107| +### `r7iz.metal-32xl` +#### Labels + | Label | Value | + |--|--| + |karpenter.k8s.aws/instance-category|r| + |karpenter.k8s.aws/instance-cpu|128| + |karpenter.k8s.aws/instance-encryption-in-transit-supported|true| + |karpenter.k8s.aws/instance-family|r7iz| + |karpenter.k8s.aws/instance-generation|7| + |karpenter.k8s.aws/instance-hypervisor|| + |karpenter.k8s.aws/instance-memory|1048576| + |karpenter.k8s.aws/instance-network-bandwidth|50000| + |karpenter.k8s.aws/instance-size|metal-32xl| + |kubernetes.io/arch|amd64| + |kubernetes.io/os|linux| + |node.kubernetes.io/instance-type|r7iz.metal-32xl| +#### Resources + | Resource | Quantity | + |--|--| + |cpu|127610m| + |ephemeral-storage|17Gi| + |memory|961470Mi| + |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## t1 Family ### `t1.micro` @@ -15858,7 +16775,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|1| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|627| - |karpenter.k8s.aws/instance-pods|4| |karpenter.k8s.aws/instance-size|micro| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15882,7 +16798,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|2| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|512| - |karpenter.k8s.aws/instance-pods|4| |karpenter.k8s.aws/instance-size|nano| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15905,7 +16820,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|2| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|1024| - |karpenter.k8s.aws/instance-pods|4| |karpenter.k8s.aws/instance-size|micro| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15928,7 +16842,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|2| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|2048| - |karpenter.k8s.aws/instance-pods|11| |karpenter.k8s.aws/instance-size|small| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15951,7 +16864,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|2| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|4096| - |karpenter.k8s.aws/instance-pods|17| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15974,7 +16886,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|2| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|8192| - |karpenter.k8s.aws/instance-pods|35| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -15997,7 +16908,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|2| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|16384| - |karpenter.k8s.aws/instance-pods|44| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16020,7 +16930,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-generation|2| |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|32768| - |karpenter.k8s.aws/instance-pods|44| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16045,7 +16954,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|512| |karpenter.k8s.aws/instance-network-bandwidth|32| - |karpenter.k8s.aws/instance-pods|4| |karpenter.k8s.aws/instance-size|nano| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16069,7 +16977,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|1024| |karpenter.k8s.aws/instance-network-bandwidth|64| - |karpenter.k8s.aws/instance-pods|4| |karpenter.k8s.aws/instance-size|micro| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16093,7 +17000,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|2048| |karpenter.k8s.aws/instance-network-bandwidth|128| - |karpenter.k8s.aws/instance-pods|11| |karpenter.k8s.aws/instance-size|small| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16117,7 +17023,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|4096| |karpenter.k8s.aws/instance-network-bandwidth|256| - |karpenter.k8s.aws/instance-pods|17| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16141,7 +17046,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|512| - |karpenter.k8s.aws/instance-pods|35| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16165,7 +17069,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|1024| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16189,7 +17092,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|2048| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16214,7 +17116,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|512| |karpenter.k8s.aws/instance-network-bandwidth|32| - |karpenter.k8s.aws/instance-pods|4| |karpenter.k8s.aws/instance-size|nano| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16238,7 +17139,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|1024| |karpenter.k8s.aws/instance-network-bandwidth|64| - |karpenter.k8s.aws/instance-pods|4| |karpenter.k8s.aws/instance-size|micro| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16262,7 +17162,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|2048| |karpenter.k8s.aws/instance-network-bandwidth|128| - |karpenter.k8s.aws/instance-pods|8| |karpenter.k8s.aws/instance-size|small| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16286,7 +17185,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|4096| |karpenter.k8s.aws/instance-network-bandwidth|256| - |karpenter.k8s.aws/instance-pods|17| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16310,7 +17208,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|512| - |karpenter.k8s.aws/instance-pods|35| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16334,7 +17231,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|1024| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16358,7 +17254,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|2048| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16383,7 +17278,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|512| |karpenter.k8s.aws/instance-network-bandwidth|32| - |karpenter.k8s.aws/instance-pods|4| |karpenter.k8s.aws/instance-size|nano| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -16407,7 +17301,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|1024| |karpenter.k8s.aws/instance-network-bandwidth|64| - |karpenter.k8s.aws/instance-pods|4| |karpenter.k8s.aws/instance-size|micro| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -16431,7 +17324,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|2048| |karpenter.k8s.aws/instance-network-bandwidth|128| - |karpenter.k8s.aws/instance-pods|11| |karpenter.k8s.aws/instance-size|small| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -16455,7 +17347,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|4096| |karpenter.k8s.aws/instance-network-bandwidth|256| - |karpenter.k8s.aws/instance-pods|17| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -16479,7 +17370,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|8192| |karpenter.k8s.aws/instance-network-bandwidth|512| - |karpenter.k8s.aws/instance-pods|35| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -16503,7 +17393,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|1024| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -16527,7 +17416,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|2048| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -16556,7 +17444,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|474| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|3125| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16586,7 +17473,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|7600| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|800000| - |karpenter.k8s.aws/instance-pods|247| |karpenter.k8s.aws/instance-size|32xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16599,6 +17485,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|481894Mi| |pods|247| + |vpc.amazonaws.com/efa|8| |vpc.amazonaws.com/pod-eni|82| ## trn1n Family ### `trn1n.32xlarge` @@ -16617,7 +17504,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|7600| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|1600000| - |karpenter.k8s.aws/instance-pods|247| |karpenter.k8s.aws/instance-size|32xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16630,6 +17516,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|481894Mi| |pods|247| + |vpc.amazonaws.com/efa|16| |vpc.amazonaws.com/pod-eni|120| ## u-12tb1 Family ### `u-12tb1.112xlarge` @@ -16644,7 +17531,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|12582912| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|112xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16669,7 +17555,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|18874368| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|112xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16694,7 +17579,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|25165824| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|112xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16719,7 +17603,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|3145728| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|56xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16745,7 +17628,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|6291456| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|56xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16769,7 +17651,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|6291456| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|112xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16794,7 +17675,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|9437184| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|112xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16819,7 +17699,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|24576| |karpenter.k8s.aws/instance-network-bandwidth|3120| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|3xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16844,7 +17723,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|49152| |karpenter.k8s.aws/instance-network-bandwidth|6250| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|6xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16869,7 +17747,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16881,6 +17758,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|173400Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## x1 Family ### `x1.16xlarge` @@ -16895,7 +17773,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|999424| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16919,7 +17796,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|1998848| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|32xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16944,7 +17820,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|124928| |karpenter.k8s.aws/instance-network-bandwidth|625| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16968,7 +17843,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|249856| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -16992,7 +17866,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|499712| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17016,7 +17889,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|999424| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17040,7 +17912,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|1998848| |karpenter.k8s.aws/instance-network-bandwidth|10000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17064,7 +17935,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|xen| |karpenter.k8s.aws/instance-memory|3997696| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|32xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17090,7 +17960,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|59| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|500| - |karpenter.k8s.aws/instance-pods|8| |karpenter.k8s.aws/instance-size|medium| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -17116,7 +17985,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|118| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|750| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -17142,7 +18010,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|237| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -17168,7 +18035,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|475| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -17194,7 +18060,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|950| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -17220,7 +18085,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1900| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|12000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -17246,7 +18110,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|2850| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|20000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -17272,7 +18135,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|1048576| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -17298,7 +18160,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|1048576| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|arm64| |kubernetes.io/os|linux| @@ -17325,7 +18186,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1900| |karpenter.k8s.aws/instance-memory|1048576| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17351,7 +18211,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|2850| |karpenter.k8s.aws/instance-memory|1572864| |karpenter.k8s.aws/instance-network-bandwidth|75000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17377,7 +18236,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|2097152| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|32xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17389,6 +18247,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|1931403Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ### `x2idn.metal` #### Labels @@ -17403,7 +18262,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|2097152| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17415,6 +18273,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|1931403Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## x2iedn Family ### `x2iedn.xlarge` @@ -17430,7 +18289,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|118| |karpenter.k8s.aws/instance-memory|131072| |karpenter.k8s.aws/instance-network-bandwidth|1875| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17456,7 +18314,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|237| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17482,7 +18339,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|475| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17508,7 +18364,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|950| |karpenter.k8s.aws/instance-memory|1048576| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17534,7 +18389,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1900| |karpenter.k8s.aws/instance-memory|2097152| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|16xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17560,7 +18414,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|2850| |karpenter.k8s.aws/instance-memory|3145728| |karpenter.k8s.aws/instance-network-bandwidth|75000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|24xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17586,7 +18439,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|4194304| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|32xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17598,6 +18450,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|3871269Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ### `x2iedn.metal` #### Labels @@ -17612,7 +18465,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|3800| |karpenter.k8s.aws/instance-memory|4194304| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17624,6 +18476,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|3871269Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## x2iezn Family ### `x2iezn.2xlarge` @@ -17638,7 +18491,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|262144| |karpenter.k8s.aws/instance-network-bandwidth|12500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17663,7 +18515,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|524288| |karpenter.k8s.aws/instance-network-bandwidth|15000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|4xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17688,7 +18539,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|786432| |karpenter.k8s.aws/instance-network-bandwidth|50000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|6xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17713,7 +18563,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|1048576| |karpenter.k8s.aws/instance-network-bandwidth|75000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|8xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17738,7 +18587,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|nitro| |karpenter.k8s.aws/instance-memory|1572864| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17750,6 +18598,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|1446437Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ### `x2iezn.metal` #### Labels @@ -17763,7 +18612,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-hypervisor|| |karpenter.k8s.aws/instance-memory|1572864| |karpenter.k8s.aws/instance-network-bandwidth|100000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17775,6 +18623,7 @@ below are the resources available with some assumptions and after the instance o |ephemeral-storage|17Gi| |memory|1446437Mi| |pods|737| + |vpc.amazonaws.com/efa|1| |vpc.amazonaws.com/pod-eni|107| ## z1d Family ### `z1d.large` @@ -17790,7 +18639,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|75| |karpenter.k8s.aws/instance-memory|16384| |karpenter.k8s.aws/instance-network-bandwidth|750| - |karpenter.k8s.aws/instance-pods|29| |karpenter.k8s.aws/instance-size|large| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17816,7 +18664,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|150| |karpenter.k8s.aws/instance-memory|32768| |karpenter.k8s.aws/instance-network-bandwidth|1250| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17842,7 +18689,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|300| |karpenter.k8s.aws/instance-memory|65536| |karpenter.k8s.aws/instance-network-bandwidth|2500| - |karpenter.k8s.aws/instance-pods|58| |karpenter.k8s.aws/instance-size|2xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17868,7 +18714,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|450| |karpenter.k8s.aws/instance-memory|98304| |karpenter.k8s.aws/instance-network-bandwidth|5000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|3xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17894,7 +18739,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|900| |karpenter.k8s.aws/instance-memory|196608| |karpenter.k8s.aws/instance-network-bandwidth|12000| - |karpenter.k8s.aws/instance-pods|234| |karpenter.k8s.aws/instance-size|6xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17920,7 +18764,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1800| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|12xlarge| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| @@ -17946,7 +18789,6 @@ below are the resources available with some assumptions and after the instance o |karpenter.k8s.aws/instance-local-nvme|1800| |karpenter.k8s.aws/instance-memory|393216| |karpenter.k8s.aws/instance-network-bandwidth|25000| - |karpenter.k8s.aws/instance-pods|737| |karpenter.k8s.aws/instance-size|metal| |kubernetes.io/arch|amd64| |kubernetes.io/os|linux| diff --git a/website/content/en/v0.30/concepts/metrics.md b/website/content/en/v0.35/reference/metrics.md similarity index 56% rename from website/content/en/v0.30/concepts/metrics.md rename to website/content/en/v0.35/reference/metrics.md index 112f63b0fe9c..b7cf1366f50f 100644 --- a/website/content/en/v0.30/concepts/metrics.md +++ b/website/content/en/v0.35/reference/metrics.md @@ -8,164 +8,186 @@ description: > --- Karpenter makes several metrics available in Prometheus format to allow monitoring cluster provisioning status. These metrics are available by default at `karpenter.karpenter.svc.cluster.local:8000/metrics` configurable via the `METRICS_PORT` environment variable documented [here](../settings) -## Controller Runtime Metrics +### `karpenter_build_info` +A metric with a constant '1' value labeled by version from which karpenter was built. -### `controller_runtime_active_workers` -Number of currently used workers per controller +## Nodepool Metrics -### `controller_runtime_max_concurrent_reconciles` -Maximum number of concurrent reconciles per controller +### `karpenter_nodepool_usage` +The nodepool usage is the amount of resources that have been provisioned by a particular nodepool. Labeled by nodepool name and resource type. -### `controller_runtime_reconcile_errors_total` -Total number of reconciliation errors per controller +### `karpenter_nodepool_limit` +The nodepool limits are the limits specified on the nodepool that restrict the quantity of resources provisioned. Labeled by nodepool name and resource type. -### `controller_runtime_reconcile_time_seconds` -Length of time per reconciliation per controller +## Nodes Metrics -### `controller_runtime_reconcile_total` -Total number of reconciliations per controller +### `karpenter_nodes_total_pod_requests` +Node total pod requests are the resources requested by non-DaemonSet pods bound to nodes. -## Consistency Metrics +### `karpenter_nodes_total_pod_limits` +Node total pod limits are the resources specified by non-DaemonSet pod limits. -### `karpenter_consistency_errors` -Number of consistency checks that have failed. +### `karpenter_nodes_total_daemon_requests` +Node total daemon requests are the resource requested by DaemonSet pods bound to nodes. + +### `karpenter_nodes_total_daemon_limits` +Node total daemon limits are the resources specified by DaemonSet pod limits. + +### `karpenter_nodes_termination_time_seconds` +The time taken between a node's deletion request and the removal of its finalizer -## Deprovisioning Metrics +### `karpenter_nodes_terminated` +Number of nodes terminated in total by Karpenter. Labeled by owning nodepool. -### `karpenter_deprovisioning_actions_performed` -Number of deprovisioning actions performed. Labeled by deprovisioner. +### `karpenter_nodes_system_overhead` +Node system daemon overhead are the resources reserved for system overhead, the difference between the node's capacity and allocatable values are reported by the status. -### `karpenter_deprovisioning_consolidation_timeouts` -Number of times the Consolidation algorithm has reached a timeout. Labeled by consolidation type. +### `karpenter_nodes_leases_deleted` +Number of deleted leaked leases. -### `karpenter_deprovisioning_eligible_machines` -Number of machines eligible for deprovisioning by Karpenter. Labeled by deprovisioner +### `karpenter_nodes_created` +Number of nodes created in total by Karpenter. Labeled by owning nodepool. -### `karpenter_deprovisioning_evaluation_duration_seconds` -Duration of the deprovisioning evaluation process in seconds. +### `karpenter_nodes_allocatable` +Node allocatable are the resources allocatable by nodes. -### `karpenter_deprovisioning_replacement_machine_initialized_seconds` -Amount of time required for a replacement machine to become initialized. +## Pods Metrics -### `karpenter_deprovisioning_replacement_machine_launch_failure_counter` -The number of times that Karpenter failed to launch a replacement node for deprovisioning. Labeled by deprovisioner. +### `karpenter_pods_state` +Pod state is the current state of pods. This metric can be used several ways as it is labeled by the pod name, namespace, owner, node, nodepool name, zone, architecture, capacity type, instance type and pod phase. -## Interruption Metrics +### `karpenter_pods_startup_time_seconds` +The time from pod creation until the pod is running. -### `karpenter_interruption_actions_performed` -Number of notification actions performed. Labeled by action +## Provisioner Metrics -### `karpenter_interruption_deleted_messages` -Count of messages deleted from the SQS queue. +### `karpenter_provisioner_scheduling_simulation_duration_seconds` +Duration of scheduling simulations used for deprovisioning and provisioning in seconds. -### `karpenter_interruption_message_latency_time_seconds` -Length of time between message creation in queue and an action taken on the message by the controller. +### `karpenter_provisioner_scheduling_duration_seconds` +Duration of scheduling process in seconds. -### `karpenter_interruption_received_messages` -Count of messages received from the SQS queue. Broken down by message type and whether the message was actionable. +## Nodeclaims Metrics -## Machines Metrics +### `karpenter_nodeclaims_terminated` +Number of nodeclaims terminated in total by Karpenter. Labeled by reason the nodeclaim was terminated and the owning nodepool. -### `karpenter_machines_created` -Number of machines created in total by Karpenter. Labeled by reason the machine was created and the owning provisioner. +### `karpenter_nodeclaims_registered` +Number of nodeclaims registered in total by Karpenter. Labeled by the owning nodepool. -### `karpenter_machines_disrupted` -Number of machines disrupted in total by Karpenter. Labeled by disruption type of the machine and the owning provisioner. +### `karpenter_nodeclaims_launched` +Number of nodeclaims launched in total by Karpenter. Labeled by the owning nodepool. -### `karpenter_machines_drifted` -Number of machine drifted reasons in total by Karpenter. Labeled by drift type of the machine and the owning provisioner.. +### `karpenter_nodeclaims_initialized` +Number of nodeclaims initialized in total by Karpenter. Labeled by the owning nodepool. -### `karpenter_machines_initialized` -Number of machines initialized in total by Karpenter. Labeled by the owning provisioner. +### `karpenter_nodeclaims_drifted` +Number of nodeclaims drifted reasons in total by Karpenter. Labeled by drift type of the nodeclaim and the owning nodepool. -### `karpenter_machines_launched` -Number of machines launched in total by Karpenter. Labeled by the owning provisioner. +### `karpenter_nodeclaims_disrupted` +Number of nodeclaims disrupted in total by Karpenter. Labeled by disruption type of the nodeclaim and the owning nodepool. -### `karpenter_machines_registered` -Number of machines registered in total by Karpenter. Labeled by the owning provisioner. +### `karpenter_nodeclaims_created` +Number of nodeclaims created in total by Karpenter. Labeled by reason the nodeclaim was created and the owning nodepool. -### `karpenter_machines_terminated` -Number of machines terminated in total by Karpenter. Labeled by reason the machine was terminated and the owning provisioner. +## Interruption Metrics -## Provisioner Metrics +### `karpenter_interruption_received_messages` +Count of messages received from the SQS queue. Broken down by message type and whether the message was actionable. -### `karpenter_provisioner_limit` -The Provisioner Limits are the limits specified on the provisioner that restrict the quantity of resources provisioned. Labeled by provisioner name and resource type. +### `karpenter_interruption_message_latency_time_seconds` +Length of time between message creation in queue and an action taken on the message by the controller. -### `karpenter_provisioner_scheduling_duration_seconds` -Duration of scheduling process in seconds. +### `karpenter_interruption_deleted_messages` +Count of messages deleted from the SQS queue. -### `karpenter_provisioner_scheduling_simulation_duration_seconds` -Duration of scheduling simulations used for deprovisioning and provisioning in seconds. +### `karpenter_interruption_actions_performed` +Number of notification actions performed. Labeled by action -### `karpenter_provisioner_usage` -The Provisioner Usage is the amount of resources that have been provisioned by a particular provisioner. Labeled by provisioner name and resource type. +## Disruption Metrics -### `karpenter_provisioner_usage_pct` -The Provisioner Usage Percentage is the percentage of each resource used based on the resources provisioned and the limits that have been configured in the range [0,100]. Labeled by provisioner name and resource type. +### `karpenter_disruption_replacement_nodeclaim_initialized_seconds` +Amount of time required for a replacement nodeclaim to become initialized. -## Nodes Metrics +### `karpenter_disruption_replacement_nodeclaim_failures_total` +The number of times that Karpenter failed to launch a replacement node for disruption. Labeled by disruption method. -### `karpenter_nodes_allocatable` -Node allocatable are the resources allocatable by nodes. +### `karpenter_disruption_queue_depth` +The number of commands currently being waited on in the disruption orchestration queue. -### `karpenter_nodes_created` -Number of nodes created in total by Karpenter. Labeled by owning provisioner. +### `karpenter_disruption_pods_disrupted_total` +Total number of reschedulable pods disrupted on nodes. Labeled by NodePool, disruption action, method, and consolidation type. -### `karpenter_nodes_leases_deleted` -Number of deleted leaked leases. +### `karpenter_disruption_nodes_disrupted_total` +Total number of nodes disrupted. Labeled by NodePool, disruption action, method, and consolidation type. -### `karpenter_nodes_system_overhead` -Node system daemon overhead are the resources reserved for system overhead, the difference between the node's capacity and allocatable values are reported by the status. +### `karpenter_disruption_evaluation_duration_seconds` +Duration of the disruption evaluation process in seconds. Labeled by method and consolidation type. -### `karpenter_nodes_terminated` -Number of nodes terminated in total by Karpenter. Labeled by owning provisioner. +### `karpenter_disruption_eligible_nodes` +Number of nodes eligible for disruption by Karpenter. Labeled by disruption method and consolidation type. -### `karpenter_nodes_termination_time_seconds` -The time taken between a node's deletion request and the removal of its finalizer +### `karpenter_disruption_consolidation_timeouts_total` +Number of times the Consolidation algorithm has reached a timeout. Labeled by consolidation type. -### `karpenter_nodes_total_daemon_limits` -Node total daemon limits are the resources specified by DaemonSet pod limits. +### `karpenter_disruption_budgets_allowed_disruptions` +The number of nodes for a given NodePool that can be disrupted at a point in time. Labeled by NodePool. Note that allowed disruptions can change very rapidly, as new nodes may be created and others may be deleted at any point. -### `karpenter_nodes_total_daemon_requests` -Node total daemon requests are the resource requested by DaemonSet pods bound to nodes. +### `karpenter_disruption_actions_performed_total` +Number of disruption actions performed. Labeled by disruption action, method, and consolidation type. -### `karpenter_nodes_total_pod_limits` -Node total pod limits are the resources specified by non-DaemonSet pod limits. +## Consistency Metrics -### `karpenter_nodes_total_pod_requests` -Node total pod requests are the resources requested by non-DaemonSet pods bound to nodes. +### `karpenter_consistency_errors` +Number of consistency checks that have failed. -## Pods Metrics +## Cluster State Metrics -### `karpenter_pods_startup_time_seconds` -The time from pod creation until the pod is running. +### `karpenter_cluster_state_synced` +Returns 1 if cluster state is synced and 0 otherwise. Synced checks that nodeclaims and nodes that are stored in the APIServer have the same representation as Karpenter's cluster state -### `karpenter_pods_state` -Pod state is the current state of pods. This metric can be used several ways as it is labeled by the pod name, namespace, owner, node, provisioner name, zone, architecture, capacity type, instance type and pod phase. +### `karpenter_cluster_state_node_count` +Current count of nodes in cluster state ## Cloudprovider Metrics -### `karpenter_cloudprovider_duration_seconds` -Duration of cloud provider method calls. Labeled by the controller, method name and provider. +### `karpenter_cloudprovider_instance_type_price_estimate` +Estimated hourly price used when making informed decisions on node cost calculation. This is updated once on startup and then every 12 hours. -### `karpenter_cloudprovider_errors_total` -Total number of errors returned from CloudProvider calls. +### `karpenter_cloudprovider_instance_type_memory_bytes` +Memory, in bytes, for a given instance type. ### `karpenter_cloudprovider_instance_type_cpu_cores` VCPUs cores for a given instance type. -### `karpenter_cloudprovider_instance_type_memory_bytes` -Memory, in bytes, for a given instance type. +### `karpenter_cloudprovider_errors_total` +Total number of errors returned from CloudProvider calls. -### `karpenter_cloudprovider_instance_type_price_estimate` -Estimated hourly price used when making informed decisions on node cost calculation. This is updated once on startup and then every 12 hours. +### `karpenter_cloudprovider_duration_seconds` +Duration of cloud provider method calls. Labeled by the controller, method name and provider. ## Cloudprovider Batcher Metrics +### `karpenter_cloudprovider_batcher_batch_time_seconds` +Duration of the batching window per batcher + ### `karpenter_cloudprovider_batcher_batch_size` Size of the request batch per batcher -### `karpenter_cloudprovider_batcher_batch_time_seconds` -Duration of the batching window per batcher +## Controller Runtime Metrics + +### `controller_runtime_reconcile_total` +Total number of reconciliations per controller + +### `controller_runtime_reconcile_time_seconds` +Length of time per reconciliation per controller + +### `controller_runtime_reconcile_errors_total` +Total number of reconciliation errors per controller + +### `controller_runtime_max_concurrent_reconciles` +Maximum number of concurrent reconciles per controller + +### `controller_runtime_active_workers` +Number of currently used workers per controller diff --git a/website/content/en/v0.35/reference/settings.md b/website/content/en/v0.35/reference/settings.md new file mode 100644 index 000000000000..c7dbc250391f --- /dev/null +++ b/website/content/en/v0.35/reference/settings.md @@ -0,0 +1,68 @@ +--- +title: "Settings" +linkTitle: "Settings" +weight: 5 +description: > + Configure Karpenter +--- + +Karpenter surfaces environment variables and CLI parameters to allow you to configure certain global settings on the controllers. These settings are described below. + +[comment]: <> (the content below is generated from hack/docs/configuration_gen_docs.go) + +| Environment Variable | CLI Flag | Description | +|--|--|--| +| ASSUME_ROLE_ARN | \-\-assume-role-arn | Role to assume for calling AWS services.| +| ASSUME_ROLE_DURATION | \-\-assume-role-duration | Duration of assumed credentials in minutes. Default value is 15 minutes. Not used unless aws.assumeRole set. (default = 15m0s)| +| BATCH_IDLE_DURATION | \-\-batch-idle-duration | The maximum amount of time with no new pending pods that if exceeded ends the current batching window. If pods arrive faster than this time, the batching window will be extended up to the maxDuration. If they arrive slower, the pods will be batched separately. (default = 1s)| +| BATCH_MAX_DURATION | \-\-batch-max-duration | The maximum length of a batch window. The longer this is, the more pods we can consider for provisioning at one time which usually results in fewer but larger nodes. (default = 10s)| +| CLUSTER_CA_BUNDLE | \-\-cluster-ca-bundle | Cluster CA bundle for nodes to use for TLS connections with the API server. If not set, this is taken from the controller's TLS configuration.| +| CLUSTER_ENDPOINT | \-\-cluster-endpoint | The external kubernetes cluster endpoint for new nodes to connect with. If not specified, will discover the cluster endpoint using DescribeCluster API.| +| CLUSTER_NAME | \-\-cluster-name | [REQUIRED] The kubernetes cluster name for resource discovery.| +| DISABLE_WEBHOOK | \-\-disable-webhook | Disable the admission and validation webhooks| +| ENABLE_PROFILING | \-\-enable-profiling | Enable the profiling on the metric endpoint| +| FEATURE_GATES | \-\-feature-gates | Optional features can be enabled / disabled using feature gates. Current options are: Drift,SpotToSpotConsolidation (default = Drift=true,SpotToSpotConsolidation=false)| +| HEALTH_PROBE_PORT | \-\-health-probe-port | The port the health probe endpoint binds to for reporting controller health (default = 8081)| +| INTERRUPTION_QUEUE | \-\-interruption-queue | Interruption queue is disabled if not specified. Enabling interruption handling may require additional permissions on the controller service account. Additional permissions are outlined in the docs.| +| ISOLATED_VPC | \-\-isolated-vpc | If true, then assume we can't reach AWS services which don't have a VPC endpoint. This also has the effect of disabling look-ups to the AWS pricing endpoint.| +| KARPENTER_SERVICE | \-\-karpenter-service | The Karpenter Service name for the dynamic webhook certificate| +| KUBE_CLIENT_BURST | \-\-kube-client-burst | The maximum allowed burst of queries to the kube-apiserver (default = 300)| +| KUBE_CLIENT_QPS | \-\-kube-client-qps | The smoothed rate of qps to kube-apiserver (default = 200)| +| LEADER_ELECT | \-\-leader-elect | Start leader election client and gain leadership before executing the main loop. Enable this when running replicated components for high availability.| +| LOG_LEVEL | \-\-log-level | Log verbosity level. Can be one of 'debug', 'info', or 'error' (default = info)| +| MEMORY_LIMIT | \-\-memory-limit | Memory limit on the container running the controller. The GC soft memory limit is set to 90% of this value. (default = -1)| +| METRICS_PORT | \-\-metrics-port | The port the metric endpoint binds to for operating metrics about the controller itself (default = 8000)| +| RESERVED_ENIS | \-\-reserved-enis | Reserved ENIs are not included in the calculations for max-pods or kube-reserved. This is most often used in the VPC CNI custom networking setup https://docs.aws.amazon.com/eks/latest/userguide/cni-custom-network.html. (default = 0)| +| VM_MEMORY_OVERHEAD_PERCENT | \-\-vm-memory-overhead-percent | The VM memory overhead as a percent that will be subtracted from the total memory for all instance types. (default = 0.075)| +| WEBHOOK_METRICS_PORT | \-\-webhook-metrics-port | The port the webhook metric endpoing binds to for operating metrics about the webhook (default = 8001)| +| WEBHOOK_PORT | \-\-webhook-port | The port the webhook endpoint binds to for validation and mutation of resources (default = 8443)| + +[comment]: <> (end docs generated content from hack/docs/configuration_gen_docs.go) + +### Feature Gates + +Karpenter uses [feature gates](https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/#feature-gates-for-alpha-or-beta-features) You can enable the feature gates through the `--feature-gates` CLI environment variable or the `FEATURE_GATES` environment variable in the Karpenter deployment. For example, you can configure drift, spotToSpotConsolidation by setting the CLI argument: `--feature-gates Drift=true,SpotToSpotConsolidation=true`. + +| Feature | Default | Stage | Since | Until | +|-------------------------|---------|-------|---------|---------| +| Drift | false | Alpha | v0.21.x | v0.32.x | +| Drift | true | Beta | v0.33.x | | +| SpotToSpotConsolidation | false | Alpha | v0.34.x | | + +### Batching Parameters + +The batching parameters control how Karpenter batches an incoming stream of pending pods. Reducing these values may trade off a slightly faster time from pending pod to node launch, in exchange for launching smaller nodes. Increasing the values can do the inverse. Karpenter provides reasonable defaults for these values, but if you have specific knowledge about your workloads you can tweak these parameters to match the expected rate of incoming pods. + +For a standard deployment scale-up, the pods arrive at the QPS setting of the `kube-controller-manager`, and the default values are typically fine. These settings are intended for use cases where other systems may create large numbers of pods over a period of many seconds or minutes and there is a desire to batch them together. + +#### Batch Idle Duration + +The batch idle duration duration is the period of time that a new pending pod extends the current batching window. This can be increased to handle scenarios where pods arrive slower than one second part, but it would be preferable if they were batched together onto a single larger node. + +This value is expressed as a string value like `10s`, `1m` or `2h45m`. The valid time units are `ns`, `us` (or `µs`), `ms`, `s`, `m`, `h`. + +#### Batch Max Duration + +The batch max duration is the maximum period of time a batching window can be extended to. Increasing this value will allow the maximum batch window size to increase to collect more pending pods into a single batch at the expense of a longer delay from when the first pending pod was created. + +This value is expressed as a string value like `10s`, `1m` or `2h45m`. The valid time units are `ns`, `us` (or `µs`), `ms`, `s`, `m`, `h`. diff --git a/website/content/en/v0.35/reference/threat-model.md b/website/content/en/v0.35/reference/threat-model.md new file mode 100644 index 000000000000..88f735fb3215 --- /dev/null +++ b/website/content/en/v0.35/reference/threat-model.md @@ -0,0 +1,120 @@ +--- +title: "Threat Model" +linkTitle: "Threat Model" +weight: 999 +--- + +Karpenter observes Kubernetes pods and launches nodes in response to those pods’ scheduling constraints. Karpenter does not perform the actual scheduling and instead waits for [kube-scheduler](https://kubernetes.io/docs/concepts/scheduling-eviction/kube-scheduler/) to schedule the pods. + +When running in AWS, Karpenter is typically installed onto EC2 instances that run in EKS Clusters. Karpenter relies on public facing AWS APIs and standard IAM Permissions. Karpenter uses AWS-SDK-Go v1, and AWS advises that credentials are provided using [IAM Roles for Service Accounts](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html). + + +## Architecture & Actors + +1. **Cluster Operator**: An identity that installs and configures Karpenter in a Kubernetes cluster, and configures Karpenter's cloud identity and permissions. +2. **Cluster Developer**: An identity that can create pods, typically through Deployments, DaemonSets, or other pod-controller types. +3. **Karpenter Controller:** The Karpenter application pod that operates inside a cluster. + +![threat-model](/threat-model.png) + +## Capabilities + +### Cluster Operator + +The Cluster Operator has full control to install and configure Karpenter including all [`NodePools`]({{}}) and [`EC2NodeClasses`]({{}}). The Cluster Operator has privileges to manage the cloud identities and permissions for Nodes, and the cloud identity and permissions for Karpenter. + +### Cluster Developer + +A Cluster Developer has the ability to create pods via `Deployments`, `ReplicaSets`, `StatefulSets`, `Jobs`, etc. This assumes that the Cluster Developer cannot modify the Karpenter pod or launch pods using Karpenter’s service account and gain access to Karpenter’s IAM role. + +### Karpenter Controller + +Karpenter has permissions to create and manage cloud instances. Karpenter has Kubernetes API permissions to create, update, and remove nodes, as well as evict pods. For a full list of the permissions, see the RBAC rules in the helm chart template. Karpenter also has AWS IAM permissions to create instances with IAM roles. + +* [aggregate-clusterrole.yaml](https://github.com/aws/karpenter/blob/v0.35.4/charts/karpenter/templates/aggregate-clusterrole.yaml) +* [clusterrole-core.yaml](https://github.com/aws/karpenter/blob/v0.35.4/charts/karpenter/templates/clusterrole-core.yaml) +* [clusterrole.yaml](https://github.com/aws/karpenter/blob/v0.35.4/charts/karpenter/templates/clusterrole.yaml) +* [rolebinding.yaml](https://github.com/aws/karpenter/blob/v0.35.4/charts/karpenter/templates/rolebinding.yaml) +* [role.yaml](https://github.com/aws/karpenter/blob/v0.35.4/charts/karpenter/templates/role.yaml) + +## Assumptions + +| Category | Assumption | Comment | +|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Generic | The Karpenter pod is operated on a node in the cluster, and uses a Service Account for authentication to the Kubernetes API | Cluster Operators may want to isolate the node running the Karpenter pod to a system-pool of nodes to mitigate the possibility of container breakout with Karpenter’s permissions. | +| Generic | Cluster Developer does not have any Kubernetes permissions to manage Karpenter running in the cluster (The deployment, pods, clusterrole, etc) | | +| Generic | Restrictions on the fields of pods a Cluster Developer can create are out of scope. | Cluster Operators can use policy frameworks to enforce restrictions on Pod capabilities | +| Generic | No sensitive data is included in non-Secret resources in the Kubernetes API. The Karpenter controller has the ability to list all pods, nodes, deployments, and many other pod-controller and storage resource types. | Karpenter does not have permission to list/watch cluster-wide ConfigMaps or Secrets | +| Generic | Karpenter has permissions to create, modify, and delete nodes from the cluster, and evict any pod. | Cluster Operators running applications with varying security profiles in the same cluster may want to configure dedicated nodes and scheduling rules for Karpenter to mitigate potential container escapes from other containers | +| AWS-Specific | The Karpenter IAM policy is encoded in the GitHub repo. Any additional permissions possibly granted to that role by the administrator are out of scope | | +| AWS-Specific | The Karpenter pod uses IRSA for AWS credentials | Setup of IRSA is out of scope for this document | + +## Generic Threats and Mitigations + +### Threat: Cluster Developer can influence creation of an arbitrary number of nodes + +**Background**: Karpenter creates new instances based on the count of pending pods. + +**Threat**: A Cluster Developer attempts to have Karpenter create more instances than intended by creating a large number of pods or by using anti-affinity to schedule one pod per node. + +**Mitigation**: In addition to [Kubernetes resource limits](https://kubernetes.io/docs/concepts/policy/resource-quotas/#object-count-quota), Cluster Operators can [configure limits on a NodePool]({{< ref "../concepts/nodepools#spec-limits" >}}) to limit the total amount of memory, CPU, or other resources provisioned across all nodes. + +## Threats + +### Threat: Using EC2 CreateTag/DeleteTag Permissions to Orchestrate Instance Creation/Deletion + +**Background**: As of `0.28.0`, Karpenter creates a mapping between CloudProvider instances and CustomResources in the cluster for capacity tracking. To ensure this mapping is consistent, Karpenter utilizes the following tag keys: + +* `karpenter.sh/managed-by` +* `karpenter.sh/nodepool` +* `kubernetes.io/cluster/${CLUSTER_NAME}` +* `karpenter.sh/provisioner-name` (prior to `0.32.0`) + +Any user that has the ability to Create/Delete tags on CloudProvider instances will have the ability to orchestrate Karpenter to Create/Delete CloudProvider instances as a side effect. + +In addition, as of `0.29.0`, Karpenter will Drift on Security Groups and Subnets. If a user has the Create/Delete tags permission for either of resources, they can orchestrate Karpenter to Create/Delete CloudProvider instances as a side effect. + +**Threat:** A Cluster Operator attempts to create or delete a tag on a resource discovered by Karpenter. If it has the ability to create a tag it can effectively create or delete CloudProvider instances associated with the tagged resources. + +**Mitigation** Cluster Operators should [enforce tag-based IAM policies](https://docs.aws.amazon.com/IAM/latest/UserGuide/access_tags.html) on these tags against any EC2 instance resource (`i-*`) for any users that might have [CreateTags](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_CreateTags.html)/[DeleteTags](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_DeleteTags.html) permissions but should not have [RunInstances](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_RunInstances.html)/[TerminateInstances](https://docs.aws.amazon.com/AWSEC2/latest/APIReference/API_TerminateInstances.html) permissions. + +### Threat: Launching EC2 instances with IAM roles not intended for Karpenter nodes + +**Background**: Many IAM roles in an AWS account may trust the EC2 service principal. IAM administrators must grant the `iam:PassRole` permission to IAM principals to allow those principals in the account to launch instances with specific roles. + +**Threat:** A Cluster Operator attempts to create an `EC2NodeClass` with an IAM role not intended for Karpenter + +**Mitigation**: Cluster Operators must enumerate the roles in the resource section of the IAM policy granted to the Karpenter role for the `iam:PassRole` action. Karpenter will fail to generate an instance profile if role that is specified in the `spec.role` section of the `EC2NodeClass` is not enumerated in the `iam:PassRole` permission. + +### Threat: Karpenter can orchestrate the creation/deletion of IAM Instance Profiles it doesn't own + +**Background**: Karpenter has permission to create/update/delete instance profiles as part of its controller permissions to ensure that it can auto-generate instance profiles when EC2NodeClasses are created. + +**Threat**: An actor who obtains control of the Karpenter pod’s IAM role may delete instance profiles not owned by Karpenter, causing workload disruption to other instances using the profile in the account. + +**Mitigation**: Karpenter's controller permissions enforce that it creates instance profiles with tags which provide ownership information. These tags include: + +* `karpenter.sh/managed-by` +* `kubernetes.io/cluster/${CLUSTER_NAME}` +* `karpenter.k8s.aws/ec2nodeclass` +* `topology.kubernetes.io/region` + +These tags ensure that instance profiles created by Karpenter in the account are unique to that cluster. Karpenter's controller permissions _only_ allow it to act on instance profiles that contain these tags which match the cluster information. + +### Threat: Karpenter can be used to create or terminate EC2 instances outside the cluster + +**Background**: EC2 instances can exist in an AWS account outside of the Kubernetes cluster. + +**Threat**: An actor who obtains control of the Karpenter pod’s IAM role may be able to create or terminate EC2 instances not part of the Kubernetes cluster managed by Karpenter. + +**Mitigation**: Karpenter creates instances with tags, several of which are enforced in the IAM policy granted to the Karpenter IAM role that restrict the instances Karpenter can terminate. One tag requires that the instance was provisioned by a Karpenter controller (`karpenter.sh/nodepool`), another tag can include a cluster name to mitigate any termination between two clusters with Karpenter in the same account (`kubernetes.io/cluster/${CLUSTER_NAME}`. Cluster Operators also can restrict the region to prevent two clusters in the same account with the same name in different regions. + +Additionally, Karpenter does not allow tags to be modified on instances unowned by Karpenter after creation, except for the `Name` and `karpenter.sh/nodeclaim` tags. Though these tags can be changed after instance creation, `aws:ResourceTag` conditions enforce that the Karpenter controller is only able to change these tags on instances that it already owns, enforced through the `karpenter.sh/nodepool` and `kubernetes.io/cluster/${CLUSTER_NAME}` tags. + +### Threat: Karpenter launches an EC2 instance using an unintended AMI + +**Background**: Cluster Developers can create Node Templates that refer to an AMI by metadata, such as a name rather than an AMI resource ID. + +**Threat:** A threat actor creates a public AMI with the same name as a customer’s AMI in an attempt to get Karpenter to select the threat actor’s AMI instead of the intended AMI. + +**Mitigation**: When selecting AMIs by name or tags, Karpenter defaults to adding an ownership filter of `self,amazon` so AMI images external to the account are not used. diff --git a/website/content/en/v0.35/tasks/_index.md b/website/content/en/v0.35/tasks/_index.md new file mode 100644 index 000000000000..7d4ac8605f4e --- /dev/null +++ b/website/content/en/v0.35/tasks/_index.md @@ -0,0 +1,10 @@ +--- +title: "Tasks" +linkTitle: "Tasks" +weight: 25 +description: > + Tasks to run with Karpenter +cascade: + type: docs +--- + diff --git a/website/content/en/v0.35/tasks/managing-amis.md b/website/content/en/v0.35/tasks/managing-amis.md new file mode 100644 index 000000000000..1ef31f141e08 --- /dev/null +++ b/website/content/en/v0.35/tasks/managing-amis.md @@ -0,0 +1,153 @@ +--- +title: "Managing AMIs" +linkTitle: "Managing AMIs" +weight: 10 +description: > + Task for managing AMIs in Karpenter +--- + +Understanding how Karpenter assigns AMIs to nodes can help ensure that your workloads will run successfully on those nodes and continue to run if the nodes are upgraded to newer AMIs. +Below we describe how Karpenter assigns AMIs to nodes when they are first deployed and how newer AMIs are assigned later when nodes are spun up to replace old ones. +Later, it describes the options you have to assert control over how AMIs are used by Karpenter for your clusters. + +Features for managing AMIs described here should be considered as part of the larger upgrade policies that you have for your clusters. +See [How do I upgrade an EKS Cluster with Karpenter]({{< relref "../faq/#how-do-i-upgrade-an-eks-cluster-with-karpenter" >}}) for details on this process. + +## How Karpenter assigns AMIs to nodes + +Here is how Karpenter assigns AMIs nodes: + +* When you create an `EC2NodeClass`, you are required to set the family of AMIs to use. For example, for the AL2 family, you would set `amiFamily: AL2`. +* With that `amiFamily` set, any time Karpenter spins up a new node, it uses the latest [Amazon EKS optimized Amazon Linux 2 AMIs](https://docs.aws.amazon.com/eks/latest/userguide/eks-optimized-ami.html) release. +* Later, if an existing node needs to be replaced, Karpenter checks to see if a newer AMI in the AL2 family is available and automatically uses the new AMI instead to spin up the new node. In other words, you may automatically get an AMI that you have not tested with your workloads. + +You can manually delete a node managed by Karpenter, which will cause the default behavior just described to take effect. +However, there are situations that will cause node replacements with newer AMIs to happen automatically. +These include: Expiration (if node expiry is set, the node is marked for deletion at a certain time after the node is created), [**Consolidation**]({{< relref "../concepts/disruption/#consolidation" >}}) (if a node is empty of workloads, or deemed to be inefficiently running workloads, nodes can be deleted and more appropriately featured nodes are brought up to consolidate workloads), [Drift]({{< relref "../concepts/disruption/#drift" >}}) (nodes are set for deletion when they drift from the desired state of the `NodeClaims` and new nodes are brought up to replace them), and [Interruption]({{< relref "../concepts/disruption/#interruption" >}}) (nodes are sometimes involuntarily disrupted by things like Spot interruption, health changes, and instance events, requiring new nodes to be deployed). + +See [**Automated Methods**]({{< relref "../concepts/disruption/#automated-methods" >}}) for details on how Karpenter uses these automated actions to replace nodes. + +With these types of automated updates in place, there is some risk that the new AMI being used when replacing instances will introduce some regressions or bugs that cause your workloads to be degraded or fail altogether. +The options described below tell you how to take more control over the ways in which Karpenter selects AMIs for your nodes. + +{{% alert title="Important" color="warning" %}} +If you are new to Karpenter, you should know that the behavior described here is different than you get with Managed Node Groups (MNG). MNG will always use the assigned AMI when it creates a new node and will never automatically upgrade to a new AMI when a new node is required. See [Updating a Managed Node Group](https://docs.aws.amazon.com/eks/latest/userguide/update-managed-node-group.html) to see how you would manually update MNG to use new AMIs. +{{% /alert %}} + +## Choosing AMI options +One of Karpenter's greatest assets is its ability to provide the right node at the right time, with little intervention from the person managing the cluster. +Its default behavior of using a later AMI if one becomes available in the selected family means you automatically get the latest security fixes and features. +However, with this comes the risk that the new AMI could break or degrade your workloads. + +As the Karpenter team looks for new ways to manage AMIs, the options below offer some means of reducing these risks, based on your own security and ease-of-use requirements. +Here are the advantages and challenges of each of the options described below: + +* [Option 1]({{< relref "#option-1-manage-how-amis-are-tested-and-rolled-out" >}}) (Test AMIs): The safest way, and the one we recommend, for ensuring that a new AMI doesn't break your workloads is to test it before putting it into production. This takes the most effort on your part, but most effectively models how your workloads will run in production, allowing you to catch issues ahead of time. Note that you can sometimes get different results from your test environment when you roll a new AMI into production, since issues like scale and other factors can elevate problems you might not see in test. So combining this with other options, that do things like slow rollouts, can allow you to catch problems before they impact your whole cluster. +* [Option 2]({{< relref "#option-2-lock-down-which-amis-are-selected" >}}) (Lock down AMIs): If workloads require a particluar AMI, this option can make sure that it is the only AMI used by Karpenter. This can be used in combination with Option 1, where you lock down the AMI in production, but allow the newest AMIs in a test cluster while you test your workloads before upgrading production. Keep in mind that this makes upgrades a manual process for you. +* [Option 3]({{< relref "#option-3-control-the-pace-of-node-disruptions" >}}) ([Disruption budgets]({{< relref "../concepts/disruption/" >}})): This option can be used as a way of mitigating the scope of impact if a new AMI causes problems with your workloads. With Disruption budgets you can slow the pace of upgrades to nodes with new AMIs or make sure that upgrades only happen during selected dates and times (using `schedule`). This doesn't prevent a bad AMI from being deployed, but it allows you to control when nodes are upgraded, and gives you more time respond to rollout issues. + +## Options + +The following lays out the options you have to impact Karpenter’s behavior as it relates to how nodes are created and AMIs are consumed. + +### Option 1: Manage how AMIs are tested and rolled out + +Instead of just avoiding AMI upgrades, you can set up test clusters where you can try out new AMI releases before they are put into production. +For example, you could have: + +* **Test clusters**: On lower environment clusters, you can run the latest AMIs for your workloads in a safe environment. The `EC2NodeClass` for these clusters could be set with a chosen `amiFamily`, but no `amiSelectorTerms` set. For example, the `NodePool` and `EC2NodeClass` could begin with the following: + + ```yaml + apiVersion: karpenter.sh/v1beta1 + kind: NodePool + metadata: + name: default + spec: + template: + spec: + nodeClassRef: + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass + name: default + --- + apiVersion: karpenter.k8s.aws/v1beta1 + kind: EC2NodeClass + metadata: + name: default + spec: + # The latest AMI in this family will be used + amiFamily: AL2 + ``` +* **Production clusters**: After you've confirmed that the AMI works in your lower environments, you can pin the latest AMIs to be deployed in your production clusters to roll out the AMI. One way to do that is to use `amiSelectorTerms` to set the tested AMI to be used in your production cluster. Refer to Option 2 for how to choose a particular AMI by `name` or `id`. Remember that it is still best practice to gradually roll new AMIs into your cluster, even if they have been tested. So consider implementing that for your production clusters as described in Option 3. + +### Option 2: Lock down which AMIs are selected + +Instead of letting Karpenter always run the latest AMI, you can change Karpenter’s default behavior. +When you configure the [**EC2NodeClass**]({{< relref "../concepts/nodeclasses" >}}), you can set a specific AMI that you want Karpenter to always choose, using the `amiSelectorTerms` field. +This prevents a new and potentially untested AMI from replacing existing nodes when those nodes are terminated. + +With the `amiSelectorTerms` field in an `EC2NodeClass`, you can set a specific AMI for Karpenter to use, based on AMI name or id (only one is required). +These examples show two different ways to identify the same AMI: + +```yaml +amiSelectorTerms: +- tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" + environment: prod +- name: al2023-ami-2023.3.20240219.0-kernel-6.1-x86_64 +``` + +or + +```yaml +amiSelectorTerms: +- tags: + karpenter.sh/discovery: "${CLUSTER_NAME}" + environment: prod +- id: ami-052c9ea013e6e3567 +``` + +See the [**spec.amiSelectorTerms**]({{< relref "../concepts/nodeclasses/#specamiselectorterms" >}}) section of the NodeClasses page for details. +Keep in mind, that this could prevent you from getting critical security patches when new AMIs are available, but it does give you control over exactly which AMI is running. + + +### Option 3: Control the pace of node disruptions + +To reduce the risk of entire workloads being immediately degraded when a new AMI is deployed, you can enable Karpenter [**Disruption Budgets**]({{< relref "../concepts/disruption/#disruption-budgets " >}}). +Disruption Budgets limit when and to what extent nodes can be disrupted. +You can prevent disruption based on nodes (a percentage or number of nodes that can be disrupted at a time) and schedule (excluding certain times from disrupting nodes). +You can set Disruption Budgets in a `NodePool` spec. +Here is an example: + +```yaml +disruption: + consolidationPolicy: WhenEmpty + expireAfter: 1440h + budgets: + - nodes: 15% + - nodes: "3" + - nodes: "0" + schedule: "0 7 * * sat-sun" + duration: 12h +``` + +The `disruption` settings define a few fields that indicate the state of a node that should be disrupted. +The `consolidationPolicy` field indicates that a node should be disrupted if the node is either underutilized (`WhenUnderutilized`) or not running any pods (`WhenEmpty`). +With `expireAfter` set to `1440` hours, the node expires after 60 days. +Extending those values causes longer times without disruption. + +Settings for budgets in the above example include the following: + +* **Percentage of nodes**: From the first `nodes` setting, only `15%` of the NodePool’s nodes can be disrupted at a time. +* **Number of nodes**: The second `nodes` setting limits the number of nodes that can be disrupted at a time to `3`. +* **Schedule**: The third `nodes` setting uses schedule to say that zero disruptions (`0`) are allowed starting at 7am on Saturday and Sunday and continues for 12 hours. +The format of the schedule follows the `crontab` format for identifying dates and times. +See the [crontab](https://man7.org/linux/man-pages/man5/crontab.5.html) page for information on the supported values for these fields. + +As with all disruption settings, keep in mind that avoiding updated AMIs for your nodes can result in not getting fixes for known security risks and bugs. +You need to balance that with your desire to not risk breaking the workloads on your cluster. + +## Follow-up + +The Karpenter project continues to add features to give you greater control over AMI upgrades on your clusters. +If you have opinions about features you would like to see to manage AMIs with Karpenter, feel free to enter a Karpenter [New Issue](https://github.com/aws/karpenter-provider-aws/issues/new/choose). diff --git a/website/content/en/v0.30/troubleshooting.md b/website/content/en/v0.35/troubleshooting.md similarity index 81% rename from website/content/en/v0.30/troubleshooting.md rename to website/content/en/v0.35/troubleshooting.md index a87ba78b23e5..6dc784007b75 100644 --- a/website/content/en/v0.30/troubleshooting.md +++ b/website/content/en/v0.35/troubleshooting.md @@ -1,7 +1,7 @@ --- title: "Troubleshooting" linkTitle: "Troubleshooting" -weight: 90 +weight: 70 description: > Troubleshoot Karpenter problems --- @@ -29,7 +29,7 @@ Update the zap-logger-config "level" and restart the Karpenter pod(s) to enable #### Debug logging via Helm -You can enable debug logging during installation with helm by setting the option `logLevel`. +You can enable debug logging during installation with Helm by setting the option `logLevel`. ``` helm upgrade --install karpenter oci://public.ecr.aws/karpenter/karpenter \ @@ -53,6 +53,18 @@ This can be resolved by creating the [Service Linked Role](https://docs.aws.amaz aws iam create-service-linked-role --aws-service-name spot.amazonaws.com ``` +### Failed Resolving STS Credentials with I/O Timeout + +```bash +Checking EC2 API connectivity, WebIdentityErr: failed to retrieve credentials\ncaused by: RequestError: send request failed\ncaused by: Post \"https://sts.us-east-1.amazonaws.com/\": dial tcp: lookup sts.us-east-1.amazonaws.com: i/o timeout +``` + +If you see the error above when you attempt to install Karpenter, this indicates that Karpenter is unable to reach out to the STS endpoint due to failed DNS resolution. This can happen when Karpenter is running with `dnsPolicy: ClusterFirst` and your in-cluster DNS service is not yet running. + +You have two mitigations to resolve this error: +1. Let Karpenter manage your in-cluster DNS service - You can let Karpenter manage your DNS application pods' capacity by changing Karpenter's `dnsPolicy` to be `Default` (run `--set dnsPolicy=Default` with a Helm installation). This ensures that Karpenter reaches out to the [VPC DNS service](https://docs.aws.amazon.com/vpc/latest/userguide/vpc-dns.html) when running its controllers, allowing Karpenter to start-up without the DNS application pods running, enabling Karpenter to manage the capacity for these pods. +2. Let MNG/Fargate manage your in-cluster DNS service - If running a cluster with MNG, ensure that your group has enough capacity to support the DNS application pods and ensure that the application has the correct tolerations to schedule against the capacity. If running a cluster with Fargate, ensure that you have a [fargate profile](https://docs.aws.amazon.com/eks/latest/userguide/fargate-profile.html) that selects against your DNS application pods. + ### Karpenter Role names exceeding 64-character limit If you use a tool such as AWS CDK to generate your Kubernetes cluster name, when you add Karpenter to your cluster you could end up with a cluster name that is too long to incorporate into your KarpenterNodeRole name (which is limited to 64 characters). @@ -77,7 +89,7 @@ Info on whether there has been a change to the CRD between versions of Karpenter ### Unable to schedule pod due to insufficient node group instances -v0.16.0 changed the default replicas from 1 to 2. +`0.16.0` changed the default replicas from 1 to 2. Karpenter won't launch capacity to run itself (log related to the `karpenter.sh/provisioner-name DoesNotExist requirement`) so it can't provision for the second Karpenter pod. @@ -89,30 +101,30 @@ To do so on AWS increase the `minimum` and `desired` parameters on the node grou ### Helm Error When Pulling the Chart -If Helm is showing an error when trying to install Karpenter helm charts: +If Helm is showing an error when trying to install Karpenter Helm charts: -- Ensure you are using a newer Helm version, Helm started supporting OCI images since v3.8.0. -- Helm does not have an `helm repo add` concept in OCI, so to install Karpenter you no longer need this +- Ensure you are using a newer Helm version, Helm started supporting OCI images since `3.8.0`. +- Helm does not have an `helm repo add` concept in OCI, so to install Karpenter you no longer need this. +- If you get an error like `Error: public.ecr.aws/karpenter/karpenter:0.34.0: not found` make sure you're adding a `v` prefix for Karpenter versions between `0.17.0` & `0.34.x`. - Verify that the image you are trying to pull actually exists in [gallery.ecr.aws/karpenter](https://gallery.ecr.aws/karpenter/karpenter) -- Sometimes Helm generates a generic error, you can add the --debug switch to any of the helm commands in this doc for more verbose error messages -- If you are getting a 403 forbidden error, you can try `docker logout public.ecr.aws` as explained [here](https://docs.aws.amazon.com/AmazonECR/latest/public/public-troubleshooting.html) -- If you are receiving this error: `Error: failed to download "oci://public.ecr.aws/karpenter/karpenter" at version "0.17.0"`, then you need to prepend a `v` to the version number: `v0.17.0`. Before Karpenter moved to OCI helm charts (pre-v0.17.0), both `v0.16.0` and `0.16.0` would work, but OCI charts require an exact version match. +- Sometimes Helm generates a generic error, you can add the --debug switch to any of the Helm commands in this doc for more verbose error messages +- If you are getting a 403 forbidden error, you can try `docker logout public.ecr.aws` as explained [here](https://docs.aws.amazon.com/AmazonECR/latest/public/public-troubleshooting.html). ### Helm Error when installing the `karpenter-crd` chart -Karpenter v0.26.1+ introduced the `karpenter-crd` helm chart. When installing this chart on your cluster, if you have previously added the Karpenter CRDs to your cluster through the `karpenter` controller chart or through `kubectl replace`, Helm will reject the install of the chart due to `invalid ownership metadata`. +Karpenter `0.26.1` introduced the `karpenter-crd` Helm chart. When installing this chart on your cluster, if you have previously added the Karpenter CRDs to your cluster through the `karpenter` controller chart or through `kubectl replace`, Helm will reject the install of the chart due to `invalid ownership metadata`. - In the case of `invalid ownership metadata; label validation error: missing key "app.kubernetes.io/managed-by": must be set to "Helm"` run: ```shell -kubectl label crd awsnodetemplates.karpenter.k8s.aws provisioners.karpenter.sh machines.karpenter.sh app.kubernetes.io/managed-by=Helm --overwrite +kubectl label crd ec2nodeclasses.karpenter.k8s.aws nodepools.karpenter.sh nodeclaims.karpenter.sh app.kubernetes.io/managed-by=Helm --overwrite ``` - In the case of `annotation validation error: missing key "meta.helm.sh/release-namespace": must be set to "karpenter"` run: ```shell -kubectl annotate crd awsnodetemplates.karpenter.k8s.aws provisioners.karpenter.sh machines.karpenter.sh meta.helm.sh/release-name=karpenter-crd --overwrite -kubectl annotate crd awsnodetemplates.karpenter.k8s.aws provisioners.karpenter.sh machines.karpenter.sh meta.helm.sh/release-namespace=karpenter --overwrite +kubectl annotate crd ec2nodeclasses.karpenter.k8s.aws nodepools.karpenter.sh nodeclaims.karpenter.sh meta.helm.sh/release-name=karpenter-crd --overwrite +kubectl annotate crd ec2nodeclasses.karpenter.k8s.aws nodepools.karpenter.sh nodeclaims.karpenter.sh meta.helm.sh/release-namespace=karpenter --overwrite ``` ## Uninstallation @@ -137,7 +149,7 @@ kubectl get nodes -ojsonpath='{range .items[*].metadata}{@.name}:{@.finalizers}{ If you are not able to create a provisioner due to `Internal error occurred: failed calling webhook "validation.webhook.provisioners.karpenter.sh":` -Webhooks were renamed in `v0.19.0`. There's a bug in ArgoCD's upgrade workflow where webhooks are leaked. This results in Provisioner's failing to be validated, since the validation server no longer corresponds to the webhook definition. +Webhooks were renamed in `0.19.0`. There's a bug in ArgoCD's upgrade workflow where webhooks are leaked. This results in Provisioner's failing to be validated, since the validation server no longer corresponds to the webhook definition. Delete the stale webhooks. @@ -148,7 +160,7 @@ kubectl delete validatingwebhookconfiguration validation.webhook.provisioners.ka ### Failed calling webhook "defaulting.webhook.karpenter.sh" -The `defaulting.webhook.karpenter.sh` mutating webhook was removed in `v0.27.3`. If you are coming from an older version of Karpenter where this webhook existed and the webhook was not managed by Helm, you may need to delete the stale webhook. +The `defaulting.webhook.karpenter.sh` mutating webhook was removed in `0.27.3`. If you are coming from an older version of Karpenter where this webhook existed and the webhook was not managed by Helm, you may need to delete the stale webhook. ```text kubectl delete mutatingwebhookconfigurations defaulting.webhook.karpenter.sh @@ -192,11 +204,11 @@ Disabling swap will allow kubelet to join the cluster successfully, however user ### DaemonSets can result in deployment failures -For Karpenter versions 0.5.3 and earlier, DaemonSets were not properly considered when provisioning nodes. +For Karpenter versions `0.5.3` and earlier, DaemonSets were not properly considered when provisioning nodes. This sometimes caused nodes to be deployed that could not meet the needs of the requested DaemonSets and workloads. -This issue no longer occurs after Karpenter version 0.5.3 (see [PR #1155](https://github.com/aws/karpenter/pull/1155)). +This issue no longer occurs after Karpenter version `0.5.3` (see [PR #1155](https://github.com/aws/karpenter/pull/1155)). -If you are using a pre-0.5.3 version of Karpenter, one workaround is to set your provisioner to only use larger instance types that you know will be big enough for the DaemonSet and the workload. +If you are using a pre `0.5.3` version of Karpenter, one workaround is to set your provisioner to only use larger instance types that you know will be big enough for the DaemonSet and the workload. For more information, see [Issue #1084](https://github.com/aws/karpenter/issues/1084). Examples of this behavior are included in [Issue #1180](https://github.com/aws/karpenter/issues/1180). @@ -213,7 +225,7 @@ See the Karpenter [Best Practices Guide](https://aws.github.io/aws-eks-best-prac ### Missing subnetSelector and securityGroupSelector tags causes provisioning failures -Starting with Karpenter v0.5.5, if you are using Karpenter-generated launch template, provisioners require that [subnetSelector]({{}}) and [securityGroupSelector]({{}}) tags be set to match your cluster. +Starting with Karpenter `0.5.5`, if you are using Karpenter-generated launch template, provisioners require that [subnetSelector]({{}}) and [securityGroupSelector]({{}}) tags be set to match your cluster. The [Provisioner]({{}}) section in the Karpenter Getting Started Guide uses the following example: ```text @@ -266,7 +278,7 @@ spec: When attempting to schedule a large number of pods with PersistentVolumes, it's possible that these pods will co-locate on the same node. Pods will report the following errors in their events using a `kubectl describe pod` call -```console +```bash Warning FailedAttachVolume pod/example-pod AttachVolume.Attach failed for volume "***" : rpc error: code = Internal desc = Could not attach volume "***" to node "***": attachment of disk "***" failed, expected device to be attached but was attaching Warning FailedMount pod/example-pod Unable to attach or mount volumes: unmounted volumes=[***], unattached volumes=[***]: timed out waiting for the condition ``` @@ -277,14 +289,30 @@ In this case, Karpenter may fail to scale-up your nodes due to these pods due to Karpenter does not support [in-tree storage plugins](https://kubernetes.io/blog/2021/12/10/storage-in-tree-to-csi-migration-status-update/) to provision PersistentVolumes, since nearly all of the in-tree plugins have been deprecated in upstream Kubernetes. This means that, if you are using a statically-provisioned PersistentVolume that references a volume source like `AWSElasticBlockStore` or a dynamically-provisioned PersistentVolume that references a StorageClass with a in-tree storage plugin provisioner like `kubernetes.io/aws-ebs`, Karpenter will fail to discover the maxiumum volume attachments for the node. Instead, Karpenter may think the node still has more schedulable space due to memory and cpu constraints when there is really no more schedulable space on the node due to volume limits. When Karpenter sees you are using an in-tree storage plugin on your pod volumes, it will print the following error message into the logs. If you see this message, upgrade your StorageClasses and statically-provisioned PersistentVolumes to use the latest CSI drivers for your cloud provider. -```console +```bash 2023-04-05T23:56:53.363Z ERROR controller.node_state PersistentVolume source 'AWSElasticBlockStore' uses an in-tree storage plugin which is unsupported by Karpenter and is deprecated by Kubernetes. Scale-ups may fail because Karpenter will not discover driver limits. Use a PersistentVolume that references the 'CSI' volume source for Karpenter auto-scaling support. {"commit": "b2af562", "node": "ip-192-168-36-137.us-west-2.compute.internal", "pod": "inflate0-6c4bdb8b75-7qmfd", "volume": "mypd", "persistent-volume": "pvc-11db7489-3c6e-46f3-a958-91f9d5009d41"} 2023-04-05T23:56:53.464Z ERROR controller.node_state StorageClass .spec.provisioner uses an in-tree storage plugin which is unsupported by Karpenter and is deprecated by Kubernetes. Scale-ups may fail because Karpenter will not discover driver limits. Create a new StorageClass with a .spec.provisioner referencing the CSI driver plugin name 'ebs.csi.aws.com'. {"commit": "b2af562", "node": "ip-192-168-36-137.us-west-2.compute.internal", "pod": "inflate0-6c4bdb8b75-7qmfd", "volume": "mypd", "storage-class": "gp2", "provisioner": "kubernetes.io/aws-ebs"} ``` #### Pods were scheduled due to a race condition in Kubernetes -Due to [this race condition in Kubernetes](https://github.com/kubernetes/kubernetes/issues/95911), it's possible that the scheduler and the CSINode can race during node registration such that the scheduler assumes that a node can mount more volumes than the node attachments support. There is currently no solve for this problem other than enforcing `toplogySpreadConstraints` and `podAntiAffinity` on your workloads that use PVCs such that you attempt to reduce the number of PVCs that schedule to a given node. +Due to [this race condition in Kubernetes](https://github.com/kubernetes/kubernetes/issues/95911), it's possible that the scheduler and the CSINode can race during node registration such that the scheduler assumes that a node can mount more volumes than the node attachments support. There is currently no universal solve for this problem other than enforcing `toplogySpreadConstraints` and `podAntiAffinity` on your workloads that use PVCs such that you attempt to reduce the number of PVCs that schedule to a given node. + +The following is a list of known CSI drivers which support a startupTaint to eliminate this issue: +- [aws-ebs-csi-driver](https://github.com/kubernetes-sigs/aws-ebs-csi-driver/blob/master/docs/install.md#configure-node-startup-taint) +- [aws-efs-csi-driver](https://github.com/kubernetes-sigs/aws-efs-csi-driver/tree/master/docs#configure-node-startup-taint) + +These taints should be configured via `startupTaints` on your `NodePool`. For example, to enable this for EBS, add the following to your `NodePool`: +```yaml +apiVersion: karpenter.sh/v1beta1 +kind: NodePool +spec: + template: + spec: + startupTaints: + - key: ebs.csi.aws.com/agent-not-ready + effect: NoExecute +``` ### CNI is unable to allocate IPs to pods @@ -300,7 +328,7 @@ time=2023-06-12T19:18:15Z type=Warning reason=FailedCreatePodSandBox from=kubele By default, the number of pods on a node is limited by both the number of networking interfaces (ENIs) that may be attached to an instance type and the number of IP addresses that can be assigned to each ENI. See [IP addresses per network interface per instance type](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/using-eni.html#AvailableIpPerENI) for a more detailed information on these instance types' limits. -If the max-pods (configured through your Provisioner [`kubeletConfiguration`]({{}})) is greater than the number of supported IPs for a given instance type, the CNI will fail to assign an IP to the pod and your pod will be left in a `ContainerCreating` state. +If the max-pods (configured through your Provisioner [`kubeletConfiguration`]({{}})) is greater than the number of supported IPs for a given instance type, the CNI will fail to assign an IP to the pod and your pod will be left in a `ContainerCreating` state. ##### Solutions @@ -308,13 +336,13 @@ To avoid this discrepancy between `maxPods` and the supported pod density of the 1. Enable [Prefix Delegation](https://www.eksworkshop.com/docs/networking/prefix/) to increase the number of allocatable IPs for the ENIs on each instance type 2. Reduce your `maxPods` value to be under the maximum pod density for the instance types assigned to your Provisioner -3. Remove the `maxPods` value from your [`kubeletConfiguration`]({{}}) if you no longer need it and instead rely on the defaulted values from Karpenter and EKS AMIs. +3. Remove the `maxPods` value from your [`kubeletConfiguration`]({{}}) if you no longer need it and instead rely on the defaulted values from Karpenter and EKS AMIs. -For more information on pod density, view the [Pod Density Conceptual Documentation]({{}}). +For more information on pod density, view the [Pod Density Section in the NodePools doc]({{}}). #### IP exhaustion in a subnet -When a node is launched by Karpenter, it is assigned to a subnet within your VPC based on the [`subnetSelector`]({{}}) value in your [`AWSNodeTemplate`]({{}})). When a subnet becomes IP address constrained, EC2 may think that it can successfully launch an instance in the subnet; however, when the CNI tries to assign IPs to the pods, there are none remaining. In this case, your pod will stay in a `ContainerCreating` state until an IP address is freed in the subnet and the CNI can assign one to the pod. +When a node is launched by Karpenter, it is assigned to a subnet within your VPC based on the [`subnetSelector`]({{}}) value in your [`AWSNodeTemplate`]({{}})). When a subnet becomes IP address constrained, EC2 may think that it can successfully launch an instance in the subnet; however, when the CNI tries to assign IPs to the pods, there are none remaining. In this case, your pod will stay in a `ContainerCreating` state until an IP address is freed in the subnet and the CNI can assign one to the pod. ##### Solutions @@ -579,7 +607,7 @@ This means that your CNI plugin is out of date. You can find instructions on how ### Node terminates before ready on failed encrypted EBS volume If you are using a custom launch template and an encrypted EBS volume, the IAM principal launching the node may not have sufficient permissions to use the KMS customer managed key (CMK) for the EC2 EBS root volume. -This issue also applies to [Block Device Mappings]({{}}) specified in the Provisioner. +This issue also applies to [Block Device Mappings]({{}}) specified in the Provisioner. In either case, this results in the node terminating almost immediately upon creation. Keep in mind that it is possible that EBS Encryption can be enabled without your knowledge. @@ -637,6 +665,21 @@ This typically occurs when the node has not been considered fully initialized fo This error indicates that the `vpc.amazonaws.com/pod-eni` resource was never reported on the node. If you've enabled Pod ENI for Karpenter nodes via the `aws.enablePodENI` setting, you will need to make the corresponding change to the VPC CNI to enable [security groups for pods](https://docs.aws.amazon.com/eks/latest/userguide/security-groups-for-pods.html) which will cause the resource to be registered. +### AWS Node Termination Handler (NTH) interactions +Karpenter [doesn't currently support draining and terminating on spot rebalance recommendations]({{< ref "concepts/disruption#interruption" >}}). Users who want support for both drain and terminate on spot interruption as well as drain and termination on spot rebalance recommendations may install Node Termination Handler (NTH) on their clusters to support this behavior. + +These two components do not share information between each other, meaning if you have drain and terminate functionality enabled on NTH, NTH may remove a node for a spot rebalance recommendation. Karpenter will replace the node to fulfill the pod capacity that was being fulfilled by the old node; however, Karpenter won't be aware of the reason that that node was terminated. This means that Karpenter may launch the same instance type that was just deprovisioned, causing a spot rebalance recommendation to be sent again. This can result in very short-lived instances where NTH continually removes nodes and Karpeneter re-launches the same instance type over and over again. + +Karpenter doesn't recommend reacting to spot rebalance recommendations when running Karpenter with spot nodes; however, if you absolutely require this functionality, note that the above scenario is possible. +Spot instances are time limited and, therefore, interruptible. When a signal is sent by AWS, it triggers actions from NTH and Karpenter, where the former signals a shutdown and the later provisions, creating a recursive situation. +This can be mitigated by either completely removing NTH or by setting the following values: + +* enableSpotInterruptionDraining: If false, do not drain nodes when the spot interruption termination notice is received. Only used in IMDS mode. +enableSpotInterruptionDraining: false + +* enableRebalanceDrainin: If true, drain nodes when the rebalance recommendation notice is received. Only used in IMDS mode. +enableRebalanceDraining: false + ## Pricing ### Stale pricing data on isolated subnet @@ -652,4 +695,4 @@ caused by: Post "https://api.pricing.us-east-1.amazonaws.com/": dial tcp 52.94.2 This network timeout occurs because there is no VPC endpoint available for the [Price List Query API.](https://docs.aws.amazon.com/awsaccountbilling/latest/aboutv2/using-pelong.html). To workaround this issue, Karpenter ships updated on-demand pricing data as part of the Karpenter binary; however, this means that pricing data will only be updated on Karpenter version upgrades. To disable pricing lookups and avoid the error messages, set the `AWS_ISOLATED_VPC` environment variable (or the `--aws-isolated-vpc` option) to true. -See [Environment Variables / CLI Flags]({{}}) for details. +See [Environment Variables / CLI Flags]({{}}) for details. diff --git a/website/content/en/v0.35/upgrading/_index.md b/website/content/en/v0.35/upgrading/_index.md new file mode 100644 index 000000000000..ef1368b22a75 --- /dev/null +++ b/website/content/en/v0.35/upgrading/_index.md @@ -0,0 +1,10 @@ +--- +title: "Upgrading" +linkTitle: "Upgrading" +weight: 30 +description: > + Upgrading Karpenter guide and reference +cascade: + type: docs +--- + diff --git a/website/content/en/v0.35/upgrading/compatibility.md b/website/content/en/v0.35/upgrading/compatibility.md new file mode 100644 index 000000000000..0f2c8d6cab94 --- /dev/null +++ b/website/content/en/v0.35/upgrading/compatibility.md @@ -0,0 +1,101 @@ +--- +title: "Compatibility" +linkTitle: "Compatibility" +weight: 20 +description: > + Compatibility issues for Karpenter +--- + +# Compatibility + +To make upgrading easier we aim to minimize the introduction of breaking changes. +Before you begin upgrading Karpenter, consider Karpenter compatibility issues related to Kubernetes and the NodePool API (previously Provisioner). + +## Compatibility Matrix + +[comment]: <> (the content below is generated from hack/docs/compataiblitymetrix_gen_docs.go) + +| KUBERNETES | 1.23 | 1.24 | 1.25 | 1.26 | 1.27 | 1.28 | 1.29 | +|------------|----------|----------|----------|----------|----------|----------|------------| +| karpenter | \>= 0.21 | \>= 0.21 | \>= 0.25 | \>= 0.28 | \>= 0.28 | \>= 0.31 | \>= 0.34.0 | + +[comment]: <> (end docs generated content from hack/docs/compataiblitymetrix_gen_docs.go) + +{{% alert title="Note" color="warning" %}} +Karpenter currently does not support the following [new `topologySpreadConstraints` keys](https://kubernetes.io/blog/2023/04/17/fine-grained-pod-topology-spread-features-beta/), promoted to beta in Kubernetes 1.27: +- `matchLabelKeys` +- `nodeAffinityPolicy` +- `nodeTaintsPolicy` + +For more information on Karpenter's support for these keys, view [this tracking issue](https://github.com/aws/karpenter-core/issues/430). +{{% /alert %}} + +{{% alert title="Note" color="warning" %}} +Karpenter supports using [Kubernetes Common Expression Language](https://kubernetes.io/docs/reference/using-api/cel/) for validating its Custom Resource Definitions out-of-the-box; however, this feature is not supported on versions of Kubernetes < 1.25. If you are running an earlier version of Kubernetes, you will need to use the Karpenter admission webhooks for validation instead. You can enable these webhooks with `--set webhook.enabled=true` when applying the Karpenter Helm chart. +{{% /alert %}} + +## Compatibility issues + +When we introduce a breaking change, we do so only as described in this document. + +Karpenter follows [Semantic Versioning 2.0.0](https://semver.org/) in its stable release versions, while in +major version zero (`0.y.z`) [anything may change at any time](https://semver.org/#spec-item-4). +However, to further protect users during this phase we will only introduce breaking changes in minor releases (releases that increment y in x.y.z). +Note this does not mean every minor upgrade has a breaking change as we will also increment the +minor version when we release a new feature. + +Users should therefore check to see if there is a breaking change every time they are upgrading to a new minor version. + +### How Do We Break Incompatibility? + +When there is a breaking change we will: + +* Increment the minor version when in major version 0 +* Add a permanent separate section named `upgrading to x.y.z+` under [release upgrade notes](#release-upgrade-notes) + clearly explaining the breaking change and what needs to be done on the user side to ensure a safe upgrade +* Add the sentence “This is a breaking change, please refer to the above link for upgrade instructions” to the top of the release notes and in all our announcements + +### How Do We Find Incompatibilities? + +Besides the peer review process for all changes to the code base we also do the followings in order to find +incompatibilities: +* (To be implemented) To check the compatibility of the application, we will automate tests for installing, uninstalling, upgrading from an older version, and downgrading to an older version +* (To be implemented) To check the compatibility of the documentation with the application, we will turn the commands in our documentation into scripts that we can automatically run + +### Security Patches + +While we are in major version 0 we will not release security patches to older versions. +Rather we provide the patches in the latest versions. +When at major version 1 we will have an EOL (end of life) policy where we provide security patches +for a subset of older versions and deprecate the others. + +## Release Types + +Karpenter offers three types of releases. This section explains the purpose of each release type and how the images for each release type are tagged in our [public image repository](https://gallery.ecr.aws/karpenter). + +### Stable Releases + +Stable releases are the most reliable releases that are released with weekly cadence. Stable releases are our only recommended versions for production environments. +Sometimes we skip a stable release because we find instability or problems that need to be fixed before having a stable release. +Stable releases are tagged with a semantic version prefixed by a `v`. For example `v0.13.0`. + +### Release Candidates + +We consider having release candidates for major and important minor versions. Our release candidates are tagged like `vx.y.z-rc.0`, `vx.y.z-rc.1`. The release candidate will then graduate to `vx.y.z` as a normal stable release. +By adopting this practice we allow our users who are early adopters to test out new releases before they are available to the wider community, thereby providing us with early feedback resulting in more stable releases. + +### Snapshot Releases + +We release a snapshot release for every commit that gets merged into [`aws/karpenter-provider-aws`](https://www.github.com/aws/karpenter-provider-aws). This enables users to immediately try a new feature or fix right after it's merged rather than waiting days or weeks for release. + +Snapshot releases are not made available in the same public ECR repository as other release types, they are instead published to a separate private ECR repository. +Helm charts are published to `oci://{{< param "snapshot_repo.account_id" >}}.dkr.ecr.{{< param "snapshot_repo.region" >}}.amazonaws.com/karpenter/snapshot/karpenter` and are tagged with the git commit hash prefixed by the Karpenter major version (e.g. `0-fc17bfc89ebb30a3b102a86012b3e3992ec08adf`). +Anyone with an AWS account can pull from this repository, but must first authenticate: + +```bash +aws ecr get-login-password --region {{< param "snapshot_repo.region" >}} | docker login --username AWS --password-stdin {{< param "snapshot_repo.account_id" >}}.dkr.ecr.{{< param "snapshot_repo.region" >}}.amazonaws.com +``` + +{{% alert title="Note" color="warning" %}} +Snapshot releases are suitable for testing, and troubleshooting but they should not be used in production environments. Snapshot releases are ephemeral and will be removed 90 days after they were published. +{{% /alert %}} diff --git a/website/content/en/v0.35/upgrading/upgrade-guide.md b/website/content/en/v0.35/upgrading/upgrade-guide.md new file mode 100644 index 000000000000..42acba982977 --- /dev/null +++ b/website/content/en/v0.35/upgrading/upgrade-guide.md @@ -0,0 +1,349 @@ +--- +title: "Upgrade Guide" +linkTitle: "Upgrade Guide" +weight: 10 +description: > + Learn about upgrading Karpenter +--- + +Karpenter is a controller that runs in your cluster, but it is not tied to a specific Kubernetes version, as the Cluster Autoscaler is. +Use your existing upgrade mechanisms to upgrade your core add-ons in Kubernetes and keep Karpenter up to date on bug fixes and new features. +This guide contains information needed to upgrade to the latest release of Karpenter, along with compatibility issues you need to be aware of when upgrading from earlier Karpenter versions. + +### CRD Upgrades + +Karpenter ships with a few Custom Resource Definitions (CRDs). These CRDs are published: +* As an independent Helm chart [karpenter-crd](https://gallery.ecr.aws/karpenter/karpenter-crd) - [source](https://github.com/aws/karpenter/blob/main/charts/karpenter-crd) that can be used by Helm to manage the lifecycle of these CRDs. To upgrade or install `karpenter-crd` run: + ```bash + KARPENTER_NAMESPACE=kube-system + helm upgrade --install karpenter-crd oci://public.ecr.aws/karpenter/karpenter-crd --version x.y.z --namespace "${KARPENTER_NAMESPACE}" --create-namespace + ``` + +{{% alert title="Note" color="warning" %}} +If you get the error `invalid ownership metadata; label validation error:` while installing the `karpenter-crd` chart from an older version of Karpenter, follow the [Troubleshooting Guide]({{}}) for details on how to resolve these errors. +{{% /alert %}} + +* As part of the helm chart [karpenter](https://gallery.ecr.aws/karpenter/karpenter) - [source](https://github.com/aws/karpenter/blob/main/charts/karpenter/crds). Helm [does not manage the lifecycle of CRDs using this method](https://helm.sh/docs/chart_best_practices/custom_resource_definitions/), the tool will only install the CRD during the first installation of the Helm chart. Subsequent chart upgrades will not add or remove CRDs, even if the CRDs have changed. When CRDs are changed, we will make a note in the version's upgrade guide. + +In general, you can reapply the CRDs in the `crds` directory of the Karpenter Helm chart: + +```shell +kubectl apply -f https://raw.githubusercontent.com/aws/karpenter/v0.35.4/pkg/apis/crds/karpenter.sh_nodepools.yaml +kubectl apply -f https://raw.githubusercontent.com/aws/karpenter/v0.35.4/pkg/apis/crds/karpenter.sh_nodeclaims.yaml +kubectl apply -f https://raw.githubusercontent.com/aws/karpenter/v0.35.4/pkg/apis/crds/karpenter.k8s.aws_ec2nodeclasses.yaml +``` + + + +### Upgrading to `0.35.0`+ + +{{% alert title="Warning" color="warning" %}} +`0.33.0`+ _only_ supports Karpenter v1beta1 APIs and will not work with existing Provisioner, AWSNodeTemplate or Machine alpha APIs. Do not upgrade to `0.35.0`+ without first [upgrading to `0.32.x`]({{}}). This version supports both the alpha and beta APIs, allowing you to migrate all of your existing APIs to beta APIs without experiencing downtime. +{{% /alert %}} + +* Karpenter OCI tags and Helm chart version are now valid semantic versions, meaning that the `v` prefix from the git tag has been removed and they now follow the `x.y.z` pattern. + +### Upgrading to `0.34.0`+ + +{{% alert title="Warning" color="warning" %}} +`0.33.0`+ _only_ supports Karpenter v1beta1 APIs and will not work with existing Provisioner, AWSNodeTemplate or Machine alpha APIs. Do not upgrade to `0.34.0`+ without first [upgrading to `0.32.x`]({{}}). This version supports both the alpha and beta APIs, allowing you to migrate all of your existing APIs to beta APIs without experiencing downtime. +{{% /alert %}} + +{{% alert title="Warning" color="warning" %}} +The Ubuntu EKS optimized AMI has moved from 20.04 to 22.04 for Kubernetes 1.29+. This new AMI version is __not currently__ supported for users relying on AMI auto-discovery with the Ubuntu AMI family. More details can be found in this [GitHub issue](https://github.com/aws/karpenter-provider-aws/issues/5572). Please review this issue before upgrading to Kubernetes 1.29 if you are using the Ubuntu AMI family. Upgrading to 1.29 without making any changes to your EC2NodeClass will result in Karpenter being unable to create new nodes. +{{% /alert %}} + +* Karpenter now supports `nodepool.spec.disruption.budgets`, which allows users to control the speed of disruption in the cluster. Since this requires an update to the Custom Resource, before upgrading, you should re-apply the new updates to the CRDs. Check out [Disruption Budgets]({{}}) for more. +* With Disruption Budgets, Karpenter will disrupt multiple batches of nodes simultaneously, which can result in overall quicker scale-down of your cluster. Before `0.34.0`, Karpenter had a hard-coded parallelism limit for each type of disruption. In `0.34.0`+, Karpenter will now disrupt at most 10% of nodes for a given NodePool. There is no setting that will be perfectly equivalent with the behavior prior to `0.34.0`. When considering how to configure your budgets, please refer to the following limits for versions prior to `0.34.0`: + * `Empty Expiration / Empty Drift / Empty Consolidation`: infinite parallelism + * `Non-Empty Expiration / Non-Empty Drift / Single-Node Consolidation`: one node at a time + * `Multi-Node Consolidation`: max 100 nodes +* To support Disruption Budgets, `0.34.0`+ includes critical changes to Karpenter's core controllers, which allows Karpenter to consider multiple batches of disrupting nodes simultaneously. This increases Karpenter's performance with the potential downside of higher CPU and memory utilization from the Karpenter pod. While the magnitude of this difference varies on a case-by-case basis, when upgrading to Karpenter `0.34.0`+, please note that you may need to increase the resources allocated to the Karpenter controller pods. +* Karpenter now adds a default `podSecurityContext` that configures the `fsgroup: 65536` of volumes in the pod. If you are using sidecar containers, you should review if this configuration is compatible for them. You can disable this default `podSecurityContext` through helm by performing `--set podSecurityContext=null` when installing/upgrading the chart. +* The `dnsPolicy` for the Karpenter controller pod has been changed back to the Kubernetes cluster default of `ClusterFirst`. Setting our `dnsPolicy` to `Default` (confusingly, this is not the Kubernetes cluster default) caused more confusion for any users running IPv6 clusters with dual-stack nodes or anyone running Karpenter with dependencies on cluster services (like clusters running service meshes). This change may be breaking for any users on Fargate or MNG who were allowing Karpenter to manage their in-cluster DNS service (`core-dns` on most clusters). If you still want the old behavior here, you can change the `dnsPolicy` to point to use `Default` by setting the helm value on install/upgrade with `--set dnsPolicy=Default`. More details on this issue can be found in the following Github issues: [#2186](https://github.com/aws/karpenter-provider-aws/issues/2186) and [#4947](https://github.com/aws/karpenter-provider-aws/issues/4947). +* Karpenter now disallows `nodepool.spec.template.spec.resources` to be set. The webhook validation never allowed `nodepool.spec.template.spec.resources`. We are now ensuring that CEL validation also disallows `nodepool.spec.template.spec.resources` to be set. If you were previously setting the resources field on your NodePool, ensure that you remove this field before upgrading to the newest version of Karpenter or else updates to the resource may fail on the new version. + +### Upgrading to `0.33.0`+ + +{{% alert title="Warning" color="warning" %}} +`0.33.0`+ _only_ supports Karpenter v1beta1 APIs and will not work with existing Provisioner, AWSNodeTemplate or Machine alpha APIs. **Do not** upgrade to `0.33.0`+ without first [upgrading to `0.32.x`]({{}}). This version supports both the alpha and beta APIs, allowing you to migrate all of your existing APIs to beta APIs without experiencing downtime. +{{% /alert %}} + +* Karpenter no longer supports using the `karpenter.sh/provisioner-name` label in NodePool labels and requirements or in application node selectors, affinities, or topologySpreadConstraints. If you were previously using this label to target applications to specific Provisioners, you should update your applications to use the `karpenter.sh/nodepool` label instead before upgrading. If you upgrade without changing these labels, you may begin to see pod scheduling failures for these applications. +* Karpenter now tags `spot-instances-request` with the same tags that it tags instances, volumes, and primary ENIs. This means that you will now need to add `ec2:CreateTags` permission for `spot-instances-request`. You can also further scope your controller policy for the `ec2:RunInstances` action to require that it launches the `spot-instances-request` with these specific tags. You can view an example of scoping these actions in the [Getting Started Guide's default CloudFormation controller policy](https://github.com/aws/karpenter/blob/v0.33.0/website/content/en/preview/getting-started/getting-started-with-karpenter/cloudformation.yaml#L61). +* We now recommend that you set the installation namespace for your Karpenter controllers to `kube-system` to denote Karpenter as a critical cluster component. This ensures that requests from the Karpenter controllers are treated with higher priority by assigning them to a different [PriorityLevelConfiguration](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/#prioritylevelconfiguration) than generic requests from other namespaces. For more details on API Priority and Fairness, read the [Kubernetes API Priority and Fairness Conceptual Docs](https://kubernetes.io/docs/concepts/cluster-administration/flow-control/). Note: Changing the namespace for your Karpenter release will cause the service account namespace to change. If you are using IRSA for authentication with AWS, you will need to change scoping set in the controller's trust policy from `karpenter:karpenter` to `kube-system:karpenter`. +* `0.33.0` disables mutating and validating webhooks by default in favor of using [Common Expression Language for CRD validation](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#validation). The Common Expression Language Validation Feature [is enabled by default on EKS 1.25](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#validation-rules). If you are using Kubernetes version >= 1.25, no further action is required. If you are using a Kubernetes version below 1.25, you now need to set `DISABLE_WEBHOOK=false` in your container environment variables or `--set webhook.enabled=true` if using Helm. View the [Webhook Support Deprecated in Favor of CEL Section of the v1beta1 Migration Guide]({{}}). +* `0.33.0` drops support for passing settings through the `karpenter-global-settings` ConfigMap. You should pass settings through the container environment variables in the Karpenter deployment manifest. View the [Global Settings Section of the v1beta1 Migration Guide]({{}}) for more details. +* `0.33.0` enables `Drift=true` by default in the `FEATURE_GATES`. If you previously didn't enable the feature gate, Karpenter will now check if there is a difference between the desired state of your nodes declared in your NodePool and the actual state of your nodes. View the [Drift Section of Disruption Conceptual Docs]({{}}) for more details. +* `0.33.0` drops looking up the `zap-logger-config` through ConfigMap discovery. Instead, Karpenter now expects the logging config to be mounted on the filesystem if you are using this to configure Zap logging. This is not enabled by default, but can be enabled through `--set logConfig.enabled=true` in the Helm values. If you are setting any values in the `logConfig` from the `0.32.x` upgrade, such as `logConfig.logEncoding`, note that you will have to explicitly set `logConfig.enabled=true` alongside it. Also, note that setting the Zap logging config is a deprecated feature in beta and is planned to be dropped at v1. View the [Logging Configuration Section of the v1beta1 Migration Guide]({{}}) for more details. +* `0.33.0` change the default `LOG_LEVEL` from `debug` to `info` by default. If you are still enabling logging configuration through the `zap-logger-config`, no action is required. +* `0.33.0` drops support for comma delimited lists on tags for `SubnetSelectorTerm`, `SecurityGroupsSelectorTerm`, and `AMISelectorTerm`. Karpenter now supports multiple terms for each of the selectors which means that we can specify a more explicit OR-based constraint through separate terms rather than a comma-delimited list of values. + +### Upgrading to `0.32.0`+ + +{{% alert title="Warning" color="warning" %}} +Karpenter `0.32.0` introduces v1beta1 APIs, including _significant_ changes to the API and installation procedures for the Karpenter controllers. **Do not** upgrade to `0.32.0`+ without referencing the [v1beta1 Migration Upgrade Procedure]({{}}). + +This version includes **dual support** for both alpha and beta APIs to ensure that you can slowly migrate your existing Provisioner, AWSNodeTemplate, and Machine alpha APIs to the newer NodePool, EC2NodeClass, and NodeClaim beta APIs. + +Note that if you are rolling back after upgrading to `0.32.0`, note that `0.31.4` is the only version that supports handling rollback after you have deployed the v1beta1 APIs to your cluster. +{{% /alert %}} + +* Karpenter now serves the webhook prometheus metrics server on port `8001`. If this port is already in-use on the pod or you are running in `hostNetworking` mode, you may need to change this port value. You can configure this port value through the `WEBHOOK_METRICS_PORT` environment variable or the `webhook.metrics.port` value if installing via Helm. +* Karpenter now exposes the ability to disable webhooks through the `webhook.enabled=false` value. This value will disable the webhook server and will prevent any permissions, mutating or validating webhook configurations from being deployed to the cluster. +* Karpenter now moves all logging configuration for the Zap logger into the `logConfig` values block. Configuring Karpenter logging with this mechanism _is_ deprecated and will be dropped at v1. Karpenter now only surfaces logLevel through the `logLevel` helm value. If you need more advanced configuration due to log parsing constraints, we recommend configuring your log parser to handle Karpenter's Zap JSON logging. +* The default log encoding changed from `console` to `json`. If you were previously not setting the type of log encoding, this default will change with the Helm chart. If you were setting the value through `logEncoding`, this value will continue to work until `0.33.x` but it is deprecated in favor of `logConfig.logEncoding` +* Karpenter now uses the `karpenter.sh/disruption:NoSchedule=disrupting` taint instead of the upstream `node.kubernetes.io/unschedulable` taint for nodes spawned with a NodePool to prevent pods from scheduling to nodes being disrupted. Pods that previously tolerated the `node.kubernetes.io/unschedulable` taint that previously weren't evicted during termination will now be evicted. This most notably affects DaemonSets, which have the `node.kubernetes.io/unschedulable` toleration by default, where Karpenter will now remove these pods during termination. If you want your specific pods to not be evicted when nodes are scaled down, you should add a toleration to the pods with the following: `Key=karpenter.sh/disruption, Effect=NoSchedule, Operator=Equals, Values=disrupting`. + * Note: Karpenter will continue to use the old `node.kubernetes.io/unschedulable` taint for nodes spawned with a Provisioner. + +### Upgrading to `0.31.0`+ + +* Karpenter moved its `securityContext` constraints from pod-wide to only applying to the Karpenter container exclusively. If you were previously relying on the pod-wide `securityContext` for your sidecar containers, you will now need to set these values explicitly in your sidecar container configuration. + +### Upgrading to `0.30.0`+ + +* Karpenter will now [statically drift]({{}}) on both Provisioner and AWSNodeTemplate Fields. For Provisioner Static Drift, the `karpenter.sh/provisioner-hash` annotation must be present on both the Provisioner and Machine. For AWSNodeTemplate drift, the `karpenter.k8s.aws/nodetemplate-hash` annotation must be present on the AWSNodeTemplate and Machine. Karpenter will not add these annotations to pre-existing nodes, so each of these nodes will need to be recycled one time for the annotations to be added. +* Karpenter will now fail validation on AWSNodeTemplates and Provisioner `spec.provider` that have `amiSelectors`, `subnetSelectors`, or `securityGroupSelectors` set with a combination of id selectors (`aws-ids`, `aws::ids`) and other selectors. +* Karpenter now statically sets the `securityContext` at both the pod and container-levels and doesn't allow override values to be passed through the Helm chart. This change was made to adhere to [Restricted Pod Security Standard](https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted), which follows pod hardening best practices. + +{{% alert title="Note" color="primary" %}} +If you have sidecar containers configured to run alongside Karpenter that cannot tolerate the [pod-wide `securityContext` constraints](https://github.com/aws/karpenter/blob/v0.30.0/charts/karpenter/templates/deployment.yaml#L40), you will need to specify overrides to the sidecar `securityContext` in your deployment. +{{% /alert %}} + +### Upgrading to `0.29.0`+ + +{{% alert title="Warning" color="warning" %}} +Karpenter `0.29.1` contains a [file descriptor and memory leak bug](https://github.com/aws/karpenter/issues/4296) that leads to Karpenter getting OOMKilled and restarting at the point that it hits its memory or file descriptor limit. Karpenter `0.29.2`+ fixes this leak. +{{% /alert %}} + +* Karpenter has changed the default metrics service port from 8080 to 8000 and the default webhook service port from 443 to 8443. In `0.28.0`, the Karpenter pod port was changed to 8000, but referenced the service by name, allowing users to scrape the service at port 8080 for metrics. `0.29.0` aligns the two ports so that service and pod metrics ports are the same. These ports are set by the `controller.metrics.port` and `webhook.port` Helm chart values, so if you have previously set these to non-default values, you may need to update your Prometheus scraper to match these new values. + +* Karpenter will now reconcile nodes that are drifted due to their Security Groups or their Subnets. If your AWSNodeTemplate's Security Groups differ from the Security Groups used for an instance, Karpenter will consider it drifted. If the Subnet used by an instance is not contained in the allowed list of Subnets for an AWSNodeTemplate, Karpenter will also consider it drifted. + * Since Karpenter uses tags for discovery of Subnets and SecurityGroups, check the [Threat Model]({{}}) to see how to manage this IAM Permission. + +### Upgrading to `0.28.0`+ + +{{% alert title="Warning" color="warning" %}} +Karpenter `0.28.0` is incompatible with Kubernetes version 1.26+, which can result in additional node scale outs when using `--cloudprovider=external`, which is the default for the EKS Optimized AMI. See: https://github.com/aws/karpenter-core/pull/375. Karpenter `0.28.1`+ fixes this issue and is compatible with Kubernetes version 1.26+. +{{% /alert %}} + +* The `extraObjects` value is now removed from the Helm chart. Having this value in the chart proved to not work in the majority of Karpenter installs and often led to anti-patterns, where the Karpenter resources installed to manage Karpenter's capacity were directly tied to the install of the Karpenter controller deployments. The Karpenter team recommends that, if you want to install Karpenter manifests alongside the Karpenter Helm chart, to do so by creating a separate chart for the manifests, creating a dependency on the controller chart. +* The `aws.nodeNameConvention` setting is now removed from the [`karpenter-global-settings`]({{}}) ConfigMap. Because Karpenter is now driving its orchestration of capacity through Machines, it no longer needs to know the node name, making this setting obsolete. Karpenter ignores configuration that it doesn't recognize in the [`karpenter-global-settings`]({{}}) ConfigMap, so leaving the `aws.nodeNameConvention` in the ConfigMap will simply cause this setting to be ignored. +* Karpenter now defines a set of "restricted tags" which can't be overridden with custom tagging in the AWSNodeTemplate or in the [`karpenter-global-settings`]({{}}) ConfigMap. If you are currently using any of these tag overrides when tagging your instances, webhook validation will now fail. These tags include: + + * `karpenter.sh/managed-by` + * `karpenter.sh/provisioner-name` + * `kubernetes.io/cluster/${CLUSTER_NAME}` + +* The following metrics changed their meaning, based on the introduction of the Machine resource: + * `karpenter_nodes_terminated`: Use `karpenter_machines_terminated` if you are interested in the reason why a Karpenter machine was deleted. `karpenter_nodes_terminated` now only tracks the count of terminated nodes without any additional labels. + * `karpenter_nodes_created`: Use `karpenter_machines_created` if you are interested in the reason why a Karpenter machine was created. `karpenter_nodes_created` now only tracks the count of created nodes without any additional labels. + * `karpenter_deprovisioning_replacement_node_initialized_seconds`: This metric has been replaced in favor of `karpenter_deprovisioning_replacement_machine_initialized_seconds`. +* `0.28.0` introduces the Machine CustomResource into the `karpenter.sh` API Group and requires this CustomResourceDefinition to run properly. Karpenter now orchestrates its CloudProvider capacity through these in-cluster Machine CustomResources. When performing a scheduling decision, Karpenter will create a Machine, resulting in launching CloudProvider capacity. The kubelet running on the new capacity will then register the node to the cluster shortly after launch. + * If you are using Helm to upgrade between versions of Karpenter, note that [Helm does not automate the process of upgrading or install the new CRDs into your cluster](https://helm.sh/docs/chart_best_practices/custom_resource_definitions/#some-caveats-and-explanations). To install or upgrade the existing CRDs, follow the guidance under the [Custom Resource Definition (CRD) Upgrades]({{< relref "#custom-resource-definition-crd-upgrades" >}}) section of the upgrade guide. + * Karpenter will hydrate Machines on startup for existing capacity managed by Karpenter into the cluster. Existing capacity launched by an older version of Karpenter is discovered by finding CloudProvider capacity with the `karpenter.sh/provisioner-name` tag or the `karpenter.sh/provisioner-name` label on nodes. +* The metrics port for the Karpenter deployment was changed from 8080 to 8000. Users who scrape the pod directly for metrics rather than the service will need to adjust the commands they use to reference port 8000. Any users who scrape metrics from the service should be unaffected. + +{{% alert title="Warning" color="warning" %}} +Karpenter creates a mapping between CloudProvider machines and CustomResources in the cluster for capacity tracking. To ensure this mapping is consistent, Karpenter utilizes the following tag keys: + +* `karpenter.sh/managed-by` +* `karpenter.sh/provisioner-name` +* `kubernetes.io/cluster/${CLUSTER_NAME}` + +Because Karpenter takes this dependency, any user that has the ability to Create/Delete these tags on CloudProvider machines will have the ability to orchestrate Karpenter to Create/Delete CloudProvider machines as a side effect. Check the [Threat Model]({{}}) to see how this might affect you, and ways to mitigate this. +{{% /alert %}} + +{{% alert title="Rolling Back" color="warning" %}} +If, after upgrading to `0.28.0`+, a rollback to an older version of Karpenter needs to be performed, Karpenter will continue to function normally, though you will still have the Machine CustomResources on your cluster. You will need to manually delete the Machines and patch out the finalizers to fully complete the rollback. + +Karpenter marks CloudProvider capacity as "managed by" a Machine using the `karpenter-sh/managed-by` tag on the CloudProvider machine. It uses this tag to ensure that the Machine CustomResources in the cluster match the CloudProvider capacity managed by Karpenter. If these states don't match, Karpenter will garbage collect the capacity. Because of this, if performing an upgrade, followed by a rollback, followed by another upgrade to `0.28.0`+, ensure you remove the `karpenter.sh/managed-by` tags from existing capacity; otherwise, Karpenter will deprovision the capacity without a Machine CR counterpart. +{{% /alert %}} + +### Upgrading to `0.27.3`+ + +* The `defaulting.webhook.karpenter.sh` mutating webhook was removed in `0.27.3`. If you are coming from an older version of Karpenter where this webhook existed and the webhook was not managed by Helm, you may need to delete the stale webhook. + +```bash +kubectl delete mutatingwebhookconfigurations defaulting.webhook.karpenter.sh +``` + +### Upgrading to `0.27.0`+ + +* The Karpenter controller pods now deploy with `kubernetes.io/hostname` self anti-affinity by default. If you are running Karpenter in HA (high-availability) mode and you do not have enough nodes to match the number of pod replicas you are deploying with, you will need to scale-out your nodes for Karpenter. +* The following controller metrics changed and moved under the `controller_runtime` metrics namespace: + * `karpenter_metricscraper_...` + * `karpenter_deprovisioning_...` + * `karpenter_provisioner_...` + * `karpenter_interruption_...` +* The following controller metric names changed, affecting the `controller` label value under `controller_runtime_...` metrics. These metrics include: + * `podmetrics` -> `pod_metrics` + * `provisionermetrics` -> `provisioner_metrics` + * `metricscraper` -> `metric_scraper` + * `provisioning` -> `provisioner_trigger` + * `node-state` -> `node_state` + * `pod-state` -> `pod_state` + * `provisioner-state` -> `provisioner_state` +* The `karpenter_allocation_controller_scheduling_duration_seconds` metric name changed to `karpenter_provisioner_scheduling_duration_seconds` + +### Upgrading to `0.26.0`+ + +* The `karpenter.sh/do-not-evict` annotation no longer blocks node termination when running `kubectl delete node`. This annotation on pods will only block automatic deprovisioning that is considered "voluntary," that is, disruptions that can be avoided. Disruptions that Karpenter deems as "involuntary" and will ignore the `karpenter.sh/do-not-evict` annotation include spot interruption and manual deletion of the node. See [Disabling Deprovisioning]({{}}) for more details. +* Default resources `requests` and `limits` are removed from the Karpenter's controller deployment through the Helm chart. If you have not set custom resource `requests` or `limits` in your Helm values and are using Karpenter's defaults, you will now need to set these values in your Helm chart deployment. +* The `controller.image` value in the Helm chart has been broken out to a map consisting of `controller.image.repository`, `controller.image.tag`, and `controller.image.digest`. If manually overriding the `controller.image`, you will need to update your values to the new design. + +### Upgrading to `0.25.0`+ + +* Cluster Endpoint can now be automatically discovered. If you are using Amazon Elastic Kubernetes Service (EKS), you can now omit the `clusterEndpoint` field in your configuration. In order to allow the resolving, you have to add the permission `eks:DescribeCluster` to the Karpenter Controller IAM role. + +### Upgrading to `0.24.0`+ + +* Settings are no longer updated dynamically while Karpenter is running. If you manually make a change to the [`karpenter-global-settings`]({{}}) ConfigMap, you will need to reload the containers by restarting the deployment with `kubectl rollout restart -n karpenter deploy/karpenter` +* Karpenter no longer filters out instance types internally. Previously, `g2` (not supported by the NVIDIA device plugin) and FPGA instance types were filtered. The only way to filter instance types now is to set requirements on your provisioner or pods using well-known node labels described [here]({{}}). If you are currently using overly broad requirements that allows all of the `g` instance-category, you will want to tighten the requirement, or add an instance-generation requirement. +* `aws.tags` in [`karpenter-global-settings`]({{}}) ConfigMap is now a top-level field and expects the value associated with this key to be a JSON object of string to string. This is change from previous versions where keys were given implicitly by providing the key-value pair `aws.tags.: value` in the ConfigMap. + +### Upgrading to `0.22.0`+ + +* Do not upgrade to this version unless you are on Kubernetes >= v1.21. Karpenter no longer supports Kubernetes v1.20, but now supports Kubernetes v1.25. This change is due to the v1 PDB API, which was introduced in K8s v1.20 and subsequent removal of the v1beta1 API in K8s v1.25. + +### Upgrading to `0.20.0`+ + +* Prior to `0.20.0`, Karpenter would prioritize certain instance type categories absent of any requirements in the Provisioner. `0.20.0`+ removes prioritizing these instance type categories ("m", "c", "r", "a", "t", "i") in code. Bare Metal and GPU instance types are still deprioritized and only used if no other instance types are compatible with the node requirements. Since Karpenter does not prioritize any instance types, if you do not want exotic instance types and are not using the runtime Provisioner defaults, you will need to specify this in the Provisioner. + +### Upgrading to `0.19.0`+ + +* The karpenter webhook and controller containers are combined into a single binary, which requires changes to the Helm chart. If your Karpenter installation (Helm or otherwise) currently customizes the karpenter webhook, your deployment tooling may require minor changes. +* Karpenter now supports native interruption handling. If you were previously using Node Termination Handler for spot interruption handling and health events, you will need to remove the component from your cluster before enabling `aws.interruptionQueueName`. For more details on Karpenter's interruption handling, see the [Interruption Handling Docs]({{< ref "../concepts/disruption/#interruption" >}}). +* Instance category defaults are now explicitly persisted in the Provisioner, rather than handled implicitly in memory. By default, Provisioners will limit instance category to c,m,r. If any instance type constraints are applied, it will override this default. If you have created Provisioners in the past with unconstrained instance type, family, or category, Karpenter will now more flexibly use instance types than before. If you would like to apply these constraints, they must be included in the Provisioner CRD. +* Karpenter CRD raw YAML URLs have migrated from `https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.19.3/charts/karpenter/crds/...` to `https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.19.3/pkg/apis/crds/...`. If you reference static Karpenter CRDs or rely on `kubectl replace -f` to apply these CRDs from their remote location, you will need to migrate to the new location. +* Pods without an ownerRef (also called "controllerless" or "naked" pods) will now be evicted by default during node termination and consolidation. Users can prevent controllerless pods from being voluntarily disrupted by applying the `karpenter.sh/do-not-evict: "true"` annotation to the pods in question. +* The following CLI options/environment variables are now removed and replaced in favor of pulling settings dynamically from the [`karpenter-global-settings`]({{}}) ConfigMap. See the [Settings docs]({{}}) for more details on configuring the new values in the ConfigMap. + + * `CLUSTER_NAME` -> `settings.aws.clusterName` + * `CLUSTER_ENDPOINT` -> `settings.aws.clusterEndpoint` + * `AWS_DEFAULT_INSTANCE_PROFILE` -> `settings.aws.defaultInstanceProfile` + * `AWS_ENABLE_POD_ENI` -> `settings.aws.enablePodENI` + * `AWS_ENI_LIMITED_POD_DENSITY` -> `settings.aws.enableENILimitedPodDensity` + * `AWS_ISOLATED_VPC` -> `settings.aws.isolatedVPC` + * `AWS_NODE_NAME_CONVENTION` -> `settings.aws.nodeNameConvention` + * `VM_MEMORY_OVERHEAD` -> `settings.aws.vmMemoryOverheadPercent` + +### Upgrading to `0.18.0`+ + +* `0.18.0` removes the `karpenter_consolidation_nodes_created` and `karpenter_consolidation_nodes_terminated` prometheus metrics in favor of the more generic `karpenter_nodes_created` and `karpenter_nodes_terminated` metrics. You can still see nodes created and terminated by consolidation by checking the `reason` label on the metrics. Check out all the metrics published by Karpenter [here]({{}}). + +### Upgrading to `0.17.0`+ + +Karpenter's Helm chart package is now stored in [Karpenter's OCI (Open Container Initiative) registry](https://gallery.ecr.aws/karpenter/karpenter). The Helm CLI supports the new format since [v3.8.0+](https://helm.sh/docs/topics/registries/). +With this change [charts.karpenter.sh](https://charts.karpenter.sh/) is no longer updated but preserved to allow using older Karpenter versions. For examples on working with the Karpenter Helm charts look at [Install Karpenter Helm Chart]({{< ref "../getting-started/getting-started-with-karpenter/#install-karpenter-helm-chart" >}}). + +Users who have scripted the installation or upgrading of Karpenter need to adjust their scripts with the following changes: +1. There is no longer a need to add the Karpenter Helm repo with `helm repo add` +2. The full URL of the Helm chart needs to be present when using the `helm` CLI +3. If you were not prepending a `v` to the version (i.e. `0.17.0`), you will need to do so with the OCI chart (i.e `v0.17.0`). + +### Upgrading to `0.16.2`+ + +* `0.16.2` adds new kubeletConfiguration fields to the `provisioners.karpenter.sh` v1alpha5 CRD. The CRD will need to be updated to use the new parameters: +```bash +kubectl replace -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.16.2/charts/karpenter/crds/karpenter.sh_provisioners.yaml +``` + +### Upgrading to `0.16.0`+ + +* `0.16.0` adds a new weight field to the `provisioners.karpenter.sh` v1alpha5 CRD. The CRD will need to be updated to use the new parameters: +```bash +kubectl replace -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.16.0/charts/karpenter/crds/karpenter.sh_provisioners.yaml +``` + +### Upgrading to `0.15.0`+ + +* `0.15.0` adds a new consolidation field to the `provisioners.karpenter.sh` v1alpha5 CRD. The CRD will need to be updated to use the new parameters: +```bash +kubectl replace -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.15.0/charts/karpenter/crds/karpenter.sh_provisioners.yaml +``` + +### Upgrading to `0.14.0`+ + +* `0.14.0` adds new fields to the `provisioners.karpenter.sh` v1alpha5 and `awsnodetemplates.karpenter.k8s.aws` v1alpha1 CRDs. The CRDs will need to be updated to use the new parameters: + +```bash +kubectl replace -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.14.0/charts/karpenter/crds/karpenter.sh_provisioners.yaml + +kubectl replace -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.14.0/charts/karpenter/crds/karpenter.k8s.aws_awsnodetemplates.yaml +``` + +* `0.14.0` changes the way Karpenter discovers its dynamically generated AWS launch templates to use a tag rather than a Name scheme. The previous name scheme was `Karpenter-${CLUSTER_NAME}-*` which could collide with user created launch templates that Karpenter should not manage. The new scheme uses a tag on the launch template `karpenter.k8s.aws/cluster: ${CLUSTER_NAME}`. As a result, Karpenter will not clean-up dynamically generated launch templates using the old name scheme. You can manually clean these up with the following commands: + +```bash +## Find launch templates that match the naming pattern and you do not want to keep +aws ec2 describe-launch-templates --filters="Name=launch-template-name,Values=Karpenter-${CLUSTER_NAME}-*" + +## Delete launch template(s) that match the name but do not have the "karpenter.k8s.aws/cluster" tag +aws ec2 delete-launch-template --launch-template-id +``` + +* `0.14.0` introduces additional instance type filtering if there are no `node.kubernetes.io/instance-type` or `karpenter.k8s.aws/instance-family` or `karpenter.k8s.aws/instance-category` requirements that restrict instance types specified on the provisioner. This prevents Karpenter from launching bare metal and some older non-current generation instance types unless the provisioner has been explicitly configured to allow them. If you specify an instance type or family requirement that supplies a list of instance-types or families, that list will be used regardless of filtering. The filtering can also be completely eliminated by adding an `Exists` requirement for instance type or family. +```yaml + - key: node.kubernetes.io/instance-type + operator: Exists +``` + +* `0.14.0` introduces support for custom AMIs without the need for an entire launch template. You must add the `ec2:DescribeImages` permission to the Karpenter Controller Role for this feature to work. This permission is needed for Karpenter to discover custom images specified. Read the [Custom AMI documentation here]({{}}) to get started +* `0.14.0` adds an an additional default toleration (CriticalAddonOnly=Exists) to the Karpenter Helm chart. This may cause Karpenter to run on nodes with that use this Taint which previously would not have been schedulable. This can be overridden by using `--set tolerations[0]=null`. + +* `0.14.0` deprecates the `AWS_ENI_LIMITED_POD_DENSITY` environment variable in-favor of specifying `spec.kubeletConfiguration.maxPods` on the Provisioner. `AWS_ENI_LIMITED_POD_DENSITY` will continue to work when `maxPods` is not set on the Provisioner. If `maxPods` is set, it will override `AWS_ENI_LIMITED_POD_DENSITY` on that specific Provisioner. + +### Upgrading to `0.13.0`+ + +* `0.13.0` introduces a new CRD named `AWSNodeTemplate` which can be used to specify AWS Cloud Provider parameters. Everything that was previously specified under `spec.provider` in the Provisioner resource, can now be specified in the spec of the new resource. The use of `spec.provider` is deprecated but will continue to function to maintain backwards compatibility for the current API version (v1alpha5) of the Provisioner resource. `0.13.0` also introduces support for custom user data that doesn't require the use of a custom launch template. The user data can be specified in-line in the AWSNodeTemplate resource. + + If you are upgrading from `0.10.1` - `0.11.1`, a new CRD `awsnodetemplate` was added. In `0.12.0`, this crd was renamed to `awsnodetemplates`. Since Helm does not manage the lifecycle of CRDs, you will need to perform a few manual steps for this CRD upgrade: + 1. Make sure any `awsnodetemplate` manifests are saved somewhere so that they can be reapplied to the cluster. + 2. `kubectl delete crd awsnodetemplate` + 3. `kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.13.2/charts/karpenter/crds/karpenter.k8s.aws_awsnodetemplates.yaml` + 4. Perform the Karpenter upgrade to `0.13.0`+, which will install the new `awsnodetemplates` CRD. + 5. Reapply the `awsnodetemplate` manifests you saved from step 1, if applicable. +* `0.13.0` also adds EC2/spot price fetching to Karpenter to allow making more accurate decisions regarding node deployments. Our [getting started guide]({{< ref "../getting-started/getting-started-with-karpenter" >}}) documents this, but if you are upgrading Karpenter you will need to modify your Karpenter controller policy to add the `pricing:GetProducts` and `ec2:DescribeSpotPriceHistory` permissions. + +### Upgrading to `0.12.0`+ + +* `0.12.0` adds an OwnerReference to each Node created by a provisioner. Previously, deleting a provisioner would orphan nodes. Now, deleting a provisioner will cause Kubernetes [cascading delete](https://kubernetes.io/docs/concepts/architecture/garbage-collection/#cascading-deletion) logic to gracefully terminate the nodes using the Karpenter node finalizer. You may still orphan nodes by removing the owner reference. +* If you are upgrading from `0.10.1` - `0.11.1`, a new CRD `awsnodetemplate` was added. In `0.12.0`, this crd was renamed to `awsnodetemplates`. Since Helm does not manage the lifecycle of CRDs, you will need to perform a few manual steps for this CRD upgrade: + 1. Make sure any `awsnodetemplate` manifests are saved somewhere so that they can be reapplied to the cluster. + 2. `kubectl delete crd awsnodetemplate` + 3. `kubectl apply -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.12.1/charts/karpenter/crds/karpenter.k8s.aws_awsnodetemplates.yaml` + 4. Perform the Karpenter upgrade to `0.12.0`+, which will install the new `awsnodetemplates` CRD. + 5. Reapply the `awsnodetemplate` manifests you saved from step 1, if applicable. + +### Upgrading to `0.11.0`+ + +`0.11.0` changes the way that the `vpc.amazonaws.com/pod-eni` resource is reported. Instead of being reported for all nodes that could support the resources regardless of if the cluster is configured to support it, it is now controlled by a command line flag or environment variable. The parameter defaults to false and must be set if your cluster uses [security groups for pods](https://docs.aws.amazon.com/eks/latest/userguide/security-groups-for-pods.html). This can be enabled by setting the environment variable `AWS_ENABLE_POD_ENI` to true via the helm value `controller.env`. + +Other extended resources must be registered on nodes by their respective device plugins which are typically installed as DaemonSets (e.g. the `nvidia.com/gpu` resource will be registered by the [NVIDIA device plugin](https://github.com/NVIDIA/k8s-device-plugin). Previously, Karpenter would register these resources on nodes at creation and they would be zeroed out by `kubelet` at startup. By allowing the device plugins to register the resources, pods will not bind to the nodes before any device plugin initialization has occurred. + +`0.11.0` adds a `providerRef` field in the Provisioner CRD. To use this new field you will need to replace the Provisioner CRD manually: + +```shell +kubectl replace -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.11.0/charts/karpenter/crds/karpenter.sh_provisioners.yaml +``` + +### Upgrading to `0.10.0`+ + +`0.10.0` adds a new field, `startupTaints` to the provisioner spec. Standard Helm upgrades [do not upgrade CRDs](https://helm.sh/docs/chart_best_practices/custom_resource_definitions/#some-caveats-and-explanations) so the field will not be available unless the CRD is manually updated. This can be performed prior to the standard upgrade by applying the new CRD manually: + +```shell +kubectl replace -f https://raw.githubusercontent.com/aws/karpenter-provider-aws/v0.10.0/charts/karpenter/crds/karpenter.sh_provisioners.yaml +``` + +📝 If you don't perform this manual CRD update, Karpenter will work correctly except for rejecting the creation/update of provisioners that use `startupTaints`. + +### Upgrading to `0.6.2`+ + +If using Helm, the variable names have changed for the cluster's name and endpoint. You may need to update any configuration +that sets the old variable names. + +- `controller.clusterName` is now `clusterName` +- `controller.clusterEndpoint` is now `clusterEndpoint` diff --git a/website/go.mod b/website/go.mod index 6d12c9ed5d1c..a0532639df1c 100644 --- a/website/go.mod +++ b/website/go.mod @@ -1,6 +1,6 @@ module github.com/aws/karpenter-provider-aws/website -go 1.21 +go 1.22 require ( github.com/google/docsy v0.6.0 // indirect diff --git a/website/hugo.yaml b/website/hugo.yaml index 60e9a2f03b91..1bbb068ef082 100644 --- a/website/hugo.yaml +++ b/website/hugo.yaml @@ -46,8 +46,8 @@ imaging: quality: 75 anchor: smart params: - copyright: 'Amazon.com, Inc. or its affiliates.' - github_repo: 'https://github.com/aws/karpenter-provider-aws' + copyright: "Amazon.com, Inc. or its affiliates." + github_repo: "https://github.com/aws/karpenter-provider-aws" github_subdir: website github_branch: main images: @@ -69,28 +69,28 @@ params: links: developer: - name: GitHub - url: 'https://github.com/aws/karpenter-provider-aws' + url: "https://github.com/aws/karpenter-provider-aws" icon: fab fa-github desc: Development takes place here! - name: Slack - url: 'https://slack.k8s.io/' + url: "https://slack.k8s.io/" icon: fab fa-slack - desc: 'Chat with us on Slack in the #aws-provider channel' - latest_release_version: v0.33.1 - latest_k8s_version: 1.28 + desc: "Chat with us on Slack in the #aws-provider channel" + latest_release_version: 0.35.4 + latest_k8s_version: 1.29 versions: + - v0.35 + - v0.34 - v0.33 - v0.32 - - v0.31 - - v0.30 - preview menu: main: - name: GitHub weight: 99 - url: 'https://github.com/aws/karpenter-provider-aws' + url: "https://github.com/aws/karpenter-provider-aws" pre: - name: Docs weight: 20 - url: 'docs' + url: "docs" pre: diff --git a/website/layouts/partials/navbar-version-selector.html b/website/layouts/partials/navbar-version-selector.html index 5f4380d5ab16..08b4d896b907 100644 --- a/website/layouts/partials/navbar-version-selector.html +++ b/website/layouts/partials/navbar-version-selector.html @@ -1,7 +1,7 @@ {{ $pageDir := index (split (path.Dir .Page.RelPermalink) "/") 1 }} {{ if or (eq $pageDir "docs") (eq $pageDir "") }} - {{ $pageDir = printf "%s.%s" ( index ( split .Site.Params.latest_release_version "." ) 0 ) ( index ( split .Site.Params.latest_release_version "." ) 1 ) }} + {{ $pageDir = printf "v%s.%s" ( index ( split .Site.Params.latest_release_version "." ) 0 ) ( index ( split .Site.Params.latest_release_version "." ) 1 ) }} {{ end }} {{ $pagePath := replace .Page.RelPermalink "docs" "" }} {{ $pagePath = replace $pagePath $pageDir "" }} diff --git a/website/package-lock.json b/website/package-lock.json index 0833a1418c42..e1522aed9821 100644 --- a/website/package-lock.json +++ b/website/package-lock.json @@ -10,8 +10,8 @@ "devDependencies": { "ansi-regex": ">=5.0.1", "autoprefixer": "^10.4.17", - "hugo-extended": "0.121.2", - "postcss": "^8.4.33", + "hugo-extended": "0.122.0", + "postcss": "^8.4.35", "postcss-cli": "^11.0.0" } }, @@ -257,9 +257,9 @@ } }, "node_modules/autoprefixer": { - "version": "10.4.17", - "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.4.17.tgz", - "integrity": "sha512-/cpVNRLSfhOtcGflT13P2794gVSgmPgTR+erw5ifnMLZb0UnSlkK4tquLmkd3BhA+nLo5tX8Cu0upUsGKvKbmg==", + "version": "10.4.18", + "resolved": "https://registry.npmjs.org/autoprefixer/-/autoprefixer-10.4.18.tgz", + "integrity": "sha512-1DKbDfsr6KUElM6wg+0zRNkB/Q7WcKYAaK+pzXn+Xqmszm/5Xa9coeNdtP88Vi+dPzZnMjhge8GIV49ZQkDa+g==", "dev": true, "funding": [ { @@ -276,8 +276,8 @@ } ], "dependencies": { - "browserslist": "^4.22.2", - "caniuse-lite": "^1.0.30001578", + "browserslist": "^4.23.0", + "caniuse-lite": "^1.0.30001591", "fraction.js": "^4.3.7", "normalize-range": "^0.1.2", "picocolors": "^1.0.0", @@ -345,9 +345,9 @@ } }, "node_modules/browserslist": { - "version": "4.22.2", - "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.22.2.tgz", - "integrity": "sha512-0UgcrvQmBDvZHFGdYUehrCNIazki7/lUP3kkoi/r3YB2amZbFM9J43ZRkJTXBUZK4gmx56+Sqk9+Vs9mwZx9+A==", + "version": "4.23.0", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.23.0.tgz", + "integrity": "sha512-QW8HiM1shhT2GuzkvklfjcKDiWFXHOeFCIA/huJPwHsslwcydgk7X+z2zXpEijP98UCY7HbubZt5J2Zgvf0CaQ==", "dev": true, "funding": [ { @@ -364,8 +364,8 @@ } ], "dependencies": { - "caniuse-lite": "^1.0.30001565", - "electron-to-chromium": "^1.4.601", + "caniuse-lite": "^1.0.30001587", + "electron-to-chromium": "^1.4.668", "node-releases": "^2.0.14", "update-browserslist-db": "^1.0.13" }, @@ -471,9 +471,9 @@ } }, "node_modules/caniuse-lite": { - "version": "1.0.30001579", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001579.tgz", - "integrity": "sha512-u5AUVkixruKHJjw/pj9wISlcMpgFWzSrczLZbrqBSxukQixmg0SJ5sZTpvaFvxU0HoQKd4yoyAogyrAz9pzJnA==", + "version": "1.0.30001593", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001593.tgz", + "integrity": "sha512-UWM1zlo3cZfkpBysd7AS+z+v007q9G1+fLTUU42rQnY6t2axoogPW/xol6T7juU5EUoOhML4WgBIdG+9yYqAjQ==", "dev": true, "funding": [ { @@ -776,9 +776,9 @@ } }, "node_modules/electron-to-chromium": { - "version": "1.4.640", - "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.640.tgz", - "integrity": "sha512-z/6oZ/Muqk4BaE7P69bXhUhpJbUM9ZJeka43ZwxsDshKtePns4mhBlh8bU5+yrnOnz3fhG82XLzGUXazOmsWnA==", + "version": "1.4.690", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.4.690.tgz", + "integrity": "sha512-+2OAGjUx68xElQhydpcbqH50hE8Vs2K6TkAeLhICYfndb67CVH0UsZaijmRUE3rHlIxU1u0jxwhgVe6fK3YANA==", "dev": true }, "node_modules/emoji-regex": { @@ -1121,9 +1121,9 @@ } }, "node_modules/hugo-extended": { - "version": "0.121.2", - "resolved": "https://registry.npmjs.org/hugo-extended/-/hugo-extended-0.121.2.tgz", - "integrity": "sha512-kb5XX5b9COxI88PDoH8+n4nmt/pe8ylyFLIBSdCaAGfH9/fEvk88Bv2MabyDCwHOGIAa8M6WpwulJG007vXwWg==", + "version": "0.122.0", + "resolved": "https://registry.npmjs.org/hugo-extended/-/hugo-extended-0.122.0.tgz", + "integrity": "sha512-f9kPVSKxk5mq62wmw1tbhg5CV7n93Tbt7jZoy+C3yfRlEZhGqBlxaEJ3MeeNoilz3IPy5STHB7R0Bdhuap7mHA==", "dev": true, "hasInstallScript": true, "dependencies": { @@ -1678,9 +1678,9 @@ } }, "node_modules/postcss": { - "version": "8.4.33", - "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.33.tgz", - "integrity": "sha512-Kkpbhhdjw2qQs2O2DGX+8m5OVqEcbB9HRBvuYM9pgrjEFUg30A9LmXNlTAUj4S9kgtGyrMbTzVjH7E+s5Re2yg==", + "version": "8.4.35", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.4.35.tgz", + "integrity": "sha512-u5U8qYpBCpN13BsiEB0CbR1Hhh4Gc0zLFuedrHJKMctHCHAGrMdG0PRM/KErzAL3CU6/eckEtmHNB3x6e3c0vA==", "dev": true, "funding": [ { diff --git a/website/package.json b/website/package.json index df5c55f0a784..5ea24c81ab13 100644 --- a/website/package.json +++ b/website/package.json @@ -4,9 +4,9 @@ "devDependencies": { "ansi-regex": ">=5.0.1", "autoprefixer": "^10.4.17", - "postcss": "^8.4.33", + "postcss": "^8.4.35", "postcss-cli": "^11.0.0", - "hugo-extended": "0.121.2" + "hugo-extended": "0.122.0" }, "scripts": { diff --git a/website/static/_redirects b/website/static/_redirects index b154d7cded6a..01b76800454c 100644 --- a/website/static/_redirects +++ b/website/static/_redirects @@ -77,4 +77,10 @@ /v0.34/concepts/threat-model /v0.34/reference/threat-model # Redirect any v1beta1-migration reference to the v0.32 version -/*/upgrading/v1beta1-migration /v0.32/upgrading/v1beta1-migration \ No newline at end of file +/*/upgrading/v1beta1-migration /v0.32/upgrading/v1beta1-migration + +# Redirect old "Managing AMIs" Task path +/preview/tasks/amitasks /preview/tasks/managing-amis +/docs/tasks/amitasks /docs/tasks/managing-amis +/v0.34/tasks/amitasks /v0.34/tasks/managing-amis +/v0.35/tasks/amitasks /v0.35/tasks/managing-amis \ No newline at end of file