From 6687fa083f81130eb1bf98693337499baf9df426 Mon Sep 17 00:00:00 2001 From: Reed Schalo Date: Tue, 29 Oct 2024 00:20:16 -0700 Subject: [PATCH] fix: Cherry pick chart change v0.34.x (#7296) --- .../actions/e2e/install-karpenter/action.yaml | 31 +---------- .../karpenter.k8s.aws_ec2nodeclasses.yaml | 2 + .../templates/karpenter.sh_nodeclaims.yaml | 2 + .../templates/karpenter.sh_nodepools.yaml | 2 + hack/mutation/conversion_webhook_injection.sh | 52 ++----------------- hack/mutation/ec2nodeclasses.sh | 26 ++++++++++ hack/mutation/nodeclaims.sh | 26 ++++++++++ hack/mutation/nodepools.sh | 26 ++++++++++ test/hack/e2e_scripts/install_karpenter.sh | 40 ++++++++++++++ 9 files changed, 130 insertions(+), 77 deletions(-) create mode 100755 hack/mutation/ec2nodeclasses.sh create mode 100755 hack/mutation/nodeclaims.sh create mode 100755 hack/mutation/nodepools.sh create mode 100755 test/hack/e2e_scripts/install_karpenter.sh diff --git a/.github/actions/e2e/install-karpenter/action.yaml b/.github/actions/e2e/install-karpenter/action.yaml index 0fc49e0c1b77..a9d7b7542c9d 100644 --- a/.github/actions/e2e/install-karpenter/action.yaml +++ b/.github/actions/e2e/install-karpenter/action.yaml @@ -58,36 +58,7 @@ runs: K8S_VERSION: ${{ inputs.k8s_version }} WEBHOOKS_ENABLED: ${{ inputs.webhooks_enabled }} run: | - aws eks update-kubeconfig --name "$CLUSTER_NAME" - - # Parse minor version to determine whether to enable the webhooks - RELEASE_VERSION_MINOR="${K8S_VERSION#*.}" - - # Remove service account annotation when dropping support for 1.23 - helm upgrade --install karpenter "oci://$ECR_ACCOUNT_ID.dkr.ecr.$ECR_REGION.amazonaws.com/karpenter/snapshot/karpenter" \ - -n kube-system \ - --version "v0-$(git rev-parse HEAD)" \ - --set logLevel=debug \ - --set webhook.enabled=${WEBHOOKS_ENABLED} \ - --set serviceAccount.annotations."eks\.amazonaws\.com/role-arn"="arn:aws:iam::$ACCOUNT_ID:role/karpenter-irsa-$CLUSTER_NAME" \ - --set settings.clusterName="$CLUSTER_NAME" \ - --set settings.interruptionQueue="$CLUSTER_NAME" \ - --set settings.featureGates.spotToSpotConsolidation=true \ - --set controller.resources.requests.cpu=3 \ - --set controller.resources.requests.memory=3Gi \ - --set controller.resources.limits.cpu=3 \ - --set controller.resources.limits.memory=3Gi \ - --set serviceMonitor.enabled=true \ - --set serviceMonitor.additionalLabels.scrape=enabled \ - --set "serviceMonitor.endpointConfig.relabelings[0].targetLabel=clusterName" \ - --set "serviceMonitor.endpointConfig.relabelings[0].replacement=$CLUSTER_NAME" \ - --set "serviceMonitor.endpointConfig.relabelings[1].targetLabel=gitRef" \ - --set "serviceMonitor.endpointConfig.relabelings[1].replacement=$(git rev-parse HEAD)" \ - --set "serviceMonitor.endpointConfig.relabelings[2].targetLabel=mostRecentTag" \ - --set "serviceMonitor.endpointConfig.relabelings[2].replacement=$(git describe --abbrev=0 --tags)" \ - --set "serviceMonitor.endpointConfig.relabelings[3].targetLabel=commitsAfterTag" \ - --set "serviceMonitor.endpointConfig.relabelings[3].replacement=\"$(git describe --tags | cut -d '-' -f 2)\"" \ - --wait + ./test/hack/e2e_scripts/install_karpenter.sh - name: diff-karpenter shell: bash env: diff --git a/charts/karpenter-crd/templates/karpenter.k8s.aws_ec2nodeclasses.yaml b/charts/karpenter-crd/templates/karpenter.k8s.aws_ec2nodeclasses.yaml index 386e13de6ca9..ace6d1776560 100644 --- a/charts/karpenter-crd/templates/karpenter.k8s.aws_ec2nodeclasses.yaml +++ b/charts/karpenter-crd/templates/karpenter.k8s.aws_ec2nodeclasses.yaml @@ -19,6 +19,7 @@ spec: singular: ec2nodeclass scope: Cluster versions: +{{- if .Values.webhook.enabled }} - additionalPrinterColumns: - jsonPath: .status.conditions[?(@.type=="Ready")].status name: Ready @@ -735,6 +736,7 @@ spec: storage: false subresources: status: {} +{{- end }} - name: v1beta1 schema: openAPIV3Schema: diff --git a/charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml b/charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml index d03f99572890..9a00f213e91e 100644 --- a/charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml +++ b/charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml @@ -16,6 +16,7 @@ spec: singular: nodeclaim scope: Cluster versions: +{{- if .Values.webhook.enabled }} - additionalPrinterColumns: - jsonPath: .metadata.labels.node\.kubernetes\.io/instance-type name: Type @@ -379,6 +380,7 @@ spec: storage: false subresources: status: {} +{{- end }} - additionalPrinterColumns: - jsonPath: .metadata.labels.node\.kubernetes\.io/instance-type name: Type diff --git a/charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml b/charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml index 673c7642d66d..92db80558a2e 100644 --- a/charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml +++ b/charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml @@ -16,6 +16,7 @@ spec: singular: nodepool scope: Cluster versions: +{{- if .Values.webhook.enabled }} - additionalPrinterColumns: - jsonPath: .spec.template.spec.nodeClassRef.name name: NodeClass @@ -502,6 +503,7 @@ spec: storage: false subresources: status: {} +{{- end }} - additionalPrinterColumns: - jsonPath: .spec.template.spec.nodeClassRef.name name: NodeClass diff --git a/hack/mutation/conversion_webhook_injection.sh b/hack/mutation/conversion_webhook_injection.sh index ef57c2e6fd12..ec878cfdcbbc 100755 --- a/hack/mutation/conversion_webhook_injection.sh +++ b/hack/mutation/conversion_webhook_injection.sh @@ -6,54 +6,12 @@ yq eval '.spec.conversion = {"strategy": "Webhook", "webhook": {"conversionRevie yq eval '.spec.conversion = {"strategy": "Webhook", "webhook": {"conversionReviewVersions": ["v1beta1", "v1"], "clientConfig": {"service": {"name": "karpenter", "namespace": "kube-system", "port": 8443}}}}' -i pkg/apis/crds/karpenter.sh_nodepools.yaml # Update to the karpenter-crd charts - -# Remove the copied over conversion stanzas from CRD spec +# Remove the copied conversion stanzas from CRD specs yq eval 'del(.spec.conversion)' -i charts/karpenter-crd/templates/karpenter.k8s.aws_ec2nodeclasses.yaml yq eval 'del(.spec.conversion)' -i charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml yq eval 'del(.spec.conversion)' -i charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml -# Add the conversion stanza template to the CRD spec to enable conversion via webhook -echo "{{- if .Values.webhook.enabled }} - conversion: - strategy: Webhook - webhook: - conversionReviewVersions: - - v1beta1 - - v1 - clientConfig: - service: - name: {{ .Values.webhook.serviceName }} - namespace: {{ .Values.webhook.serviceNamespace | default .Release.Namespace }} - port: {{ .Values.webhook.port }} -{{- end }} -" >> charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml - -echo "{{- if .Values.webhook.enabled }} - conversion: - strategy: Webhook - webhook: - conversionReviewVersions: - - v1beta1 - - v1 - clientConfig: - service: - name: {{ .Values.webhook.serviceName }} - namespace: {{ .Values.webhook.serviceNamespace | default .Release.Namespace }} - port: {{ .Values.webhook.port }} -{{- end }} -" >> charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml - -echo "{{- if .Values.webhook.enabled }} - conversion: - strategy: Webhook - webhook: - conversionReviewVersions: - - v1beta1 - - v1 - clientConfig: - service: - name: {{ .Values.webhook.serviceName }} - namespace: {{ .Values.webhook.serviceNamespace | default .Release.Namespace }} - port: {{ .Values.webhook.port }} -{{- end }} -" >> charts/karpenter-crd/templates/karpenter.k8s.aws_ec2nodeclasses.yaml \ No newline at end of file +# Template the v1 version and the conversion strategy of the spec +hack/mutation/ec2nodeclasses.sh +hack/mutation/nodepools.sh +hack/mutation/nodeclaims.sh diff --git a/hack/mutation/ec2nodeclasses.sh b/hack/mutation/ec2nodeclasses.sh new file mode 100755 index 000000000000..5d442b182b67 --- /dev/null +++ b/hack/mutation/ec2nodeclasses.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +VERSION_START="$(cat charts/karpenter-crd/templates/karpenter.k8s.aws_ec2nodeclasses.yaml | yq '.spec.versions.[0] | line')" +VERSION_END="$(cat charts/karpenter-crd/templates/karpenter.k8s.aws_ec2nodeclasses.yaml | yq '.spec.versions.[1] | line')" +VERSION_END=$(($VERSION_END+1)) +TEMP=$(mktemp) + +cat charts/karpenter-crd/templates/karpenter.k8s.aws_ec2nodeclasses.yaml | awk -v n=$VERSION_START 'NR==n {sub(/$/,"\n{{- if .Values.webhook.enabled }}")} 1' \ +| awk -v n=$VERSION_END 'NR==n {sub(/$/,"\n{{- end }}")} 1' > $TEMP + +cat $TEMP > charts/karpenter-crd/templates/karpenter.k8s.aws_ec2nodeclasses.yaml + +echo "{{- if .Values.webhook.enabled }} + conversion: + strategy: Webhook + webhook: + conversionReviewVersions: + - v1beta1 + - v1 + clientConfig: + service: + name: {{ .Values.webhook.serviceName }} + namespace: {{ .Values.webhook.serviceNamespace | default .Release.Namespace }} + port: {{ .Values.webhook.port }} +{{- end }} +" >> charts/karpenter-crd/templates/karpenter.k8s.aws_ec2nodeclasses.yaml \ No newline at end of file diff --git a/hack/mutation/nodeclaims.sh b/hack/mutation/nodeclaims.sh new file mode 100755 index 000000000000..77afccdf8a47 --- /dev/null +++ b/hack/mutation/nodeclaims.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +VERSION_START="$(cat charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml | yq '.spec.versions.[0] | line')" +VERSION_END="$(cat charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml | yq '.spec.versions.[1] | line')" +VERSION_END=$(($VERSION_END+1)) +TEMP=$(mktemp) + +cat charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml | awk -v n=$VERSION_START 'NR==n {sub(/$/,"\n{{- if .Values.webhook.enabled }}")} 1' \ +| awk -v n=$VERSION_END 'NR==n {sub(/$/,"\n{{- end }}")} 1' > $TEMP + +cat $TEMP > charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml + +echo "{{- if .Values.webhook.enabled }} + conversion: + strategy: Webhook + webhook: + conversionReviewVersions: + - v1beta1 + - v1 + clientConfig: + service: + name: {{ .Values.webhook.serviceName }} + namespace: {{ .Values.webhook.serviceNamespace | default .Release.Namespace }} + port: {{ .Values.webhook.port }} +{{- end }} +" >> charts/karpenter-crd/templates/karpenter.sh_nodeclaims.yaml \ No newline at end of file diff --git a/hack/mutation/nodepools.sh b/hack/mutation/nodepools.sh new file mode 100755 index 000000000000..9ad1a6d279fa --- /dev/null +++ b/hack/mutation/nodepools.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash + +VERSION_START="$(cat charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml | yq '.spec.versions.[0] | line')" +VERSION_END="$(cat charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml | yq '.spec.versions.[1] | line')" +VERSION_END=$(($VERSION_END+1)) +TEMP=$(mktemp) + +cat charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml | awk -v n=$VERSION_START 'NR==n {sub(/$/,"\n{{- if .Values.webhook.enabled }}")} 1' \ +| awk -v n=$VERSION_END 'NR==n {sub(/$/,"\n{{- end }}")} 1' > $TEMP + +cat $TEMP > charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml + +echo "{{- if .Values.webhook.enabled }} + conversion: + strategy: Webhook + webhook: + conversionReviewVersions: + - v1beta1 + - v1 + clientConfig: + service: + name: {{ .Values.webhook.serviceName }} + namespace: {{ .Values.webhook.serviceNamespace | default .Release.Namespace }} + port: {{ .Values.webhook.port }} +{{- end }} +" >> charts/karpenter-crd/templates/karpenter.sh_nodepools.yaml \ No newline at end of file diff --git a/test/hack/e2e_scripts/install_karpenter.sh b/test/hack/e2e_scripts/install_karpenter.sh new file mode 100755 index 000000000000..45da7a21b66f --- /dev/null +++ b/test/hack/e2e_scripts/install_karpenter.sh @@ -0,0 +1,40 @@ +aws eks update-kubeconfig --name "$CLUSTER_NAME" + +# First, conditionally install the webhook stanza and CRDs +if (( "$WEBHOOKS_ENABLED" == false )); then +helm upgrade --install karpenter-crd oci://$ECR_ACCOUNT_ID.dkr.ecr.$ECR_REGION.amazonaws.com/karpenter/snapshot/karpenter-crd \ + --namespace kube-system \ + --version "v0-$(git rev-parse HEAD)" \ + --set webhook.enabled=${WEBHOOKS_ENABLED} \ + --wait +fi + +CHART="oci://$ECR_ACCOUNT_ID.dkr.ecr.$ECR_REGION.amazonaws.com/karpenter/snapshot/karpenter" +ADDITIONAL_FLAGS="" + + +# Remove service account annotation when dropping support for 1.23 +helm upgrade --install karpenter "${CHART}" \ + -n kube-system \ + --version "v0-$(git rev-parse HEAD)" \ + --set logLevel=debug \ + --set webhook.enabled=${WEBHOOKS_ENABLED} \ + --set serviceAccount.annotations."eks\.amazonaws\.com/role-arn"="arn:aws:iam::$ACCOUNT_ID:role/karpenter-irsa-$CLUSTER_NAME" \ + --set settings.clusterName="$CLUSTER_NAME" \ + --set settings.interruptionQueue="$CLUSTER_NAME" \ + --set settings.featureGates.spotToSpotConsolidation=true \ + --set controller.resources.requests.cpu=3 \ + --set controller.resources.requests.memory=3Gi \ + --set controller.resources.limits.cpu=3 \ + --set controller.resources.limits.memory=3Gi \ + --set serviceMonitor.enabled=true \ + --set serviceMonitor.additionalLabels.scrape=enabled \ + --set "serviceMonitor.endpointConfig.relabelings[0].targetLabel=clusterName" \ + --set "serviceMonitor.endpointConfig.relabelings[0].replacement=$CLUSTER_NAME" \ + --set "serviceMonitor.endpointConfig.relabelings[1].targetLabel=gitRef" \ + --set "serviceMonitor.endpointConfig.relabelings[1].replacement=$(git rev-parse HEAD)" \ + --set "serviceMonitor.endpointConfig.relabelings[2].targetLabel=mostRecentTag" \ + --set "serviceMonitor.endpointConfig.relabelings[2].replacement=$(git describe --abbrev=0 --tags)" \ + --set "serviceMonitor.endpointConfig.relabelings[3].targetLabel=commitsAfterTag" \ + --set "serviceMonitor.endpointConfig.relabelings[3].replacement=\"$(git describe --tags | cut -d '-' -f 2)\"" \ + --wait