From 7718c483bf26a8ca05d70e39814ec30f99d5a12e Mon Sep 17 00:00:00 2001 From: Reed Schalo Date: Tue, 15 Oct 2024 10:08:46 -0700 Subject: [PATCH 1/6] fix: smoke test for webhook e2e (#7218) --- .github/workflows/e2e.yaml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 95cc85246409..c5af96718b5b 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -138,6 +138,28 @@ jobs: prometheus_region: ${{ vars.PROMETHEUS_REGION }} enable_local_zones: ${{ inputs.suite == 'LocalZone' }} cleanup: ${{ inputs.cleanup }} + codebuild_role: ${{ vars[format('{0}_CODEBUILD_ROLE', inputs.codebuild_region)] }} + webhooks_enabled: ${{ inputs.suite != 'Webhooks' && true }} # Set webhooks_enabled to false if running webhook smoke test suite + - name: run tests for private cluster + if: ${{ inputs.workflow_trigger == 'private_cluster' }} + uses: ./.github/actions/e2e/run-tests-private-cluster + with: + cluster_name: ${{ steps.generate-cluster-name.outputs.CLUSTER_NAME }} + suite: ${{ inputs.suite }} + prometheus_region: ${{ vars.PROMETHEUS_REGION }} + prometheus_workspace_id: ${{ vars.WORKSPACE_ID }} + region: ${{ inputs.region }} + account_id: ${{ vars.CI_ACCOUNT_ID }} + k8s_version: ${{ inputs.k8s_version }} + ecr_account_id: ${{ vars.SNAPSHOT_ACCOUNT_ID }} + ecr_region: ${{ vars.SNAPSHOT_REGION }} + private_cluster: ${{ inputs.workflow_trigger == 'private_cluster' }} + enable_metrics: ${{ inputs.enable_metrics }} + metrics_region: ${{ vars.TIMESTREAM_REGION }} + node_role: ${{ env.NODE_ROLE }} + cleanup: ${{ inputs.cleanup }} + codebuild_sg: ${{ vars[format('{0}_CODEBUILD_SG', inputs.codebuild_region)] }} + codebuild_vpc: ${{ vars[format('{0}_CODEBUILD_VPC', inputs.codebuild_region)] }} - name: run the ${{ inputs.suite }} test suite env: SUITE: ${{ inputs.suite }} From e4a25cd36320dfc3cdd85072dbb9c002d1f57d43 Mon Sep 17 00:00:00 2001 From: Reed Schalo Date: Tue, 15 Oct 2024 10:27:03 -0700 Subject: [PATCH 2/6] reduce diff --- .github/workflows/e2e.yaml | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index c5af96718b5b..9cdf6cb9af64 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -138,28 +138,7 @@ jobs: prometheus_region: ${{ vars.PROMETHEUS_REGION }} enable_local_zones: ${{ inputs.suite == 'LocalZone' }} cleanup: ${{ inputs.cleanup }} - codebuild_role: ${{ vars[format('{0}_CODEBUILD_ROLE', inputs.codebuild_region)] }} webhooks_enabled: ${{ inputs.suite != 'Webhooks' && true }} # Set webhooks_enabled to false if running webhook smoke test suite - - name: run tests for private cluster - if: ${{ inputs.workflow_trigger == 'private_cluster' }} - uses: ./.github/actions/e2e/run-tests-private-cluster - with: - cluster_name: ${{ steps.generate-cluster-name.outputs.CLUSTER_NAME }} - suite: ${{ inputs.suite }} - prometheus_region: ${{ vars.PROMETHEUS_REGION }} - prometheus_workspace_id: ${{ vars.WORKSPACE_ID }} - region: ${{ inputs.region }} - account_id: ${{ vars.CI_ACCOUNT_ID }} - k8s_version: ${{ inputs.k8s_version }} - ecr_account_id: ${{ vars.SNAPSHOT_ACCOUNT_ID }} - ecr_region: ${{ vars.SNAPSHOT_REGION }} - private_cluster: ${{ inputs.workflow_trigger == 'private_cluster' }} - enable_metrics: ${{ inputs.enable_metrics }} - metrics_region: ${{ vars.TIMESTREAM_REGION }} - node_role: ${{ env.NODE_ROLE }} - cleanup: ${{ inputs.cleanup }} - codebuild_sg: ${{ vars[format('{0}_CODEBUILD_SG', inputs.codebuild_region)] }} - codebuild_vpc: ${{ vars[format('{0}_CODEBUILD_VPC', inputs.codebuild_region)] }} - name: run the ${{ inputs.suite }} test suite env: SUITE: ${{ inputs.suite }} From 0478d9053610714e4c3cad1d743db4df82c6af54 Mon Sep 17 00:00:00 2001 From: Reed Schalo Date: Tue, 15 Oct 2024 12:59:27 -0700 Subject: [PATCH 3/6] add webhook test --- .../actions/e2e/install-karpenter/action.yaml | 7 +- .github/actions/e2e/setup-cluster/action.yaml | 3 + .github/workflows/e2e-matrix.yaml | 2 + .github/workflows/e2e.yaml | 1 + test/suites/webhooks/suite_test.go | 68 +++++++++++++++++++ 5 files changed, 79 insertions(+), 2 deletions(-) create mode 100644 test/suites/webhooks/suite_test.go diff --git a/.github/actions/e2e/install-karpenter/action.yaml b/.github/actions/e2e/install-karpenter/action.yaml index b0c8ee550c00..c1e5a6a7f6f6 100644 --- a/.github/actions/e2e/install-karpenter/action.yaml +++ b/.github/actions/e2e/install-karpenter/action.yaml @@ -22,6 +22,9 @@ inputs: k8s_version: description: 'Version of Kubernetes to use for the launched cluster' default: "1.29" + webhooks_enabled: + description: "Whether webhooks are enabled or not. Valid values are true or false" + default: true git_ref: description: "The git commit, tag, or branch to check out. Requires a corresponding Karpenter snapshot release" runs: @@ -53,19 +56,19 @@ runs: ACCOUNT_ID: ${{ inputs.account_id }} CLUSTER_NAME: ${{ inputs.cluster_name }} K8S_VERSION: ${{ inputs.k8s_version }} + WEBHOOKS_ENABLED: ${{ inputs.webhooks_enabled }} run: | aws eks update-kubeconfig --name "$CLUSTER_NAME" # Parse minor version to determine whether to enable the webhooks K8S_VERSION_MINOR="${K8S_VERSION#*.}" - WEBHOOK_ENABLED=true # Remove service account annotation when dropping support for 1.23 helm upgrade --install karpenter "oci://$ECR_ACCOUNT_ID.dkr.ecr.$ECR_REGION.amazonaws.com/karpenter/snapshot/karpenter" \ -n kube-system \ --version "0-$(git rev-parse HEAD)" \ --set logLevel=debug \ - --set webhook.enabled=${WEBHOOK_ENABLED} \ + --set webhook.enabled=${WEBHOOKS_ENABLED} \ --set serviceAccount.annotations."eks\.amazonaws\.com/role-arn"="arn:aws:iam::$ACCOUNT_ID:role/karpenter-irsa-$CLUSTER_NAME" \ --set settings.clusterName="$CLUSTER_NAME" \ --set settings.interruptionQueue="$CLUSTER_NAME" \ diff --git a/.github/actions/e2e/setup-cluster/action.yaml b/.github/actions/e2e/setup-cluster/action.yaml index 5a78144a0d30..b08cb9f4c4ff 100644 --- a/.github/actions/e2e/setup-cluster/action.yaml +++ b/.github/actions/e2e/setup-cluster/action.yaml @@ -45,6 +45,9 @@ inputs: cleanup: description: "Whether to cleanup resources on failure" default: 'false' + webhooks_enabled: + description: "Whether webhooks are enabled or not. Valid values are 'true' or 'false'" + default: 'true' runs: using: "composite" steps: diff --git a/.github/workflows/e2e-matrix.yaml b/.github/workflows/e2e-matrix.yaml index 5d5be7392c5c..bd33fe06c885 100644 --- a/.github/workflows/e2e-matrix.yaml +++ b/.github/workflows/e2e-matrix.yaml @@ -76,6 +76,8 @@ jobs: region: ${{ inputs.region }} - name: IPv6 region: ${{ inputs.region }} + - name: Webhooks + region: ${{ inputs.region }} - name: LocalZone # LAX is the only local zone available in the CI account, therefore only use us-west-2 region: us-west-2 diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 9cdf6cb9af64..32abe7f09bc0 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -27,6 +27,7 @@ on: - Scale - PrivateCluster - LocalZone + - Webhooks k8s_version: type: choice options: diff --git a/test/suites/webhooks/suite_test.go b/test/suites/webhooks/suite_test.go new file mode 100644 index 000000000000..3b550d86670f --- /dev/null +++ b/test/suites/webhooks/suite_test.go @@ -0,0 +1,68 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package webhooks_test + +import ( + "context" + "testing" + + v1beta1 "github.com/aws/karpenter-provider-aws/pkg/apis/v1beta1" + "github.com/aws/karpenter-provider-aws/test/pkg/environment/aws" + + karpv1beta1 "sigs.k8s.io/karpenter/pkg/apis/v1beta1" + + karptest "sigs.k8s.io/karpenter/pkg/test" + + "github.com/aws/karpenter-provider-aws/pkg/test" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + . "knative.dev/pkg/logging/testing" + . "sigs.k8s.io/karpenter/pkg/test/expectations" +) + +var ctx context.Context +var env *aws.Environment +var nodeClass *v1beta1.EC2NodeClass +var nodePool *karpv1beta1.NodePool + +func TestWebhook(t *testing.T) { + RegisterFailHandler(Fail) + + ctx = TestContextWithLogger(t) + BeforeSuite(func() { + env = aws.NewEnvironment(t) + }) + AfterSuite(func() { + env.Stop() + }) + RunSpecs(t, "Webhook") +} + +var _ = BeforeEach(func() { + env.BeforeEach() + nodeClass = env.DefaultEC2NodeClass() + nodePool = env.DefaultNodePool(nodeClass) +}) + +var _ = Describe("Webhook", func() { + It("should schedule pods when webhooks are disabled", func() { + nodeClass := test.EC2NodeClass() + env.ExpectCreated(nodeClass, nodePool) + ExpectApplied(ctx, env.Client, nodePool, nodeClass) + pod := karptest.UnschedulablePod() + ExpectScheduled(ctx, env.Client, pod) + }) +}) From 53921b0c3d485b5fbf96c7cb658cb779cf85fc9e Mon Sep 17 00:00:00 2001 From: Reed Schalo Date: Tue, 15 Oct 2024 13:00:32 -0700 Subject: [PATCH 4/6] adjust bool --- .github/actions/e2e/install-karpenter/action.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/e2e/install-karpenter/action.yaml b/.github/actions/e2e/install-karpenter/action.yaml index c1e5a6a7f6f6..36d2ef0b9dc8 100644 --- a/.github/actions/e2e/install-karpenter/action.yaml +++ b/.github/actions/e2e/install-karpenter/action.yaml @@ -23,8 +23,8 @@ inputs: description: 'Version of Kubernetes to use for the launched cluster' default: "1.29" webhooks_enabled: - description: "Whether webhooks are enabled or not. Valid values are true or false" - default: true + description: "Whether webhooks are enabled or not. Valid values are 'true' or 'false'" + default: 'true' git_ref: description: "The git commit, tag, or branch to check out. Requires a corresponding Karpenter snapshot release" runs: From f5732c18649dd224789ccc9ef489671aaacd2f62 Mon Sep 17 00:00:00 2001 From: Reed Schalo Date: Tue, 15 Oct 2024 13:15:53 -0700 Subject: [PATCH 5/6] add input field --- .github/actions/e2e/setup-cluster/action.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/e2e/setup-cluster/action.yaml b/.github/actions/e2e/setup-cluster/action.yaml index b08cb9f4c4ff..06de1503449f 100644 --- a/.github/actions/e2e/setup-cluster/action.yaml +++ b/.github/actions/e2e/setup-cluster/action.yaml @@ -284,3 +284,4 @@ runs: cluster_name: ${{ inputs.cluster_name }} k8s_version: ${{ inputs.k8s_version }} git_ref: ${{ inputs.git_ref }} + webhooks_enabled: ${{ inputs.webhooks_enabled }} From d4371a2f464a404faa50b66b7b8323ce99bffce9 Mon Sep 17 00:00:00 2001 From: Reed Schalo Date: Tue, 15 Oct 2024 15:38:12 -0700 Subject: [PATCH 6/6] fix test --- test/suites/webhooks/suite_test.go | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/test/suites/webhooks/suite_test.go b/test/suites/webhooks/suite_test.go index 3b550d86670f..b66ce2fa82aa 100644 --- a/test/suites/webhooks/suite_test.go +++ b/test/suites/webhooks/suite_test.go @@ -25,12 +25,9 @@ import ( karptest "sigs.k8s.io/karpenter/pkg/test" - "github.com/aws/karpenter-provider-aws/pkg/test" - . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" . "knative.dev/pkg/logging/testing" - . "sigs.k8s.io/karpenter/pkg/test/expectations" ) var ctx context.Context @@ -38,7 +35,7 @@ var env *aws.Environment var nodeClass *v1beta1.EC2NodeClass var nodePool *karpv1beta1.NodePool -func TestWebhook(t *testing.T) { +func TestWebhooks(t *testing.T) { RegisterFailHandler(Fail) ctx = TestContextWithLogger(t) @@ -48,7 +45,7 @@ func TestWebhook(t *testing.T) { AfterSuite(func() { env.Stop() }) - RunSpecs(t, "Webhook") + RunSpecs(t, "Webhooks") } var _ = BeforeEach(func() { @@ -56,13 +53,14 @@ var _ = BeforeEach(func() { nodeClass = env.DefaultEC2NodeClass() nodePool = env.DefaultNodePool(nodeClass) }) +var _ = AfterEach(func() { env.Cleanup() }) +var _ = AfterEach(func() { env.AfterEach() }) -var _ = Describe("Webhook", func() { +var _ = Describe("Webhooks", func() { It("should schedule pods when webhooks are disabled", func() { - nodeClass := test.EC2NodeClass() - env.ExpectCreated(nodeClass, nodePool) - ExpectApplied(ctx, env.Client, nodePool, nodeClass) - pod := karptest.UnschedulablePod() - ExpectScheduled(ctx, env.Client, pod) + pod := karptest.Pod() + env.ExpectCreated(pod, nodeClass, nodePool) + env.EventuallyExpectHealthy(pod) + env.ExpectCreatedNodeCount("==", 1) }) })