From 8a2207a731c3f116997159c63885b212cdb8d2e8 Mon Sep 17 00:00:00 2001 From: Amanuel Engeda Date: Sat, 28 Oct 2023 00:04:11 -0700 Subject: [PATCH] Soak testing --- .github/actions/e2e/cleanup/action.yaml | 2 +- .github/actions/e2e/slack/notify/action.yaml | 7 ++- .github/workflows/e2e-soak-trigger.yaml | 51 ++++++++++++++++ .github/workflows/e2e-upgrade.yaml | 1 + .github/workflows/e2e.yaml | 38 +++++++++--- .github/workflows/resource-count.yaml | 2 +- .github/workflows/sweeper.yaml | 10 ++- test/hack/resource/clean/main.go | 22 ++++--- test/hack/resource/go.mod | 9 ++- test/hack/resource/go.sum | 19 +++--- test/hack/soak/get_clusters.go | 64 ++++++++++++++++++++ test/hack/soak/go.mod | 27 +++++++++ test/hack/soak/go.sum | 47 ++++++++++++++ test/suites/integration/ami_test.go | 1 + test/suites/integration/scheduling_test.go | 4 +- test/suites/integration/utilization_test.go | 2 +- 16 files changed, 266 insertions(+), 40 deletions(-) create mode 100644 .github/workflows/e2e-soak-trigger.yaml create mode 100644 test/hack/soak/get_clusters.go create mode 100644 test/hack/soak/go.mod create mode 100644 test/hack/soak/go.sum diff --git a/.github/actions/e2e/cleanup/action.yaml b/.github/actions/e2e/cleanup/action.yaml index 8b1951995665..0034440341a9 100644 --- a/.github/actions/e2e/cleanup/action.yaml +++ b/.github/actions/e2e/cleanup/action.yaml @@ -39,6 +39,6 @@ runs: cache: false - name: "Run cleanup script" run: | - go run main.go ${{ inputs.cluster_name }} + go run main.go --cluster-name ${{ inputs.cluster_name }} working-directory: ./test/hack/resource/clean shell: bash diff --git a/.github/actions/e2e/slack/notify/action.yaml b/.github/actions/e2e/slack/notify/action.yaml index ad317d1ffeb9..28ba71aebc46 100644 --- a/.github/actions/e2e/slack/notify/action.yaml +++ b/.github/actions/e2e/slack/notify/action.yaml @@ -1,6 +1,9 @@ name: SlackNotify description: 'Notifies slack of the success or failure of the suite' inputs: + cluster_name: + description: "Name of the cluster" + required: false suite: description: "Suite that's running" required: true @@ -18,8 +21,10 @@ runs: - id: get-run-name shell: bash run: | - if [[ ${{ github.event_name }} == "schedule" ]]; then + if [[ ${{ github.event_name }} == "schedule" && inputs.suite != "soak" ]]; then RUN_NAME="${{ inputs.suite }}-periodic" + elif [[ ${{ github.event_name }} == "schedule" ]]; then + RUN_NAME="soak-periodic" else RUN_NAME="${{ inputs.suite }}-${GITHUB_SHA::7}" fi diff --git a/.github/workflows/e2e-soak-trigger.yaml b/.github/workflows/e2e-soak-trigger.yaml new file mode 100644 index 000000000000..0d3b0f225655 --- /dev/null +++ b/.github/workflows/e2e-soak-trigger.yaml @@ -0,0 +1,51 @@ +name: E2ESoakTrigger +on: + schedule: + - cron: '0 */3 * * *' +jobs: + resolve_cluster: + permissions: + id-token: write # aws-actions/configure-aws-credentials@v4.0.1 + # rmif: github.repository == 'aws/karpenter-provider-aws' + runs-on: ubuntu-latest + outputs: + PREEXISTING_CLUSTERS: ${{ steps.list_clusters.outputs.PREEXISTING_CLUSTERS }} + steps: + - uses: actions/checkout@v4 + - name: configure aws credentials + uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a + with: + role-to-assume: arn:aws:iam::${{ vars.ACCOUNT_ID }}:role/${{ vars.ROLE_NAME }} + aws-region: eu-north-1 + - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0 + with: + go-version-file: test/hack/soak/go.mod + cache-dependency-path: test/hack/soak/go.sum + check-latest: true + cache: false + # Grab all the names of the soak testing clusters, and identify clusters that will need to be deleted + # Empty cluster_name will indicate that a new cluster will be created + - id: list_clusters + name: "Run list clusters script" + run: | + PREEXISTING_CLUSTERS="$(go run get_clusters.go)" + echo PREEXISTING_CLUSTERS="$PREEXISTING_CLUSTERS" >> "$GITHUB_OUTPUT" + working-directory: ./test/hack/soak + shell: bash + soak: + permissions: + id-token: write # aws-actions/configure-aws-credentials@v4.0.1 + statuses: write # required by e2e.yaml + needs: [resolve_cluster] + strategy: + fail-fast: false + matrix: ${{ fromJson(needs.resolve_cluster.outputs.PREEXISTING_CLUSTERS) }} + uses: ./.github/workflows/e2e.yaml + with: + suite: Integration + region: eu-north-1 + workflow_trigger: "soak" + cluster_name: ${{ matrix.cluster_name }} + cleanup: ${{ matrix.cluster_cleanup }} + secrets: + SLACK_WEBHOOK_SOAK_URL: ${{ secrets.SLACK_WEBHOOK_SOAK_URL }} diff --git a/.github/workflows/e2e-upgrade.yaml b/.github/workflows/e2e-upgrade.yaml index bf01668c6a60..6b83c9160b0a 100644 --- a/.github/workflows/e2e-upgrade.yaml +++ b/.github/workflows/e2e-upgrade.yaml @@ -130,6 +130,7 @@ jobs: uses: ./.github/actions/e2e/slack/notify if: (success() || failure()) && github.event_name != 'workflow_run' && inputs.workflow_trigger != 'versionCompatibility' with: + cluster_name: ${{ steps.generate-cluster-name.outputs.CLUSTER_NAME }} url: ${{ secrets.SLACK_WEBHOOK_URL }} suite: Upgrade git_ref: ${{ inputs.to_git_ref }} diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 387fcb844639..7f51203024d6 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -36,13 +36,15 @@ on: - "1.27" - "1.28" default: "1.28" - enable_metrics: - type: boolean - default: false + cluster_name: + type: string cleanup: type: boolean required: true default: true + enable_metrics: + type: boolean + default: false workflow_call: inputs: git_ref: @@ -64,9 +66,14 @@ on: required: true workflow_trigger: type: string + cluster_name: + type: string + description: If cluster_name is empty, a new cluster will be created. Otherwise, tests will run on an existing cluster secrets: SLACK_WEBHOOK_URL: - required: true + required: false + SLACK_WEBHOOK_SOAK_URL: + required: false jobs: run-suite: permissions: @@ -97,10 +104,18 @@ jobs: - id: generate-cluster-name name: generate cluster name run: | - CLUSTER_NAME="$(echo ${{ inputs.suite }}-"$RANDOM$RANDOM" | awk '{print tolower($0)}' | tr / -)" - echo Using cluster name "$CLUSTER_NAME" + CLUSTER_NAME='' + if [[ '${{ inputs.cluster_name }}' == '' ]] && [[ '${{ inputs.workflow_trigger }}' == 'soak' ]]; then + CLUSTER_NAME=$(echo soak-periodic-$RANDOM$RANDOM | awk '{print tolower($0)}' | tr / -) + elif [[ '${{ inputs.cluster_name }}' == '' ]] && [[ '${{ inputs.workflow_trigger }}' != 'soak' ]]; then + CLUSTER_NAME=$(echo ${{ inputs.suite }}-$RANDOM$RANDOM | awk '{print tolower($0)}' | tr / -) + else + CLUSTER_NAME='${{ inputs.cluster_name }}' + fi + echo "Using cluster name \"$CLUSTER_NAME\"" echo CLUSTER_NAME="$CLUSTER_NAME" >> "$GITHUB_OUTPUT" - name: setup eks cluster '${{ steps.generate-cluster-name.outputs.CLUSTER_NAME }}' + if: inputs.cluster_name == '' uses: ./.github/actions/e2e/setup-cluster with: account_id: ${{ vars.ACCOUNT_ID }} @@ -124,6 +139,12 @@ jobs: TEST_SUITE="Integration" fi aws eks update-kubeconfig --name ${{ steps.generate-cluster-name.outputs.CLUSTER_NAME }} + # Clean up the cluster before running all tests + kubectl delete nodepool --all + kubectl delete ec2nodeclass --all + kubectl delete deployment --all + + # Run test Suite TEST_SUITE="$TEST_SUITE" ENABLE_METRICS=${{ inputs.enable_metrics }} METRICS_REGION=${{ vars.TIMESTREAM_REGION }} GIT_REF="$(git rev-parse HEAD)" \ CLUSTER_NAME="${{ steps.generate-cluster-name.outputs.CLUSTER_NAME }}" CLUSTER_ENDPOINT="$(aws eks describe-cluster --name ${{ steps.generate-cluster-name.outputs.CLUSTER_NAME }} --query "cluster.endpoint" --output text)" \ INTERRUPTION_QUEUE="${{ steps.generate-cluster-name.outputs.CLUSTER_NAME }}" make e2etests @@ -131,8 +152,9 @@ jobs: uses: ./.github/actions/e2e/slack/notify if: (success() || failure()) && github.event_name != 'workflow_run' && inputs.workflow_trigger != 'versionCompatibility' with: - url: ${{ secrets.SLACK_WEBHOOK_URL }} - suite: ${{ inputs.suite }} + cluster_name: ${{ steps.generate-cluster-name.outputs.CLUSTER_NAME }} + url: ${{ inputs.workflow_trigger == 'soak' && secrets.SLACK_WEBHOOK_SOAK_URL || secrets.SLACK_WEBHOOK_URL }} + suite: ${{ inputs.workflow_trigger == 'soak' && 'soak' || inputs.suite }} git_ref: ${{ inputs.git_ref }} - name: dump logs on failure uses: ./.github/actions/e2e/dump-logs diff --git a/.github/workflows/resource-count.yaml b/.github/workflows/resource-count.yaml index b9f29b361923..6720f62f2039 100644 --- a/.github/workflows/resource-count.yaml +++ b/.github/workflows/resource-count.yaml @@ -11,7 +11,7 @@ jobs: strategy: fail-fast: false matrix: - region: [us-east-2, us-west-2, eu-west-1] + region: [us-east-2, us-west-2, eu-west-1, eu-north-1] runs-on: ubuntu-latest steps: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 diff --git a/.github/workflows/sweeper.yaml b/.github/workflows/sweeper.yaml index d19edc5f6139..42ef09ceb3b6 100644 --- a/.github/workflows/sweeper.yaml +++ b/.github/workflows/sweeper.yaml @@ -11,7 +11,7 @@ jobs: strategy: fail-fast: false matrix: - region: [us-east-2, us-west-2, eu-west-1] + region: [us-east-2, us-west-2, eu-west-1, eu-north-1] runs-on: ubuntu-latest steps: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 @@ -25,6 +25,12 @@ jobs: go-version-file: test/hack/resource/go.mod check-latest: true cache-dependency-path: "test/hack/resource/go.sum" - - run: go run main.go + # eu-north-1 contains the soak tests and resources should be swept if they are older than 8 days + - run: | + if [[ "${{ matrix.region }}" == "eu-north-1" ]]; then + go run main.go --expiration 192h + else + go run main.go --expiration 12h + fi working-directory: ./test/hack/resource/clean name: "Run cleanup script" diff --git a/test/hack/resource/clean/main.go b/test/hack/resource/clean/main.go index 6b78e61b393d..be115a6b855b 100644 --- a/test/hack/resource/clean/main.go +++ b/test/hack/resource/clean/main.go @@ -16,8 +16,8 @@ package main import ( "context" + "flag" "fmt" - "os" "time" "github.com/aws/aws-sdk-go-v2/config" @@ -31,19 +31,22 @@ import ( "github.com/aws/karpenter-provider-aws/test/hack/resource/pkg/resourcetypes" ) -const expirationTTL = time.Hour * 12 const sweeperCleanedResourcesTableName = "sweeperCleanedResources" func main() { - var clusterName string - if len(os.Args) == 2 { - clusterName = os.Args[1] - } + expiration := flag.String("expiration", "12h", "define the expirationTTL of the resources") + clusterName := flag.String("cluster-name", "", "define cluster name to cleanup") + flag.Parse() + ctx := context.Background() cfg := lo.Must(config.LoadDefaultConfig(ctx)) logger := lo.Must(zap.NewProduction()).Sugar() + expirationTTL, err := time.ParseDuration(lo.FromPtr(expiration)) + if err != nil { + logger.Fatalln("need a valid expiration duration", err) + } expirationTime := time.Now().Add(-expirationTTL) logger.With("expiration-time", expirationTime.String()).Infof("resolved expiration time for all resourceTypes") @@ -73,10 +76,11 @@ func main() { resourceLogger := logger.With("type", resourceTypes[i].String()) var ids []string var err error - if clusterName == "" { + // If there's no cluster defined, clean up all expired resources. otherwise, only cleanup the resources associated with the cluster + if lo.FromPtr(clusterName) == "" { ids, err = resourceTypes[i].GetExpired(ctx, expirationTime) } else { - ids, err = resourceTypes[i].Get(ctx, clusterName) + ids, err = resourceTypes[i].Get(ctx, lo.FromPtr(clusterName)) } if err != nil { resourceLogger.Errorf("%v", err) @@ -88,7 +92,7 @@ func main() { resourceLogger.Errorf("%v", err) } // Should only fire metrics if the resource have expired - if clusterName == "" { + if lo.FromPtr(clusterName) == "" { if err = metricsClient.FireMetric(ctx, sweeperCleanedResourcesTableName, fmt.Sprintf("%sDeleted", resourceTypes[i].String()), float64(len(cleaned)), lo.Ternary(resourceTypes[i].Global(), "global", cfg.Region)); err != nil { resourceLogger.Errorf("%v", err) } diff --git a/test/hack/resource/go.mod b/test/hack/resource/go.mod index 00c35d96a277..30966932745d 100644 --- a/test/hack/resource/go.mod +++ b/test/hack/resource/go.mod @@ -3,7 +3,7 @@ module github.com/aws/karpenter-provider-aws/test/hack/resource go 1.21 require ( - github.com/aws/aws-sdk-go-v2 v1.21.0 + github.com/aws/aws-sdk-go-v2 v1.22.1 github.com/aws/aws-sdk-go-v2/config v1.18.27 github.com/aws/aws-sdk-go-v2/service/cloudformation v1.30.0 github.com/aws/aws-sdk-go-v2/service/ec2 v1.102.0 @@ -17,16 +17,15 @@ require ( require ( github.com/aws/aws-sdk-go-v2/credentials v1.13.26 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.1 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.1 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35 // indirect github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.7.32 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.12.12 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.19.2 // indirect - github.com/aws/smithy-go v1.14.2 // indirect - github.com/google/go-cmp v0.5.9 // indirect + github.com/aws/smithy-go v1.16.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/stretchr/testify v1.8.1 // indirect diff --git a/test/hack/resource/go.sum b/test/hack/resource/go.sum index 9321752dc0d9..e03db99812ba 100644 --- a/test/hack/resource/go.sum +++ b/test/hack/resource/go.sum @@ -1,7 +1,7 @@ github.com/aws/aws-sdk-go-v2 v1.18.1/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= github.com/aws/aws-sdk-go-v2 v1.20.1/go.mod h1:NU06lETsFm8fUC6ZjhgDpVBcGZTFQ6XM+LZWZxMI4ac= -github.com/aws/aws-sdk-go-v2 v1.21.0 h1:gMT0IW+03wtYJhRqTVYn0wLzwdnK9sRMcxmtfGzRdJc= -github.com/aws/aws-sdk-go-v2 v1.21.0/go.mod h1:/RfNgGmRxI+iFOB1OeJUyxiU+9s88k3pfHvDagGEp0M= +github.com/aws/aws-sdk-go-v2 v1.22.1 h1:sjnni/AuoTXxHitsIdT0FwmqUuNUuHtufcVDErVFT9U= +github.com/aws/aws-sdk-go-v2 v1.22.1/go.mod h1:Kd0OJtkW3Q0M0lUWGszapWjEvrXDzRW+D21JNsroB+c= github.com/aws/aws-sdk-go-v2/config v1.18.27 h1:Az9uLwmssTE6OGTpsFqOnaGpLnKDqNYOJzWuC6UAYzA= github.com/aws/aws-sdk-go-v2/config v1.18.27/go.mod h1:0My+YgmkGxeqjXZb5BYme5pc4drjTnM+x1GJ3zv42Nw= github.com/aws/aws-sdk-go-v2/credentials v1.13.26 h1:qmU+yhKmOCyujmuPY7tf5MxR/RKyZrOPO3V4DobiTUk= @@ -10,12 +10,12 @@ github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4 h1:LxK/bitrAr4lnh9LnIS6i7z github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4/go.mod h1:E1hLXN/BL2e6YizK1zFlYd8vsfi2GTjbjBazinMmeaM= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34/go.mod h1:wZpTEecJe0Btj3IYnDx/VlUzor9wm3fJHyvLpQF0VwY= github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.38/go.mod h1:qggunOChCMu9ZF/UkAfhTz25+U2rLVb3ya0Ua6TTfCA= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 h1:22dGT7PneFMx4+b3pz7lMTRyN8ZKH7M2cW4GP9yUS2g= -github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41/go.mod h1:CrObHAuPneJBlfEJ5T3szXOUkLEThaGfvnhTf33buas= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.1 h1:fi1ga6WysOyYb5PAf3Exd6B5GiSNpnZim4h1rhlBqx0= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.1/go.mod h1:V5CY8wNurvPUibTi9mwqUqpiFZ5LnioKWIFUDtIzdI8= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28/go.mod h1:7VRpKQQedkfIEXb4k52I7swUnZP0wohVajJMRn3vsUw= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.32/go.mod h1:0ZXSqrty4FtQ7p8TEuRde/SZm9X05KT18LAUlR40Ln0= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 h1:SijA0mgjV8E+8G45ltVHs0fvKpTj8xmZJ3VwhGKtUSI= -github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35/go.mod h1:SJC1nEVVva1g3pHAIdCp7QsRIkMmLAgoDquQ9Rr8kYw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.1 h1:ZpaV/j48RlPc4AmOZuPv22pJliXjXq8/reL63YzyFnw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.1/go.mod h1:R8aXraabD2e3qv1csxM14/X9WF4wFMIY0kH4YEtYD5M= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35 h1:LWA+3kDM8ly001vJ1X1waCuLJdtTl48gwkPKWy9sosI= github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35/go.mod h1:0Eg1YjxE0Bhn56lx+SHJwCzhW+2JGtizsrx+lCqrfm0= github.com/aws/aws-sdk-go-v2/service/cloudformation v1.30.0 h1:XbDkc4FLeg1RfnqeblfbJvaEabqq9ByZl4zqyPFkfSc= @@ -38,16 +38,15 @@ github.com/aws/aws-sdk-go-v2/service/timestreamwrite v1.18.2 h1:5QyvAYyr+ZibpVxf github.com/aws/aws-sdk-go-v2/service/timestreamwrite v1.18.2/go.mod h1:3ZCiyyNF7myh/a7DcOjcqRsLmSF9EdhEZSr00Qlui4s= github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/aws/smithy-go v1.14.1/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= -github.com/aws/smithy-go v1.14.2 h1:MJU9hqBGbvWZdApzpvoF2WAIJDbtjK2NDJSiJP7HblQ= -github.com/aws/smithy-go v1.14.2/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= +github.com/aws/smithy-go v1.16.0 h1:gJZEH/Fqh+RsvlJ1Zt4tVAtV6bKkp3cC+R6FCZMNzik= +github.com/aws/smithy-go v1.16.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= diff --git a/test/hack/soak/get_clusters.go b/test/hack/soak/get_clusters.go new file mode 100644 index 000000000000..7afbd69c0ceb --- /dev/null +++ b/test/hack/soak/get_clusters.go @@ -0,0 +1,64 @@ +/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/eks" + "github.com/aws/aws-sdk-go/aws" + "github.com/samber/lo" +) + +type cluster struct { + Name string `json:"cluster_name"` + Cleanup bool `json:"cluster_cleanup"` +} + +const expirationTTL = time.Hour * 168 // 7 days + +func main() { + ctx := context.Background() + cfg := lo.Must(config.LoadDefaultConfig(ctx)) + eksClient := eks.NewFromConfig(cfg) + + var outputList []*cluster + createNewCluster := true + expirationTime := time.Now().Add(-expirationTTL) + + clusters := lo.Must(eksClient.ListClusters(ctx, &eks.ListClustersInput{})) + + for _, c := range clusters.Clusters { + clusterDetails := lo.Must(eksClient.DescribeCluster(ctx, &eks.DescribeClusterInput{Name: aws.String(c)})) + if clusterDetails.Cluster.CreatedAt.YearDay() == time.Unix(time.Now().Unix(), 0).YearDay() { + createNewCluster = false + } + + if strings.HasPrefix(c, "soak-periodic-") { + outputList = append(outputList, &cluster{Name: c, Cleanup: clusterDetails.Cluster.CreatedAt.Before(expirationTime)}) + } + } + + if createNewCluster { + outputList = append(outputList, &cluster{Name: "", Cleanup: false}) + } + + fmt.Println(string(lo.Must(json.Marshal(map[string][]*cluster{"include": outputList})))) +} diff --git a/test/hack/soak/go.mod b/test/hack/soak/go.mod new file mode 100644 index 000000000000..2a9582b9fc72 --- /dev/null +++ b/test/hack/soak/go.mod @@ -0,0 +1,27 @@ +module github.com/aws/karpenter/test/hack/soak + +go 1.21 + +require ( + github.com/aws/aws-sdk-go v1.47.9 + github.com/aws/aws-sdk-go-v2/config v1.23.0 + github.com/aws/aws-sdk-go-v2/service/eks v1.32.0 + github.com/samber/lo v1.38.1 +) + +require ( + github.com/aws/aws-sdk-go-v2 v1.22.2 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.15.2 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.3 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.2 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.2 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.6.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.2 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.17.1 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.25.1 // indirect + github.com/aws/smithy-go v1.16.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 // indirect +) diff --git a/test/hack/soak/go.sum b/test/hack/soak/go.sum new file mode 100644 index 000000000000..b2102b04b21d --- /dev/null +++ b/test/hack/soak/go.sum @@ -0,0 +1,47 @@ +github.com/aws/aws-sdk-go v1.47.9 h1:rarTsos0mA16q+huicGx0e560aYRtOucV5z2Mw23JRY= +github.com/aws/aws-sdk-go v1.47.9/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= +github.com/aws/aws-sdk-go-v2 v1.22.2 h1:lV0U8fnhAnPz8YcdmZVV60+tr6CakHzqA6P8T46ExJI= +github.com/aws/aws-sdk-go-v2 v1.22.2/go.mod h1:Kd0OJtkW3Q0M0lUWGszapWjEvrXDzRW+D21JNsroB+c= +github.com/aws/aws-sdk-go-v2/config v1.23.0 h1:kqzEfGGDIrRJpfJckgwuZfFTbU9NB1jZnRcaO9MpOqE= +github.com/aws/aws-sdk-go-v2/config v1.23.0/go.mod h1:p7wbxKXXjS1GGQOss7VXOazVMFF9bjUGq85/4wR/fSw= +github.com/aws/aws-sdk-go-v2/credentials v1.15.2 h1:rKH7khRMxPdD0u3dHecd0Q7NOVw3EUe7AqdkUOkiOGI= +github.com/aws/aws-sdk-go-v2/credentials v1.15.2/go.mod h1:tXM8wmaeAhfC7nZoCxb0FzM/aRaB1m1WQ7x0qlBLq80= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.3 h1:G5KawTAkyHH6WyKQCdHiW4h3PmAXNJpOgwKg3H7sDRE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.14.3/go.mod h1:hugKmSFnZB+HgNI1sYGT14BUPZkO6alC/e0AWu+0IAQ= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.2 h1:AaQsr5vvGR7rmeSWBtTCcw16tT9r51mWijuCQhzLnq8= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.2.2/go.mod h1:o1IiRn7CWocIFTXJjGKJDOwxv1ibL53NpcvcqGWyRBA= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.2 h1:UZx8SXZ0YtzRiALzYAWcjb9Y9hZUR7MBKaBQ5ouOjPs= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.5.2/go.mod h1:ipuRpcSaklmxR6C39G187TpBAO132gUfleTGccUPs8c= +github.com/aws/aws-sdk-go-v2/internal/ini v1.6.0 h1:hwZB07/beLiCopuRKF0t+dEHmP39iN4YtDh3X5d3hrg= +github.com/aws/aws-sdk-go-v2/internal/ini v1.6.0/go.mod h1:rdAuXeHWhI/zkpYcO5n8WCpaIgY9MUxFyBsuqq3kjyA= +github.com/aws/aws-sdk-go-v2/service/eks v1.32.0 h1:w8O52S+rH0026BqntZs55OWbV8Huq8VhXL9bSixCRYc= +github.com/aws/aws-sdk-go-v2/service/eks v1.32.0/go.mod h1:l13fsSUzq27egNlyKBw4l0+mOSwa5Kt6r3JmJblYt14= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.2 h1:h7j73yuAVVjic8pqswh+L/7r2IHP43QwRyOu6zcCDDE= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.10.2/go.mod h1:H07AHdK5LSy8F7EJUQhoxyiCNkePoHj2D8P2yGTWafo= +github.com/aws/aws-sdk-go-v2/service/sso v1.17.1 h1:km+ZNjtLtpXYf42RdaDZnNHm9s7SYAuDGTafy6nd89A= +github.com/aws/aws-sdk-go-v2/service/sso v1.17.1/go.mod h1:aHBr3pvBSD5MbzOvQtYutyPLLRPbl/y9x86XyJJnUXQ= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.1 h1:iRFNqZH4a67IqPvK8xxtyQYnyrlsvwmpHOe9r55ggBA= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.19.1/go.mod h1:pTy5WM+6sNv2tB24JNKFtn6EvciQ5k40ZJ0pq/Iaxj0= +github.com/aws/aws-sdk-go-v2/service/sts v1.25.1 h1:txgVXIXWPXyqdiVn92BV6a/rgtpX31HYdsOYj0sVQQQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.25.1/go.mod h1:VAiJiNaoP1L89STFlEMgmHX1bKixY+FaP+TpRFrmyZ4= +github.com/aws/smithy-go v1.16.0 h1:gJZEH/Fqh+RsvlJ1Zt4tVAtV6bKkp3cC+R6FCZMNzik= +github.com/aws/smithy-go v1.16.0/go.mod h1:NukqUGpCZIILqqiV0NIjeFh24kd/FAa4beRb6nbIUPE= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/samber/lo v1.38.1 h1:j2XEAqXKb09Am4ebOg31SpvzUTTs6EN3VfgeLUhPdXM= +github.com/samber/lo v1.38.1/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 h1:3MTrJm4PyNL9NBqvYDSj3DHl46qQakyfqfWo4jgfaEM= +golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/test/suites/integration/ami_test.go b/test/suites/integration/ami_test.go index 53eb65e0d6b7..72f6fc06b503 100644 --- a/test/suites/integration/ami_test.go +++ b/test/suites/integration/ami_test.go @@ -102,6 +102,7 @@ var _ = Describe("AMI", func() { Expect(pod.Spec.NodeName).To(Equal("")) }) It("should support ami selector Name with default owners", func() { + Skip("failing in forked account due to a private AMI") output, err := env.EC2API.DescribeImages(&ec2.DescribeImagesInput{ ImageIds: []*string{aws.String(customAMI)}, }) diff --git a/test/suites/integration/scheduling_test.go b/test/suites/integration/scheduling_test.go index 6f1ba7654b39..ff6b16dbca88 100644 --- a/test/suites/integration/scheduling_test.go +++ b/test/suites/integration/scheduling_test.go @@ -405,7 +405,7 @@ var _ = Describe("Scheduling", Ordered, ContinueOnFailure, func() { { Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, - Values: []string{"c4.large"}, + Values: []string{"c5.large"}, }, }, }, @@ -416,7 +416,7 @@ var _ = Describe("Scheduling", Ordered, ContinueOnFailure, func() { env.ExpectCreated(pod, nodeClass, nodePoolLowPri, nodePoolHighPri) env.EventuallyExpectHealthy(pod) env.ExpectCreatedNodeCount("==", 1) - Expect(ptr.StringValue(env.GetInstance(pod.Spec.NodeName).InstanceType)).To(Equal("c4.large")) + Expect(ptr.StringValue(env.GetInstance(pod.Spec.NodeName).InstanceType)).To(Equal("c5.large")) Expect(env.GetNode(pod.Spec.NodeName).Labels[corev1beta1.NodePoolLabelKey]).To(Equal(nodePoolHighPri.Name)) }) }) diff --git a/test/suites/integration/utilization_test.go b/test/suites/integration/utilization_test.go index 3e159873014a..73e59386e56d 100644 --- a/test/suites/integration/utilization_test.go +++ b/test/suites/integration/utilization_test.go @@ -35,7 +35,7 @@ var _ = Describe("Utilization", Label(debug.NoWatch), Label(debug.NoEvents), fun v1.NodeSelectorRequirement{ Key: v1.LabelInstanceTypeStable, Operator: v1.NodeSelectorOpIn, - Values: []string{"t3a.small"}, + Values: []string{"t3.small"}, }, v1.NodeSelectorRequirement{ Key: v1beta1.LabelInstanceCategory,