fix SG and OICD leak

engedaam · Jul 26, 2023 · bda02b1 · bda02b1
1 parent 4cecea2
commit bda02b1
Show file tree

Hide file tree

Showing 7 changed files with 239 additions and 100 deletions.
diff --git a/.github/actions/e2e/cleanup/action.yaml b/.github/actions/e2e/cleanup/action.yaml
@@ -27,32 +27,45 @@ runs:
     - uses: actions/checkout@v3
       with:
         ref: ${{ inputs.git_ref }}
-    - uses: ./.github/actions/e2e/install-eksctl
-      with:
-        eksctl_version: v0.147.0
-    - name: delete-cluster
-      shell: bash
-      run: |
-        eksctl delete cluster --name ${{ inputs.cluster_name }} --timeout 60m --wait || true
-    - name: delete-iam-policies-stack
-      shell: bash
-      run: |
-        aws cloudformation delete-stack --stack-name iam-${{ inputs.cluster_name }}
-        aws cloudformation wait stack-delete-complete --stack-name iam-${{ inputs.cluster_name }}
-    - name: delete-cluster-stack
-      shell: bash
-      run: |
-        aws cloudformation delete-stack --stack-name eksctl-${{ inputs.cluster_name }}-cluster || true
-        aws cloudformation wait stack-delete-complete --stack-name eksctl-${{ inputs.cluster_name }}-cluster || true
-    - name: delete-launch-templates
+    # - uses: ./.github/actions/e2e/install-eksctl
+    #   with:
+    #     eksctl_version: v0.147.0
+    - name: delete-security-group
       shell: bash
-      run: |
-        aws ec2 describe-launch-templates \
-          --filter Name=tag:karpenter.k8s.aws/cluster,Values=${{ inputs.cluster_name }} \
-          --query "LaunchTemplates[*].LaunchTemplateId" \
-          --output text |
-        xargs \
-          -n 1 \
-          -r \
-          aws ec2 delete-launch-template \
-            --launch-template-id
+      run: | 
+        aws ec2 describe-security-groups \
+          --group-names "security-group-drift"
+          --filters Name=tag:karpenter.sh/discovery,Values=${{ inputs.cluster_name }} \
+          --query "SecurityGroups[*].{ID:GroupId}" \
+          --output text
+    # - name: delete-cluster
+    #   shell: bash
+    #   run: |
+    #     eksctl delete cluster --name ${{ inputs.cluster_name }} --timeout 60m --wait || true
+    # - name: delete-iam-policies-stack
+    #   shell: bash
+    #   run: |
+    #     aws cloudformation delete-stack --stack-name iam-${{ inputs.cluster_name }}
+    #     aws cloudformation wait stack-delete-complete --stack-name iam-${{ inputs.cluster_name }}
+    # - name: delete-cluster-stack
+    #   shell: bash
+    #   run: |
+    #     aws cloudformation delete-stack --stack-name eksctl-${{ inputs.cluster_name }}-cluster || true
+    #     aws cloudformation wait stack-delete-complete --stack-name eksctl-${{ inputs.cluster_name }}-cluster || true
+    # - name: delete-launch-templates
+    #   shell: bash
+    #   run: |
+    #     aws ec2 describe-launch-templates \
+    #       --filter Name=tag:karpenter.k8s.aws/cluster,Values=${{ inputs.cluster_name }} \
+    #       --query "LaunchTemplates[*].LaunchTemplateId" \
+    #       --output text |
+    #     xargs \
+    #       -n 1 \
+    #       -r \
+    #       aws ec2 delete-launch-template \
+    #         --launch-template-id
+
+          #   aws ec2 describe-security-groups \                      
+          # --filters Name=tag:karpenter.sh/discovery,Values=${{ inputs.cluster_name }} \
+          # --query "SecurityGroups[*].{ID:GroupId}" \
+          # --output text
diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
@@ -80,74 +80,74 @@ jobs:
           role-to-assume: arn:aws:iam::${{ vars.ACCOUNT_ID }}:role/${{ vars.ROLE_NAME }}
           aws-region: ${{ inputs.region }}
           role-duration-seconds: 21600
-      - name: add jitter on cluster creation
-        run: |
-          # Creating jitter so that we can stagger cluster creation to avoid throttling 
-          sleep $(( $RANDOM % 60 + 1 ))
-      - name: generate cluster name
-        run: |
-          CLUSTER_NAME=$(echo ${{ inputs.suite }}-$RANDOM$RANDOM | awk '{print tolower($0)}')
-          echo "Using cluster name \"$CLUSTER_NAME\""
-          echo CLUSTER_NAME=$CLUSTER_NAME >> $GITHUB_ENV
-      - name: create eks cluster '${{ env.CLUSTER_NAME }}'
-        uses: ./.github/actions/e2e/create-cluster
-        with:
-          account_id: ${{ vars.ACCOUNT_ID }}
-          role: ${{ vars.ROLE_NAME }}
-          region: ${{ inputs.region }}
-          cluster_name: ${{ env.CLUSTER_NAME }}
-          k8s_version: ${{ inputs.k8s_version }}
-          ip_family: ${{ inputs.suite == 'IPv6' && 'IPv6' || 'IPv4' }} # Set the value to IPv6 if IPv6 suite, else IPv4
-          git_ref: ${{ inputs.git_ref }}
-      - name: install prometheus
-        uses: ./.github/actions/e2e/install-prometheus
-        with:
-          account_id: ${{ vars.ACCOUNT_ID }}
-          role: ${{ vars.ROLE_NAME }}
-          region: ${{ vars.PROMETHEUS_REGION }}
-          cluster_name: ${{ env.CLUSTER_NAME }}
-          workspace_id: ${{ vars.WORKSPACE_ID }}
-          git_ref: ${{ inputs.git_ref }}
-      - name: install karpenter
-        uses: ./.github/actions/e2e/install-karpenter
-        with:
-          account_id: ${{ vars.ACCOUNT_ID }}
-          role: ${{ vars.ROLE_NAME }}
-          region: ${{ inputs.region }}
-          cluster_name: ${{ env.CLUSTER_NAME }}
-          git_ref: ${{ inputs.git_ref }}
-      - name: run the ${{ inputs.suite }} test suite
-        run: |
-          aws eks update-kubeconfig --name ${{ env.CLUSTER_NAME }}
-          TEST_SUITE="${{ inputs.suite }}" ENABLE_METRICS=${{ inputs.enable_metrics }} METRICS_REGION=${{ vars.TIMESTREAM_REGION }} GIT_REF="$(git rev-parse HEAD)" make e2etests
-      - name: notify slack of success or failure
-        uses: ./.github/actions/e2e/slack/notify
-        if: (success() || failure()) && inputs.event_name != 'workflow_run' && inputs.event_name != 'conformance'
-        with:
-          url: ${{ secrets.SLACK_WEBHOOK_URL }}
-          suite: ${{ inputs.suite }}
-          k8s_version: ${{ inputs.k8s_version }}
-          event_name: ${{ inputs.event_name }}
-          git_ref: ${{ inputs.git_ref }}
-      - name: dump logs on failure
-        uses: ./.github/actions/e2e/dump-logs
-        if: failure() || cancelled()
-        with:
-          account_id: ${{ vars.ACCOUNT_ID }}
-          role: ${{ vars.ROLE_NAME }}
-          region: ${{ inputs.region }}
-          cluster_name: ${{ env.CLUSTER_NAME }}
+      # - name: add jitter on cluster creation
+      #   run: |
+      #     # Creating jitter so that we can stagger cluster creation to avoid throttling 
+      #     sleep $(( $RANDOM % 60 + 1 ))
+      # - name: generate cluster name
+      #   run: |
+      #     CLUSTER_NAME=$(echo ${{ inputs.suite }}-$RANDOM$RANDOM | awk '{print tolower($0)}')
+      #     echo "Using cluster name \"$CLUSTER_NAME\""
+      #     echo CLUSTER_NAME=$CLUSTER_NAME >> $GITHUB_ENV
+      # - name: create eks cluster '${{ env.CLUSTER_NAME }}'
+      #   uses: ./.github/actions/e2e/create-cluster
+      #   with:
+      #     account_id: ${{ vars.ACCOUNT_ID }}
+      #     role: ${{ vars.ROLE_NAME }}
+      #     region: ${{ inputs.region }}
+      #     cluster_name: ${{ env.CLUSTER_NAME }}
+      #     k8s_version: ${{ inputs.k8s_version }}
+      #     ip_family: ${{ inputs.suite == 'IPv6' && 'IPv6' || 'IPv4' }} # Set the value to IPv6 if IPv6 suite, else IPv4
+      #     git_ref: ${{ inputs.git_ref }}
+      # - name: install prometheus
+      #   uses: ./.github/actions/e2e/install-prometheus
+      #   with:
+      #     account_id: ${{ vars.ACCOUNT_ID }}
+      #     role: ${{ vars.ROLE_NAME }}
+      #     region: ${{ vars.PROMETHEUS_REGION }}
+      #     cluster_name: ${{ env.CLUSTER_NAME }}
+      #     workspace_id: ${{ vars.WORKSPACE_ID }}
+      #     git_ref: ${{ inputs.git_ref }}
+      # - name: install karpenter
+      #   uses: ./.github/actions/e2e/install-karpenter
+      #   with:
+      #     account_id: ${{ vars.ACCOUNT_ID }}
+      #     role: ${{ vars.ROLE_NAME }}
+      #     region: ${{ inputs.region }}
+      #     cluster_name: ${{ env.CLUSTER_NAME }}
+      #     git_ref: ${{ inputs.git_ref }}
+      # - name: run the ${{ inputs.suite }} test suite
+      #   run: |
+      #     aws eks update-kubeconfig --name ${{ env.CLUSTER_NAME }}
+      #     FOCUS="should deprovision nodes that have drifted due to securitygroup" ENABLE_METRICS=${{ inputs.enable_metrics }} METRICS_REGION=${{ vars.TIMESTREAM_REGION }} GIT_REF="$(git rev-parse HEAD)" make e2etests
+      # - name: notify slack of success or failure
+      #   uses: ./.github/actions/e2e/slack/notify
+      #   if: (success() || failure()) && inputs.event_name != 'workflow_run' && inputs.event_name != 'conformance'
+      #   with:
+      #     url: ${{ secrets.SLACK_WEBHOOK_URL }}
+      #     suite: ${{ inputs.suite }}
+      #     k8s_version: ${{ inputs.k8s_version }}
+      #     event_name: ${{ inputs.event_name }}
+      #     git_ref: ${{ inputs.git_ref }}
+      # - name: dump logs on failure
+      #   uses: ./.github/actions/e2e/dump-logs
+      #   if: failure() || cancelled()
+      #   with:
+      #     account_id: ${{ vars.ACCOUNT_ID }}
+      #     role: ${{ vars.ROLE_NAME }}
+      #     region: ${{ inputs.region }}
+      #     cluster_name: ${{ env.CLUSTER_NAME }}
       - name: cleanup karpenter and cluster '${{ env.CLUSTER_NAME }}' resources
         uses: ./.github/actions/e2e/cleanup
         if: always()
         with:
           account_id: ${{ vars.ACCOUNT_ID }}
           role: ${{ vars.ROLE_NAME }}
           region: ${{ inputs.region }}
-          cluster_name: ${{ env.CLUSTER_NAME }}
+          cluster_name: aengeda-karpenter-playground-us-west-2
           git_ref: ${{ inputs.git_ref }}
-      - if: always() && inputs.event_name == 'workflow_run'
-        uses: ./.github/actions/commit-status/end
-        with:
-          name: "${{ github.workflow }} / e2e (${{ inputs.suite }}) / ${{ github.job }} (snapshot)"
-          git_ref: ${{ inputs.git_ref }}
+      # - if: always() && inputs.event_name == 'workflow_run'
+      #   uses: ./.github/actions/commit-status/end
+      #   with:
+      #     name: "${{ github.workflow }} / e2e (${{ inputs.suite }}) / ${{ github.job }} (snapshot)"
+      #     git_ref: ${{ inputs.git_ref }}
diff --git a/.github/workflows/sweeper.yaml b/.github/workflows/sweeper.yaml
@@ -9,14 +9,18 @@ permissions:
 jobs:
   sweeper:
     if: github.repository == 'aws/karpenter' || github.event_name == 'workflow_dispatch'
+    strategy:
+      fail-fast: false
+      matrix:
+        region: [us-east-2, us-west-2, eu-west-1]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
       - name: configure aws credentials
         uses: aws-actions/configure-aws-credentials@v2
         with:
           role-to-assume: arn:aws:iam::${{ vars.ACCOUNT_ID }}:role/${{ vars.ROLE_NAME }}
-          aws-region: ${{ vars.AWS_REGION }}
+          aws-region: ${{ matrix.region }}
       - uses: actions/setup-go@v4
         with:
           go-version-file: test/hack/cleanup/go.mod

diff --git a/test/hack/cleanup/go.mod b/test/hack/cleanup/go.mod
@@ -7,6 +7,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/cloudformation v1.30.0
 	github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.26.2
 	github.com/aws/aws-sdk-go-v2/service/ec2 v1.102.0
+	github.com/aws/aws-sdk-go-v2/service/iam v1.21.0
 	github.com/samber/lo v1.38.1
 	go.uber.org/zap v1.24.0
 )

diff --git a/test/hack/cleanup/go.sum b/test/hack/cleanup/go.sum
@@ -18,6 +18,8 @@ github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.26.2 h1:PWGu2JhCb/XJlJ7SSFJq7
 github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.26.2/go.mod h1:2KOZkkzMDZCo/aLzPhys06mHNkiU74u85aMJA3PLRvg=
 github.com/aws/aws-sdk-go-v2/service/ec2 v1.102.0 h1:P4dyjm49F2kKws0FpouBC6fjVImACXKt752+CWa01lM=
 github.com/aws/aws-sdk-go-v2/service/ec2 v1.102.0/go.mod h1:tIctCeX9IbzsUTKHt53SVEcgyfxV2ElxJeEB+QUbc4M=
+github.com/aws/aws-sdk-go-v2/service/iam v1.21.0 h1:8hEpu60CWlrp7iEBUFRZhgPoX6+gadaGL1sD4LoRYS0=
+github.com/aws/aws-sdk-go-v2/service/iam v1.21.0/go.mod h1:aQZ8BI+reeaY7RI/QQp7TKCSUHOesTdrzzylp3CW85c=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28 h1:bkRyG4a929RCnpVSTvLM2j/T4ls015ZhhYApbmYs15s=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28/go.mod h1:jj7znCIg05jXlaGBlFMGP8+7UN3VtCkRBG2spnmRQkU=
 github.com/aws/aws-sdk-go-v2/service/sso v1.12.12 h1:nneMBM2p79PGWBQovYO/6Xnc2ryRMw3InnDJq1FHkSY=