Skip to content

Commit

Permalink
Merge pull request #116 from engedaam/use-sweeper-to-clean-up-resources
Browse files Browse the repository at this point in the history 
Update E2E clean-up
  • Loading branch information
@engedaam
engedaam authored Oct 30, 2023
2 parents d8af77a + bdd50d1 commit 5fd1d98
Show file tree
Hide file tree
Showing 5 changed files with 96 additions and 104 deletions.
67 changes: 4 additions & 63 deletions .github/actions/e2e/cleanup/action.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,71 +27,12 @@ runs:
- uses: ./.github/actions/e2e/install-eksctl
with:
version: ${{ inputs.eksctl_version }}
- name: delete-instance-profiles
shell:
run: |
for name in $(aws iam list-instance-profiles --query "InstanceProfiles[*].{Name:InstanceProfileName}" --output text); do
tags=$(aws iam list-instance-profile-tags --instance-profile-name $name --output json || true)
if [[ $(echo $tags | jq -r '.Tags[] | select(.Key == "testing/cluster") | .Value') == "${{ inputs.cluster_name }}" ]]; then
echo "Deleting instance profile '$name'..."
roleName=$(aws iam get-instance-profile --instance-profile-name $name --query "InstanceProfile.Roles[*].{Name:RoleName}" --output text)
aws iam remove-role-from-instance-profile --instance-profile-name $name --role-name $roleName
aws iam delete-instance-profile --instance-profile-name $name
fi
done
- name: delete-cluster
shell: bash
run: |
eksctl delete cluster --name ${{ inputs.cluster_name }} --timeout 60m --wait || true
- name: delete-network-interfaces
shell: bash
run: |
aws ec2 describe-network-interfaces \
--filter Name=tag:cluster.k8s.amazonaws.com/name,Values=${{ inputs.cluster_name }} \
--query "NetworkInterfaces[*].NetworkInterfaceId" \
--output text |
xargs \
-n 1 \
-r \
aws ec2 delete-network-interface \
--network-interface-id
- name: delete-security-group
shell: bash
# For drift testing, we create a security group and need to clean it up here
# to avoid leaks if the tests is not fully completed
run: |
aws ec2 describe-security-groups \
--filters Name=group-name,Values=security-group-drift Name=tag:karpenter.sh/discovery,Values=${{ inputs.cluster_name }} \
--query "SecurityGroups[*].{ID:GroupId}" \
--output text |
xargs \
-n 1 \
-r \
aws ec2 delete-security-group \
--group-id
- name: delete-iam-alpha-policy
shell: bash
run: |
aws iam delete-policy --policy-arn "arn:aws:iam::${{ inputs.account_id }}:policy/KarpenterControllerPolicy-Alpha-${{ inputs.cluster_name }}" || true
- name: delete-iam-policies-stack
shell: bash
run: |
aws cloudformation delete-stack --stack-name iam-${{ inputs.cluster_name }}
aws cloudformation wait stack-delete-complete --stack-name iam-${{ inputs.cluster_name }}
- name: delete-cluster-stack
- run: |
go run main.go ${{ inputs.cluster_name }}
working-directory: ./test/hack/cleanup
shell: bash
run: |
aws cloudformation delete-stack --stack-name eksctl-${{ inputs.cluster_name }}-cluster || true
aws cloudformation wait stack-delete-complete --stack-name eksctl-${{ inputs.cluster_name }}-cluster || true
- name: delete-launch-templates
shell: bash
run: |
aws ec2 describe-launch-templates \
--filter Name=tag:karpenter.k8s.aws/cluster,Values=${{ inputs.cluster_name }} \
--query "LaunchTemplates[*].LaunchTemplateId" \
--output text |
xargs \
-n 1 \
-r \
aws ec2 delete-launch-template \
--launch-template-id
name: "Run cleanup script"
10 changes: 5 additions & 5 deletions .github/actions/install-deps/action.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,14 @@ inputs:
runs:
using: "composite"
steps:
- uses: actions/setup-go@v4
with:
go-version-file: go.mod
check-latest: true
cache-dependency-path: "**/go.sum"
# Root path permission workaround for caching https://github.com/actions/cache/issues/845#issuecomment-1252594999
- run: sudo chown "$USER" /usr/local
shell: bash
- uses: actions/setup-go@v4
with:
go-version-file: test/hack/cleanup/go.mod
check-latest: true
cache-dependency-path: "test/hack/cleanup/go.sum"
- uses: actions/cache@v3
id: cache-toolchain
with:
Expand Down
8 changes: 4 additions & 4 deletions test/hack/cleanup/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,18 +16,18 @@ require (
)

require (
github.com/aws/aws-sdk-go-v2 v1.20.1 // indirect
github.com/aws/aws-sdk-go-v2 v1.21.0 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.13.26 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.38 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.32 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.7.32 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.12.12 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.19.2 // indirect
github.com/aws/smithy-go v1.14.1 // indirect
github.com/aws/smithy-go v1.14.2 // indirect
github.com/go-logr/logr v1.2.3 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
go.uber.org/atomic v1.7.0 // indirect
Expand Down
12 changes: 8 additions & 4 deletions test/hack/cleanup/go.sum
View file
Original file line number Diff line number Diff line change
@@ -1,20 +1,23 @@
github.com/aws/aws-sdk-go v1.44.309 h1:IPJOFBzXekakxmEpDwd4RTKmmBR6LIAiXgNsM51bWbU=
github.com/aws/aws-sdk-go v1.44.309/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
github.com/aws/aws-sdk-go-v2 v1.18.1/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw=
github.com/aws/aws-sdk-go-v2 v1.20.1 h1:rZBf5DWr7YGrnlTK4kgDQGn1ltqOg5orCYb/UhOFZkg=
github.com/aws/aws-sdk-go-v2 v1.20.1/go.mod h1:NU06lETsFm8fUC6ZjhgDpVBcGZTFQ6XM+LZWZxMI4ac=
github.com/aws/aws-sdk-go-v2 v1.21.0 h1:gMT0IW+03wtYJhRqTVYn0wLzwdnK9sRMcxmtfGzRdJc=
github.com/aws/aws-sdk-go-v2 v1.21.0/go.mod h1:/RfNgGmRxI+iFOB1OeJUyxiU+9s88k3pfHvDagGEp0M=
github.com/aws/aws-sdk-go-v2/config v1.18.27 h1:Az9uLwmssTE6OGTpsFqOnaGpLnKDqNYOJzWuC6UAYzA=
github.com/aws/aws-sdk-go-v2/config v1.18.27/go.mod h1:0My+YgmkGxeqjXZb5BYme5pc4drjTnM+x1GJ3zv42Nw=
github.com/aws/aws-sdk-go-v2/credentials v1.13.26 h1:qmU+yhKmOCyujmuPY7tf5MxR/RKyZrOPO3V4DobiTUk=
github.com/aws/aws-sdk-go-v2/credentials v1.13.26/go.mod h1:GoXt2YC8jHUBbA4jr+W3JiemnIbkXOfxSXcisUsZ3os=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4 h1:LxK/bitrAr4lnh9LnIS6i7zWbCOdMsfzKFBI6LUCS0I=
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4/go.mod h1:E1hLXN/BL2e6YizK1zFlYd8vsfi2GTjbjBazinMmeaM=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34/go.mod h1:wZpTEecJe0Btj3IYnDx/VlUzor9wm3fJHyvLpQF0VwY=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.38 h1:c8ed/T9T2K5I+h/JzmF5tpI46+OODQ74dzmdo+QnaMg=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.38/go.mod h1:qggunOChCMu9ZF/UkAfhTz25+U2rLVb3ya0Ua6TTfCA=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 h1:22dGT7PneFMx4+b3pz7lMTRyN8ZKH7M2cW4GP9yUS2g=
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41/go.mod h1:CrObHAuPneJBlfEJ5T3szXOUkLEThaGfvnhTf33buas=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28/go.mod h1:7VRpKQQedkfIEXb4k52I7swUnZP0wohVajJMRn3vsUw=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.32 h1:hNeAAymUY5gu11WrrmFb3CVIp9Dar9hbo44yzzcQpzA=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.32/go.mod h1:0ZXSqrty4FtQ7p8TEuRde/SZm9X05KT18LAUlR40Ln0=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 h1:SijA0mgjV8E+8G45ltVHs0fvKpTj8xmZJ3VwhGKtUSI=
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35/go.mod h1:SJC1nEVVva1g3pHAIdCp7QsRIkMmLAgoDquQ9Rr8kYw=
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35 h1:LWA+3kDM8ly001vJ1X1waCuLJdtTl48gwkPKWy9sosI=
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35/go.mod h1:0Eg1YjxE0Bhn56lx+SHJwCzhW+2JGtizsrx+lCqrfm0=
github.com/aws/aws-sdk-go-v2/service/cloudformation v1.30.0 h1:XbDkc4FLeg1RfnqeblfbJvaEabqq9ByZl4zqyPFkfSc=
Expand All @@ -36,8 +39,9 @@ github.com/aws/aws-sdk-go-v2/service/sts v1.19.2/go.mod h1:dp0yLPsLBOi++WTxzCjA/
github.com/aws/aws-sdk-go-v2/service/timestreamwrite v1.18.2 h1:5QyvAYyr+ZibpVxfovzd5JMTZ8miv9s3zT4jG4PJkIA=
github.com/aws/aws-sdk-go-v2/service/timestreamwrite v1.18.2/go.mod h1:3ZCiyyNF7myh/a7DcOjcqRsLmSF9EdhEZSr00Qlui4s=
github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
github.com/aws/smithy-go v1.14.1 h1:EFKMUmH/iHMqLiwoEDx2rRjRQpI1YCn5jTysoaDujFs=
github.com/aws/smithy-go v1.14.1/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
github.com/aws/smithy-go v1.14.2 h1:MJU9hqBGbvWZdApzpvoF2WAIJDbtjK2NDJSiJP7HblQ=
github.com/aws/smithy-go v1.14.2/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
Expand Down
103 changes: 75 additions & 28 deletions test/hack/cleanup/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,15 @@ package main
import (
"context"
"fmt"
"os"
"time"

"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/service/cloudformation"
cloudformationtypes "github.com/aws/aws-sdk-go-v2/service/cloudformation/types"
"github.com/aws/aws-sdk-go-v2/service/ec2"
ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types"

"github.com/aws/aws-sdk-go-v2/service/iam"
"github.com/aws/aws-sdk-go-v2/service/timestreamwrite"
timestreamtypes "github.com/aws/aws-sdk-go-v2/service/timestreamwrite/types"
Expand Down Expand Up @@ -52,7 +54,7 @@ const (

type CleanableResourceType interface {
Type() string
Get(context.Context, time.Time) ([]string, error)
Get(context.Context, string, time.Time) ([]string, error)
Cleanup(context.Context, []string) ([]string, error)
}

Expand All @@ -61,6 +63,10 @@ type MetricsClient interface {
}

func main() {
clusterName := ""
if len(os.Args) == 2 {
clusterName = os.Args[1]
}
ctx := context.Background()
cfg := lo.Must(config.LoadDefaultConfig(ctx))

Expand All @@ -78,15 +84,16 @@ func main() {

resources := []CleanableResourceType{
&eni{ec2Client: ec2Client},
&instance{ec2Client: ec2Client},
&securitygroup{ec2Client: ec2Client},
&instance{ec2Client: ec2Client},
&stack{cloudFormationClient: cloudFormationClient},
&launchtemplate{ec2Client: ec2Client},
&oidc{iamClient: iamClient},
&instanceProfile{iamClient: iamClient},
}

workqueue.ParallelizeUntil(ctx, len(resources), len(resources), func(i int) {
ids, err := resources[i].Get(ctx, expirationTime)
ids, err := resources[i].Get(ctx, clusterName, expirationTime)
if err != nil {
logger.With("type", resources[i].Type()).Errorf("%v", err)
}
Expand All @@ -112,7 +119,7 @@ func (i *instance) Type() string {
return "Instances"
}

func (i *instance) Get(ctx context.Context, expirationTime time.Time) (ids []string, err error) {
func (i *instance) Get(ctx context.Context, _ string, expirationTime time.Time) (ids []string, err error) {
var nextToken *string
for {
out, err := i.ec2Client.DescribeInstances(ctx, &ec2.DescribeInstancesInput{
Expand Down Expand Up @@ -169,16 +176,29 @@ func (sg *securitygroup) Type() string {
return "SecurityGroup"
}

func (sg *securitygroup) Get(ctx context.Context, expirationTime time.Time) (ids []string, err error) {
func (sg *securitygroup) Get(ctx context.Context, clusterName string, expirationTime time.Time) (ids []string, err error) {
var nextToken *string
var ec2Filter []ec2types.Filter

if clusterName == "" {
ec2Filter = []ec2types.Filter{
{
Name: lo.ToPtr("group-name"),
Values: []string{"security-group-drift"},
},
}
} else {
ec2Filter = []ec2types.Filter{
{
Name: lo.ToPtr("tag:" + karpenterSecurityGroupTag),
Values: []string{clusterName},
},
}
}

for {
out, err := sg.ec2Client.DescribeSecurityGroups(ctx, &ec2.DescribeSecurityGroupsInput{
Filters: []ec2types.Filter{
{
Name: lo.ToPtr("group-name"),
Values: []string{"security-group-drift"},
},
},
Filters: ec2Filter,
NextToken: nextToken,
})
if err != nil {
Expand Down Expand Up @@ -234,8 +254,11 @@ func (s *stack) Type() string {
return "CloudformationStacks"
}

func (s *stack) Get(ctx context.Context, expirationTime time.Time) (names []string, err error) {
func (s *stack) Get(ctx context.Context, clusterName string, expirationTime time.Time) (names []string, err error) {
var nextToken *string
if clusterName != "" {
return []string{fmt.Sprintf("iam-%s", clusterName), fmt.Sprintf("eksctl-%s-cluster", clusterName)}, nil
}
for {
out, err := s.cloudFormationClient.DescribeStacks(ctx, &cloudformation.DescribeStacksInput{
NextToken: nextToken,
Expand Down Expand Up @@ -290,16 +313,28 @@ func (lt *launchtemplate) Type() string {
return "LaunchTemplates"
}

func (lt *launchtemplate) Get(ctx context.Context, expirationTime time.Time) (names []string, err error) {
func (lt *launchtemplate) Get(ctx context.Context, clusterName string, expirationTime time.Time) (names []string, err error) {
var nextToken *string
var ec2Filter []ec2types.Filter
if clusterName == "" {
ec2Filter = []ec2types.Filter{
{
Name: lo.ToPtr("tag-key"),
Values: []string{karpenterLaunchTemplateTag},
},
}
} else {
ec2Filter = []ec2types.Filter{
{
Name: lo.ToPtr("tag:" + karpenterLaunchTemplateTag),
Values: []string{clusterName},
},
}
}

for {
out, err := lt.ec2Client.DescribeLaunchTemplates(ctx, &ec2.DescribeLaunchTemplatesInput{
Filters: []ec2types.Filter{
{
Name: lo.ToPtr("tag-key"),
Values: []string{karpenterLaunchTemplateTag},
},
},
Filters: ec2Filter,
NextToken: nextToken,
})
if err != nil {
Expand Down Expand Up @@ -346,7 +381,7 @@ func (o *oidc) Type() string {
return "OpenIDConnectProvider"
}

func (o *oidc) Get(ctx context.Context, expirationTime time.Time) (names []string, err error) {
func (o *oidc) Get(ctx context.Context, _ string, expirationTime time.Time) (names []string, err error) {
out, err := o.iamClient.ListOpenIDConnectProviders(ctx, &iam.ListOpenIDConnectProvidersInput{})
if err != nil {
return names, err
Expand Down Expand Up @@ -398,7 +433,7 @@ func (ip *instanceProfile) Type() string {
return "InstanceProfile"
}

func (ip *instanceProfile) Get(ctx context.Context, expirationTime time.Time) (names []string, err error) {
func (ip *instanceProfile) Get(ctx context.Context, _ string, expirationTime time.Time) (names []string, err error) {
out, err := ip.iamClient.ListInstanceProfiles(ctx, &iam.ListInstanceProfilesInput{})
if err != nil {
return names, err
Expand Down Expand Up @@ -456,16 +491,28 @@ func (e *eni) Type() string {
return "ElasticNetworkInterface"
}

func (e *eni) Get(ctx context.Context, expirationTime time.Time) (ids []string, err error) {
func (e *eni) Get(ctx context.Context, clusterName string, expirationTime time.Time) (ids []string, err error) {
var nextToken *string
var ec2Filter []ec2types.Filter
if clusterName == "" {
ec2Filter = []ec2types.Filter{
{
Name: lo.ToPtr("tag-key"),
Values: []string{k8sClusterTag},
},
}
} else {
ec2Filter = []ec2types.Filter{
{
Name: lo.ToPtr("tag:" + k8sClusterTag),
Values: []string{clusterName},
},
}
}

for {
out, err := e.ec2Client.DescribeNetworkInterfaces(ctx, &ec2.DescribeNetworkInterfacesInput{
Filters: []ec2types.Filter{
{
Name: lo.ToPtr("tag-key"),
Values: []string{k8sClusterTag},
},
},
Filters: ec2Filter,
NextToken: nextToken,
})
if err != nil {
Expand Down

0 comments on commit 5fd1d98

Please sign in to comment.