diff --git a/.github/actions/e2e/cleanup/action.yaml b/.github/actions/e2e/cleanup/action.yaml index 2e0c4a56d497..2f4c4600cb5d 100644 --- a/.github/actions/e2e/cleanup/action.yaml +++ b/.github/actions/e2e/cleanup/action.yaml @@ -42,6 +42,22 @@ runs: break fi done + - name: delete-cluster + shell: bash + run: | + eksctl delete cluster --name ${{ inputs.cluster_name }} --timeout 60m --wait || true + - name: delete-network-interfaces + shell: bash + run: | + aws ec2 describe-network-interfaces \ + --filter Name=tag:cluster.k8s.amazonaws.com/name,Values=${{ inputs.cluster_name }} \ + --query "NetworkInterfaces[*].NetworkInterfaceId" \ + --output text | + xargs \ + -n 1 \ + -r \ + aws ec2 delete-network-interface \ + --network-interface-id - name: delete-security-group shell: bash # For drift testing, we create a security group and need to clean it up here @@ -56,10 +72,6 @@ runs: -r \ aws ec2 delete-security-group \ --group-id - - name: delete-cluster - shell: bash - run: | - eksctl delete cluster --name ${{ inputs.cluster_name }} --timeout 60m --wait || true - name: delete-iam-policies-stack shell: bash run: | diff --git a/test/hack/cleanup/main.go b/test/hack/cleanup/main.go index 244c13828914..2493bcdf423d 100644 --- a/test/hack/cleanup/main.go +++ b/test/hack/cleanup/main.go @@ -44,6 +44,7 @@ const ( karpenterLaunchTemplateTag = "karpenter.k8s.aws/cluster" karpenterSecurityGroupTag = "karpenter.sh/discovery" karpenterTestingTag = "testing.karpenter.sh/cluster" + k8sClusterTag = "cluster.k8s.amazonaws.com/name" githubRunURLTag = "github.com/run-url" ) @@ -74,6 +75,7 @@ func main() { metricsClient := MetricsClient(×tream{timestreamClient: timestreamwrite.NewFromConfig(cfg, WithRegion(karpenterMetricRegion))}) resources := []CleanableResourceType{ + &eni{ec2Client: ec2Client}, &instance{ec2Client: ec2Client}, &securitygroup{ec2Client: ec2Client}, &stack{cloudFormationClient: cloudFormationClient}, @@ -441,6 +443,71 @@ func (ip *instanceProfile) Cleanup(ctx context.Context, names []string) ([]strin return deleted, errs } +type eni struct { + ec2Client *ec2.Client +} + +func (e *eni) Type() string { + return "ElasticNetworkInterface" +} + +func (e *eni) Get(ctx context.Context, expirationTime time.Time) (ids []string, err error) { + var nextToken *string + for { + out, err := e.ec2Client.DescribeNetworkInterfaces(ctx, &ec2.DescribeNetworkInterfacesInput{ + Filters: []ec2types.Filter{ + { + Name: lo.ToPtr("tag-key"), + Values: []string{k8sClusterTag}, + }, + }, + NextToken: nextToken, + }) + if err != nil { + return ids, err + } + + for _, ni := range out.NetworkInterfaces { + creationDate, found := lo.Find(ni.TagSet, func(tag ec2types.Tag) bool { + return *tag.Key == "node.k8s.amazonaws.com/createdAt" + }) + if !found { + continue + } + time, err := time.Parse(time.RFC3339, *creationDate.Value) + if err != nil { + continue + } + if ni.Status == ec2types.NetworkInterfaceStatusAvailable && time.Before(expirationTime) { + ids = append(ids, lo.FromPtr(ni.NetworkInterfaceId)) + } + } + + nextToken = out.NextToken + if nextToken == nil { + break + } + } + return ids, err +} + +func (e *eni) Cleanup(ctx context.Context, ids []string) ([]string, error) { + deleted := []string{} + var errs error + for i := range ids { + _, err := e.ec2Client.DeleteNetworkInterface(ctx, &ec2.DeleteNetworkInterfaceInput{ + NetworkInterfaceId: aws.String(ids[i]), + }) + if err != nil { + errs = multierr.Append(errs, err) + continue + } + deleted = append(deleted, ids[i]) + } + + return deleted, errs +} + type timestream struct { timestreamClient *timestreamwrite.Client }