chore: Resource clean-up move to Timestream (#4444)

test/cloudformation/README.md

@@ -13,7 +13,7 @@ aws cloudformation deploy \
 aws cloudformation deploy \
    --stack-name GithubActionsTimestream \
    --template-file timestream_cloudformation.yaml \
-   --parameter-overrides "DatabaseName=karpenterTesting" "TableName=scaleTestDurations"
+   --parameter-overrides "DatabaseName=karpenterTesting" "TableName=scaleTestDurations" "SweeperTableName=sweeperCleanedResources"
 ```
 
 ### [Optional] Deploying ManagedGrafana and its Policy
@@ -30,7 +30,7 @@ aws cloudformation deploy \
 ```console
 aws cloudformation deploy --stack-name GithubActionsIAM \
     --template-file iam_cloudformation.yaml \
-    --parameter-overrides "DatabaseName=karpenterTesting" "TableName=scaleTestDurations" "Repository=<repository>" Branches="*" "PrometheusWorkspaceID=<workspace-id>" Regions="us-east-2,us-west-2,..." \
+    --parameter-overrides "DatabaseName=karpenterTesting" "TableName=scaleTestDurations" "SweeperTableName=sweeperCleanedResources" "Repository=<repository>" Branches="*" "PrometheusWorkspaceID=<workspace-id>" Regions="us-east-2,us-west-2,..." \
     --capabilities CAPABILITY_NAMED_IAM
 ```
 

test/cloudformation/iam_cloudformation.yaml

@@ -19,6 +19,9 @@ Parameters:
   TableName:
     Type: String
     Description: "Timestream table to forward test metrics to"
+  SweeperTableName:
+    Type: String
+    Description: "Timestream table to forward leaked resources to"
 Resources:
   GithubOIDCProvider:
     Type: AWS::IAM::OIDCProvider
@@ -239,7 +242,9 @@ Resources:
                   Ref: Regions
           - Effect: Allow
             Action: timestream:WriteRecords
-            Resource: !Sub "arn:${AWS::Partition}:timestream:${AWS::Region}:${AWS::AccountId}:database/${DatabaseName}/table/${TableName}"
+            Resource:
+              - !Sub "arn:${AWS::Partition}:timestream:${AWS::Region}:${AWS::AccountId}:database/${DatabaseName}/table/${TableName}"
+              - !Sub "arn:${AWS::Partition}:timestream:${AWS::Region}:${AWS::AccountId}:database/${DatabaseName}/table/${SweeperTableName}"
           - Effect: Allow
             Action: timestream:DescribeEndpoints
             Resource: "*"

test/cloudformation/timestream_cloudformation.yaml

@@ -6,7 +6,10 @@ Parameters:
     Description: "Timestream database to forward test metrics to"
   TableName:
     Type: String
-    Description: "Timestream table to forward test metrics to"
+    Description: "Timestream table to forward scale test metrics to "
+  SweeperTableName:
+    Type: String
+    Description: "Timestream table to forward leaked resources to"
   BackupDisasterRecoveryRegion:
     Type: String
     Default: "us-east-1"
@@ -26,11 +29,21 @@ Resources:
       RetentionProperties:
         MemoryStoreRetentionPeriodInHours: "2160" # Three months of memory store retention
         MagneticStoreRetentionPeriodInDays: "1825" # 5 years of magnetic store retention
+  SweeperTimestreamTable:
+    Type: "AWS::Timestream::Table"
+    Properties:
+      DatabaseName: !Ref "TimestreamDatabase"
+      TableName: !Sub "${SweeperTableName}"
+      MagneticStoreWriteProperties:
+        EnableMagneticStoreWrites: true
+      RetentionProperties:
+        MemoryStoreRetentionPeriodInHours: "2160" # Three months of memory store retention
+        MagneticStoreRetentionPeriodInDays: "1825" # 5 years of magnetic store retention
   TimestreamBackupPlan:
     Type: "AWS::Backup::BackupPlan"
     Properties:
       BackupPlan:
-        BackupPlanName: !Sub "timestream-${DatabaseName}-${TableName}-backup"
+        BackupPlanName: !Sub "timestream-${DatabaseName}-backup"
         BackupPlanRule:
           - RuleName: "DailyBackups"
             TargetBackupVault: "Default"
@@ -47,12 +60,16 @@ Resources:
     Type: "AWS::Backup::BackupSelection"
     Properties:
       BackupSelection:
-        SelectionName: !Sub "timestream_${DatabaseName}_${TableName}_selection"
+        SelectionName: !Sub "timestream_${DatabaseName}_selection"
         IamRoleArn: !Sub "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/service-role/AWSBackupDefaultServiceRole"
         Resources:
           - !GetAtt "TimestreamTable.Arn"
+          - !GetAtt "SweeperTimestreamTable.Arn"
       BackupPlanId: !Ref "TimestreamBackupPlan"
 Outputs:
-  TimestreamTableARN:
-    Description: "Timestream Table ARN"
-    Value: !GetAtt TimestreamTable.Arn
+  ScaleTimestreamTable:
+    Description: "Scale Test Timestream Table ARN"
+    Value: !GetAtt TimestreamTable.Arn
+  SweeperTimestreamTable:
+    Description: "Account Resource Timestream Table ARN"
+    Value: !GetAtt SweeperTimestreamTable.Arn

test/hack/cleanup/go.mod

@@ -6,27 +6,28 @@ require (
 	github.com/aws/aws-sdk-go v1.44.309
 	github.com/aws/aws-sdk-go-v2/config v1.18.27
 	github.com/aws/aws-sdk-go-v2/service/cloudformation v1.30.0
-	github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.26.2
 	github.com/aws/aws-sdk-go-v2/service/ec2 v1.102.0
 	github.com/aws/aws-sdk-go-v2/service/iam v1.21.0
+	github.com/aws/aws-sdk-go-v2/service/timestreamwrite v1.18.2
 	github.com/samber/lo v1.38.1
 	go.uber.org/multierr v1.11.0
 	go.uber.org/zap v1.24.0
 	k8s.io/client-go v0.27.4
 )
 
 require (
-	github.com/aws/aws-sdk-go-v2 v1.18.1 // indirect
+	github.com/aws/aws-sdk-go-v2 v1.20.1 // indirect
 	github.com/aws/aws-sdk-go-v2/credentials v1.13.26 // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.38 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.32 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.7.32 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sso v1.12.12 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.19.2 // indirect
-	github.com/aws/smithy-go v1.13.5 // indirect
+	github.com/aws/smithy-go v1.14.1 // indirect
 	github.com/go-logr/logr v1.2.3 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	go.uber.org/atomic v1.7.0 // indirect

test/hack/cleanup/go.sum

@@ -1,27 +1,30 @@
 github.com/aws/aws-sdk-go v1.44.309 h1:IPJOFBzXekakxmEpDwd4RTKmmBR6LIAiXgNsM51bWbU=
 github.com/aws/aws-sdk-go v1.44.309/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
-github.com/aws/aws-sdk-go-v2 v1.18.1 h1:+tefE750oAb7ZQGzla6bLkOwfcQCEtC5y2RqoqCeqKo=
 github.com/aws/aws-sdk-go-v2 v1.18.1/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw=
+github.com/aws/aws-sdk-go-v2 v1.20.1 h1:rZBf5DWr7YGrnlTK4kgDQGn1ltqOg5orCYb/UhOFZkg=
+github.com/aws/aws-sdk-go-v2 v1.20.1/go.mod h1:NU06lETsFm8fUC6ZjhgDpVBcGZTFQ6XM+LZWZxMI4ac=
 github.com/aws/aws-sdk-go-v2/config v1.18.27 h1:Az9uLwmssTE6OGTpsFqOnaGpLnKDqNYOJzWuC6UAYzA=
 github.com/aws/aws-sdk-go-v2/config v1.18.27/go.mod h1:0My+YgmkGxeqjXZb5BYme5pc4drjTnM+x1GJ3zv42Nw=
 github.com/aws/aws-sdk-go-v2/credentials v1.13.26 h1:qmU+yhKmOCyujmuPY7tf5MxR/RKyZrOPO3V4DobiTUk=
 github.com/aws/aws-sdk-go-v2/credentials v1.13.26/go.mod h1:GoXt2YC8jHUBbA4jr+W3JiemnIbkXOfxSXcisUsZ3os=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4 h1:LxK/bitrAr4lnh9LnIS6i7zWbCOdMsfzKFBI6LUCS0I=
 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.4/go.mod h1:E1hLXN/BL2e6YizK1zFlYd8vsfi2GTjbjBazinMmeaM=
-github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34 h1:A5UqQEmPaCFpedKouS4v+dHCTUo2sKqhoKO9U5kxyWo=
 github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.34/go.mod h1:wZpTEecJe0Btj3IYnDx/VlUzor9wm3fJHyvLpQF0VwY=
-github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28 h1:srIVS45eQuewqz6fKKu6ZGXaq6FuFg5NzgQBAM6g8Y4=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.38 h1:c8ed/T9T2K5I+h/JzmF5tpI46+OODQ74dzmdo+QnaMg=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.38/go.mod h1:qggunOChCMu9ZF/UkAfhTz25+U2rLVb3ya0Ua6TTfCA=
 github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.28/go.mod h1:7VRpKQQedkfIEXb4k52I7swUnZP0wohVajJMRn3vsUw=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.32 h1:hNeAAymUY5gu11WrrmFb3CVIp9Dar9hbo44yzzcQpzA=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.32/go.mod h1:0ZXSqrty4FtQ7p8TEuRde/SZm9X05KT18LAUlR40Ln0=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35 h1:LWA+3kDM8ly001vJ1X1waCuLJdtTl48gwkPKWy9sosI=
 github.com/aws/aws-sdk-go-v2/internal/ini v1.3.35/go.mod h1:0Eg1YjxE0Bhn56lx+SHJwCzhW+2JGtizsrx+lCqrfm0=
 github.com/aws/aws-sdk-go-v2/service/cloudformation v1.30.0 h1:XbDkc4FLeg1RfnqeblfbJvaEabqq9ByZl4zqyPFkfSc=
 github.com/aws/aws-sdk-go-v2/service/cloudformation v1.30.0/go.mod h1:SwQFcCs9Rog8hSHm+81KBkAK+UKLXErA/1ChaEI8mLE=
-github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.26.2 h1:PWGu2JhCb/XJlJ7SSFJq76pxk4xWsN76nZxh7TzMHx0=
-github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.26.2/go.mod h1:2KOZkkzMDZCo/aLzPhys06mHNkiU74u85aMJA3PLRvg=
 github.com/aws/aws-sdk-go-v2/service/ec2 v1.102.0 h1:P4dyjm49F2kKws0FpouBC6fjVImACXKt752+CWa01lM=
 github.com/aws/aws-sdk-go-v2/service/ec2 v1.102.0/go.mod h1:tIctCeX9IbzsUTKHt53SVEcgyfxV2ElxJeEB+QUbc4M=
 github.com/aws/aws-sdk-go-v2/service/iam v1.21.0 h1:8hEpu60CWlrp7iEBUFRZhgPoX6+gadaGL1sD4LoRYS0=
 github.com/aws/aws-sdk-go-v2/service/iam v1.21.0/go.mod h1:aQZ8BI+reeaY7RI/QQp7TKCSUHOesTdrzzylp3CW85c=
+github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.7.32 h1:ltFklFRb78MNetqtmqZ/6Tc6i76QRMXxDe0LXYl/jd8=
+github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.7.32/go.mod h1:jBlPRKTAedLFuhO71Wm5dgN9x+/pJ6TtwfQmq7RLvNk=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28 h1:bkRyG4a929RCnpVSTvLM2j/T4ls015ZhhYApbmYs15s=
 github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.28/go.mod h1:jj7znCIg05jXlaGBlFMGP8+7UN3VtCkRBG2spnmRQkU=
 github.com/aws/aws-sdk-go-v2/service/sso v1.12.12 h1:nneMBM2p79PGWBQovYO/6Xnc2ryRMw3InnDJq1FHkSY=
@@ -30,8 +33,11 @@ github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12 h1:2qTR7IFk7/0IN/adSFhYu9X
 github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.12/go.mod h1:E4VrHCPzmVB/KFXtqBGKb3c8zpbNBgKe3fisDNLAW5w=
 github.com/aws/aws-sdk-go-v2/service/sts v1.19.2 h1:XFJ2Z6sNUUcAz9poj+245DMkrHE4h2j5I9/xD50RHfE=
 github.com/aws/aws-sdk-go-v2/service/sts v1.19.2/go.mod h1:dp0yLPsLBOi++WTxzCjA/oZqi6NPIhoR+uF7GeMU9eg=
-github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8=
+github.com/aws/aws-sdk-go-v2/service/timestreamwrite v1.18.2 h1:5QyvAYyr+ZibpVxfovzd5JMTZ8miv9s3zT4jG4PJkIA=
+github.com/aws/aws-sdk-go-v2/service/timestreamwrite v1.18.2/go.mod h1:3ZCiyyNF7myh/a7DcOjcqRsLmSF9EdhEZSr00Qlui4s=
 github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
+github.com/aws/smithy-go v1.14.1 h1:EFKMUmH/iHMqLiwoEDx2rRjRQpI1YCn5jTysoaDujFs=
+github.com/aws/smithy-go v1.14.1/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA=
 github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

test/hack/cleanup/main.go

@@ -22,11 +22,11 @@ import (
 	"github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/service/cloudformation"
 	cloudformationtypes "github.com/aws/aws-sdk-go-v2/service/cloudformation/types"
-	"github.com/aws/aws-sdk-go-v2/service/cloudwatch"
-	cloudwatchtypes "github.com/aws/aws-sdk-go-v2/service/cloudwatch/types"
 	"github.com/aws/aws-sdk-go-v2/service/ec2"
 	ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types"
 	"github.com/aws/aws-sdk-go-v2/service/iam"
+	"github.com/aws/aws-sdk-go-v2/service/timestreamwrite"
+	timestreamtypes "github.com/aws/aws-sdk-go-v2/service/timestreamwrite/types"
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/samber/lo"
 	"go.uber.org/multierr"
@@ -36,7 +36,9 @@ import (
 
 const (
 	expirationTTL            = time.Hour * 12
-	karpenterMetricNamespace = "testing.karpenter.sh/cleanup"
+	karpenterMetricRegion    = "us-east-2"
+	karpenterMetricDatabase  = "karpenterTesting"
+	karpenterMetricTableName = "sweeperCleanedResources"
 
 	karpenterProvisionerNameTag = "karpenter.sh/provisioner-name"
 	karpenterLaunchTemplateTag  = "karpenter.k8s.aws/cluster"
@@ -51,6 +53,10 @@ type CleanableResourceType interface {
 	Cleanup(context.Context, []string) ([]string, error)
 }
 
+type MetricsClient interface {
+	FireMetric(context.Context, string, float64, string) error
+}
+
 func main() {
 	ctx := context.Background()
 	cfg := lo.Must(config.LoadDefaultConfig(ctx))
@@ -63,9 +69,10 @@ func main() {
 
 	ec2Client := ec2.NewFromConfig(cfg)
 	cloudFormationClient := cloudformation.NewFromConfig(cfg)
-	cloudWatchClient := cloudwatch.NewFromConfig(cfg)
 	iamClient := iam.NewFromConfig(cfg)
 
+	metricsClient := MetricsClient(&timestream{timestreamClient: timestreamwrite.NewFromConfig(cfg, WithRegion(karpenterMetricRegion))})
+
 	resources := []CleanableResourceType{
 		&instance{ec2Client: ec2Client},
 		&securitygroup{ec2Client: ec2Client},
@@ -85,7 +92,7 @@ func main() {
 			if err != nil {
 				logger.With("type", resources[i].Type()).Errorf("%v", err)
 			}
-			if err = fireMetric(ctx, cloudWatchClient, fmt.Sprintf("%sDeleted", resources[i].Type()), float64(len(cleaned))); err != nil {
+			if err = metricsClient.FireMetric(ctx, fmt.Sprintf("%sDeleted", resources[i].Type()), float64(len(cleaned)), cfg.Region); err != nil {
 				logger.With("type", resources[i].Type()).Errorf("%v", err)
 			}
 			logger.With("type", resources[i].Type(), "ids", cleaned, "count", len(cleaned)).Infof("deleted resources")
@@ -207,6 +214,7 @@ func (sg *securitygroup) Cleanup(ctx context.Context, ids []string) ([]string, e
 		})
 		if err != nil {
 			errs = multierr.Append(errs, err)
+			continue
 		}
 		deleted = append(deleted, ids[i])
 	}
@@ -263,6 +271,7 @@ func (s *stack) Cleanup(ctx context.Context, names []string) ([]string, error) {
 		})
 		if err != nil {
 			errs = multierr.Append(errs, err)
+			continue
 		}
 		deleted = append(deleted, names[i])
 	}
@@ -318,6 +327,7 @@ func (lt *launchtemplate) Cleanup(ctx context.Context, names []string) ([]string
 		})
 		if err != nil {
 			errs = multierr.Append(errs, err)
+			continue
 		}
 		deleted = append(deleted, names[i])
 	}
@@ -368,7 +378,9 @@ func (o *oidc) Cleanup(ctx context.Context, arns []string) ([]string, error) {
 		})
 		if err != nil {
 			errs = multierr.Append(errs, err)
+			continue
 		}
+		deleted = append(deleted, arns[i])
 	}
 	return deleted, errs
 }
@@ -427,15 +439,33 @@ func (ip *instanceProfile) Cleanup(ctx context.Context, names []string) ([]strin
 	return deleted, errs
 }
 
-func fireMetric(ctx context.Context, cloudWatchClient *cloudwatch.Client, name string, value float64) error {
-	_, err := cloudWatchClient.PutMetricData(ctx, &cloudwatch.PutMetricDataInput{
-		Namespace: lo.ToPtr(karpenterMetricNamespace),
-		MetricData: []cloudwatchtypes.MetricDatum{
+type timestream struct {
+	timestreamClient *timestreamwrite.Client
+}
+
+func (t *timestream) FireMetric(ctx context.Context, name string, value float64, region string) error {
+	_, err := t.timestreamClient.WriteRecords(ctx, &timestreamwrite.WriteRecordsInput{
+		DatabaseName: aws.String(karpenterMetricDatabase),
+		TableName:    aws.String(karpenterMetricTableName),
+		Records: []timestreamtypes.Record{
 			{
-				MetricName: lo.ToPtr(name),
-				Value:      lo.ToPtr(value),
+				MeasureName:  aws.String(name),
+				MeasureValue: aws.String(fmt.Sprintf("%f", value)),
+				Time:         aws.String(fmt.Sprintf("%d", time.Now().UnixMilli())),
+				Dimensions: []timestreamtypes.Dimension{
+					{
+						Name:  aws.String("region"),
+						Value: aws.String(region),
+					},
+				},
 			},
 		},
 	})
 	return err
 }
+
+func WithRegion(region string) func(*timestreamwrite.Options) {
+	return func(o *timestreamwrite.Options) {
+		o.Region = region
+	}
+}