feat: workflow aws nightly cleanup for all test regions

.github/workflows/aws_nightly_cleanup.yml

@@ -0,0 +1,92 @@
+---
+  name: AWS Nightly Cleanup of test regions
+
+  on:
+    schedule:
+      - cron: '0 5 * * *'
+    workflow_dispatch:
+    pull_request:
+      paths:
+        - .github/workflows/aws_nightly_cleanup.yml
+
+  env:
+    AWS_PROFILE: "infex"
+    # remember to also update tests.yml!
+    AWS_REGION: "eu-west-2"
+    CLEANUP_OLDER_THAN: "12h"
+    # renovate: datasource=github-tags depName=gruntwork-io/cloud-nuke
+    CLOUD_NUKE_VERSION: v0.36.0
+
+  jobs:
+    aws-nightly-cleanup:
+      runs-on: ubuntu-latest
+      strategy:
+        fail-fast: false # don't propagate failing jobs
+        matrix:
+          aws_test_regions: ["eu-west-2", "eu-west-3"]
+      env:
+        AWS_REGION: ${{ matrix.aws_test_regions }}
+      steps:
+        - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4
+
+        - name: Import Secrets
+          id: secrets
+          uses: hashicorp/vault-action@d1720f055e0635fd932a1d2a48f87a666a57906c # v3
+          with:
+            url: ${{ secrets.VAULT_ADDR }}
+            method: approle
+            roleId: ${{ secrets.VAULT_ROLE_ID }}
+            secretId: ${{ secrets.VAULT_SECRET_ID }}
+            exportEnv: false
+            secrets: |
+              secret/data/products/infrastructure-experience/ci/common AWS_ACCESS_KEY;
+              secret/data/products/infrastructure-experience/ci/common AWS_SECRET_KEY;
+
+        # Official action does not support profiles
+        - name: Add profile credentials to ~/.aws/credentials
+          run: |
+            aws configure set aws_access_key_id ${{ steps.secrets.outputs.AWS_ACCESS_KEY }} --profile ${{ env.AWS_PROFILE }}
+            aws configure set aws_secret_access_key ${{ steps.secrets.outputs.AWS_SECRET_KEY }} --profile ${{ env.AWS_PROFILE }}
+            aws configure set region ${{ env.AWS_REGION }} --profile ${{ env.AWS_PROFILE }}
+
+        - name: Install Cloud Nuke
+          run: |
+            wget https://github.com/gruntwork-io/cloud-nuke/releases/download/${{ env.CLOUD_NUKE_VERSION }}/cloud-nuke_linux_amd64
+            chmod +x cloud-nuke_linux_amd64
+
+        # This is likely to fail, therefore we ignore the error
+        # We're ignoring ec2_dhcp_option as they couldn't be deleted
+        # cloudtrail is managed by IT and can't be deleted either
+        - name: Run Cloud Nuke
+          timeout-minutes: 45
+          env:
+            DISABLE_TELEMETRY: "true"
+          run: |
+            ./cloud-nuke_linux_amd64 aws \
+            --region ${{ env.AWS_REGION }} \
+            --force \
+            --older-than ${{ env.CLEANUP_OLDER_THAN }} \
+            --exclude-resource-type ec2_dhcp_option \
+            --exclude-resource-type ec2-keypairs \
+            --exclude-resource-type s3 \
+            --exclude-resource-type cloudtrail || true
+
+        # Following will delete global resources and things that cloud-nuke does not support
+        - name: Delete additional AWS resources
+          timeout-minutes: 15
+          run: .github/workflows/scripts/aws_cleanup.sh "${{ env.AWS_REGION }}"
+
+        # The second run should remove the remaining resources (VPCs) and fail if there's anything left
+        - name: Run Cloud Nuke
+          timeout-minutes: 45
+          env:
+            DISABLE_TELEMETRY: "true"
+          run: |
+            ./cloud-nuke_linux_amd64 aws \
+            --region ${{ env.AWS_REGION }} \
+            --force \
+            --older-than ${{ env.CLEANUP_OLDER_THAN }} \
+            --exclude-resource-type ec2_dhcp_option \
+            --exclude-resource-type cloudtrail \
+            --exclude-resource-type ec2-keypairs \
+            --exclude-resource-type s3

.github/workflows/scripts/aws_cleanup.sh

@@ -0,0 +1,94 @@
+#!/bin/bash
+
+# This script deletes additional AWS resources based on specified criteria.
+
+# Check if the region argument is provided
+if [ -z "$1" ]; then
+    echo "Please provide the AWS region as the first argument."
+    exit 1
+fi
+
+region="$1"
+
+echo "Deleting additional resources in the $region region..."
+
+
+echo "Deleting additional resources..."
+# KMS keys can't be deleted due to resource policies, requires manual intervention
+
+echo "Deleting IAM Roles"
+# Detach permissions and profile instances and delete IAM roles
+role_arns=$(aws iam list-roles --query "Roles[?contains(RoleName, 'nightly')].RoleName" --output text)
+
+read -r -a role_arns_array <<< "$role_arns"
+
+for role_arn in "${role_arns_array[@]}"
+do
+    echo "Removing instance profiles and policies of role: $role_arn"
+    attached_policy_arns=$(aws iam list-attached-role-policies --role-name "$role_arn" --query 'AttachedPolicies[].PolicyArn' --output text)
+    read -r -a attached_policy_arns_array <<< "$attached_policy_arns"
+
+    for policy_arn in "${attached_policy_arns_array[@]}"
+    do
+        echo "Removing attached policy: $policy_arn"
+        aws iam detach-role-policy --role-name "$role_arn" --policy-arn "$policy_arn"
+    done
+
+    policy_arns=$(aws iam list-role-policies --role-name "$role_arn" --query 'PolicyNames' --output text)
+    read -r -a policy_arns_array <<< "$policy_arns"
+
+    for policy_arn in "${policy_arns_array[@]}"
+    do
+        echo "Deleting policy: $policy_arn"
+        aws iam delete-role-policy --role-name "$role_arn" --policy-name "$policy_arn"
+    done
+
+    instance_profile_arns=$(aws iam list-instance-profiles-for-role --role-name "$role_arn" --query 'InstanceProfiles[].InstanceProfileName' --output text)
+    read -r -a instance_profile_arns_array <<< "$instance_profile_arns"
+
+    for instance_profile_arn in "${instance_profile_arns_array[@]}"
+    do
+        echo "Removing instance profile: $instance_profile_arn"
+        aws iam remove-role-from-instance-profile --instance-profile-name "$instance_profile_arn" --role-name "$role_arn"
+    done
+
+    echo "Deleting role: $role_arn"
+    aws iam delete-role --role-name "$role_arn"
+
+done
+
+echo "Deleting IAM Policies"
+# Delete Policies
+iam_policies=$(aws iam list-policies --query "Policies[?contains(PolicyName, 'nightly')].Arn" --output text)
+
+read -r -a iam_policies_array <<< "$iam_policies"
+
+for iam_policy in "${iam_policies_array[@]}"
+do
+    echo "Deleting policy: $iam_policy"
+    aws iam delete-policy --policy-arn "$iam_policy"
+done
+
+echo "Deleting OIDC Providers"
+# Delete OIDC Provider
+oidc_providers=$(aws iam list-open-id-connect-providers --query "OpenIDConnectProviderList[?contains(Arn, 'eu-west-2') || contains(Arn, 'eu-west-3')].Arn" --output text)
+
+read -r -a oidc_providers_array <<< "$oidc_providers"
+
+for oidc_provider in "${oidc_providers_array[@]}"
+do
+    echo "Deleting OIDC Provider: $oidc_provider"
+    aws iam delete-open-id-connect-provider --open-id-connect-provider-arn "$oidc_provider"
+done
+
+echo "Deleting VPC Peering Connections"
+# Delete VPC Peering Connection
+peering_connection_ids=$(aws ec2 describe-vpc-peering-connections --region "$region" --query "VpcPeeringConnections[?Status.Code == 'active' && Tags[?contains(Value, 'nightly')]]".VpcPeeringConnectionId --output text)
+
+read -r -a peering_connection_ids_array <<< "$peering_connection_ids"
+
+for peering_connection_id in "${peering_connection_ids_array[@]}"
+do
+    echo "Deleting VPC Peering Connection: $peering_connection_id"
+    aws ec2 delete-vpc-peering-connection --region "$region" --vpc-peering-connection-id "$peering_connection_id"
+done