Add S3 replication and mediaflux

bin/aws/ecs_shell

@@ -6,7 +6,7 @@ TASK_NAME="${1^}"
 shift
 
 echo "Getting cluster..."
-CLUSTER=$(aws ecs list-clusters | jq -r '.clusterArns | .[]' | grep nabu)
+CLUSTER=$(aws ecs list-clusters | jq -r '.clusterArns | .[]' | grep 'nabu$')
 
 if [ -z "$TASK_NAME" ]; then
   TASKS=$(aws ecs list-services --cluster "$CLUSTER")

cdk/docker/mediaflux/Dockerfile

@@ -0,0 +1,43 @@
+FROM ubuntu:24.04
+
+ARG MF_CLIENT_URL=https://gitlab.unimelb.edu.au/resplat-mediaflux/releases/raw/master/mediaflux/unimelb-mf-clients-0.7.9-linux-x64.tar.gz
+ARG AWS_CLI_URL=https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip
+
+RUN apt-get update && apt-get install -y \
+        wget unzip \
+        && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+
+RUN wget $MF_CLIENT_URL \
+        && tar -xzf unimelb-mf-clients-0.7.9-linux-x64.tar.gz \
+        && rm unimelb-mf-clients-0.7.9-linux-x64.tar.gz \
+        && mv * mf
+
+
+RUN <<EOF
+        for i in mf/bin/unix/unimelb-mf-*; do
+                name=$(basename $i | sed 's/unimelb-mf-//')
+                ln -s $i $name
+        done
+EOF
+
+RUN cd /tmp && \
+        wget $AWS_CLI_URL && \
+        unzip awscli-exe-linux-x86_64.zip && \
+        ./aws/install && \
+        rm -rf /tmp/awscli-exe-linux-x86_64.zip /tmp/aws
+
+
+# NOTE: https://gitlab.unimelb.edu.au/resplat-mediaflux/unimelb-mf-clients/-/blob/master/README.md#system-environment-variables
+ENV MFLUX_HOST=mediaflux.researchsoftware.unimelb.edu.au
+ENV MFLUX_PORT=443
+ENV MFLUX_DOMAIN=local
+
+ENV MFLUX_TRANSPORT=https
+
+COPY archive.sh /app/archive.sh
+RUN chmod +x /app/archive.sh
+
+CMD ["/app/archive.sh"]

cdk/docker/mediaflux/archive.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+echo "$@"
+
+if [ -z "$S3_BUCKET" ] || [ -z "$S3_KEY" ]; then
+        echo "Error: S3_BUCKET and S3_KEY environment variables must be set"
+        exit 1
+fi
+
+# Download file from S3 to /tmp
+if ! aws s3 cp "s3://${S3_BUCKET}/${S3_KEY}" /tmp/downloaded_file; then
+        echo "Error: Failed to download file from S3"
+        exit 1
+fi
+
+# Process the downloaded file with MediaFlux script
+/app/upload --dest "/projects/proj-1190_paradisec_backup-1128.4.248/TEST/$S3_KEY" --create-parents /tmp/downloaded_file
+
+# Remove the downloaded file
+rm /tmp/downloaded_file

cdk/lib/app-stack.ts

@@ -6,15 +6,19 @@ import type { Construct } from 'constructs';
 import * as autoscaling from 'aws-cdk-lib/aws-autoscaling';
 import * as backup from 'aws-cdk-lib/aws-backup';
 import * as ec2 from 'aws-cdk-lib/aws-ec2';
+import * as ecrAssets from 'aws-cdk-lib/aws-ecr-assets';
 import * as ecs from 'aws-cdk-lib/aws-ecs';
 import * as elbv2 from 'aws-cdk-lib/aws-elasticloadbalancingv2';
-import * as opensearch from 'aws-cdk-lib/aws-opensearchservice';
+import * as events from 'aws-cdk-lib/aws-events';
+import * as eventbridge from 'aws-cdk-lib/aws-events';
+import * as targets from 'aws-cdk-lib/aws-events-targets';
 import * as iam from 'aws-cdk-lib/aws-iam';
+import * as opensearch from 'aws-cdk-lib/aws-opensearchservice';
 import * as rds from 'aws-cdk-lib/aws-rds';
 import * as route53 from 'aws-cdk-lib/aws-route53';
+import * as secretsmanager from 'aws-cdk-lib/aws-secretsmanager';
 import * as ses from 'aws-cdk-lib/aws-ses';
 import * as ssm from 'aws-cdk-lib/aws-ssm';
-import * as secretsmanager from 'aws-cdk-lib/aws-secretsmanager';
 
 import { NagSuppressions } from 'cdk-nag';
 
@@ -473,6 +477,86 @@ export class AppStack extends cdk.Stack {
       resources: [backup.BackupResource.fromRdsDatabaseInstance(db)],
     });
 
+    // ////////////////////////
+    // S3 Event Handling
+    // ////////////////////////
+
+    if (env === 'prod') {
+      const image = new ecrAssets.DockerImageAsset(this, 'CopyToMediafluxImage', {
+        directory: 'docker/mediaflux',
+      });
+
+      const taskDefinition = new ecs.FargateTaskDefinition(this, 'CopyToMediaFluxTaskDefinition', {
+        cpu: 16384,
+        memoryLimitMiB: 32768,
+        ephemeralStorageGiB: 200,
+      });
+      NagSuppressions.addResourceSuppressions(searchDomain, [
+        { id: 'AwsSolutions-IAM5', reason: 'Star on S3 get is fine' },
+      ]);
+
+      const mediafluxSecrets = new secretsmanager.Secret(this, 'MediaFluxSecrets', {
+        secretName: '/nabu/mediaflux',
+        secretObjectValue: {
+          username: cdk.SecretValue.unsafePlainText('secret'),
+          password: cdk.SecretValue.unsafePlainText('secret'),
+        },
+      });
+      NagSuppressions.addResourceSuppressions(mediafluxSecrets, [
+        { id: 'AwsSolutions-SMG4', reason: 'No auto rotation needed' },
+      ]);
+
+      taskDefinition.addContainer('MediafluxContainer', {
+        image: ecs.ContainerImage.fromDockerImageAsset(image),
+        logging: new ecs.AwsLogDriver({ streamPrefix: 'copy-to-mediaflux' }),
+        pseudoTerminal: true,
+        secrets: {
+          MFLUX_USER: ecs.Secret.fromSecretsManager(mediafluxSecrets, 'username'),
+          MFLUX_PASSWORD: ecs.Secret.fromSecretsManager(mediafluxSecrets, 'password'),
+        },
+      });
+      catalogBucket.grantRead(taskDefinition.taskRole);
+
+      const cluster = new ecs.Cluster(this, 'NabuCluster', {
+        vpc,
+        containerInsights: true,
+      });
+
+      const mediaFluxTask = new targets.EcsTask({
+        cluster,
+        enableExecuteCommand: true,
+        subnetSelection: {
+          subnets: appSubnets,
+        },
+        taskDefinition,
+        containerOverrides: [
+          {
+            containerName: 'MediafluxContainer',
+            environment: [
+              { name: 'S3_BUCKET', value: events.EventField.fromPath('$.detail.bucket.name') },
+              { name: 'S3_KEY', value: events.EventField.fromPath('$.detail.object.key') },
+            ],
+          },
+        ],
+      });
+
+      new events.Rule(this, 'S3PutEventRule', {
+        description: 'Rule to trigger Fargate task on S3 put event',
+        eventPattern: {
+          source: ['aws.s3'],
+          detailType: ['Object Created'],
+          detail: {
+            bucket: {
+              name: [catalogBucket.bucketName],
+            },
+            object: {
+              key: [{ prefix: '' }],
+            },
+          },
+        },
+        targets: [mediaFluxTask],
+      });
+    }
     cdk.Tags.of(this).add('uni:billing:application', 'para');
   }
 }

cdk/lib/dr-stack.ts

@@ -0,0 +1,81 @@
+import * as cdk from 'aws-cdk-lib';
+import type { Construct } from 'constructs';
+
+import * as s3 from 'aws-cdk-lib/aws-s3';
+
+import type { Environment } from './types';
+import { NagSuppressions } from 'cdk-nag';
+
+export class DrStack extends cdk.Stack {
+  public drBucket: s3.IBucket;
+
+  constructor(scope: Construct, id: string, environment: Environment, props?: cdk.StackProps) {
+    super(scope, id, props);
+
+    const {
+      appName,
+      // account,
+      region,
+      env,
+    } = environment;
+
+    if (env !== 'prod') {
+      throw new Error('DR stack can only be deployed to prod');
+    }
+    if (region !== 'ap-southeast-4') {
+      console.log(region);
+      throw new Error('DR stack can only be deployed to ap-southeast-4');
+    }
+
+    // ////////////////////////
+    // DR Meta Bucket
+    // ////////////////////////
+    const metaDrBucket = new s3.Bucket(this, 'MetaBucket', {
+      bucketName: `${appName}-metadr-${env}`,
+      encryption: s3.BucketEncryption.S3_MANAGED,
+      blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL,
+      enforceSSL: true,
+      removalPolicy: cdk.RemovalPolicy.RETAIN,
+    });
+    NagSuppressions.addResourceSuppressions(metaDrBucket, [
+      { id: 'AwsSolutions-S1', reason: "This bucket holds logs for other buckets and we don't want a loop" },
+    ]);
+
+    // ////////////////////////
+    // Dr bucket
+    // ////////////////////////
+
+    this.drBucket = new s3.Bucket(this, 'DrBucket', {
+      bucketName: `${appName}-catalogdr-${env}`,
+      encryption: s3.BucketEncryption.S3_MANAGED,
+      blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL,
+      enforceSSL: true,
+      lifecycleRules: [{ abortIncompleteMultipartUploadAfter: cdk.Duration.days(7) }],
+      versioned: env === 'prod',
+      inventories: [
+        {
+          destination: {
+            bucket: metaDrBucket,
+            prefix: 'inventories/catalogdr',
+          },
+          frequency: s3.InventoryFrequency.WEEKLY,
+          includeObjectVersions: s3.InventoryObjectVersion.ALL,
+          optionalFields: [
+            'Size',
+            'LastModifiedDate',
+            'StorageClass',
+            'ReplicationStatus',
+            'IntelligentTieringAccessTier',
+            'ChecksumAlgorithm',
+            'ETag',
+          ],
+        },
+      ],
+      removalPolicy: cdk.RemovalPolicy.RETAIN,
+      serverAccessLogsBucket: metaDrBucket,
+      serverAccessLogsPrefix: `s3-access-logs/${appName}-catalogdr-${env}`,
+    });
+
+    cdk.Tags.of(this).add('uni:billing:application', 'para');
+  }
+}

cdk/lib/main-stack.ts

@@ -7,9 +7,9 @@ import * as route53 from 'aws-cdk-lib/aws-route53';
 import * as s3 from 'aws-cdk-lib/aws-s3';
 import * as ssm from 'aws-cdk-lib/aws-ssm';
 
-import type { Environment } from './types';
 import { NagSuppressions } from 'cdk-nag';
-import { CfnBucket } from 'aws-cdk-lib/aws-s3';
+
+import type { Environment } from './types';
 
 export class MainStack extends cdk.Stack {
   public catalogBucket: s3.IBucket;
@@ -130,7 +130,12 @@ export class MainStack extends cdk.Stack {
       ],
       serverAccessLogsBucket: metaBucket,
       serverAccessLogsPrefix: `s3-access-logs/${appName}-catalog-${env}`,
+      eventBridgeEnabled: true,
     });
+    NagSuppressions.addStackSuppressions(this, [
+      { id: 'AwsSolutions-IAM4', reason: 'OK with * resources' },
+      { id: 'AwsSolutions-IAM5', reason: 'OK with * resources' },
+    ]);
 
     if (env === 'prod') {
       if (!drBucket) {
@@ -184,7 +189,7 @@ export class MainStack extends cdk.Stack {
         }),
       );
 
-      const cfnBucket = this.catalogBucket.node.defaultChild as CfnBucket;
+      const cfnBucket = this.catalogBucket.node.defaultChild as s3.CfnBucket;
       cfnBucket.replicationConfiguration = {
         role: replicationRole.roleArn,
         rules: [