From c18339c1ed1b4783be963594ce4039c4546a4e23 Mon Sep 17 00:00:00 2001 From: stekern Date: Tue, 18 Apr 2023 12:35:23 +0200 Subject: [PATCH 1/3] build: add support for library feature flags This allows us to hide experimental behavior in the library behind feature flags that consumers opt in to by setting the flag in their project's `cdk.json`. This is similar to how AWS is doing it with their CDK feature flags. --- src/feature-flags.ts | 66 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 src/feature-flags.ts diff --git a/src/feature-flags.ts b/src/feature-flags.ts new file mode 100644 index 00000000..aab2fd56 --- /dev/null +++ b/src/feature-flags.ts @@ -0,0 +1,66 @@ +import * as constructs from "constructs" + +interface FeatureFlagInfo { + /** + * The default value for the feature flag. + * + * NOTE: This will be the value used for consumers that have not + * explicitly set the feature flag (which will be most of them!), + * so we should make sure that the default value does NOT lead + * to any breaking behavior. + */ + default: boolean + /** + * A short description of the feature flag. + */ + description: string +} + +// Custom feature flags for liflig-cdk +export const FEATURE_FLAG_CDK_PIPELINES_SPEED_UP = + "@liflig-cdk/cdk-pipelines:enableExperimentalSpeedUp" + +const FLAGS: { [key: string]: FeatureFlagInfo } = { + [FEATURE_FLAG_CDK_PIPELINES_SPEED_UP]: { + default: false, + description: + "Reduce execution time of CDK Pipelines by making various tweaks (e.g., skip creation of CloudFormation changesets, disable CodePipeline S3 polling).", + }, +} + +const getFeatureFlagDefault = (flagName: string) => { + return FLAGS[flagName]?.default ?? false +} + +/** + * Exposes feature flags we can use in liflig-cdk to allow consumers to opt-in + * to experimental functionality without affecting current consumers and having + * to pollute the official library API with experimental properties and behavior. + * + * NOTE: We should only use these flags temporarily and very sparingly as they lead + * to a brittle and more complex codebase with a lot of branching logic. + * Once an experiment has concluded we should remove them and update the + * official library API. + */ +export class FeatureFlags { + private constructor(private readonly scope: constructs.IConstruct) {} + public static of(scope: constructs.Construct) { + return new FeatureFlags(scope) + } + public isEnabled(flagName: string) { + if (!Object.keys(FLAGS).includes(flagName)) { + throw new Error(`Unsupported feature flag ${flagName}`) + } + const contextValue = this.scope.node.tryGetContext(flagName) as unknown + if (contextValue === undefined) { + return getFeatureFlagDefault(flagName) + } else if ( + Object.prototype.toString.call(contextValue) === "[object Boolean]" + ) { + return Boolean(contextValue) + } + throw new Error( + `Unsupported value for feature flag ${flagName}. Only boolean values are supported.`, + ) + } +} From 2385131485b473f2ca9f7dc5ee0c8ad3e20948e8 Mon Sep 17 00:00:00 2001 From: stekern Date: Tue, 18 Apr 2023 12:40:25 +0200 Subject: [PATCH 2/3] perf: enable s3 event notifications for ci bucket This is a non-breaking change that in the future will allow us to move away from the default trigger mechanism for Liflig CDK Pipelines which uses S3 polling (which can be slow), and use EventBridge rules instead. --- .../app/build-artifacts.template.json | 132 ++++++++++++++++++ __snapshots__/app/manifest.json | 24 ++++ src/build-artifacts/index.ts | 1 + 3 files changed, 157 insertions(+) diff --git a/__snapshots__/app/build-artifacts.template.json b/__snapshots__/app/build-artifacts.template.json index dee5a5c5..453f1826 100644 --- a/__snapshots__/app/build-artifacts.template.json +++ b/__snapshots__/app/build-artifacts.template.json @@ -53,6 +53,27 @@ "aws:cdk:path": "build-artifacts/BuildArtifacts/S3Bucket/Resource" } }, + "BuildArtifactsS3BucketNotifications8F606582": { + "Type": "Custom::S3BucketNotifications", + "Properties": { + "ServiceToken": { + "Fn::GetAtt": [ + "BucketNotificationsHandler050a0587b7544547bf325f094a3db8347ECC3691", + "Arn" + ] + }, + "BucketName": { + "Ref": "BuildArtifactsS3BucketF34A0F91" + }, + "NotificationConfiguration": { + "EventBridgeConfiguration": {} + }, + "Managed": true + }, + "Metadata": { + "aws:cdk:path": "build-artifacts/BuildArtifacts/S3Bucket/Notifications/Resource" + } + }, "BuildArtifactsS3BucketPolicy963F6593": { "Type": "AWS::S3::BucketPolicy", "Properties": { @@ -297,6 +318,117 @@ "Metadata": { "aws:cdk:path": "build-artifacts/BuildArtifacts/EcrRepository/Resource" } + }, + "BucketNotificationsHandler050a0587b7544547bf325f094a3db834RoleB6FB88EC": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + ] + ] + } + ], + "Tags": [ + { + "Key": "Project", + "Value": "my-project" + }, + { + "Key": "SourceRepo", + "Value": "github/capralifecycle/liflig-cdk" + }, + { + "Key": "StackName", + "Value": "build-artifacts" + } + ] + }, + "Metadata": { + "aws:cdk:path": "build-artifacts/BucketNotificationsHandler050a0587b7544547bf325f094a3db834/Role/Resource" + } + }, + "BucketNotificationsHandler050a0587b7544547bf325f094a3db834RoleDefaultPolicy2CF63D36": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": "s3:PutBucketNotification", + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "BucketNotificationsHandler050a0587b7544547bf325f094a3db834RoleDefaultPolicy2CF63D36", + "Roles": [ + { + "Ref": "BucketNotificationsHandler050a0587b7544547bf325f094a3db834RoleB6FB88EC" + } + ] + }, + "Metadata": { + "aws:cdk:path": "build-artifacts/BucketNotificationsHandler050a0587b7544547bf325f094a3db834/Role/DefaultPolicy/Resource" + } + }, + "BucketNotificationsHandler050a0587b7544547bf325f094a3db8347ECC3691": { + "Type": "AWS::Lambda::Function", + "Properties": { + "Description": "AWS CloudFormation handler for \"Custom::S3BucketNotifications\" resources (@aws-cdk/aws-s3)", + "Code": { + "ZipFile": "import boto3 # type: ignore\nimport json\nimport logging\nimport urllib.request\n\ns3 = boto3.client(\"s3\")\n\nEVENTBRIDGE_CONFIGURATION = 'EventBridgeConfiguration'\n\nCONFIGURATION_TYPES = [\"TopicConfigurations\", \"QueueConfigurations\", \"LambdaFunctionConfigurations\"]\n\ndef handler(event: dict, context):\n response_status = \"SUCCESS\"\n error_message = \"\"\n try:\n props = event[\"ResourceProperties\"]\n bucket = props[\"BucketName\"]\n notification_configuration = props[\"NotificationConfiguration\"]\n request_type = event[\"RequestType\"]\n managed = props.get('Managed', 'true').lower() == 'true'\n stack_id = event['StackId']\n\n if managed:\n config = handle_managed(request_type, notification_configuration)\n else:\n config = handle_unmanaged(bucket, stack_id, request_type, notification_configuration)\n\n put_bucket_notification_configuration(bucket, config)\n except Exception as e:\n logging.exception(\"Failed to put bucket notification configuration\")\n response_status = \"FAILED\"\n error_message = f\"Error: {str(e)}. \"\n finally:\n submit_response(event, context, response_status, error_message)\n\ndef handle_managed(request_type, notification_configuration):\n if request_type == 'Delete':\n return {}\n return notification_configuration\n\ndef handle_unmanaged(bucket, stack_id, request_type, notification_configuration):\n external_notifications = find_external_notifications(bucket, stack_id)\n\n if request_type == 'Delete':\n return external_notifications\n\n def with_id(notification):\n notification['Id'] = f\"{stack_id}-{hash(json.dumps(notification, sort_keys=True))}\"\n return notification\n\n notifications = {}\n for t in CONFIGURATION_TYPES:\n external = external_notifications.get(t, [])\n incoming = [with_id(n) for n in notification_configuration.get(t, [])]\n notifications[t] = external + incoming\n\n if EVENTBRIDGE_CONFIGURATION in notification_configuration:\n notifications[EVENTBRIDGE_CONFIGURATION] = notification_configuration[EVENTBRIDGE_CONFIGURATION]\n elif EVENTBRIDGE_CONFIGURATION in external_notifications:\n notifications[EVENTBRIDGE_CONFIGURATION] = external_notifications[EVENTBRIDGE_CONFIGURATION]\n\n return notifications\n\ndef find_external_notifications(bucket, stack_id):\n existing_notifications = get_bucket_notification_configuration(bucket)\n external_notifications = {}\n for t in CONFIGURATION_TYPES:\n external_notifications[t] = [n for n in existing_notifications.get(t, []) if not n['Id'].startswith(f\"{stack_id}-\")]\n\n if EVENTBRIDGE_CONFIGURATION in existing_notifications:\n external_notifications[EVENTBRIDGE_CONFIGURATION] = existing_notifications[EVENTBRIDGE_CONFIGURATION]\n\n return external_notifications\n\ndef get_bucket_notification_configuration(bucket):\n return s3.get_bucket_notification_configuration(Bucket=bucket)\n\ndef put_bucket_notification_configuration(bucket, notification_configuration):\n s3.put_bucket_notification_configuration(Bucket=bucket, NotificationConfiguration=notification_configuration)\n\ndef submit_response(event: dict, context, response_status: str, error_message: str):\n response_body = json.dumps(\n {\n \"Status\": response_status,\n \"Reason\": f\"{error_message}See the details in CloudWatch Log Stream: {context.log_stream_name}\",\n \"PhysicalResourceId\": event.get(\"PhysicalResourceId\") or event[\"LogicalResourceId\"],\n \"StackId\": event[\"StackId\"],\n \"RequestId\": event[\"RequestId\"],\n \"LogicalResourceId\": event[\"LogicalResourceId\"],\n \"NoEcho\": False,\n }\n ).encode(\"utf-8\")\n headers = {\"content-type\": \"\", \"content-length\": str(len(response_body))}\n try:\n req = urllib.request.Request(url=event[\"ResponseURL\"], headers=headers, data=response_body, method=\"PUT\")\n with urllib.request.urlopen(req) as response:\n print(response.read().decode(\"utf-8\"))\n print(\"Status code: \" + response.reason)\n except Exception as e:\n print(\"send(..) failed executing request.urlopen(..): \" + str(e))\n" + }, + "Handler": "index.handler", + "Role": { + "Fn::GetAtt": [ + "BucketNotificationsHandler050a0587b7544547bf325f094a3db834RoleB6FB88EC", + "Arn" + ] + }, + "Runtime": "python3.9", + "Timeout": 300, + "Tags": [ + { + "Key": "Project", + "Value": "my-project" + }, + { + "Key": "SourceRepo", + "Value": "github/capralifecycle/liflig-cdk" + }, + { + "Key": "StackName", + "Value": "build-artifacts" + } + ] + }, + "DependsOn": [ + "BucketNotificationsHandler050a0587b7544547bf325f094a3db834RoleDefaultPolicy2CF63D36", + "BucketNotificationsHandler050a0587b7544547bf325f094a3db834RoleB6FB88EC" + ], + "Metadata": { + "aws:cdk:path": "build-artifacts/BucketNotificationsHandler050a0587b7544547bf325f094a3db834/Resource" + } } }, "Outputs": { diff --git a/__snapshots__/app/manifest.json b/__snapshots__/app/manifest.json index fceca704..3ba7459f 100644 --- a/__snapshots__/app/manifest.json +++ b/__snapshots__/app/manifest.json @@ -76,6 +76,12 @@ "data": "BuildArtifactsS3BucketF34A0F91" } ], + "/build-artifacts/BuildArtifacts/S3Bucket/Notifications/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "BuildArtifactsS3BucketNotifications8F606582" + } + ], "/build-artifacts/BuildArtifacts/S3Bucket/Policy/Resource": [ { "type": "aws:cdk:logicalId", @@ -118,6 +124,24 @@ "data": "BuildArtifactsCiRoleArn6A5A560C" } ], + "/build-artifacts/BucketNotificationsHandler050a0587b7544547bf325f094a3db834/Role/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "BucketNotificationsHandler050a0587b7544547bf325f094a3db834RoleB6FB88EC" + } + ], + "/build-artifacts/BucketNotificationsHandler050a0587b7544547bf325f094a3db834/Role/DefaultPolicy/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "BucketNotificationsHandler050a0587b7544547bf325f094a3db834RoleDefaultPolicy2CF63D36" + } + ], + "/build-artifacts/BucketNotificationsHandler050a0587b7544547bf325f094a3db834/Resource": [ + { + "type": "aws:cdk:logicalId", + "data": "BucketNotificationsHandler050a0587b7544547bf325f094a3db8347ECC3691" + } + ], "/build-artifacts/CDKMetadata/Default": [ { "type": "aws:cdk:logicalId", diff --git a/src/build-artifacts/index.ts b/src/build-artifacts/index.ts index 73a1a0f9..49ba4b4f 100644 --- a/src/build-artifacts/index.ts +++ b/src/build-artifacts/index.ts @@ -96,6 +96,7 @@ export class BuildArtifacts extends constructs.Construct { bucket = new s3.Bucket(this, "S3Bucket", { bucketName: props.bucketName, encryption: s3.BucketEncryption.S3_MANAGED, + eventBridgeEnabled: true, blockPublicAccess: s3.BlockPublicAccess.BLOCK_ALL, versioned: true, lifecycleRules: [ From 7bf724c1cdbdaa5a5b1bf4a0baab802c7552afcb Mon Sep 17 00:00:00 2001 From: stekern Date: Tue, 18 Apr 2023 13:03:23 +0200 Subject: [PATCH 3/3] feat: add experimental cdk pipelines speed up --- .../__tests__/liflig-cdk-pipeline.test.ts | 74 +++++++++++++++++++ src/cdk-pipelines/liflig-cdk-pipeline.ts | 32 ++++++++ 2 files changed, 106 insertions(+) create mode 100644 src/cdk-pipelines/__tests__/liflig-cdk-pipeline.test.ts diff --git a/src/cdk-pipelines/__tests__/liflig-cdk-pipeline.test.ts b/src/cdk-pipelines/__tests__/liflig-cdk-pipeline.test.ts new file mode 100644 index 00000000..085febee --- /dev/null +++ b/src/cdk-pipelines/__tests__/liflig-cdk-pipeline.test.ts @@ -0,0 +1,74 @@ +import * as assertions from "aws-cdk-lib/assertions" +import { App, CfnOutput, Stack, Stage } from "aws-cdk-lib" +import { LifligCdkPipeline } from "../liflig-cdk-pipeline" +import { FEATURE_FLAG_CDK_PIPELINES_SPEED_UP } from "../../feature-flags" + +test("liflig-cdk-pipeline-with-feature-flag", () => { + const app = new App({ + context: { + [FEATURE_FLAG_CDK_PIPELINES_SPEED_UP]: true, + }, + }) + + const stage = new Stage(app, "Stage") + const stack = new Stack(stage, "Stack") + new CfnOutput(stack, "ExampleOutput", { + value: "hello world", + }) + + const pipelineStack = new Stack(app, "PipelineStack") + + const pipeline = new LifligCdkPipeline(pipelineStack, "Pipeline", { + pipelineName: "test-pipeline", + sourceType: "cdk-source", + }) + + pipeline.cdkPipeline.addStage(stage) + const template = assertions.Template.fromStack(pipelineStack) + + // Assert that S3 polling is deactivated + template.hasResourceProperties("AWS::CodePipeline::Pipeline", { + Stages: assertions.Match.arrayWith([ + assertions.Match.objectLike({ + Actions: assertions.Match.arrayWith([ + assertions.Match.objectLike({ + ActionTypeId: assertions.Match.objectEquals({ + Category: "Source", + Owner: "AWS", + Provider: "S3", + Version: "1", + }), + Configuration: assertions.Match.objectLike({ + PollForSourceChanges: false, + }), + }), + ]), + }), + ]), + }) + + // Assert that there are no actions that creates changesets + template.hasResourceProperties("AWS::CodePipeline::Pipeline", { + Stages: assertions.Match.not( + assertions.Match.arrayWith([ + assertions.Match.objectLike({ + Actions: assertions.Match.arrayWith([ + assertions.Match.objectLike({ + ActionTypeId: assertions.Match.objectEquals({ + Category: "Deploy", + Owner: "AWS", + Provider: "CloudFormation", + Version: "1", + }), + Configuration: assertions.Match.objectLike({ + ActionMode: "CHANGE_SET_REPLACE", + }), + }), + ]), + }), + ]), + ), + }) + // Assert that there's an EventBridge rule set up to trigger the pipeline + template.hasResource("AWS::Events::Rule", {}) +}) diff --git a/src/cdk-pipelines/liflig-cdk-pipeline.ts b/src/cdk-pipelines/liflig-cdk-pipeline.ts index dcbee775..160888fa 100644 --- a/src/cdk-pipelines/liflig-cdk-pipeline.ts +++ b/src/cdk-pipelines/liflig-cdk-pipeline.ts @@ -3,6 +3,8 @@ import * as codepipeline from "aws-cdk-lib/aws-codepipeline" import * as codepipelineActions from "aws-cdk-lib/aws-codepipeline-actions" import * as iam from "aws-cdk-lib/aws-iam" import * as lambda from "aws-cdk-lib/aws-lambda" +import * as events from "aws-cdk-lib/aws-events" +import * as targets from "aws-cdk-lib/aws-events-targets" import * as s3 from "aws-cdk-lib/aws-s3" import * as cdk from "aws-cdk-lib" import * as pipelines from "aws-cdk-lib/pipelines" @@ -14,6 +16,10 @@ import { CloudAssemblyLookupUserParameters, } from "./cloud-assembly-lookup-handler" import { SlackNotification, SlackNotificationProps } from "./slack-notification" +import { + FeatureFlags, + FEATURE_FLAG_CDK_PIPELINES_SPEED_UP, +} from "../feature-flags" export interface LifligCdkPipelineProps { /** @@ -168,6 +174,11 @@ export class LifligCdkPipeline extends constructs.Construct { new codepipelineActions.S3SourceAction({ actionName: "source", bucket: artifactsBucket, + trigger: FeatureFlags.of(scope).isEnabled( + FEATURE_FLAG_CDK_PIPELINES_SPEED_UP, + ) + ? codepipelineActions.S3Trigger.NONE + : undefined, bucketKey: LifligCdkPipeline.pipelineS3TriggerKey( props.pipelineName, ), @@ -180,8 +191,29 @@ export class LifligCdkPipeline extends constructs.Construct { restartExecutionOnUpdate: true, }) + if (FeatureFlags.of(scope).isEnabled(FEATURE_FLAG_CDK_PIPELINES_SPEED_UP)) { + new events.Rule(this, "PipelineTrigger", { + eventPattern: { + source: ["aws.s3"], + detailType: ["Object Created"], + detail: { + bucket: { + name: [artifactsBucket.bucketName], + }, + object: { + key: [LifligCdkPipeline.pipelineS3TriggerKey(props.pipelineName)], + }, + }, + }, + targets: [new targets.CodePipeline(this.codePipeline)], + }) + } + this.cdkPipeline = new pipelines.CodePipeline(this, "CdkPipeline", { synth, + useChangeSets: !FeatureFlags.of(scope).isEnabled( + FEATURE_FLAG_CDK_PIPELINES_SPEED_UP, + ), codePipeline: this.codePipeline, }) }