From a83027fca22a8cb84f35bd5d5da0f408ff646bb2 Mon Sep 17 00:00:00 2001 From: Ru Chern Chong Date: Thu, 23 May 2024 16:55:11 +0800 Subject: [PATCH] Refactor and clean up Pulumi stacks --- .github/workflows/pulumi.preview.yml | 2 + Pulumi.dev.yaml | 11 +- __main__.py | 193 +++++++++++---------------- 3 files changed, 85 insertions(+), 121 deletions(-) diff --git a/.github/workflows/pulumi.preview.yml b/.github/workflows/pulumi.preview.yml index 487bdad..7fa7e81 100644 --- a/.github/workflows/pulumi.preview.yml +++ b/.github/workflows/pulumi.preview.yml @@ -56,6 +56,8 @@ jobs: stack-name: staging env: PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }} + MONGODB_URI: ${{ secrets.MONGODB_URI }} + MONGODB_DB_NAME: ${{ secrets.MONGODB_DB_NAME }} # - name: Deploy to AWS Lambda - Production # if: env.ENVIRONMENT == 'Production' diff --git a/Pulumi.dev.yaml b/Pulumi.dev.yaml index 1edc45d..6edabb6 100644 --- a/Pulumi.dev.yaml +++ b/Pulumi.dev.yaml @@ -1,4 +1,7 @@ -config: - lta-datasets-updater:MONGODB_DB_NAME: main - lta-datasets-updater:MONGODB_URI: - secure: AAABAD/e3y5xSMuYF1ew0Um4TwbTYPu2+rX1Din6225V5YpPKHgoxLn3VzfkHEMdssmUp1TaRzWtMUOfjEmbjS6olmNInRp2NDxEfTTURAFeeXlKGxMN5yL3kDEktydtjfO/OrFIcEcAHeURhUZDE0TMp4s0ehshluGp+t2p3+1mcepoeGq97AnQpsEfN59ZuSZSOy0u1sC9v+UgJUacBexHN18iclcYDNU= +{ + config: { + 'pulumi:tags': { + 'pulumi:template': aws-python + } + } +} diff --git a/__main__.py b/__main__.py index 689d6a2..7956b13 100644 --- a/__main__.py +++ b/__main__.py @@ -1,12 +1,25 @@ import json +import os import pulumi import pulumi_aws as aws +from dotenv import load_dotenv -project_name = f"{pulumi.get_stack()}-lta-datasets-updater" +load_dotenv() + +PROJECT_NAME = f"{pulumi.get_stack()}-{pulumi.get_project()}" + +MEMORY_SIZE = 1024 +TIMEOUT = 15 +RUNTIME = aws.lambda_.Runtime.PYTHON3D12 +TIMEZONE = "Asia/Singapore" + +config = pulumi.Config() +MONGODB_URI = os.environ.get("MONGODB_URI") +MONGODB_DB_NAME = os.environ.get("MONGODB_DB_NAME") role = aws.iam.Role( - f"{project_name}-role", + f"{PROJECT_NAME}-role", assume_role_policy=json.dumps( { "Version": "2012-10-17", @@ -22,138 +35,84 @@ ) aws.iam.RolePolicyAttachment( - f"{project_name}-policy", + f"{PROJECT_NAME}-policy", role=role.name, policy_arn=aws.iam.ManagedPolicy.AWS_LAMBDA_BASIC_EXECUTION_ROLE, ) -# lta_datasets_updater_bucket = aws.s3.Bucket(f"{project_name}-bucket") - -# packages = aws.s3.BucketObjectv2( -# f"{project_name}-packages", -# bucket=lta_datasets_updater_bucket.id, -# source=pulumi.FileAsset("package.zip"), -# ) - -packages_layer = aws.lambda_.LayerVersion( - f"{project_name}-layer", - layer_name=f"{project_name}-layer", +package_layer = aws.lambda_.LayerVersion( + f"{PROJECT_NAME}-layer", + layer_name=f"{PROJECT_NAME}-layer", code=pulumi.FileArchive("package.zip"), - # s3_bucket=lta_datasets_updater_bucket.id, - # s3_key=packages.key, - compatible_runtimes=[aws.lambda_.Runtime.PYTHON3D12], -) - -update_cars_function = aws.lambda_.Function( - f"{project_name}-cars-function", - code=pulumi.AssetArchive( - { - "utils": pulumi.FileArchive("utils"), - "download_file.py": pulumi.FileAsset("download_file.py"), - "update_cars.py": pulumi.FileAsset("update_cars.py"), - "updater.py": pulumi.FileAsset("updater.py"), - } - ), - role=role.arn, - environment=aws.lambda_.FunctionEnvironmentArgs( - variables={ - "TZ": "Asia/Singapore", - "MONGODB_URI": pulumi.Config().require_secret("MONGODB_URI"), - "MONGODB_DB_NAME": pulumi.Config().require("MONGODB_DB_NAME"), - } - ), - handler="update_cars.handler", - runtime=aws.lambda_.Runtime.PYTHON3D12, - memory_size=1024, - timeout=15, - layers=[packages_layer.arn], + compatible_runtimes=[RUNTIME], ) -update_coe_function = aws.lambda_.Function( - f"{project_name}-coe-function", - code=pulumi.AssetArchive( - { - "utils": pulumi.FileArchive("utils"), - "download_file.py": pulumi.FileAsset("download_file.py"), - "update_coe.py": pulumi.FileAsset("update_coe.py"), - "updater.py": pulumi.FileAsset("updater.py"), - } - ), - role=role.arn, - environment=aws.lambda_.FunctionEnvironmentArgs( - variables={ - "TZ": "Asia/Singapore", - "MONGODB_URI": pulumi.Config().require_secret("MONGODB_URI"), - "MONGODB_DB_NAME": pulumi.Config().require("MONGODB_DB_NAME"), - } - ), - handler="update_coe.handler", - runtime=aws.lambda_.Runtime.PYTHON3D12, - memory_size=1024, - timeout=15, - layers=[packages_layer.arn], -) -update_cars_rule = aws.cloudwatch.EventRule( - f"{project_name}-cars-rule", - schedule_expression="cron(0/60 0-10 ? * MON-FRI *)", +def create_lambda_function(name, handler, code): + return aws.lambda_.Function(f"{PROJECT_NAME}-{name}-function", + code=code, + role=role.arn, + environment=aws.lambda_.FunctionEnvironmentArgs( + variables={ + "TZ": "Asia/Singapore", + "MONGODB_URI": MONGODB_URI, + "MONGODB_DB_NAME": MONGODB_DB_NAME + } + ), handler=handler, runtime=RUNTIME, memory_size=MEMORY_SIZE, timeout=TIMEOUT, + layers=[package_layer.arn]) + + +def create_asset_archive(update_file): + files = {"utils": pulumi.FileArchive("utils"), + "download_file.py": pulumi.FileAsset("download_file.py"), + "updater.py": pulumi.FileAsset("updater.py"), + f"{update_file}.py": pulumi.FileAsset(f"{update_file}.py")} + return pulumi.AssetArchive(files) + + +update_cars_function = create_lambda_function( + "cars", + "update_cars.handler", + code=create_asset_archive("update_cars"), ) -update_coe_rule = aws.cloudwatch.EventRule( - f"{project_name}-coe-rule", - schedule_expression="cron(0/60 0-10 ? * MON-FRI *)", +update_coe_function = create_lambda_function( + "coe", + "update_coe.handler", + code=create_asset_archive("update_coe"), ) -update_coe_first_bidding_rule = aws.cloudwatch.EventRule( - f"{project_name}-coe-first-bidding-rule", - schedule_expression="cron(0/10 8-10 ? * 4#1 *)", -) -update_coe_second_bidding_rule = aws.cloudwatch.EventRule( - f"{project_name}-coe-second-bidding-rule", - schedule_expression="cron(0/10 8-10 ? * 4#3 *)", -) +def create_event_rule(name, scheduled_expression, target_lambda_function): + rule = aws.cloudwatch.EventRule(f"{PROJECT_NAME}-{name}-rule", schedule_expression=scheduled_expression) -aws.lambda_.Permission( - f"{project_name}-permission", - action="lambda:InvokeFunction", - function=update_cars_function.name, - principal="events.amazonaws.com", - source_arn=update_cars_rule.arn, -) + aws.cloudwatch.EventTarget( + f"{PROJECT_NAME}-{name}-target", + rule=rule.name, + arn=target_lambda_function.arn, + ) -aws.lambda_.Permission( - f"{project_name}-coe-permission", - action="lambda:InvokeFunction", - function=update_coe_function.name, - principal="events.amazonaws.com", - source_arn=update_coe_rule.arn, -) + aws.lambda_.Permission( + f"{PROJECT_NAME}-{name}-permission", + action="lambda:InvokeFunction", + function=target_lambda_function.name, + principal="events.amazonaws.com", + source_arn=rule.arn, + ) -update_cars_target = aws.cloudwatch.EventTarget( - f"{project_name}-cars-target", - rule=update_cars_rule.name, - arn=update_cars_function.arn, -) + return rule -pulumi.export("update_cars_function", update_cars_function.name) -pulumi.export("update_cars_rule", update_cars_rule.schedule_expression) -update_coe_target = aws.cloudwatch.EventTarget( - f"{project_name}-coe-target", - rule=update_coe_rule.name, - arn=update_coe_function.arn, -) -update_coe_first_bidding_target = aws.cloudwatch.EventTarget( - f"{project_name}-coe-first-bidding-target", - rule=update_coe_first_bidding_rule.name, - arn=update_coe_function.arn, -) +cron_schedulers = { + "cars": {"cron": "cron(0/60 0-10 ? * MON-FRI *)", "function": update_cars_function}, + "coe": {"cron": "cron(0/60 0-10 ? * MON-FRI *)", "function": update_coe_function}, + "coe-first-bidding": {"cron": "cron(0/10 8-10 ? * 4#1 *)", "function": update_coe_function}, + "coe-second-bidding": {"cron": "cron(0/10 8-10 ? * 4#3 *)", "function": update_coe_function}, +} -update_coe_second_bidding_target = aws.cloudwatch.EventTarget( - f"{project_name}-coe-second-bidding-target", - rule=update_coe_second_bidding_rule.name, - arn=update_coe_function.arn, -) +for name, scheduler in cron_schedulers.items(): + create_event_rule(name, scheduler['cron'], scheduler['function']) -pulumi.export("packages_layer", packages_layer.source_code_size) +pulumi.export("update_cars_function", update_cars_function.name) +pulumi.export("update_coe_function", update_coe_function.name) +pulumi.export("packages_layer", package_layer.source_code_size)