Skip to content

Commit

Permalink
Refactor and clean up Pulumi stacks
Browse files Browse the repository at this point in the history
  • Loading branch information
Ru Chern Chong committed May 23, 2024
1 parent 99a1ca1 commit a83027f
Show file tree
Hide file tree
Showing 3 changed files with 85 additions and 121 deletions.
2 changes: 2 additions & 0 deletions .github/workflows/pulumi.preview.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,8 @@ jobs:
stack-name: staging
env:
PULUMI_ACCESS_TOKEN: ${{ secrets.PULUMI_ACCESS_TOKEN }}
MONGODB_URI: ${{ secrets.MONGODB_URI }}
MONGODB_DB_NAME: ${{ secrets.MONGODB_DB_NAME }}

# - name: Deploy to AWS Lambda - Production
# if: env.ENVIRONMENT == 'Production'
Expand Down
11 changes: 7 additions & 4 deletions Pulumi.dev.yaml
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
config:
lta-datasets-updater:MONGODB_DB_NAME: main
lta-datasets-updater:MONGODB_URI:
secure: AAABAD/e3y5xSMuYF1ew0Um4TwbTYPu2+rX1Din6225V5YpPKHgoxLn3VzfkHEMdssmUp1TaRzWtMUOfjEmbjS6olmNInRp2NDxEfTTURAFeeXlKGxMN5yL3kDEktydtjfO/OrFIcEcAHeURhUZDE0TMp4s0ehshluGp+t2p3+1mcepoeGq97AnQpsEfN59ZuSZSOy0u1sC9v+UgJUacBexHN18iclcYDNU=
{
config: {
'pulumi:tags': {
'pulumi:template': aws-python
}
}
}
193 changes: 76 additions & 117 deletions __main__.py
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,25 @@
import json
import os

import pulumi
import pulumi_aws as aws
from dotenv import load_dotenv

project_name = f"{pulumi.get_stack()}-lta-datasets-updater"
load_dotenv()

PROJECT_NAME = f"{pulumi.get_stack()}-{pulumi.get_project()}"

MEMORY_SIZE = 1024
TIMEOUT = 15
RUNTIME = aws.lambda_.Runtime.PYTHON3D12
TIMEZONE = "Asia/Singapore"

config = pulumi.Config()
MONGODB_URI = os.environ.get("MONGODB_URI")
MONGODB_DB_NAME = os.environ.get("MONGODB_DB_NAME")

role = aws.iam.Role(
f"{project_name}-role",
f"{PROJECT_NAME}-role",
assume_role_policy=json.dumps(
{
"Version": "2012-10-17",
Expand All @@ -22,138 +35,84 @@
)

aws.iam.RolePolicyAttachment(
f"{project_name}-policy",
f"{PROJECT_NAME}-policy",
role=role.name,
policy_arn=aws.iam.ManagedPolicy.AWS_LAMBDA_BASIC_EXECUTION_ROLE,
)

# lta_datasets_updater_bucket = aws.s3.Bucket(f"{project_name}-bucket")

# packages = aws.s3.BucketObjectv2(
# f"{project_name}-packages",
# bucket=lta_datasets_updater_bucket.id,
# source=pulumi.FileAsset("package.zip"),
# )

packages_layer = aws.lambda_.LayerVersion(
f"{project_name}-layer",
layer_name=f"{project_name}-layer",
package_layer = aws.lambda_.LayerVersion(
f"{PROJECT_NAME}-layer",
layer_name=f"{PROJECT_NAME}-layer",
code=pulumi.FileArchive("package.zip"),
# s3_bucket=lta_datasets_updater_bucket.id,
# s3_key=packages.key,
compatible_runtimes=[aws.lambda_.Runtime.PYTHON3D12],
)

update_cars_function = aws.lambda_.Function(
f"{project_name}-cars-function",
code=pulumi.AssetArchive(
{
"utils": pulumi.FileArchive("utils"),
"download_file.py": pulumi.FileAsset("download_file.py"),
"update_cars.py": pulumi.FileAsset("update_cars.py"),
"updater.py": pulumi.FileAsset("updater.py"),
}
),
role=role.arn,
environment=aws.lambda_.FunctionEnvironmentArgs(
variables={
"TZ": "Asia/Singapore",
"MONGODB_URI": pulumi.Config().require_secret("MONGODB_URI"),
"MONGODB_DB_NAME": pulumi.Config().require("MONGODB_DB_NAME"),
}
),
handler="update_cars.handler",
runtime=aws.lambda_.Runtime.PYTHON3D12,
memory_size=1024,
timeout=15,
layers=[packages_layer.arn],
compatible_runtimes=[RUNTIME],
)

update_coe_function = aws.lambda_.Function(
f"{project_name}-coe-function",
code=pulumi.AssetArchive(
{
"utils": pulumi.FileArchive("utils"),
"download_file.py": pulumi.FileAsset("download_file.py"),
"update_coe.py": pulumi.FileAsset("update_coe.py"),
"updater.py": pulumi.FileAsset("updater.py"),
}
),
role=role.arn,
environment=aws.lambda_.FunctionEnvironmentArgs(
variables={
"TZ": "Asia/Singapore",
"MONGODB_URI": pulumi.Config().require_secret("MONGODB_URI"),
"MONGODB_DB_NAME": pulumi.Config().require("MONGODB_DB_NAME"),
}
),
handler="update_coe.handler",
runtime=aws.lambda_.Runtime.PYTHON3D12,
memory_size=1024,
timeout=15,
layers=[packages_layer.arn],
)

update_cars_rule = aws.cloudwatch.EventRule(
f"{project_name}-cars-rule",
schedule_expression="cron(0/60 0-10 ? * MON-FRI *)",
def create_lambda_function(name, handler, code):
return aws.lambda_.Function(f"{PROJECT_NAME}-{name}-function",
code=code,
role=role.arn,
environment=aws.lambda_.FunctionEnvironmentArgs(
variables={
"TZ": "Asia/Singapore",
"MONGODB_URI": MONGODB_URI,
"MONGODB_DB_NAME": MONGODB_DB_NAME
}
), handler=handler, runtime=RUNTIME, memory_size=MEMORY_SIZE, timeout=TIMEOUT,
layers=[package_layer.arn])


def create_asset_archive(update_file):
files = {"utils": pulumi.FileArchive("utils"),
"download_file.py": pulumi.FileAsset("download_file.py"),
"updater.py": pulumi.FileAsset("updater.py"),
f"{update_file}.py": pulumi.FileAsset(f"{update_file}.py")}
return pulumi.AssetArchive(files)


update_cars_function = create_lambda_function(
"cars",
"update_cars.handler",
code=create_asset_archive("update_cars"),
)

update_coe_rule = aws.cloudwatch.EventRule(
f"{project_name}-coe-rule",
schedule_expression="cron(0/60 0-10 ? * MON-FRI *)",
update_coe_function = create_lambda_function(
"coe",
"update_coe.handler",
code=create_asset_archive("update_coe"),
)

update_coe_first_bidding_rule = aws.cloudwatch.EventRule(
f"{project_name}-coe-first-bidding-rule",
schedule_expression="cron(0/10 8-10 ? * 4#1 *)",
)

update_coe_second_bidding_rule = aws.cloudwatch.EventRule(
f"{project_name}-coe-second-bidding-rule",
schedule_expression="cron(0/10 8-10 ? * 4#3 *)",
)
def create_event_rule(name, scheduled_expression, target_lambda_function):
rule = aws.cloudwatch.EventRule(f"{PROJECT_NAME}-{name}-rule", schedule_expression=scheduled_expression)

aws.lambda_.Permission(
f"{project_name}-permission",
action="lambda:InvokeFunction",
function=update_cars_function.name,
principal="events.amazonaws.com",
source_arn=update_cars_rule.arn,
)
aws.cloudwatch.EventTarget(
f"{PROJECT_NAME}-{name}-target",
rule=rule.name,
arn=target_lambda_function.arn,
)

aws.lambda_.Permission(
f"{project_name}-coe-permission",
action="lambda:InvokeFunction",
function=update_coe_function.name,
principal="events.amazonaws.com",
source_arn=update_coe_rule.arn,
)
aws.lambda_.Permission(
f"{PROJECT_NAME}-{name}-permission",
action="lambda:InvokeFunction",
function=target_lambda_function.name,
principal="events.amazonaws.com",
source_arn=rule.arn,
)

update_cars_target = aws.cloudwatch.EventTarget(
f"{project_name}-cars-target",
rule=update_cars_rule.name,
arn=update_cars_function.arn,
)
return rule

pulumi.export("update_cars_function", update_cars_function.name)
pulumi.export("update_cars_rule", update_cars_rule.schedule_expression)
update_coe_target = aws.cloudwatch.EventTarget(
f"{project_name}-coe-target",
rule=update_coe_rule.name,
arn=update_coe_function.arn,
)

update_coe_first_bidding_target = aws.cloudwatch.EventTarget(
f"{project_name}-coe-first-bidding-target",
rule=update_coe_first_bidding_rule.name,
arn=update_coe_function.arn,
)
cron_schedulers = {
"cars": {"cron": "cron(0/60 0-10 ? * MON-FRI *)", "function": update_cars_function},
"coe": {"cron": "cron(0/60 0-10 ? * MON-FRI *)", "function": update_coe_function},
"coe-first-bidding": {"cron": "cron(0/10 8-10 ? * 4#1 *)", "function": update_coe_function},
"coe-second-bidding": {"cron": "cron(0/10 8-10 ? * 4#3 *)", "function": update_coe_function},
}

update_coe_second_bidding_target = aws.cloudwatch.EventTarget(
f"{project_name}-coe-second-bidding-target",
rule=update_coe_second_bidding_rule.name,
arn=update_coe_function.arn,
)
for name, scheduler in cron_schedulers.items():
create_event_rule(name, scheduler['cron'], scheduler['function'])

pulumi.export("packages_layer", packages_layer.source_code_size)
pulumi.export("update_cars_function", update_cars_function.name)
pulumi.export("update_coe_function", update_coe_function.name)
pulumi.export("packages_layer", package_layer.source_code_size)

0 comments on commit a83027f

Please sign in to comment.