From fd69ba95d22c90938301faec2d154fdfcdb321b9 Mon Sep 17 00:00:00 2001 From: Ru Chern Chong Date: Fri, 24 May 2024 13:24:57 +0800 Subject: [PATCH] Add create of python package for lambda layer --- .github/workflows/pulumi.preview.yml | 4 ++-- Pulumi.yaml | 2 +- __main__.py | 21 +++++++++++++++------ create_package_zip.py | 28 ++++++++++++++++++++++++++++ db.py | 3 --- update_cars.py | 12 ++++-------- update_coe.py | 12 ++++-------- updater.py | 2 -- 8 files changed, 54 insertions(+), 30 deletions(-) create mode 100644 create_package_zip.py diff --git a/.github/workflows/pulumi.preview.yml b/.github/workflows/pulumi.preview.yml index 7fa7e81..67c7410 100644 --- a/.github/workflows/pulumi.preview.yml +++ b/.github/workflows/pulumi.preview.yml @@ -48,8 +48,8 @@ jobs: run: | python -m venv python source python/bin/activate - pip install -r requirements.txt - zip -r package.zip python/lib/python3.12/site-packages -x "**/pulumi*/*" "**/pip*/*" "**/pytest*/*" + pip install -r requirements.txt -t python + python create_package_zip.py - uses: pulumi/actions@v5 with: command: up diff --git a/Pulumi.yaml b/Pulumi.yaml index 18d50af..5454e79 100644 --- a/Pulumi.yaml +++ b/Pulumi.yaml @@ -2,7 +2,7 @@ name: lta-datasets-updater runtime: name: python options: - virtualenv: python + virtualenv: venv config: pulumi:tags: value: diff --git a/__main__.py b/__main__.py index 7956b13..2859dc4 100644 --- a/__main__.py +++ b/__main__.py @@ -1,10 +1,13 @@ import json import os +import tempfile import pulumi import pulumi_aws as aws from dotenv import load_dotenv +import create_package_zip + load_dotenv() PROJECT_NAME = f"{pulumi.get_stack()}-{pulumi.get_project()}" @@ -40,12 +43,16 @@ policy_arn=aws.iam.ManagedPolicy.AWS_LAMBDA_BASIC_EXECUTION_ROLE, ) -package_layer = aws.lambda_.LayerVersion( - f"{PROJECT_NAME}-layer", - layer_name=f"{PROJECT_NAME}-layer", - code=pulumi.FileArchive("package.zip"), - compatible_runtimes=[RUNTIME], -) +temp_dir = tempfile.TemporaryDirectory().name +zip_file = 'python.zip' + +create_package_zip.main(temp_dir, zip_file) + +package_layer = aws.lambda_.LayerVersion(f'{PROJECT_NAME}-layer', + layer_name=f"{PROJECT_NAME}-layer", + code=pulumi.AssetArchive({".": pulumi.FileArchive(f"{temp_dir}/{zip_file}")}), + compatible_runtimes=[aws.lambda_.Runtime.PYTHON3D12] + ) def create_lambda_function(name, handler, code): @@ -63,7 +70,9 @@ def create_lambda_function(name, handler, code): def create_asset_archive(update_file): + # TODO: Need a more recursive way to handle these file dependencies files = {"utils": pulumi.FileArchive("utils"), + "db.py": pulumi.FileAsset("db.py"), "download_file.py": pulumi.FileAsset("download_file.py"), "updater.py": pulumi.FileAsset("updater.py"), f"{update_file}.py": pulumi.FileAsset(f"{update_file}.py")} diff --git a/create_package_zip.py b/create_package_zip.py new file mode 100644 index 0000000..179fd57 --- /dev/null +++ b/create_package_zip.py @@ -0,0 +1,28 @@ +import os +import subprocess +import tempfile + + +def main(temp_dir, zip_file): + package_dir = 'python' + excluded_patterns = ["**/__pycache__/*", "**/bin/*", "**/black*/", "**/pip*/*", "**/pulumi*/*", "**/pytest*/*"] + + # Ensure the 'python' directory exists + os.makedirs(package_dir, exist_ok=True) + os.makedirs(temp_dir, exist_ok=True) + + print(f"Temporary directory created at: {temp_dir}") + zip_path = os.path.join(temp_dir, zip_file) + + # Construct the exclusion part of the zip command + exclusion_args = sum([["-x", pattern] for pattern in excluded_patterns], []) + + # Zip the 'python' directory into the temporary directory + subprocess.run(["zip", "-r", zip_path, package_dir, *exclusion_args], stdout=subprocess.DEVNULL) + + print(f"Zipped content is available at {zip_path}") + print(f"Zipped content size {os.path.getsize(zip_path)}") + + +if __name__ == "__main__": + main(tempfile.TemporaryDirectory().name, 'python.zip') diff --git a/db.py b/db.py index d8b134e..0a6a055 100644 --- a/db.py +++ b/db.py @@ -14,6 +14,3 @@ def __init__(self, connection_string=os.environ.get("MONGODB_URI")): @property def database(self): return self.db - - def close_connection(self): - self.client.close() diff --git a/update_cars.py b/update_cars.py index 5af7bbe..b340418 100644 --- a/update_cars.py +++ b/update_cars.py @@ -15,21 +15,17 @@ async def main(): db = MongoDBConnection().database collection = db[collection_name] + + # Create indexes collection.create_index({"month": 1, "make": 1}); collection.create_index({"make": 1}); collection.create_index({"fuel_type": 1}); collection.create_index({"make": 1, "fuel_type": 1}); collection.create_index({"number": 1}); - db.client.close() - response = await updater.main( - collection_name=collection_name, - zip_url=zip_url, - zip_file_name=zip_file_name, - key_fields=key_fields, - ) + db.client.close() - return response + return await updater.main(collection_name, zip_file_name, zip_url, key_fields) def handler(event, context): diff --git a/update_coe.py b/update_coe.py index 53ff42d..04c9232 100644 --- a/update_coe.py +++ b/update_coe.py @@ -15,21 +15,17 @@ async def main(): db = MongoDBConnection().database collection = db[collection_name] + + # Create indexes collection.create_index({"month": 1, "vehicle_class": 1}); collection.create_index({"vehicle_class": 1}); collection.create_index({"month": 1, "bidding_no": 1}); collection.create_index({"premium": 1}); collection.create_index({"bids_success": 1, "bids_received": 1}); - db.client.close() - response = await updater.main( - collection_name=collection_name, - zip_url=zip_url, - zip_file_name=zip_file_name, - key_fields=key_fields, - ) + db.client.close() - return response + return await updater.main(collection_name, zip_file_name, zip_url, key_fields) def handler(event, context): diff --git a/updater.py b/updater.py index 30670ec..00bd2d4 100644 --- a/updater.py +++ b/updater.py @@ -18,8 +18,6 @@ async def updater( collection_name: str, zip_file_name: str, zip_url: str, key_fields: List[str] ) -> str: - # client = MongoClient(os.environ.get("MONGODB_URI")) - # db = client[os.environ.get("MONGODB_DB_NAME")] db = MongoDBConnection().database collection = db[collection_name]