Add create of python package for lambda layer

.github/workflows/pulumi.preview.yml

@@ -48,8 +48,8 @@ jobs:
         run: |
           python -m venv python
           source python/bin/activate
-          pip install -r requirements.txt
-          zip -r package.zip python/lib/python3.12/site-packages -x "**/pulumi*/*" "**/pip*/*" "**/pytest*/*"
+          pip install -r requirements.txt -t python
+          python create_package_zip.py
       - uses: pulumi/actions@v5
         with:
           command: up

Pulumi.yaml

@@ -2,7 +2,7 @@ name: lta-datasets-updater
 runtime:
   name: python
   options:
-    virtualenv: python
+    virtualenv: venv
 config:
   pulumi:tags:
     value:

__main__.py

@@ -1,10 +1,13 @@
 import json
 import os
+import tempfile
 
 import pulumi
 import pulumi_aws as aws
 from dotenv import load_dotenv
 
+import create_package_zip
+
 load_dotenv()
 
 PROJECT_NAME = f"{pulumi.get_stack()}-{pulumi.get_project()}"
@@ -40,12 +43,16 @@
     policy_arn=aws.iam.ManagedPolicy.AWS_LAMBDA_BASIC_EXECUTION_ROLE,
 )
 
-package_layer = aws.lambda_.LayerVersion(
-    f"{PROJECT_NAME}-layer",
-    layer_name=f"{PROJECT_NAME}-layer",
-    code=pulumi.FileArchive("package.zip"),
-    compatible_runtimes=[RUNTIME],
-)
+temp_dir = tempfile.TemporaryDirectory().name
+zip_file = 'python.zip'
+
+create_package_zip.main(temp_dir, zip_file)
+
+package_layer = aws.lambda_.LayerVersion(f'{PROJECT_NAME}-layer',
+                                         layer_name=f"{PROJECT_NAME}-layer",
+                                         code=pulumi.AssetArchive({".": pulumi.FileArchive(f"{temp_dir}/{zip_file}")}),
+                                         compatible_runtimes=[aws.lambda_.Runtime.PYTHON3D12]
+                                         )
 
 
 def create_lambda_function(name, handler, code):
@@ -63,7 +70,9 @@ def create_lambda_function(name, handler, code):
 
 
 def create_asset_archive(update_file):
+    # TODO: Need a more recursive way to handle these file dependencies
     files = {"utils": pulumi.FileArchive("utils"),
+             "db.py": pulumi.FileAsset("db.py"),
              "download_file.py": pulumi.FileAsset("download_file.py"),
              "updater.py": pulumi.FileAsset("updater.py"),
              f"{update_file}.py": pulumi.FileAsset(f"{update_file}.py")}

create_package_zip.py

@@ -0,0 +1,28 @@
+import os
+import subprocess
+import tempfile
+
+
+def main(temp_dir, zip_file):
+    package_dir = 'python'
+    excluded_patterns = ["**/__pycache__/*", "**/bin/*", "**/black*/", "**/pip*/*", "**/pulumi*/*", "**/pytest*/*"]
+
+    # Ensure the 'python' directory exists
+    os.makedirs(package_dir, exist_ok=True)
+    os.makedirs(temp_dir, exist_ok=True)
+
+    print(f"Temporary directory created at: {temp_dir}")
+    zip_path = os.path.join(temp_dir, zip_file)
+
+    # Construct the exclusion part of the zip command
+    exclusion_args = sum([["-x", pattern] for pattern in excluded_patterns], [])
+
+    # Zip the 'python' directory into the temporary directory
+    subprocess.run(["zip", "-r", zip_path, package_dir, *exclusion_args], stdout=subprocess.DEVNULL)
+
+    print(f"Zipped content is available at {zip_path}")
+    print(f"Zipped content size {os.path.getsize(zip_path)}")
+
+
+if __name__ == "__main__":
+    main(tempfile.TemporaryDirectory().name, 'python.zip')

db.py

@@ -14,6 +14,3 @@ def __init__(self, connection_string=os.environ.get("MONGODB_URI")):
     @property
     def database(self):
         return self.db
-
-    def close_connection(self):
-        self.client.close()

update_cars.py

@@ -15,21 +15,17 @@ async def main():
 
     db = MongoDBConnection().database
     collection = db[collection_name]
+
+    # Create indexes
     collection.create_index({"month": 1, "make": 1});
     collection.create_index({"make": 1});
     collection.create_index({"fuel_type": 1});
     collection.create_index({"make": 1, "fuel_type": 1});
     collection.create_index({"number": 1});
-    db.client.close()
 
-    response = await updater.main(
-        collection_name=collection_name,
-        zip_url=zip_url,
-        zip_file_name=zip_file_name,
-        key_fields=key_fields,
-    )
+    db.client.close()
 
-    return response
+    return await updater.main(collection_name, zip_file_name, zip_url, key_fields)
 
 
 def handler(event, context):

update_coe.py

@@ -15,21 +15,17 @@ async def main():
 
     db = MongoDBConnection().database
     collection = db[collection_name]
+
+    # Create indexes
     collection.create_index({"month": 1, "vehicle_class": 1});
     collection.create_index({"vehicle_class": 1});
     collection.create_index({"month": 1, "bidding_no": 1});
     collection.create_index({"premium": 1});
     collection.create_index({"bids_success": 1, "bids_received": 1});
-    db.client.close()
 
-    response = await updater.main(
-        collection_name=collection_name,
-        zip_url=zip_url,
-        zip_file_name=zip_file_name,
-        key_fields=key_fields,
-    )
+    db.client.close()
 
-    return response
+    return await updater.main(collection_name, zip_file_name, zip_url, key_fields)
 
 
 def handler(event, context):

updater.py

@@ -18,8 +18,6 @@
 async def updater(
         collection_name: str, zip_file_name: str, zip_url: str, key_fields: List[str]
 ) -> str:
-    # client = MongoClient(os.environ.get("MONGODB_URI"))
-    # db = client[os.environ.get("MONGODB_DB_NAME")]
     db = MongoDBConnection().database
     collection = db[collection_name]