From 73eea1bdf82c1c69f3614b0621e64a5da060c4f9 Mon Sep 17 00:00:00 2001
From: alexobaseki <alex@pluralpolicy.com>
Date: Wed, 5 Jun 2024 21:31:37 -0400
Subject: [PATCH] Set archiving to false by default

---
 app.py | 41 ++++++++++++++++++++---------------------
 1 file changed, 20 insertions(+), 21 deletions(-)

diff --git a/app.py b/app.py
index 01319b0..151f90a 100644
--- a/app.py
+++ b/app.py
@@ -54,13 +54,12 @@ def process_import_function(event, context):
     if not messages:
         return
 
-    bucket = messages[0].get("bucket")
-
     for message in messages:
         bucket = message.get("bucket")
         key = message.get("file_path")
         jurisdiction_id = message.get("jurisdiction_id")
         jurisdiction_name = message.get("jurisdiction_name")
+        is_archive_files = message.get("is_archive_files", False)
 
         # for some reason, the key is url encoded sometimes
         key = urllib.parse.unquote(key, encoding="utf-8")
@@ -102,26 +101,26 @@ def process_import_function(event, context):
             logger.error(f"Error downloading file: {e}")
             all_files.remove(filedir)
             continue
-
     # Process imports for all files per jurisdiction in a batch
-    for abbreviation, juris in unique_jurisdictions.items():
-        logger.info(f"importing {juris['id']}...")
-
-        try:
-            do_import(juris["id"], f"{datadir}{abbreviation}")
-            stats.send_last_run(
-                "last_collection_run_time",
-                {
-                    "jurisdiction": juris["name"],
-                    "scrape_type": "import",
-                },
-            )
-            archive_files(bucket, juris["keys"])
-        except Exception as e:
-            logger.error(
-                f"Error importing jurisdiction {juris['id']}: {e}"
-            )  # noqa: E501
-            continue
+        for abbreviation, juris in unique_jurisdictions.items():
+            logger.info(f"importing {juris['id']}...")
+
+            try:
+                do_import(juris["id"], f"{datadir}{abbreviation}")
+                stats.send_last_run(
+                    "last_collection_run_time",
+                    {
+                        "jurisdiction": juris["name"],
+                        "scrape_type": "import",
+                    },
+                )
+                if is_archive_files:
+                    archive_files(bucket, juris["keys"])
+            except Exception as e:
+                logger.error(
+                    f"Error importing jurisdiction {juris['id']}: {e}"
+                )  # noqa: E501
+                continue
 
     logger.info(f"{len(all_files)} files processed")
     stats.close()