Replace ebola with marburg

globaldothealth · Mar 30, 2023 · 25ce110 · 25ce110
1 parent 24aff2c
commit 25ce110
Show file tree

Hide file tree

Showing 12 changed files with 227 additions and 207 deletions.
diff --git a/LICENSE b/LICENSE
@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2022 Global.health
+Copyright (c) 2023 Global.health
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

diff --git a/s3_ui/README.md b/s3_ui/README.md
@@ -1,8 +1,8 @@
-# Ebola Data Navigator
+# Marburg 2023 Data Navigator
 
 ## What it does
 
-This folder contains the code needed to run a web service that exposes archived Ebola files.
+This folder contains the code needed to run a web service that exposes archived Marburg 2023 files.
 
 Users can use their web browsers to navigate and download files by using templated links.
 The server finds files in desired folders and exposes on-demand presigned URLs to S3 objects.

diff --git a/s3_ui/pyproject.toml b/s3_ui/pyproject.toml
@@ -1,20 +1,20 @@
 [tool.poetry]
-name = "ebola-data"
-version = "0.1.0"
-description = ""
-authors = ["Global.health team <[email protected]>"]
+  name = "marburg-data"
+  version = "0.1.0"
+  description = ""
+  authors = ["Global.health team <[email protected]>"]
 
-[tool.poetry.dependencies]
-python = "^3.9"
-requests = "^2.28.1"
-boto3 = "^1.24.39"
-Faker = "^13.15.1"
-Flask = "^2.1.3"
-serverless-wsgi = "^3.0.0"
+  [tool.poetry.dependencies]
+    python = "^3.9"
+    requests = "^2.28.1"
+    boto3 = "^1.24.39"
+    Faker = "^13.15.1"
+    Flask = "^2.1.3"
+    serverless-wsgi = "^3.0.0"
 
-[tool.poetry.dev-dependencies]
-pytest = "^7.1.2"
+  [tool.poetry.dev-dependencies]
+    pytest = "^7.1.2"
 
 [build-system]
-requires = ["poetry-core>=1.0.0"]
-build-backend = "poetry.core.masonry.api"
+  requires = ["poetry-core>=1.0.0"]
+  build-backend = "poetry.core.masonry.api"
diff --git a/s3_ui/setup_localstack.py b/s3_ui/setup_localstack.py
@@ -14,65 +14,65 @@
 
 
 LOCALSTACK_URL = os.environ.get("LOCALSTACK_URL", "http://localstack:4566")
-S3_BUCKET = os.environ.get("S3_BUCKET", "monkeypox")
+S3_BUCKET = os.environ.get("S3_BUCKET", "marburg")
 S3_CLIENT = boto3.client("s3", endpoint_url=LOCALSTACK_URL)
 
 FAKE = Faker()
 
 
 def wait_for_localstack():
-	logging.info("Waiting for localstack")
-	healthcheck_url = "".join([LOCALSTACK_URL, "/health"])
-	counter = 0
-	while counter < 42:
-		try:
-			response = requests.get(healthcheck_url)
-			s3_status = response.json().get("services", {}).get("s3")
-			if s3_status == "running":
-				return
-		except requests.exceptions.ConnectionError:
-			pass
-		counter += 1
-		sleep(5)
-	raise Exception("Localstack not available")
+    logging.info("Waiting for localstack")
+    healthcheck_url = "".join([LOCALSTACK_URL, "/health"])
+    counter = 0
+    while counter < 42:
+        try:
+            response = requests.get(healthcheck_url)
+            s3_status = response.json().get("services", {}).get("s3")
+            if s3_status == "running":
+                return
+        except requests.exceptions.ConnectionError:
+            pass
+        counter += 1
+        sleep(5)
+    raise Exception("Localstack not available")
 
 
 def create_bucket(bucket_name: str) -> None:
-	logging.info(f"Creating bucket {bucket_name}")
-	S3_CLIENT.create_bucket(Bucket=bucket_name)
+    logging.info(f"Creating bucket {bucket_name}")
+    S3_CLIENT.create_bucket(Bucket=bucket_name)
 
 
 def create_fake_data() -> list[dict]:
-	logging.info("Creating fake data")
-	return [FAKE.profile() for _ in range(0, 42)]
+    logging.info("Creating fake data")
+    return [FAKE.profile() for _ in range(0, 42)]
 
 
-def create_fake_file(file_name: str=FAKE.file_name(), data: list[dict]=[]) -> str:
-	print(f"File name: {file_name}")
-	fn_w_ext = f"{file_name}.csv"
-	logging.info(f"Using fake data to create file {fn_w_ext}")
-	with open(fn_w_ext, "w") as fh:
-		fields = list(data[0].keys())
-		writer = csv.DictWriter(fh, fieldnames=fields)
-		writer.writeheader()
-		for row in data:
-			writer.writerow(row)
-	return fn_w_ext
+def create_fake_file(file_name: str = FAKE.file_name(), data: list[dict] = []) -> str:
+    print(f"File name: {file_name}")
+    fn_w_ext = f"{file_name}.csv"
+    logging.info(f"Using fake data to create file {fn_w_ext}")
+    with open(fn_w_ext, "w") as fh:
+        fields = list(data[0].keys())
+        writer = csv.DictWriter(fh, fieldnames=fields)
+        writer.writeheader()
+        for row in data:
+            writer.writerow(row)
+    return fn_w_ext
 
 
 def upload_file(folder: str, file_name: str) -> None:
-	logging.info(f"Uploading file {file_name} to folder {S3_BUCKET}/{folder}")
-	S3_CLIENT.upload_file(file_name, S3_BUCKET, f"{folder}/{file_name}")
+    logging.info(f"Uploading file {file_name} to folder {S3_BUCKET}/{folder}")
+    S3_CLIENT.upload_file(file_name, S3_BUCKET, f"{folder}/{file_name}")
 
 
 if __name__ == "__main__":
-	setup_logger()
-	logging.info("Starting script")
-	wait_for_localstack()
-	create_bucket(S3_BUCKET)
-	for folder in FOLDERS:
-		print(f"Folder in setup: {folder}")
-		for _ in range(0, 3):
-			data = create_fake_data()
-			file_name = create_fake_file(data=data)
-			upload_file(folder, file_name)
+    setup_logger()
+    logging.info("Starting script")
+    wait_for_localstack()
+    create_bucket(S3_BUCKET)
+    for folder in FOLDERS:
+        print(f"Folder in setup: {folder}")
+        for _ in range(0, 3):
+            data = create_fake_data()
+            file_name = create_fake_file(data=data)
+            upload_file(folder, file_name)
diff --git a/s3_ui/templates/folder.html b/s3_ui/templates/folder.html
@@ -1,21 +1,24 @@
 <!DOCTYPE html>
 <html lang="en">
+
 <head>
     <meta charset="UTF-8">
-    <title>Ebola Data Files</title>
+    <title>Marburg 2023 Data Files</title>
 </head>
+
 <body>
     <div>
         <h2>Contents of {{ folder }}</h2>
     </div>
     {% for f in files %}
-        <div>
-            <a href="{{ url_for('get_presigned_url', folder=folder, file_name=f) }}">{{ f }}</a>
-        </div>
+    <div>
+        <a href="{{ url_for('get_presigned_url', folder=folder, file_name=f) }}">{{ f }}</a>
+    </div>
     {% endfor %}
     <br>
     <div>
         <a href="{{ url_for('home') }}">Home</a>
     </div>
 </body>
-</html>
+
+</html>
diff --git a/s3_ui/templates/index.html b/s3_ui/templates/index.html
@@ -1,18 +1,21 @@
 <!DOCTYPE html>
 <html lang="en">
+
 <head>
     <meta charset="UTF-8">
-    <title>Ebola Data Files</title>
+    <title>Marburg 2023 Data Files</title>
 </head>
+
 <body>
     <div>
         <h2>Contents of G.h Ebola data bucket</h2>
     </div>
     <div><a href="{{ url_for('get_archive_files') }}">Line List Archives</a></div>
     {% for f in files %}
-        <div>
-            <a href="{{ url_for('get_presigned_url', folder='', file_name=f) }}">{{ f }}</a>
-        </div>
+    <div>
+        <a href="{{ url_for('get_presigned_url', folder='', file_name=f) }}">{{ f }}</a>
+    </div>
     {% endfor %}
 </body>
-</html>
+
+</html>
diff --git a/scripts/ingestion/docker-compose-test.yml b/scripts/ingestion/docker-compose-test.yml
@@ -10,7 +10,7 @@ services:
       LOCALSTACK_URL: "http://localstack:4566"
       S3_BUCKET: "fake"
       DB_CONNECTION: "mongodb://mongo:27017"
-      DATABASE_NAME: "ebola"
+      DATABASE_NAME: "marburg"
       GH_COLLECTION: "gh"
       AWS_ACCESS_KEY_ID: "fake"
       AWS_SECRET_ACCESS_KEY: "fake"
@@ -34,4 +34,4 @@ services:
     ports:
       - "27017:27017"
     environment:
-      MONGO_INITDB_DATABASE: "ebola"
+      MONGO_INITDB_DATABASE: "marburg"
diff --git a/scripts/ingestion/docker-compose.yml b/scripts/ingestion/docker-compose.yml
@@ -10,7 +10,7 @@ services:
       LOCALSTACK_URL: "http://localstack:4566"
       S3_BUCKET: "fake"
       DB_CONNECTION: "mongodb://mongo:27017"
-      DATABASE_NAME: "ebola"
+      DATABASE_NAME: "marburg"
       GH_COLLECTION: "gh"
       AWS_ACCESS_KEY_ID: "fake"
       AWS_SECRET_ACCESS_KEY: "fake"
@@ -34,4 +34,4 @@ services:
     ports:
       - "27017:27017"
     environment:
-      MONGO_INITDB_DATABASE: "ebola"
+      MONGO_INITDB_DATABASE: "marburg"
diff --git a/scripts/ingestion/pyproject.toml b/scripts/ingestion/pyproject.toml
@@ -1,18 +1,18 @@
 [tool.poetry]
-name = "gh_ebola_ingestion"
-version = "0.1.0"
-description = "Transfers line list data from Google Sheets to storage"
-authors = ["Global.health team <[email protected]>"]
+  name = "gh_marburg_ingestion"
+  version = "0.1.0"
+  description = "Transfers line list data from Google Sheets to storage"
+  authors = ["Global.health team <[email protected]>"]
 
-[tool.poetry.dependencies]
-python = "^3.10"
-boto3 = "^1.25.1"
-pymongo = "^4.3.2"
-pygsheets = "^2.0.5"
+  [tool.poetry.dependencies]
+    python = "^3.10"
+    boto3 = "^1.25.1"
+    pymongo = "^4.3.2"
+    pygsheets = "^2.0.5"
 
-[tool.poetry.dev-dependencies]
-pytest = "^7.2.0"
+  [tool.poetry.dev-dependencies]
+    pytest = "^7.2.0"
 
 [build-system]
-requires = ["poetry-core>=1.0.0"]
-build-backend = "poetry.core.masonry.api"
+  requires = ["poetry-core>=1.0.0"]
+  build-backend = "poetry.core.masonry.api"