From 3e8a9752144c9399e20a4ede21ed0f3615b3c264 Mon Sep 17 00:00:00 2001 From: Peter Desmet Date: Tue, 22 Aug 2023 10:08:37 +0200 Subject: [PATCH] Use production inventory --- src/vptstools/bin/vph5_to_vpts.py | 2 +- tests/conftest.py | 10 +++++----- tests/test_s3.py | 14 +++++++------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/vptstools/bin/vph5_to_vpts.py b/src/vptstools/bin/vph5_to_vpts.py index 20086c7..be1dc1b 100644 --- a/src/vptstools/bin/vph5_to_vpts.py +++ b/src/vptstools/bin/vph5_to_vpts.py @@ -19,8 +19,8 @@ # (load_dotenv doesn't override existing environment variables) load_dotenv() -INVENTORY_BUCKET = os.environ.get("INVENTORY_BUCKET", "aloft-inventory") S3_BUCKET = os.environ.get("DESTINATION_BUCKET", "aloftdata") +INVENTORY_BUCKET = os.environ.get("INVENTORY_BUCKET", "inbo-aloft-prod-eu-west-1-inventory") AWS_SNS_TOPIC = os.environ.get("SNS_TOPIC") AWS_PROFILE = os.environ.get("AWS_PROFILE", None) AWS_REGION = os.environ.get("AWS_REGION", "eu-west-1") diff --git a/tests/conftest.py b/tests/conftest.py index f596e44..a8b5a3d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -269,20 +269,20 @@ def s3_inventory(aws_credentials, path_inventory): s3 = boto3.client("s3") # Add S3 inventory setup s3.create_bucket( - Bucket="aloft-inventory", + Bucket="inbo-aloft-prod-eu-west-1-inventory", CreateBucketConfiguration={"LocationConstraint": "eu-west-1"}, ) with open(manifest, "rb") as manifest_file: s3.upload_fileobj( manifest_file, - "aloft-inventory", - "aloft/aloft-hdf5-files-inventory/2023-02-01T01-00Z/manifest.json", + "inbo-aloft-prod-eu-west-1-inventory", + "aloftdata/aloftdata-hdf5-files-inventory/2023-02-01T01-00Z/manifest.json", ) with open(inventory, "rb") as inventory_file: s3.upload_fileobj( inventory_file, - "aloft-inventory", - "aloft/aloft-hdf5-files-inventory/data/dummy_inventory.csv.gz", + "inbo-aloft-prod-eu-west-1-inventory", + "aloftdata/aloftdata-hdf5-files-inventory/data/dummy_inventory.csv.gz", ) # Add example data to aloft mocked S3 bucket diff --git a/tests/test_s3.py b/tests/test_s3.py index ad389b1..118af6e 100644 --- a/tests/test_s3.py +++ b/tests/test_s3.py @@ -428,12 +428,12 @@ def test_list_manifest_file_keys(self, s3_inventory): """Individual inventory items are correctly parsed from manifest file""" inventory_files = list( list_manifest_file_keys( - "aloft-inventory/aloft/aloft-hdf5-files-inventory/2023-02-01T01-00Z/manifest.json" + "inbo-aloft-prod-eu-west-1-inventory/aloftdata/aloftdata-hdf5-files-inventory/2023-02-01T01-00Z/manifest.json" ) ) assert len(inventory_files) == 1 assert ( - inventory_files[0]["key"] == "aloft/aloft-hdf5-files-inventory/data/" + inventory_files[0]["key"] == "aloftdata/aloftdata-hdf5-files-inventory/data/" "dummy_inventory.csv.gz" ) @@ -452,13 +452,13 @@ def test_list_manifest_file_keys_with_profile(self, s3_inventory, tmp_path): # run inventory with alternative profile inventory_files = list( list_manifest_file_keys( - "aloft-inventory/aloft/aloft-hdf5-files-inventory/2023-02-01T01-00Z/manifest.json", + "inbo-aloft-prod-eu-west-1-inventory/aloftdata/aloftdata-hdf5-files-inventory/2023-02-01T01-00Z/manifest.json", storage_options={"profile": "my-aws-profile"}, ) ) assert len(inventory_files) == 1 assert ( - inventory_files[0]["key"] == "aloft/aloft-hdf5-files-inventory/data/" + inventory_files[0]["key"] == "aloftdata/aloftdata-hdf5-files-inventory/data/" "dummy_inventory.csv.gz" ) # clean up env variable @@ -472,7 +472,7 @@ def test_handle_manifest_all(self, s3_inventory): return_value=pd.Timestamp("2023-02-01 00:00:00", tz="UTC"), ): df_cov, days_to_create_vpts = handle_manifest( - "s3://aloft-inventory/aloft/aloft-hdf5-files-inventory/2023-02-01T01-00Z/manifest.json", + "s3://inbo-aloft-prod-eu-west-1-inventory/aloftdata/aloftdata-hdf5-files-inventory/2023-02-01T01-00Z/manifest.json", modified_days_ago="60days", ) # large enough number to get all inventory 'modified' items # When date-modified implies full scan, df_cov and days_to_create_vpts are the same @@ -495,7 +495,7 @@ def test_handle_manifest_subset(self, s3_inventory): return_value=pd.Timestamp("2023-02-01 00:00:00", tz="UTC"), ): df_cov, days_to_create_vpts = handle_manifest( - "s3://aloft-inventory/aloft/aloft-hdf5-files-inventory/2023-02-01T01-00Z/manifest.json", + "s3://inbo-aloft-prod-eu-west-1-inventory/aloftdata/aloftdata-hdf5-files-inventory/2023-02-01T01-00Z/manifest.json", modified_days_ago="5days", ) # only subset of files is within the time window of days # Coverage returns the full inventory overview @@ -517,7 +517,7 @@ def test_handle_manifest_none(self, s3_inventory): return_value=pd.Timestamp("2023-03-01 00:00:00", tz="UTC"), ): df_cov, days_to_create_vpts = handle_manifest( - "s3://aloft-inventory/aloft/aloft-hdf5-files-inventory/2023-02-01T01-00Z/manifest.json", + "s3://inbo-aloft-prod-eu-west-1-inventory/aloftdata/aloftdata-hdf5-files-inventory/2023-02-01T01-00Z/manifest.json", modified_days_ago="1days", ) # only subset of files is within the time window of days # Coverage returns the full inventory overview