From 28d710c65ee21d769a0b2d7b0020d10d83a9798b Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Wed, 1 May 2024 13:48:41 -0700 Subject: [PATCH 01/66] fix type in readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a50d2a2..13f9c37 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ See an example of how to use this cumulus module in [browse_image_workflow.tf](/ ## MacOS 1. Install miniconda (or conda) and [poetry](https://python-poetry.org/) -2. Run `conda env create -f conda-environment.yml` to install GDAL +2. Run `conda env create -f conda-environment.yaml` to install GDAL 3. Activate the bignbit conda environment `conda activate bignbit` 4. Install python package and dependencies `poetry install` 5. Verify tests pass `poetry run pytest tests/` From f398c3a8564f5176dc69b0c0d1394bea42d25c48 Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Wed, 1 May 2024 14:00:53 -0700 Subject: [PATCH 02/66] change task token to uuid in send to gitc --- bignbit/send_to_gitc.py | 21 +++++++++++---------- poetry.lock | 15 ++++++++++++--- pyproject.toml | 1 + 3 files changed, 24 insertions(+), 13 deletions(-) diff --git a/bignbit/send_to_gitc.py b/bignbit/send_to_gitc.py index 66ac7f1..0091385 100644 --- a/bignbit/send_to_gitc.py +++ b/bignbit/send_to_gitc.py @@ -3,6 +3,7 @@ import json import logging import os +import uuid from datetime import datetime, timezone import boto3 @@ -46,19 +47,19 @@ def process(self): """ notification_id = "" - token = self.config.get('token') + gitc_id = uuid.uuid4() if self.input is not None: # Send ImageSet(s) to GITC for processing collection_name = self.input.get('collection_name') cmr_provider = self.input.get('cmr_provider') image_set = ImageSet(**self.input['image_set']) - notification_id = notify_gitc(image_set, cmr_provider, token, collection_name) + notification_id = notify_gitc(image_set, cmr_provider, gitc_id, collection_name) return notification_id -def notify_gitc(image_set: ImageSet, cmr_provider: str, token: str, collection_name: str): +def notify_gitc(image_set: ImageSet, cmr_provider: str, gitc_id: str, collection_name: str): """ Builds and sends a CNM message to GITC @@ -68,8 +69,8 @@ def notify_gitc(image_set: ImageSet, cmr_provider: str, token: str, collection_n The image set to send cmr_provider: str The provider sent in the CNM message - token: str - The token identifying this particular request to GITC + gitc_id: str + The uuid identifying this particular request to GITC collection_name: str Collection that this image set belongs to @@ -82,7 +83,7 @@ def notify_gitc(image_set: ImageSet, cmr_provider: str, token: str, collection_n queue_url = os.environ.get(GIBS_SQS_URL_ENV_NAME) CUMULUS_LOGGER.info(f'Sending SQS message to GITC for image {image_set.name}') - cnm = construct_cnm(image_set, cmr_provider, token, collection_name) + cnm = construct_cnm(image_set, cmr_provider, gitc_id, collection_name) cnm_json = json.dumps(cnm) sqs_message_params = { @@ -102,7 +103,7 @@ def notify_gitc(image_set: ImageSet, cmr_provider: str, token: str, collection_n return cnm['identifier'] -def construct_cnm(image_set: ImageSet, cmr_provider: str, token: str, collection_name: str): +def construct_cnm(image_set: ImageSet, cmr_provider: str, gitc_id: str, collection_name: str): """ Construct the CNM message for GITC @@ -112,8 +113,8 @@ def construct_cnm(image_set: ImageSet, cmr_provider: str, token: str, collection ImageSet for one image to be sent to gibs cmr_provider: str The provider sent in the CNM message - token: str - The token identifying this particular request to GITC + gitc_id: str + The uuid identifying this particular request to GITC collection_name: str Collection that this image set belongs to @@ -131,7 +132,7 @@ def construct_cnm(image_set: ImageSet, cmr_provider: str, token: str, collection "duplicationid": image_set.name, "collection": new_collection, "submissionTime": submission_time, - "identifier": token, + "identifier": gitc_id, "product": product, 'provider': cmr_provider } diff --git a/poetry.lock b/poetry.lock index 1f00f5d..e9d5466 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. [[package]] name = "aiohttp" @@ -1462,7 +1462,6 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, @@ -1629,6 +1628,16 @@ brotli = ["brotli (==1.0.9)", "brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotl secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] +[[package]] +name = "uuid" +version = "1.30" +description = "UUID object and generation functions (Python 2.3 or higher)" +optional = false +python-versions = "*" +files = [ + {file = "uuid-1.30.tar.gz", hash = "sha256:1f87cc004ac5120466f36c5beae48b4c48cc411968eed0eaecd3da82aa96193f"}, +] + [[package]] name = "websockets" version = "12.0" @@ -1938,4 +1947,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "60ff4a221e14d420c4a35caea4f048cfb7fcd32baaee00ee7d11337c7fa5b3b8" +content-hash = "0d469a1b4a8302d64063c24a39779b2b93129354ea3b50b0058e8ce277c98447" diff --git a/pyproject.toml b/pyproject.toml index 49a4fe3..cd18e73 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,7 @@ python-graphql-client = "^0.4.3" requests = "^2.28.2" gdal = "3.6.3" importlib-resources = "^5.12.0" +uuid = "^1.30" [tool.poetry.group.dev.dependencies] pytest = "^7.4.0" From 1f9cf7133f06042697b3ba80814c7ce7ebc6fa3e Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Wed, 1 May 2024 16:10:59 -0700 Subject: [PATCH 03/66] remove wait for task token & gitc response handler from state machine --- terraform/state_machine_definition.tpl | 49 ++------------------------ 1 file changed, 2 insertions(+), 47 deletions(-) diff --git a/terraform/state_machine_definition.tpl b/terraform/state_machine_definition.tpl index ad30a50..ccbbfe7 100644 --- a/terraform/state_machine_definition.tpl +++ b/terraform/state_machine_definition.tpl @@ -438,7 +438,6 @@ "task_config": { "collection": "{$.collection}", "cmr_provider": "{$.cmr_provider}", - "token.$": "$$.Task.Token", "cumulus_message": { "input": "{$}" } @@ -447,25 +446,11 @@ } }, "Type": "Task", - "Resource": "arn:aws:states:::lambda:invoke.waitForTaskToken", + "Resource": "arn:aws:states:::lambda:invoke", "TimeoutSeconds": 86400, "End": true, - "ResultPath": "$.gitc_response", - "Catch": [ - { - "ErrorEquals": [ - "States.Timeout" - ], - "ResultPath": "$.gitc_response", - "Next": "GITC Timeout" - } - ] + "ResultPath": "$.gitc_response" }, - "GITC Timeout": { - "Type": "Pass", - "End": true, - "Comment": "No response was received from GITC within the configured timeout" - } } }, "ResultPath": "$.payload.pobit", @@ -487,36 +472,6 @@ "MaxAttempts": 1 } ], - "Next": "Save CMA Message" - }, - "Save CMA Message": { - "Type": "Task", - "Resource": "${SaveCMAMessageLambda}", - "Parameters": { - "cma": { - "event.$": "$", - "task_config": { - "pobit_audit_bucket": "${PobitAuditBucket}", - "cma_key_name.$": "States.Format('${PobitAuditPath}/{}/{}.{}.cma.json', $.meta.collection.name, $.payload.granules[0].granuleId, $$.State.EnteredTime)", - "cumulus_message": { - "input": "{$.payload}" - } - } - } - }, - "Retry": [ - { - "ErrorEquals": [ - "Lambda.ServiceException", - "Lambda.AWSLambdaException", - "Lambda.SdkClientException", - "Lambda.TooManyRequestsException" - ], - "IntervalSeconds": 2, - "MaxAttempts": 6, - "BackoffRate": 2 - } - ], "Next": "WorkflowSucceeded" }, "WorkflowSucceeded": { From c587bb02980f3561bfd9e7d80b1e5d87ae5bff06 Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Wed, 1 May 2024 17:35:43 -0700 Subject: [PATCH 04/66] change gitc response handler to invoke save cma message --- bignbit/handle_gitc_response.py | 26 ++++++++++---- bignbit/save_cma_message.py | 60 +++++---------------------------- terraform/lambda_functions.tf | 3 ++ 3 files changed, 31 insertions(+), 58 deletions(-) diff --git a/bignbit/handle_gitc_response.py b/bignbit/handle_gitc_response.py index 1c40a37..eb947c5 100644 --- a/bignbit/handle_gitc_response.py +++ b/bignbit/handle_gitc_response.py @@ -3,6 +3,7 @@ import logging import os from json import loads +from datetime import datetime, timezone import boto3 from botocore.exceptions import ClientError @@ -32,13 +33,26 @@ def handler(event, _): for message in event["Records"]: message_body = loads(message["body"]) - task_token = message_body["identifier"] - client = boto3.client('stepfunctions') + gitc_id = message_body["identifier"] + collection = message_body["collection"] + cma_key = "{}/{}/{}.{}.cma.json" + + received_time = datetime.now(timezone.utc).isoformat()[:-9] + 'Z', + + client = boto3.client('lambda') + + cma_event = ('{"pobit_audit_bucket": "' + os.environ['POBIT_AUDIT_BUCKET_NAME'] + + '", "cma_key_name": "' + cma_key.format(os.environ['POBIT_AUDIT_PATH_NAME'], collection, received_time) + + '", "cma_content": "' + json.dumps(message_body) + '"}') + try: - client.send_task_success(taskToken=task_token, output=json.dumps(message_body)) - logger.info("Step function triggered for task token %s", task_token) + client.invoke( + FunctionName=os.environ['SAVE_CMA_LAMBDA_FUNCTION_NAME'], + InvocationType='Event', + Payload=cma_event) + logger.info("Save CMA message lambda invoked for uuid %s", gitc_id) except ClientError: - logger.warning("Error sending task success for messageId %s task token %s", - message['messageId'], task_token, + logger.warning("Error invoking save cma lambda for messageId %s gitcID %s", + message['messageId'], gitc_id, exc_info=True) return {"statusCode": 200, "body": "All good"} diff --git a/bignbit/save_cma_message.py b/bignbit/save_cma_message.py index dca1d8e..31e42d6 100644 --- a/bignbit/save_cma_message.py +++ b/bignbit/save_cma_message.py @@ -1,40 +1,8 @@ """lambda function that stores the CMA message into a s3 bucket""" import json import logging -import os import boto3 -from cumulus_logger import CumulusLogger -from cumulus_process import Process - -CUMULUS_LOGGER = CumulusLogger('save_cma_message') - - -class CMA(Process): - """ - A cumulus message adapter - """ - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.logger = CUMULUS_LOGGER - - def process(self): - """ - Upload CMA message into a s3 bucket - - Returns - ------- - dict - Same input sent to this function - - """ - pobit_audit_bucket = self.config['pobit_audit_bucket'] - cma_key_name = self.config['cma_key_name'] - - upload_cma(pobit_audit_bucket, cma_key_name, self.input) - - return self.input def upload_cma(pobit_audit_bucket: str, cma_key_name: str, cma_content: dict): @@ -72,27 +40,15 @@ def lambda_handler(event, context): event from a lambda call context: dictionary context from a lambda call - Returns - ---------- - dict - A CMA json message - """ - # pylint: disable=duplicate-code - levels = { - 'critical': logging.CRITICAL, - 'error': logging.ERROR, - 'warn': logging.WARNING, - 'warning': logging.WARNING, - 'info': logging.INFO, - 'debug': logging.DEBUG - } - logging_level = os.environ.get('LOGGING_LEVEL', 'info') - CUMULUS_LOGGER.logger.level = levels.get(logging_level, 'info') - CUMULUS_LOGGER.setMetadata(event, context) + """ + logger = logging.getLogger('save_cma_message') - return CMA.cumulus_handler(event, context=context) + pobit_audit_bucket = event['pobit_audit_bucket'] + cma_key_name = event['cma_key_name'] + cma_content = event['cma_content'] + gitc_id = cma_content['identifier'] -if __name__ == "__main__": - CMA() + logger.info("Uploading CMA message to S3 for uuid %s", gitc_id) + upload_cma(pobit_audit_bucket, cma_key_name, cma_content) diff --git a/terraform/lambda_functions.tf b/terraform/lambda_functions.tf index 6c5d947..a6aa1c0 100644 --- a/terraform/lambda_functions.tf +++ b/terraform/lambda_functions.tf @@ -438,6 +438,9 @@ resource "aws_lambda_function" "handle_gitc_response" { STACK_NAME = local.lambda_resources_name CUMULUS_MESSAGE_ADAPTER_DIR = "/opt/" REGION = var.region + SAVE_CMA_LAMBDA_FUNCTION_NAME = aws_lambda_function.save_cma_message.function_name + POBIT_AUDIT_BUCKET_NAME = var.pobit_audit_bucket + POBIT_AUDIT_PATH_NAME = var.pobit_audit_path } } From 42a87066f64f00814b74bd00f52d5db549f71354 Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Wed, 1 May 2024 17:40:41 -0700 Subject: [PATCH 05/66] fix typo --- bignbit/handle_gitc_response.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bignbit/handle_gitc_response.py b/bignbit/handle_gitc_response.py index eb947c5..c9ba3a8 100644 --- a/bignbit/handle_gitc_response.py +++ b/bignbit/handle_gitc_response.py @@ -37,7 +37,7 @@ def handler(event, _): collection = message_body["collection"] cma_key = "{}/{}/{}.{}.cma.json" - received_time = datetime.now(timezone.utc).isoformat()[:-9] + 'Z', + received_time = datetime.now(timezone.utc).isoformat()[:-9] + 'Z' client = boto3.client('lambda') From 53c6d1e4e3aed8dc1352d5166db1d4af997c97fa Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Wed, 1 May 2024 17:43:42 -0700 Subject: [PATCH 06/66] fix context parameter --- bignbit/save_cma_message.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bignbit/save_cma_message.py b/bignbit/save_cma_message.py index 31e42d6..7aa73bf 100644 --- a/bignbit/save_cma_message.py +++ b/bignbit/save_cma_message.py @@ -32,7 +32,7 @@ def upload_cma(pobit_audit_bucket: str, cma_key_name: str, cma_content: dict): ) -def lambda_handler(event, context): +def lambda_handler(event, _context): """handler that gets called by aws lambda Parameters ---------- From 168013eb0ab9ed1e374e222034dced23d07f3bca Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Thu, 2 May 2024 09:02:02 -0700 Subject: [PATCH 07/66] convert uuid to string --- bignbit/send_to_gitc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bignbit/send_to_gitc.py b/bignbit/send_to_gitc.py index 0091385..e253fde 100644 --- a/bignbit/send_to_gitc.py +++ b/bignbit/send_to_gitc.py @@ -47,7 +47,7 @@ def process(self): """ notification_id = "" - gitc_id = uuid.uuid4() + gitc_id = str(uuid.uuid4()) if self.input is not None: # Send ImageSet(s) to GITC for processing From d264ee394f1c4fb648733b0ffaae2b9af1ba8dc9 Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Thu, 2 May 2024 09:32:17 -0700 Subject: [PATCH 08/66] fix json typo --- terraform/state_machine_definition.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/state_machine_definition.tpl b/terraform/state_machine_definition.tpl index ccbbfe7..1f01293 100644 --- a/terraform/state_machine_definition.tpl +++ b/terraform/state_machine_definition.tpl @@ -450,7 +450,7 @@ "TimeoutSeconds": 86400, "End": true, "ResultPath": "$.gitc_response" - }, + } } }, "ResultPath": "$.payload.pobit", From fc48c276444db4c7db51a87c66001e7bf256b6dd Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Thu, 2 May 2024 09:57:44 -0700 Subject: [PATCH 09/66] remove uuid lib to use stdlib uuid --- poetry.lock | 12 +----------- pyproject.toml | 1 - 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/poetry.lock b/poetry.lock index e9d5466..8b86158 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1628,16 +1628,6 @@ brotli = ["brotli (==1.0.9)", "brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotl secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] -[[package]] -name = "uuid" -version = "1.30" -description = "UUID object and generation functions (Python 2.3 or higher)" -optional = false -python-versions = "*" -files = [ - {file = "uuid-1.30.tar.gz", hash = "sha256:1f87cc004ac5120466f36c5beae48b4c48cc411968eed0eaecd3da82aa96193f"}, -] - [[package]] name = "websockets" version = "12.0" @@ -1947,4 +1937,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "0d469a1b4a8302d64063c24a39779b2b93129354ea3b50b0058e8ce277c98447" +content-hash = "60ff4a221e14d420c4a35caea4f048cfb7fcd32baaee00ee7d11337c7fa5b3b8" diff --git a/pyproject.toml b/pyproject.toml index cd18e73..49a4fe3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,6 @@ python-graphql-client = "^0.4.3" requests = "^2.28.2" gdal = "3.6.3" importlib-resources = "^5.12.0" -uuid = "^1.30" [tool.poetry.group.dev.dependencies] pytest = "^7.4.0" From f3ddcae65536d39800be02a5010b0d4eedf9b2c5 Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Thu, 2 May 2024 10:33:42 -0700 Subject: [PATCH 10/66] change identifier to image set name instead of uuid --- bignbit/handle_gitc_response.py | 2 +- bignbit/send_to_gitc.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/bignbit/handle_gitc_response.py b/bignbit/handle_gitc_response.py index c9ba3a8..3322a8c 100644 --- a/bignbit/handle_gitc_response.py +++ b/bignbit/handle_gitc_response.py @@ -42,7 +42,7 @@ def handler(event, _): client = boto3.client('lambda') cma_event = ('{"pobit_audit_bucket": "' + os.environ['POBIT_AUDIT_BUCKET_NAME'] - + '", "cma_key_name": "' + cma_key.format(os.environ['POBIT_AUDIT_PATH_NAME'], collection, received_time) + + '", "cma_key_name": "' + cma_key.format(os.environ['POBIT_AUDIT_PATH_NAME'], collection, gitc_id, received_time) + '", "cma_content": "' + json.dumps(message_body) + '"}') try: diff --git a/bignbit/send_to_gitc.py b/bignbit/send_to_gitc.py index e253fde..89ced35 100644 --- a/bignbit/send_to_gitc.py +++ b/bignbit/send_to_gitc.py @@ -47,13 +47,13 @@ def process(self): """ notification_id = "" - gitc_id = str(uuid.uuid4()) if self.input is not None: # Send ImageSet(s) to GITC for processing collection_name = self.input.get('collection_name') cmr_provider = self.input.get('cmr_provider') image_set = ImageSet(**self.input['image_set']) + gitc_id = image_set.name notification_id = notify_gitc(image_set, cmr_provider, gitc_id, collection_name) return notification_id @@ -70,7 +70,7 @@ def notify_gitc(image_set: ImageSet, cmr_provider: str, gitc_id: str, collection cmr_provider: str The provider sent in the CNM message gitc_id: str - The uuid identifying this particular request to GITC + The unique identifier for this particular request to GITC collection_name: str Collection that this image set belongs to @@ -114,7 +114,7 @@ def construct_cnm(image_set: ImageSet, cmr_provider: str, gitc_id: str, collecti cmr_provider: str The provider sent in the CNM message gitc_id: str - The uuid identifying this particular request to GITC + The unique identifier for this particular request to GITC collection_name: str Collection that this image set belongs to From 49f0af2ecfd53cfa45c7bf3d385ee47eed813fa7 Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Thu, 2 May 2024 10:34:07 -0700 Subject: [PATCH 11/66] remove uuid import --- bignbit/send_to_gitc.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bignbit/send_to_gitc.py b/bignbit/send_to_gitc.py index 89ced35..cd719b7 100644 --- a/bignbit/send_to_gitc.py +++ b/bignbit/send_to_gitc.py @@ -3,7 +3,6 @@ import json import logging import os -import uuid from datetime import datetime, timezone import boto3 From 191f76dec1198e26ef9be0f3207beda492b21cc4 Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Thu, 2 May 2024 13:06:41 -0700 Subject: [PATCH 12/66] fix json formatting of cma invoke --- bignbit/handle_gitc_response.py | 4 ++-- bignbit/save_cma_message.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bignbit/handle_gitc_response.py b/bignbit/handle_gitc_response.py index 3322a8c..de18be1 100644 --- a/bignbit/handle_gitc_response.py +++ b/bignbit/handle_gitc_response.py @@ -43,14 +43,14 @@ def handler(event, _): cma_event = ('{"pobit_audit_bucket": "' + os.environ['POBIT_AUDIT_BUCKET_NAME'] + '", "cma_key_name": "' + cma_key.format(os.environ['POBIT_AUDIT_PATH_NAME'], collection, gitc_id, received_time) - + '", "cma_content": "' + json.dumps(message_body) + '"}') + + '", "cma_content": ' + json.dumps(message_body) + '}') try: client.invoke( FunctionName=os.environ['SAVE_CMA_LAMBDA_FUNCTION_NAME'], InvocationType='Event', Payload=cma_event) - logger.info("Save CMA message lambda invoked for uuid %s", gitc_id) + logger.info("Save CMA message lambda invoked for id %s", gitc_id) except ClientError: logger.warning("Error invoking save cma lambda for messageId %s gitcID %s", message['messageId'], gitc_id, diff --git a/bignbit/save_cma_message.py b/bignbit/save_cma_message.py index 7aa73bf..ce9dc8a 100644 --- a/bignbit/save_cma_message.py +++ b/bignbit/save_cma_message.py @@ -50,5 +50,5 @@ def lambda_handler(event, _context): gitc_id = cma_content['identifier'] - logger.info("Uploading CMA message to S3 for uuid %s", gitc_id) + logger.info("Uploading CMA message to S3 for id %s", gitc_id) upload_cma(pobit_audit_bucket, cma_key_name, cma_content) From 86492da9b13b4000cf670d0a56e18a397f5e1885 Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Thu, 2 May 2024 15:23:23 -0700 Subject: [PATCH 13/66] add granule concept ID to identifier --- bignbit/send_to_gitc.py | 2 +- terraform/state_machine_definition.tpl | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/bignbit/send_to_gitc.py b/bignbit/send_to_gitc.py index cd719b7..f86bdf4 100644 --- a/bignbit/send_to_gitc.py +++ b/bignbit/send_to_gitc.py @@ -52,7 +52,7 @@ def process(self): collection_name = self.input.get('collection_name') cmr_provider = self.input.get('cmr_provider') image_set = ImageSet(**self.input['image_set']) - gitc_id = image_set.name + gitc_id = image_set.name + '_' + self.input.get('granule_conceptId') notification_id = notify_gitc(image_set, cmr_provider, gitc_id, collection_name) return notification_id diff --git a/terraform/state_machine_definition.tpl b/terraform/state_machine_definition.tpl index 1f01293..33a7d4f 100644 --- a/terraform/state_machine_definition.tpl +++ b/terraform/state_machine_definition.tpl @@ -438,6 +438,7 @@ "task_config": { "collection": "{$.collection}", "cmr_provider": "{$.cmr_provider}", + "granule_conceptId": "{$.payload.granules.cmrConceptId} "cumulus_message": { "input": "{$}" } From ca46a1630c0860ffcc7ae1060c9f4fb66f9536ba Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Mon, 13 May 2024 16:45:30 -0700 Subject: [PATCH 14/66] update image set name with granule conceptid --- bignbit/build_image_sets.py | 2 ++ bignbit/send_to_gitc.py | 2 +- terraform/state_machine_definition.tpl | 1 - 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/bignbit/build_image_sets.py b/bignbit/build_image_sets.py index 2d13c86..cedba46 100644 --- a/bignbit/build_image_sets.py +++ b/bignbit/build_image_sets.py @@ -53,6 +53,8 @@ def process(self): response_payload['pobit'] = [] for image_set in image_sets: + image_set.name = image_set.name + '_' + self.input['granules']['cmrConceptId'] + response_payload['pobit'].append({ 'image_set': image_set._asdict(), 'cmr_provider': self.config.get('cmr_provider'), diff --git a/bignbit/send_to_gitc.py b/bignbit/send_to_gitc.py index f86bdf4..cd719b7 100644 --- a/bignbit/send_to_gitc.py +++ b/bignbit/send_to_gitc.py @@ -52,7 +52,7 @@ def process(self): collection_name = self.input.get('collection_name') cmr_provider = self.input.get('cmr_provider') image_set = ImageSet(**self.input['image_set']) - gitc_id = image_set.name + '_' + self.input.get('granule_conceptId') + gitc_id = image_set.name notification_id = notify_gitc(image_set, cmr_provider, gitc_id, collection_name) return notification_id diff --git a/terraform/state_machine_definition.tpl b/terraform/state_machine_definition.tpl index 33a7d4f..1f01293 100644 --- a/terraform/state_machine_definition.tpl +++ b/terraform/state_machine_definition.tpl @@ -438,7 +438,6 @@ "task_config": { "collection": "{$.collection}", "cmr_provider": "{$.cmr_provider}", - "granule_conceptId": "{$.payload.granules.cmrConceptId} "cumulus_message": { "input": "{$}" } From 5953f73462ee3767e50d5e33d5165eec622747ef Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Tue, 14 May 2024 15:32:48 -0700 Subject: [PATCH 15/66] remove whitespace --- bignbit/build_image_sets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bignbit/build_image_sets.py b/bignbit/build_image_sets.py index cedba46..e8a4409 100644 --- a/bignbit/build_image_sets.py +++ b/bignbit/build_image_sets.py @@ -54,7 +54,7 @@ def process(self): for image_set in image_sets: image_set.name = image_set.name + '_' + self.input['granules']['cmrConceptId'] - + response_payload['pobit'].append({ 'image_set': image_set._asdict(), 'cmr_provider': self.config.get('cmr_provider'), From 2969c68743775d883f4482f49300f1132b05e2e9 Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Tue, 14 May 2024 17:02:52 -0700 Subject: [PATCH 16/66] move get umm json to utils and save gitc outgoing cnm --- bignbit/get_granule_umm_json.py | 31 +------------- bignbit/send_to_gitc.py | 22 +++++++++- bignbit/utils.py | 57 ++++++++++++++++++++++++++ terraform/state_machine_definition.tpl | 2 + 4 files changed, 81 insertions(+), 31 deletions(-) diff --git a/bignbit/get_granule_umm_json.py b/bignbit/get_granule_umm_json.py index b2f05e8..37def1c 100644 --- a/bignbit/get_granule_umm_json.py +++ b/bignbit/get_granule_umm_json.py @@ -32,39 +32,12 @@ def process(self): """ cmr_environment = self.config['cmr_environment'] - cmr_link = self.input['granules'][0]['cmrLink'] + cmr_concept_id = self.input['granules'][0]['cmrConceptId'] - self.input['granule_umm_json'] = download_umm_json(cmr_link, cmr_environment) + self.input['granule_umm_json'] = utils.get_umm_json(cmr_concept_id, cmr_environment) return self.input -def download_umm_json(cmr_link: str, cmr_environment: str) -> dict: - """ - Retrieve the umm-json document from the given cmr_link - - Parameters - ---------- - cmr_link: str - Link to the umm-g for downloading - - cmr_environment: str - CMR environment used to retrieve user token - - Returns - ------- - dict - The umm-json document - """ - edl_user, edl_pass = utils.get_edl_creds() - token = utils.get_cmr_user_token(edl_user, edl_pass, cmr_environment) - - umm_json_response = requests.get(cmr_link, headers={'Authorization': f'Bearer {token}'}, timeout=10) - umm_json_response.raise_for_status() - umm_json = umm_json_response.json() - - return umm_json - - def lambda_handler(event, context): """handler that gets called by aws lambda Parameters diff --git a/bignbit/send_to_gitc.py b/bignbit/send_to_gitc.py index cd719b7..b8fe821 100644 --- a/bignbit/send_to_gitc.py +++ b/bignbit/send_to_gitc.py @@ -9,6 +9,7 @@ from cumulus_logger import CumulusLogger from cumulus_process import Process +from bignbit import utils from bignbit.image_set import ImageSet, to_cnm_product_dict REGION_NAME = 'us-west-2' @@ -52,13 +53,16 @@ def process(self): collection_name = self.input.get('collection_name') cmr_provider = self.input.get('cmr_provider') image_set = ImageSet(**self.input['image_set']) + pobit_audit_bucket = self.input.get('pobit_audit_bucket') + cmr_env = self.input.get('cmr_environment') gitc_id = image_set.name - notification_id = notify_gitc(image_set, cmr_provider, gitc_id, collection_name) + + notification_id = notify_gitc(image_set, cmr_provider, gitc_id, collection_name, pobit_audit_bucket, cmr_env) return notification_id -def notify_gitc(image_set: ImageSet, cmr_provider: str, gitc_id: str, collection_name: str): +def notify_gitc(image_set: ImageSet, cmr_provider: str, gitc_id: str, collection_name: str, audit_bucket: str, cmr_env: str): """ Builds and sends a CNM message to GITC @@ -72,6 +76,10 @@ def notify_gitc(image_set: ImageSet, cmr_provider: str, gitc_id: str, collection The unique identifier for this particular request to GITC collection_name: str Collection that this image set belongs to + audit_bucket: str + The name of the S3 bucket where a copy of the outgoing CNM will be saved + cmr_env: str + The CMR environment to use for getting granule metadata Returns ------- @@ -99,6 +107,16 @@ def notify_gitc(image_set: ImageSet, cmr_provider: str, gitc_id: str, collection response = sqs.send_message(**sqs_message_params) CUMULUS_LOGGER.debug(f'SQS send_message output: {response}') + + granule_concept_id = image_set.name[-19:] + umm_json = utils.get_umm_json(granule_concept_id, cmr_env) + granule_ur = umm_json['GranuleUR'] + + cnm_key_name = collection_name + "/" + granule_ur + "." + cnm_json['submission_time'] + "." + "cnm.json" + + utils.upload_cnm(audit_bucket, cnm_key_name, cnm_json) + CUMULUS_LOGGER.debug('CNM uploaded to s3 audit bucket') + return cnm['identifier'] diff --git a/bignbit/utils.py b/bignbit/utils.py index 7844acf..ea8a503 100644 --- a/bignbit/utils.py +++ b/bignbit/utils.py @@ -110,6 +110,35 @@ def get_cmr_user_token(edl_user: str, edl_pass: str, cmr_env: str) -> str: return EDL_USER_TOKEN['access_token'] +def get_umm_json(granule_concept_id: str, cmr_environment): + """ + Get the granuleUR for the given concept ID + + Parameters + ---------- + granule_concept_id: str + the concept ID for the granule to find + + cmr_environment: str + CMR environment used to retrieve user token + + Returns + ------- + dict + The umm-json document + """ + + edl_user, edl_pass = get_edl_creds() + token = get_cmr_user_token(edl_user, edl_pass, cmr_environment) + + cmr_link = f'https://cmr.{"uat." if cmr_environment == "UAT" else ""}earthdata.nasa.gov/search/concepts/{granule_concept_id}.umm_json' + umm_json_response = requests.get(cmr_link, headers={'Authorization': f'Bearer {token}'}, timeout=10) + umm_json_response.raise_for_status() + umm_json = umm_json_response.json() + + return umm_json + + def sha512sum(filepath: pathlib.Path): """ Generate a SHA512 hash for the given file @@ -131,6 +160,34 @@ def sha512sum(filepath: pathlib.Path): return hash512.hexdigest() +def upload_cnm(bucket_name: str, key_name: str, cnm_content: dict): + """ + Upload CNM message into a s3 bucket + + Parameters + ---------- + bucket_name: str + Bucket name containing where CNM should be uploaded + + key_name: str + Key to object location in bucket + + cnm_content: dict + The CNM message to upload + + Returns + ------- + None + """ + s3_client = boto3.client('s3') + s3_client.put_object( + Body=cnm_content, + Bucket=bucket_name, + Key=key_name + ) + return f's3://{bucket_name}/{key_name}' + + def upload_to_s3(filepath: pathlib.Path, bucket_name: str, object_key: str): """ Uploads a file to S3 diff --git a/terraform/state_machine_definition.tpl b/terraform/state_machine_definition.tpl index 1f01293..6e18172 100644 --- a/terraform/state_machine_definition.tpl +++ b/terraform/state_machine_definition.tpl @@ -438,6 +438,8 @@ "task_config": { "collection": "{$.collection}", "cmr_provider": "{$.cmr_provider}", + "cmr_environment":"{$.meta.cmr.cmrEnvironment}", + "pobit_audit_bucket": "${PobitAuditBucket}", "cumulus_message": { "input": "{$}" } From e84af9faad539513e7adb99a9482e45aa39aff82 Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Tue, 14 May 2024 17:03:21 -0700 Subject: [PATCH 17/66] remove unused import --- bignbit/get_granule_umm_json.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bignbit/get_granule_umm_json.py b/bignbit/get_granule_umm_json.py index 37def1c..2e45e8e 100644 --- a/bignbit/get_granule_umm_json.py +++ b/bignbit/get_granule_umm_json.py @@ -4,7 +4,6 @@ import logging import os -import requests from cumulus_logger import CumulusLogger from cumulus_process import Process From da12a51c0efb28282023a309986e1a45a913aed7 Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Tue, 14 May 2024 17:30:27 -0700 Subject: [PATCH 18/66] try setting cmr query env based on stage --- bignbit/handle_gitc_response.py | 39 +++++++++++++-------------------- terraform/lambda_functions.tf | 2 +- 2 files changed, 16 insertions(+), 25 deletions(-) diff --git a/bignbit/handle_gitc_response.py b/bignbit/handle_gitc_response.py index de18be1..90dfcef 100644 --- a/bignbit/handle_gitc_response.py +++ b/bignbit/handle_gitc_response.py @@ -3,9 +3,8 @@ import logging import os from json import loads -from datetime import datetime, timezone -import boto3 -from botocore.exceptions import ClientError + +from bignbit import utils def handler(event, _): @@ -34,25 +33,17 @@ def handler(event, _): for message in event["Records"]: message_body = loads(message["body"]) gitc_id = message_body["identifier"] - collection = message_body["collection"] - cma_key = "{}/{}/{}.{}.cma.json" - - received_time = datetime.now(timezone.utc).isoformat()[:-9] + 'Z' - - client = boto3.client('lambda') - - cma_event = ('{"pobit_audit_bucket": "' + os.environ['POBIT_AUDIT_BUCKET_NAME'] - + '", "cma_key_name": "' + cma_key.format(os.environ['POBIT_AUDIT_PATH_NAME'], collection, gitc_id, received_time) - + '", "cma_content": ' + json.dumps(message_body) + '}') - - try: - client.invoke( - FunctionName=os.environ['SAVE_CMA_LAMBDA_FUNCTION_NAME'], - InvocationType='Event', - Payload=cma_event) - logger.info("Save CMA message lambda invoked for id %s", gitc_id) - except ClientError: - logger.warning("Error invoking save cma lambda for messageId %s gitcID %s", - message['messageId'], gitc_id, - exc_info=True) + collection_name = message_body["collection"] + cmr_env = os.environ['CMR_ENVIRONMENT'] + + granule_concept_id = gitc_id[-19:] + umm_json = utils.get_umm_json(granule_concept_id, cmr_env) + granule_ur = umm_json['GranuleUR'] + + cnm_key_name = collection_name + "/" + granule_ur + "." + message_body['submission_time'] + "." + "cnm-r.json" + + utils.upload_cnm(os.environ['POBIT_AUDIT_BUCKET_NAME'], cnm_key_name, json.dumps(message_body)) + + logging.debug('CNM-R uploaded to s3 audit bucket for id %s', gitc_id) + return {"statusCode": 200, "body": "All good"} diff --git a/terraform/lambda_functions.tf b/terraform/lambda_functions.tf index a6aa1c0..1c90216 100644 --- a/terraform/lambda_functions.tf +++ b/terraform/lambda_functions.tf @@ -438,9 +438,9 @@ resource "aws_lambda_function" "handle_gitc_response" { STACK_NAME = local.lambda_resources_name CUMULUS_MESSAGE_ADAPTER_DIR = "/opt/" REGION = var.region - SAVE_CMA_LAMBDA_FUNCTION_NAME = aws_lambda_function.save_cma_message.function_name POBIT_AUDIT_BUCKET_NAME = var.pobit_audit_bucket POBIT_AUDIT_PATH_NAME = var.pobit_audit_path + CMR_ENVIRONMENT = local.environment != "OPS" ? "UAT" : "" } } From e4fb9d7017de5143cba81dc4d3ef4dc2d5380e35 Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Tue, 14 May 2024 17:43:48 -0700 Subject: [PATCH 19/66] remove task token from tests --- tests/test_send_to_gibs_moto.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_send_to_gibs_moto.py b/tests/test_send_to_gibs_moto.py index edbd2b1..e451c79 100644 --- a/tests/test_send_to_gibs_moto.py +++ b/tests/test_send_to_gibs_moto.py @@ -106,7 +106,6 @@ def test_process_sends_message(fake_response_sqs_queue, mock_gitc_success, cnm_v sub_event = event.copy() del sub_event['cma']['event']['payload']['pobit'] sub_event['cma']['event']['payload'] = imageset - sub_event['cma']['task_config']['token'] = str(uuid.uuid4()) bignbit.send_to_gitc.lambda_handler(sub_event, {}) sent_messages = mock_gitc_success.wait_for_messages(count=1) From 921e057b2898a7bcbaceb1afba8fe4e37a382f8f Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Tue, 14 May 2024 17:52:43 -0700 Subject: [PATCH 20/66] fix granule index --- bignbit/build_image_sets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bignbit/build_image_sets.py b/bignbit/build_image_sets.py index e8a4409..a3d56c9 100644 --- a/bignbit/build_image_sets.py +++ b/bignbit/build_image_sets.py @@ -53,7 +53,7 @@ def process(self): response_payload['pobit'] = [] for image_set in image_sets: - image_set.name = image_set.name + '_' + self.input['granules']['cmrConceptId'] + image_set.name = image_set.name + '_' + self.input['granules'][0]['cmrConceptId'] response_payload['pobit'].append({ 'image_set': image_set._asdict(), From 5f6239020dbb2bbeda6b1bfde6b499ece5a5937d Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Tue, 14 May 2024 18:11:24 -0700 Subject: [PATCH 21/66] change how image set name defined --- bignbit/build_image_sets.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/bignbit/build_image_sets.py b/bignbit/build_image_sets.py index a3d56c9..349a708 100644 --- a/bignbit/build_image_sets.py +++ b/bignbit/build_image_sets.py @@ -6,7 +6,7 @@ from cumulus_logger import CumulusLogger from cumulus_process import Process -from bignbit.image_set import from_big_output, IncompleteImageSet +from bignbit.image_set import from_big_output, IncompleteImageSet, ImageSet CUMULUS_LOGGER = CumulusLogger('build_image_sets') @@ -52,11 +52,15 @@ def process(self): del response_payload['big'] response_payload['pobit'] = [] - for image_set in image_sets: - image_set.name = image_set.name + '_' + self.input['granules'][0]['cmrConceptId'] + for big_image_set in image_sets: + pobit_image_set = ImageSet( + name=big_image_set.name + '_' + self.input['granules'][0]['cmrConceptId'], + image=big_image_set.image, + image_metadata=big_image_set.image_metadata, + world_file=big_image_set.world_file) response_payload['pobit'].append({ - 'image_set': image_set._asdict(), + 'image_set': pobit_image_set._asdict(), 'cmr_provider': self.config.get('cmr_provider'), 'collection_name': self.config.get('collection').get('name'), }) From 38651767101ea2534255bd96eb6ee6a29d583848 Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Wed, 15 May 2024 16:30:26 -0700 Subject: [PATCH 22/66] add region to ssm client in utils --- bignbit/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bignbit/utils.py b/bignbit/utils.py index ea8a503..55a1156 100644 --- a/bignbit/utils.py +++ b/bignbit/utils.py @@ -27,7 +27,7 @@ def get_edl_creds() -> (str, str): global ED_USER # pylint: disable=W0603 global ED_PASS # pylint: disable=W0603 - ssm = boto3.client('ssm') + ssm = boto3.client('ssm', region_name='us-west-2') if not ED_USER: edl_user_ssm_name = os.environ.get('EDL_USER_SSM') From 7f28ba97a5e2184b3ff7d65d0344be78ebd8878f Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Wed, 15 May 2024 16:45:18 -0700 Subject: [PATCH 23/66] add ssm parameters to gitc lambdas --- bignbit/utils.py | 3 ++- terraform/lambda_functions.tf | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/bignbit/utils.py b/bignbit/utils.py index 55a1156..c25d1e0 100644 --- a/bignbit/utils.py +++ b/bignbit/utils.py @@ -27,7 +27,8 @@ def get_edl_creds() -> (str, str): global ED_USER # pylint: disable=W0603 global ED_PASS # pylint: disable=W0603 - ssm = boto3.client('ssm', region_name='us-west-2') + region = os.environ.get('REGION') + ssm = boto3.client('ssm', region_name=region) if not ED_USER: edl_user_ssm_name = os.environ.get('EDL_USER_SSM') diff --git a/terraform/lambda_functions.tf b/terraform/lambda_functions.tf index 1c90216..e724c20 100644 --- a/terraform/lambda_functions.tf +++ b/terraform/lambda_functions.tf @@ -406,6 +406,8 @@ resource "aws_lambda_function" "send_to_gitc" { REGION = var.region GIBS_REGION = var.gibs_region GIBS_SQS_URL = "https://sqs.${var.gibs_region}.amazonaws.com/${var.gibs_account_id}/${var.gibs_queue_name}" + EDL_USER_SSM = var.edl_user_ssm + EDL_PASS_SSM = var.edl_pass_ssm } } @@ -441,6 +443,8 @@ resource "aws_lambda_function" "handle_gitc_response" { POBIT_AUDIT_BUCKET_NAME = var.pobit_audit_bucket POBIT_AUDIT_PATH_NAME = var.pobit_audit_path CMR_ENVIRONMENT = local.environment != "OPS" ? "UAT" : "" + EDL_USER_SSM = var.edl_user_ssm + EDL_PASS_SSM = var.edl_pass_ssm } } From cf90d3d0de118907a38aef480e75f92dffa58e28 Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Wed, 15 May 2024 16:49:38 -0700 Subject: [PATCH 24/66] change region reference --- bignbit/utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bignbit/utils.py b/bignbit/utils.py index c25d1e0..55a1156 100644 --- a/bignbit/utils.py +++ b/bignbit/utils.py @@ -27,8 +27,7 @@ def get_edl_creds() -> (str, str): global ED_USER # pylint: disable=W0603 global ED_PASS # pylint: disable=W0603 - region = os.environ.get('REGION') - ssm = boto3.client('ssm', region_name=region) + ssm = boto3.client('ssm', region_name='us-west-2') if not ED_USER: edl_user_ssm_name = os.environ.get('EDL_USER_SSM') From 8ee95eeb69789da9c77701f8c31dad9b0deb5104 Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Thu, 16 May 2024 13:41:49 -0700 Subject: [PATCH 25/66] update how granule name referenced in send to gitc --- bignbit/build_image_sets.py | 1 + bignbit/send_to_gitc.py | 16 ++++++++-------- tests/test_send_to_gibs_moto.py | 1 - 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/bignbit/build_image_sets.py b/bignbit/build_image_sets.py index 349a708..29ec56e 100644 --- a/bignbit/build_image_sets.py +++ b/bignbit/build_image_sets.py @@ -63,6 +63,7 @@ def process(self): 'image_set': pobit_image_set._asdict(), 'cmr_provider': self.config.get('cmr_provider'), 'collection_name': self.config.get('collection').get('name'), + 'granule_ur': self.input['granules'][0]['granuleID'] }) return response_payload diff --git a/bignbit/send_to_gitc.py b/bignbit/send_to_gitc.py index b8fe821..5b4c454 100644 --- a/bignbit/send_to_gitc.py +++ b/bignbit/send_to_gitc.py @@ -9,8 +9,8 @@ from cumulus_logger import CumulusLogger from cumulus_process import Process -from bignbit import utils from bignbit.image_set import ImageSet, to_cnm_product_dict +from bignbit import utils REGION_NAME = 'us-west-2' CUMULUS_LOGGER = CumulusLogger('send_to_gitc') @@ -53,16 +53,18 @@ def process(self): collection_name = self.input.get('collection_name') cmr_provider = self.input.get('cmr_provider') image_set = ImageSet(**self.input['image_set']) + gitc_id = image_set.name + + granule_ur = self.input.get('granule_ur') pobit_audit_bucket = self.input.get('pobit_audit_bucket') cmr_env = self.input.get('cmr_environment') - gitc_id = image_set.name - notification_id = notify_gitc(image_set, cmr_provider, gitc_id, collection_name, pobit_audit_bucket, cmr_env) + notification_id = notify_gitc(image_set, cmr_provider, gitc_id, collection_name, granule_ur, pobit_audit_bucket, cmr_env) return notification_id -def notify_gitc(image_set: ImageSet, cmr_provider: str, gitc_id: str, collection_name: str, audit_bucket: str, cmr_env: str): +def notify_gitc(image_set: ImageSet, cmr_provider: str, gitc_id: str, collection_name: str, granule_ur: str, audit_bucket, cmr_environment): """ Builds and sends a CNM message to GITC @@ -76,6 +78,8 @@ def notify_gitc(image_set: ImageSet, cmr_provider: str, gitc_id: str, collection The unique identifier for this particular request to GITC collection_name: str Collection that this image set belongs to + granuleUR: str + The granuleUR for this image set audit_bucket: str The name of the S3 bucket where a copy of the outgoing CNM will be saved cmr_env: str @@ -108,10 +112,6 @@ def notify_gitc(image_set: ImageSet, cmr_provider: str, gitc_id: str, collection CUMULUS_LOGGER.debug(f'SQS send_message output: {response}') - granule_concept_id = image_set.name[-19:] - umm_json = utils.get_umm_json(granule_concept_id, cmr_env) - granule_ur = umm_json['GranuleUR'] - cnm_key_name = collection_name + "/" + granule_ur + "." + cnm_json['submission_time'] + "." + "cnm.json" utils.upload_cnm(audit_bucket, cnm_key_name, cnm_json) diff --git a/tests/test_send_to_gibs_moto.py b/tests/test_send_to_gibs_moto.py index e451c79..50828f8 100644 --- a/tests/test_send_to_gibs_moto.py +++ b/tests/test_send_to_gibs_moto.py @@ -8,7 +8,6 @@ import os import threading import urllib.request -import uuid import boto3 import jsonschema From 46a9395e7dc662e2fe10806717ec9012fe1d7bc5 Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Thu, 16 May 2024 13:44:34 -0700 Subject: [PATCH 26/66] remove unused cmr var --- bignbit/send_to_gitc.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/bignbit/send_to_gitc.py b/bignbit/send_to_gitc.py index 5b4c454..db83a7e 100644 --- a/bignbit/send_to_gitc.py +++ b/bignbit/send_to_gitc.py @@ -57,14 +57,13 @@ def process(self): granule_ur = self.input.get('granule_ur') pobit_audit_bucket = self.input.get('pobit_audit_bucket') - cmr_env = self.input.get('cmr_environment') - notification_id = notify_gitc(image_set, cmr_provider, gitc_id, collection_name, granule_ur, pobit_audit_bucket, cmr_env) + notification_id = notify_gitc(image_set, cmr_provider, gitc_id, collection_name, granule_ur, pobit_audit_bucket) return notification_id -def notify_gitc(image_set: ImageSet, cmr_provider: str, gitc_id: str, collection_name: str, granule_ur: str, audit_bucket, cmr_environment): +def notify_gitc(image_set: ImageSet, cmr_provider: str, gitc_id: str, collection_name: str, granule_ur: str, audit_bucket): """ Builds and sends a CNM message to GITC @@ -82,8 +81,6 @@ def notify_gitc(image_set: ImageSet, cmr_provider: str, gitc_id: str, collection The granuleUR for this image set audit_bucket: str The name of the S3 bucket where a copy of the outgoing CNM will be saved - cmr_env: str - The CMR environment to use for getting granule metadata Returns ------- From 8944b3552f439868125b8c47bd3e6b27e4562d3a Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Thu, 16 May 2024 13:48:52 -0700 Subject: [PATCH 27/66] fix case --- bignbit/build_image_sets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bignbit/build_image_sets.py b/bignbit/build_image_sets.py index 29ec56e..381bf18 100644 --- a/bignbit/build_image_sets.py +++ b/bignbit/build_image_sets.py @@ -63,7 +63,7 @@ def process(self): 'image_set': pobit_image_set._asdict(), 'cmr_provider': self.config.get('cmr_provider'), 'collection_name': self.config.get('collection').get('name'), - 'granule_ur': self.input['granules'][0]['granuleID'] + 'granule_ur': self.input['granules'][0]['granuleId'] }) return response_payload From bf86736fdc18246c9b929c32a7b7137693c6a201 Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Thu, 16 May 2024 13:52:55 -0700 Subject: [PATCH 28/66] fix cnm parsing --- bignbit/send_to_gitc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bignbit/send_to_gitc.py b/bignbit/send_to_gitc.py index db83a7e..77c3e25 100644 --- a/bignbit/send_to_gitc.py +++ b/bignbit/send_to_gitc.py @@ -109,7 +109,7 @@ def notify_gitc(image_set: ImageSet, cmr_provider: str, gitc_id: str, collection CUMULUS_LOGGER.debug(f'SQS send_message output: {response}') - cnm_key_name = collection_name + "/" + granule_ur + "." + cnm_json['submission_time'] + "." + "cnm.json" + cnm_key_name = collection_name + "/" + granule_ur + "." + cnm['submissionTime'] + "." + "cnm.json" utils.upload_cnm(audit_bucket, cnm_key_name, cnm_json) CUMULUS_LOGGER.debug('CNM uploaded to s3 audit bucket') From b146dcc6be7b4c09447fe217fdb287ce12a0a155 Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Tue, 28 May 2024 12:57:50 -0700 Subject: [PATCH 29/66] reformat save cnm to separate step --- bignbit/save_cnm_message.py | 104 +++++++++++++++++++++++++ bignbit/send_to_gitc.py | 18 +---- terraform/state_machine_definition.tpl | 32 +++++++- 3 files changed, 136 insertions(+), 18 deletions(-) create mode 100644 bignbit/save_cnm_message.py diff --git a/bignbit/save_cnm_message.py b/bignbit/save_cnm_message.py new file mode 100644 index 0000000..90d886b --- /dev/null +++ b/bignbit/save_cnm_message.py @@ -0,0 +1,104 @@ +"""lambda function that stores the CMA message into a s3 bucket""" +import json +import logging +import os + +import boto3 +from cumulus_logger import CumulusLogger +from cumulus_process import Process + +CUMULUS_LOGGER = CumulusLogger('save_cmm_message') + + +class CNM(Process): + """ + A cumulus message adapter + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.logger = CUMULUS_LOGGER + + def process(self): + """ + Upload CNM message into a s3 bucket + + Returns + ------- + dict + Same input sent to this function + + """ + pobit_audit_bucket = self.config['pobit_audit_bucket'] + + collection_name = self.config['collection'] + granule_ur = self.input.get('granule_ur') + + cnm_content = self.input + + cnm_key_name = collection_name + "/" + granule_ur + "." + cnm_content['submissionTime'] + "." + "cnm.json" + + upload_cnm(pobit_audit_bucket, cnm_key_name, cnm_content) + + return self.input + + +def upload_cnm(pobit_audit_bucket: str, cnm_key_name: str, cnm_content: dict): + """ + Upload CNM message into a s3 bucket + + Parameters + ---------- + pobit_audit_bucket: str + Bucket name containing where CNM should be uploaded + + cnm_key_name: str + Key to object location in bucket + + cnm_content: dict + The CNM message to upload + + Returns + ------- + None + """ + s3_client = boto3.client('s3') + s3_client.put_object( + Body=json.dumps(cnm_content, default=str).encode("utf-8"), + Bucket=pobit_audit_bucket, + Key=cnm_key_name + ) + + +def lambda_handler(event, context): + """handler that gets called by aws lambda + Parameters + ---------- + event: dictionary + event from a lambda call + context: dictionary + context from a lambda call + Returns + ---------- + dict + A CNM json message + """ + # pylint: disable=duplicate-code + levels = { + 'critical': logging.CRITICAL, + 'error': logging.ERROR, + 'warn': logging.WARNING, + 'warning': logging.WARNING, + 'info': logging.INFO, + 'debug': logging.DEBUG + } + + logging_level = os.environ.get('LOGGING_LEVEL', 'info') + CUMULUS_LOGGER.logger.level = levels.get(logging_level, 'info') + CUMULUS_LOGGER.setMetadata(event, context) + + return CNM.cumulus_handler(event, context=context) + + +if __name__ == "__main__": + CNM() diff --git a/bignbit/send_to_gitc.py b/bignbit/send_to_gitc.py index 77c3e25..4d7aacc 100644 --- a/bignbit/send_to_gitc.py +++ b/bignbit/send_to_gitc.py @@ -10,9 +10,7 @@ from cumulus_process import Process from bignbit.image_set import ImageSet, to_cnm_product_dict -from bignbit import utils -REGION_NAME = 'us-west-2' CUMULUS_LOGGER = CumulusLogger('send_to_gitc') GIBS_REGION_ENV_NAME = "GIBS_REGION" @@ -55,15 +53,12 @@ def process(self): image_set = ImageSet(**self.input['image_set']) gitc_id = image_set.name - granule_ur = self.input.get('granule_ur') - pobit_audit_bucket = self.input.get('pobit_audit_bucket') - - notification_id = notify_gitc(image_set, cmr_provider, gitc_id, collection_name, granule_ur, pobit_audit_bucket) + notification_id = notify_gitc(image_set, cmr_provider, gitc_id, collection_name) return notification_id -def notify_gitc(image_set: ImageSet, cmr_provider: str, gitc_id: str, collection_name: str, granule_ur: str, audit_bucket): +def notify_gitc(image_set: ImageSet, cmr_provider: str, gitc_id: str, collection_name: str): """ Builds and sends a CNM message to GITC @@ -77,10 +72,6 @@ def notify_gitc(image_set: ImageSet, cmr_provider: str, gitc_id: str, collection The unique identifier for this particular request to GITC collection_name: str Collection that this image set belongs to - granuleUR: str - The granuleUR for this image set - audit_bucket: str - The name of the S3 bucket where a copy of the outgoing CNM will be saved Returns ------- @@ -109,11 +100,6 @@ def notify_gitc(image_set: ImageSet, cmr_provider: str, gitc_id: str, collection CUMULUS_LOGGER.debug(f'SQS send_message output: {response}') - cnm_key_name = collection_name + "/" + granule_ur + "." + cnm['submissionTime'] + "." + "cnm.json" - - utils.upload_cnm(audit_bucket, cnm_key_name, cnm_json) - CUMULUS_LOGGER.debug('CNM uploaded to s3 audit bucket') - return cnm['identifier'] diff --git a/terraform/state_machine_definition.tpl b/terraform/state_machine_definition.tpl index 6e18172..6d5f258 100644 --- a/terraform/state_machine_definition.tpl +++ b/terraform/state_machine_definition.tpl @@ -438,8 +438,6 @@ "task_config": { "collection": "{$.collection}", "cmr_provider": "{$.cmr_provider}", - "cmr_environment":"{$.meta.cmr.cmrEnvironment}", - "pobit_audit_bucket": "${PobitAuditBucket}", "cumulus_message": { "input": "{$}" } @@ -474,6 +472,36 @@ "MaxAttempts": 1 } ], + "Next": "SaveCNMMessage" + }, + "SaveCNMMessage": { + "Type": "Task", + "Resource": "${SaveCNMMessageLambda}", + "Parameters": { + "cma": { + "event.$": "$", + "task_config": { + "collection": "{$.collection}", + "pobit_audit_bucket": "${PobitAuditBucket}", + "cumulus_message": { + "input": "{$.payload}" + } + } + } + }, + "Retry": [ + { + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 2, + "MaxAttempts": 6, + "BackoffRate": 2 + } + ], "Next": "WorkflowSucceeded" }, "WorkflowSucceeded": { From ad8ed6e0fa76c87e65574bd6bf764b38ebb94c78 Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Tue, 28 May 2024 13:29:17 -0700 Subject: [PATCH 30/66] update tf vars for save cnm --- terraform/lambda_functions.tf | 6 +++--- terraform/main.tf | 2 +- terraform/outputs.tf | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/terraform/lambda_functions.tf b/terraform/lambda_functions.tf index e724c20..90d1f65 100644 --- a/terraform/lambda_functions.tf +++ b/terraform/lambda_functions.tf @@ -456,7 +456,7 @@ resource "aws_lambda_function" "handle_gitc_response" { tags = local.tags } -resource "aws_lambda_function" "save_cma_message" { +resource "aws_lambda_function" "save_cnm_message" { depends_on = [ null_resource.upload_ecr_image ] @@ -464,9 +464,9 @@ resource "aws_lambda_function" "save_cma_message" { package_type = "Image" image_uri = "${aws_ecr_repository.lambda-image-repo.repository_url}:${local.ecr_image_tag}" image_config { - command = ["bignbit.save_cma_message.lambda_handler"] + command = ["bignbit.save_cmm_message.lambda_handler"] } - function_name = "${local.lambda_resources_name}-save_cma_message-lambda" + function_name = "${local.lambda_resources_name}-save_cnm_message-lambda" role = var.lambda_role.arn timeout = 15 memory_size = 128 diff --git a/terraform/main.tf b/terraform/main.tf index c46471a..5b2dbcb 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -27,7 +27,7 @@ data "template_file" "workflow_definition" { GenerateImageMetadataLambda = aws_lambda_function.generate_image_metadata.arn, BuildImageSetsLambda = aws_lambda_function.build_image_sets.arn, SendToGITCLambda = aws_lambda_function.send_to_gitc.arn, - SaveCMAMessageLambda = aws_lambda_function.save_cma_message.arn, + SaveCNMMessageLambda = aws_lambda_function.save_cnm_message.arn, PobitAuditBucket = var.pobit_audit_bucket, PobitAuditPath = var.pobit_audit_path } diff --git a/terraform/outputs.tf b/terraform/outputs.tf index 005eb52..6e08462 100644 --- a/terraform/outputs.tf +++ b/terraform/outputs.tf @@ -74,8 +74,8 @@ output "pobit_send_to_gitc_arn" { value = aws_lambda_function.send_to_gitc.arn } -output "pobit_save_cma_message_arn" { - value = aws_lambda_function.save_cma_message.arn +output "pobit_save_cnm_message_arn" { + value = aws_lambda_function.save_cnm_message.arn } output "workflow_definition" { From 33e22251165adb28effbe9295a19cef11ebeff70 Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Tue, 28 May 2024 13:55:19 -0700 Subject: [PATCH 31/66] fix type in module definition --- terraform/lambda_functions.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/lambda_functions.tf b/terraform/lambda_functions.tf index 90d1f65..b603976 100644 --- a/terraform/lambda_functions.tf +++ b/terraform/lambda_functions.tf @@ -464,7 +464,7 @@ resource "aws_lambda_function" "save_cnm_message" { package_type = "Image" image_uri = "${aws_ecr_repository.lambda-image-repo.repository_url}:${local.ecr_image_tag}" image_config { - command = ["bignbit.save_cmm_message.lambda_handler"] + command = ["bignbit.save_cnm_message.lambda_handler"] } function_name = "${local.lambda_resources_name}-save_cnm_message-lambda" role = var.lambda_role.arn From 10d3113b26d6bb40ce525b36339fa806d0fdad19 Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Tue, 28 May 2024 16:43:26 -0700 Subject: [PATCH 32/66] fix cnm in cma --- bignbit/save_cnm_message.py | 4 ++-- bignbit/send_to_gitc.py | 13 ++++++++----- terraform/state_machine_definition.tpl | 5 +++-- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/bignbit/save_cnm_message.py b/bignbit/save_cnm_message.py index 90d886b..7a5e859 100644 --- a/bignbit/save_cnm_message.py +++ b/bignbit/save_cnm_message.py @@ -31,8 +31,8 @@ def process(self): """ pobit_audit_bucket = self.config['pobit_audit_bucket'] - collection_name = self.config['collection'] - granule_ur = self.input.get('granule_ur') + collection_name = self.config['collection_name'] + granule_ur = self.config['granule_ur'] cnm_content = self.input diff --git a/bignbit/send_to_gitc.py b/bignbit/send_to_gitc.py index 4d7aacc..d299ce2 100644 --- a/bignbit/send_to_gitc.py +++ b/bignbit/send_to_gitc.py @@ -44,8 +44,6 @@ def process(self): list of granules """ - notification_id = "" - if self.input is not None: # Send ImageSet(s) to GITC for processing collection_name = self.input.get('collection_name') @@ -53,9 +51,14 @@ def process(self): image_set = ImageSet(**self.input['image_set']) gitc_id = image_set.name - notification_id = notify_gitc(image_set, cmr_provider, gitc_id, collection_name) + response_payload = {} + response_payload = self.input.copy() + response_payload['cnm'] = [] + + cnm_message = notify_gitc(image_set, cmr_provider, gitc_id, collection_name) + response_payload['cnm'].append(cnm_message) - return notification_id + return response_payload def notify_gitc(image_set: ImageSet, cmr_provider: str, gitc_id: str, collection_name: str): @@ -100,7 +103,7 @@ def notify_gitc(image_set: ImageSet, cmr_provider: str, gitc_id: str, collection CUMULUS_LOGGER.debug(f'SQS send_message output: {response}') - return cnm['identifier'] + return cnm def construct_cnm(image_set: ImageSet, cmr_provider: str, gitc_id: str, collection_name: str): diff --git a/terraform/state_machine_definition.tpl b/terraform/state_machine_definition.tpl index 6d5f258..c89dd8a 100644 --- a/terraform/state_machine_definition.tpl +++ b/terraform/state_machine_definition.tpl @@ -481,10 +481,11 @@ "cma": { "event.$": "$", "task_config": { - "collection": "{$.collection}", + "collection": "{$.collection_name}", + "granule_ur": "{$.granule_ur}", "pobit_audit_bucket": "${PobitAuditBucket}", "cumulus_message": { - "input": "{$.payload}" + "input": "{$.payload.cnm}" } } } From 02f4d1b67ac62d4afe2d030af4698860077813ad Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Tue, 28 May 2024 17:21:01 -0700 Subject: [PATCH 33/66] fix collection reference in save cnm --- bignbit/save_cnm_message.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bignbit/save_cnm_message.py b/bignbit/save_cnm_message.py index 7a5e859..acd75f3 100644 --- a/bignbit/save_cnm_message.py +++ b/bignbit/save_cnm_message.py @@ -31,7 +31,7 @@ def process(self): """ pobit_audit_bucket = self.config['pobit_audit_bucket'] - collection_name = self.config['collection_name'] + collection_name = self.config['collection'] granule_ur = self.config['granule_ur'] cnm_content = self.input From 7b1f4a64d00c63bf1986ec38421f3e8167d35bf2 Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Tue, 28 May 2024 17:40:32 -0700 Subject: [PATCH 34/66] add debugging log statements --- bignbit/save_cnm_message.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bignbit/save_cnm_message.py b/bignbit/save_cnm_message.py index acd75f3..92adc55 100644 --- a/bignbit/save_cnm_message.py +++ b/bignbit/save_cnm_message.py @@ -34,8 +34,11 @@ def process(self): collection_name = self.config['collection'] granule_ur = self.config['granule_ur'] - cnm_content = self.input + cnm_content = self.input[0] + CUMULUS_LOGGER.info(collection_name) + CUMULUS_LOGGER.info(granule_ur) + CUMULUS_LOGGER.info(cnm_content['submissionTime']) cnm_key_name = collection_name + "/" + granule_ur + "." + cnm_content['submissionTime'] + "." + "cnm.json" upload_cnm(pobit_audit_bucket, cnm_key_name, cnm_content) From 8b56b7ae7375b3cfa8e1d62f28f381c471e09826 Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Tue, 28 May 2024 17:54:25 -0700 Subject: [PATCH 35/66] fix input --- bignbit/save_cnm_message.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bignbit/save_cnm_message.py b/bignbit/save_cnm_message.py index 92adc55..b486dc5 100644 --- a/bignbit/save_cnm_message.py +++ b/bignbit/save_cnm_message.py @@ -34,7 +34,7 @@ def process(self): collection_name = self.config['collection'] granule_ur = self.config['granule_ur'] - cnm_content = self.input[0] + cnm_content = self.input CUMULUS_LOGGER.info(collection_name) CUMULUS_LOGGER.info(granule_ur) From b02dd73bcc274838f89c49073e2b4d564d59c2e3 Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Tue, 28 May 2024 18:08:51 -0700 Subject: [PATCH 36/66] update save cnm input --- terraform/state_machine_definition.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/state_machine_definition.tpl b/terraform/state_machine_definition.tpl index c89dd8a..67ed688 100644 --- a/terraform/state_machine_definition.tpl +++ b/terraform/state_machine_definition.tpl @@ -485,7 +485,7 @@ "granule_ur": "{$.granule_ur}", "pobit_audit_bucket": "${PobitAuditBucket}", "cumulus_message": { - "input": "{$.payload.cnm}" + "input": "{$.payload.pobit.cnm}" } } } From eaddd63a090770d70dab07bb46b5af7a022b0ef1 Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Wed, 29 May 2024 09:24:01 -0700 Subject: [PATCH 37/66] debugging --- bignbit/save_cnm_message.py | 9 +++++---- bignbit/send_to_gitc.py | 3 +-- terraform/state_machine_definition.tpl | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/bignbit/save_cnm_message.py b/bignbit/save_cnm_message.py index b486dc5..74b4c2b 100644 --- a/bignbit/save_cnm_message.py +++ b/bignbit/save_cnm_message.py @@ -34,11 +34,12 @@ def process(self): collection_name = self.config['collection'] granule_ur = self.config['granule_ur'] - cnm_content = self.input + cnm_content = self.input['cnm'] - CUMULUS_LOGGER.info(collection_name) - CUMULUS_LOGGER.info(granule_ur) - CUMULUS_LOGGER.info(cnm_content['submissionTime']) + print(collection_name) + print(granule_ur) + print(cnm_content) + print(cnm_content['submissionTime']) cnm_key_name = collection_name + "/" + granule_ur + "." + cnm_content['submissionTime'] + "." + "cnm.json" upload_cnm(pobit_audit_bucket, cnm_key_name, cnm_content) diff --git a/bignbit/send_to_gitc.py b/bignbit/send_to_gitc.py index d299ce2..0021d5d 100644 --- a/bignbit/send_to_gitc.py +++ b/bignbit/send_to_gitc.py @@ -53,10 +53,9 @@ def process(self): response_payload = {} response_payload = self.input.copy() - response_payload['cnm'] = [] cnm_message = notify_gitc(image_set, cmr_provider, gitc_id, collection_name) - response_payload['cnm'].append(cnm_message) + response_payload['cnm'] = cnm_message return response_payload diff --git a/terraform/state_machine_definition.tpl b/terraform/state_machine_definition.tpl index 67ed688..6ed2eb1 100644 --- a/terraform/state_machine_definition.tpl +++ b/terraform/state_machine_definition.tpl @@ -485,7 +485,7 @@ "granule_ur": "{$.granule_ur}", "pobit_audit_bucket": "${PobitAuditBucket}", "cumulus_message": { - "input": "{$.payload.pobit.cnm}" + "input": "{$}" } } } From 8d1f031a49e7f5cb0bbc5ef76d8bc9ab574408ca Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Wed, 29 May 2024 09:49:09 -0700 Subject: [PATCH 38/66] debugging --- bignbit/save_cnm_message.py | 7 +++++-- terraform/state_machine_definition.tpl | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/bignbit/save_cnm_message.py b/bignbit/save_cnm_message.py index 74b4c2b..fc262e8 100644 --- a/bignbit/save_cnm_message.py +++ b/bignbit/save_cnm_message.py @@ -34,10 +34,13 @@ def process(self): collection_name = self.config['collection'] granule_ur = self.config['granule_ur'] - cnm_content = self.input['cnm'] - print(collection_name) print(granule_ur) + print(self.input) + + cnm_content = self.input['cnm'] + + print(cnm_content) print(cnm_content['submissionTime']) cnm_key_name = collection_name + "/" + granule_ur + "." + cnm_content['submissionTime'] + "." + "cnm.json" diff --git a/terraform/state_machine_definition.tpl b/terraform/state_machine_definition.tpl index 6ed2eb1..a78c0be 100644 --- a/terraform/state_machine_definition.tpl +++ b/terraform/state_machine_definition.tpl @@ -485,7 +485,7 @@ "granule_ur": "{$.granule_ur}", "pobit_audit_bucket": "${PobitAuditBucket}", "cumulus_message": { - "input": "{$}" + "input": "{$.payload.pobit}" } } } From 49191ca3ce38ccf75f275e98a7ee5a43dc0827b5 Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Wed, 29 May 2024 09:53:23 -0700 Subject: [PATCH 39/66] linting --- bignbit/save_cnm_message.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bignbit/save_cnm_message.py b/bignbit/save_cnm_message.py index fc262e8..08eb49f 100644 --- a/bignbit/save_cnm_message.py +++ b/bignbit/save_cnm_message.py @@ -37,10 +37,9 @@ def process(self): print(collection_name) print(granule_ur) print(self.input) - + cnm_content = self.input['cnm'] - print(cnm_content) print(cnm_content['submissionTime']) cnm_key_name = collection_name + "/" + granule_ur + "." + cnm_content['submissionTime'] + "." + "cnm.json" From 6eded09435383475d0dd72bcf1a1e3064560facc Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Wed, 29 May 2024 11:57:31 -0700 Subject: [PATCH 40/66] debugging --- bignbit/save_cnm_message.py | 2 +- bignbit/send_to_gitc.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bignbit/save_cnm_message.py b/bignbit/save_cnm_message.py index 08eb49f..9956d2b 100644 --- a/bignbit/save_cnm_message.py +++ b/bignbit/save_cnm_message.py @@ -38,7 +38,7 @@ def process(self): print(granule_ur) print(self.input) - cnm_content = self.input['cnm'] + cnm_content = self.input.get('cnm_sent') print(cnm_content) print(cnm_content['submissionTime']) diff --git a/bignbit/send_to_gitc.py b/bignbit/send_to_gitc.py index 0021d5d..f45407a 100644 --- a/bignbit/send_to_gitc.py +++ b/bignbit/send_to_gitc.py @@ -55,7 +55,7 @@ def process(self): response_payload = self.input.copy() cnm_message = notify_gitc(image_set, cmr_provider, gitc_id, collection_name) - response_payload['cnm'] = cnm_message + response_payload['cnm_sent'] = cnm_message return response_payload From ba5fc2d77148dd9da9450bc9d2338a4efd2f7f68 Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Wed, 29 May 2024 12:43:15 -0700 Subject: [PATCH 41/66] debugging --- bignbit/save_cnm_message.py | 7 ++- terraform/state_machine_definition.tpl | 69 +++++++++++++------------- 2 files changed, 40 insertions(+), 36 deletions(-) diff --git a/bignbit/save_cnm_message.py b/bignbit/save_cnm_message.py index 9956d2b..5ce1c83 100644 --- a/bignbit/save_cnm_message.py +++ b/bignbit/save_cnm_message.py @@ -30,15 +30,18 @@ def process(self): """ pobit_audit_bucket = self.config['pobit_audit_bucket'] + print(pobit_audit_bucket) + print(self.input.get('cnm_sent')) collection_name = self.config['collection'] granule_ur = self.config['granule_ur'] print(collection_name) print(granule_ur) - print(self.input) + - cnm_content = self.input.get('cnm_sent') + cnm_content = self.input + print(cnm_content) print(cnm_content) print(cnm_content['submissionTime']) diff --git a/terraform/state_machine_definition.tpl b/terraform/state_machine_definition.tpl index a78c0be..91c697e 100644 --- a/terraform/state_machine_definition.tpl +++ b/terraform/state_machine_definition.tpl @@ -448,9 +448,40 @@ "Type": "Task", "Resource": "arn:aws:states:::lambda:invoke", "TimeoutSeconds": 86400, - "End": true, + "Next": "SaveCNMMessage", "ResultPath": "$.gitc_response" - } + }, + "SaveCNMMessage": { + "Type": "Task", + "Resource": "${SaveCNMMessageLambda}", + "Parameters": { + "cma": { + "event.$": "$", + "task_config": { + "collection": "{$.collection_name}", + "granule_ur": "{$.granule_ur}", + "pobit_audit_bucket": "${PobitAuditBucket}", + "cumulus_message": { + "input": "{$.gitc_response}" + } + } + } + }, + "Retry": [ + { + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 2, + "MaxAttempts": 6, + "BackoffRate": 2 + } + ], + "Next": "WorkflowSucceeded" + }, } }, "ResultPath": "$.payload.pobit", @@ -472,39 +503,9 @@ "MaxAttempts": 1 } ], - "Next": "SaveCNMMessage" - }, - "SaveCNMMessage": { - "Type": "Task", - "Resource": "${SaveCNMMessageLambda}", - "Parameters": { - "cma": { - "event.$": "$", - "task_config": { - "collection": "{$.collection_name}", - "granule_ur": "{$.granule_ur}", - "pobit_audit_bucket": "${PobitAuditBucket}", - "cumulus_message": { - "input": "{$.payload.pobit}" - } - } - } - }, - "Retry": [ - { - "ErrorEquals": [ - "Lambda.ServiceException", - "Lambda.AWSLambdaException", - "Lambda.SdkClientException", - "Lambda.TooManyRequestsException" - ], - "IntervalSeconds": 2, - "MaxAttempts": 6, - "BackoffRate": 2 - } - ], - "Next": "WorkflowSucceeded" + "End": "True" }, + "WorkflowSucceeded": { "Type": "Succeed" }, From 9fdc7fc0b8d542b88535ee2ea54c8e71c78aee3c Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Wed, 29 May 2024 12:45:44 -0700 Subject: [PATCH 42/66] linting --- bignbit/save_cnm_message.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bignbit/save_cnm_message.py b/bignbit/save_cnm_message.py index 5ce1c83..4be1865 100644 --- a/bignbit/save_cnm_message.py +++ b/bignbit/save_cnm_message.py @@ -38,7 +38,6 @@ def process(self): print(collection_name) print(granule_ur) - cnm_content = self.input print(cnm_content) From bf25f522f981b280e0d44b60e6bf462c3792f054 Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Wed, 29 May 2024 13:09:40 -0700 Subject: [PATCH 43/66] fix state machine --- terraform/state_machine_definition.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/state_machine_definition.tpl b/terraform/state_machine_definition.tpl index 91c697e..5b993e3 100644 --- a/terraform/state_machine_definition.tpl +++ b/terraform/state_machine_definition.tpl @@ -481,7 +481,7 @@ } ], "Next": "WorkflowSucceeded" - }, + } } }, "ResultPath": "$.payload.pobit", From 03d421cebbbaabf5742a7dba480cf383405d64ab Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Wed, 29 May 2024 13:30:26 -0700 Subject: [PATCH 44/66] update state machine --- terraform/state_machine_definition.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/state_machine_definition.tpl b/terraform/state_machine_definition.tpl index 5b993e3..3e8f0f4 100644 --- a/terraform/state_machine_definition.tpl +++ b/terraform/state_machine_definition.tpl @@ -503,7 +503,7 @@ "MaxAttempts": 1 } ], - "End": "True" + "Next": "WorkflowSucceeded" }, "WorkflowSucceeded": { From 37c5f3bb63f19fa2961ed6c9c725e2bf6a215c53 Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Wed, 29 May 2024 13:59:04 -0700 Subject: [PATCH 45/66] reorg state machine --- terraform/state_machine_definition.tpl | 50 +++++++++++++------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/terraform/state_machine_definition.tpl b/terraform/state_machine_definition.tpl index 3e8f0f4..e96c918 100644 --- a/terraform/state_machine_definition.tpl +++ b/terraform/state_machine_definition.tpl @@ -448,10 +448,32 @@ "Type": "Task", "Resource": "arn:aws:states:::lambda:invoke", "TimeoutSeconds": 86400, - "Next": "SaveCNMMessage", + "End": "true", "ResultPath": "$.gitc_response" - }, - "SaveCNMMessage": { + } + }, + "ResultPath": "$.payload.pobit", + "Catch": [ + { + "ErrorEquals": [ + "States.ALL" + ], + "ResultPath": "$.exception", + "Next": "WorkflowFailed" + } + ], + "Retry": [ + { + "ErrorEquals": [ + "States.ALL" + ], + "IntervalSeconds": 2, + "MaxAttempts": 1 + } + ], + "Next": "SaveCNMMessage" + }, + "SaveCNMMessage": { "Type": "Task", "Resource": "${SaveCNMMessageLambda}", "Parameters": { @@ -483,28 +505,6 @@ "Next": "WorkflowSucceeded" } } - }, - "ResultPath": "$.payload.pobit", - "Catch": [ - { - "ErrorEquals": [ - "States.ALL" - ], - "ResultPath": "$.exception", - "Next": "WorkflowFailed" - } - ], - "Retry": [ - { - "ErrorEquals": [ - "States.ALL" - ], - "IntervalSeconds": 2, - "MaxAttempts": 1 - } - ], - "Next": "WorkflowSucceeded" - }, "WorkflowSucceeded": { "Type": "Succeed" From 74be4ddbf9baa7beb5fd979ace46ea8b6c60b237 Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Wed, 29 May 2024 14:12:12 -0700 Subject: [PATCH 46/66] missing comma --- terraform/state_machine_definition.tpl | 59 +++++++++++++------------- 1 file changed, 29 insertions(+), 30 deletions(-) diff --git a/terraform/state_machine_definition.tpl b/terraform/state_machine_definition.tpl index e96c918..10b72c0 100644 --- a/terraform/state_machine_definition.tpl +++ b/terraform/state_machine_definition.tpl @@ -474,38 +474,37 @@ "Next": "SaveCNMMessage" }, "SaveCNMMessage": { - "Type": "Task", - "Resource": "${SaveCNMMessageLambda}", - "Parameters": { - "cma": { - "event.$": "$", - "task_config": { - "collection": "{$.collection_name}", - "granule_ur": "{$.granule_ur}", - "pobit_audit_bucket": "${PobitAuditBucket}", - "cumulus_message": { - "input": "{$.gitc_response}" - } - } - } - }, - "Retry": [ - { - "ErrorEquals": [ - "Lambda.ServiceException", - "Lambda.AWSLambdaException", - "Lambda.SdkClientException", - "Lambda.TooManyRequestsException" - ], - "IntervalSeconds": 2, - "MaxAttempts": 6, - "BackoffRate": 2 - } - ], - "Next": "WorkflowSucceeded" + "Type": "Task", + "Resource": "${SaveCNMMessageLambda}", + "Parameters": { + "cma": { + "event.$": "$", + "task_config": { + "collection": "{$.collection_name}", + "granule_ur": "{$.granule_ur}", + "pobit_audit_bucket": "${PobitAuditBucket}", + "cumulus_message": { + "input": "{$.gitc_response}" + } } } - + }, + "Retry": [ + { + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 2, + "MaxAttempts": 6, + "BackoffRate": 2 + } + ], + "Next": "WorkflowSucceeded" + } + }, "WorkflowSucceeded": { "Type": "Succeed" }, From df784d275b7adca132d9d8ed9faf42df5ca65778 Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Tue, 4 Jun 2024 09:49:26 -0700 Subject: [PATCH 47/66] move save cnm into map --- terraform/state_machine_definition.tpl | 99 +++++++++++++------------- 1 file changed, 50 insertions(+), 49 deletions(-) diff --git a/terraform/state_machine_definition.tpl b/terraform/state_machine_definition.tpl index 10b72c0..a85018e 100644 --- a/terraform/state_machine_definition.tpl +++ b/terraform/state_machine_definition.tpl @@ -450,61 +450,62 @@ "TimeoutSeconds": 86400, "End": "true", "ResultPath": "$.gitc_response" - } - }, - "ResultPath": "$.payload.pobit", - "Catch": [ - { - "ErrorEquals": [ - "States.ALL" - ], - "ResultPath": "$.exception", - "Next": "WorkflowFailed" - } - ], - "Retry": [ - { - "ErrorEquals": [ - "States.ALL" + }, + "SaveCNMMessage": { + "Type": "Task", + "Resource": "${SaveCNMMessageLambda}", + "Parameters": { + "cma": { + "event.$": "$", + "task_config": { + "collection": "{$.collection_name}", + "granule_ur": "{$.granule_ur}", + "pobit_audit_bucket": "${PobitAuditBucket}", + "cumulus_message": { + "input": "{$.gitc_response}" + } + } + } + }, + "Retry": [ + { + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 2, + "MaxAttempts": 6, + "BackoffRate": 2 + } ], - "IntervalSeconds": 2, - "MaxAttempts": 1 + "Next": "WorkflowSucceeded" } - ], - "Next": "SaveCNMMessage" - }, - "SaveCNMMessage": { - "Type": "Task", - "Resource": "${SaveCNMMessageLambda}", - "Parameters": { - "cma": { - "event.$": "$", - "task_config": { - "collection": "{$.collection_name}", - "granule_ur": "{$.granule_ur}", - "pobit_audit_bucket": "${PobitAuditBucket}", - "cumulus_message": { - "input": "{$.gitc_response}" - } + }, + "ResultPath": "$.payload.pobit", + "Catch": [ + { + "ErrorEquals": [ + "States.ALL" + ], + "ResultPath": "$.exception", + "Next": "WorkflowFailed" } - } - }, - "Retry": [ - { - "ErrorEquals": [ - "Lambda.ServiceException", - "Lambda.AWSLambdaException", - "Lambda.SdkClientException", - "Lambda.TooManyRequestsException" + ], + "Retry": [ + { + "ErrorEquals": [ + "States.ALL" ], - "IntervalSeconds": 2, - "MaxAttempts": 6, - "BackoffRate": 2 - } - ], - "Next": "WorkflowSucceeded" + "IntervalSeconds": 2, + "MaxAttempts": 1 + } + ], + "Next": "SaveCNMMessage" } }, + "WorkflowSucceeded": { "Type": "Succeed" }, From 554a5b63e1f51de42059d6de96a9f73fdfe5e633 Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Tue, 4 Jun 2024 10:13:16 -0700 Subject: [PATCH 48/66] fix state machine transitions --- terraform/state_machine_definition.tpl | 62 +++++++++++++------------- 1 file changed, 30 insertions(+), 32 deletions(-) diff --git a/terraform/state_machine_definition.tpl b/terraform/state_machine_definition.tpl index a85018e..d245e76 100644 --- a/terraform/state_machine_definition.tpl +++ b/terraform/state_machine_definition.tpl @@ -448,40 +448,39 @@ "Type": "Task", "Resource": "arn:aws:states:::lambda:invoke", "TimeoutSeconds": 86400, - "End": "true", - "ResultPath": "$.gitc_response" + "Next": "SaveCNMMessage" }, "SaveCNMMessage": { - "Type": "Task", - "Resource": "${SaveCNMMessageLambda}", - "Parameters": { - "cma": { - "event.$": "$", - "task_config": { - "collection": "{$.collection_name}", - "granule_ur": "{$.granule_ur}", - "pobit_audit_bucket": "${PobitAuditBucket}", - "cumulus_message": { - "input": "{$.gitc_response}" + "Type": "Task", + "Resource": "${SaveCNMMessageLambda}", + "Parameters": { + "cma": { + "event.$": "$", + "task_config": { + "collection": "{$.collection_name}", + "granule_ur": "{$.granule_ur}", + "pobit_audit_bucket": "${PobitAuditBucket}", + "cumulus_message": { + "input": "{$}" + } } } - } - }, - "Retry": [ - { - "ErrorEquals": [ - "Lambda.ServiceException", - "Lambda.AWSLambdaException", - "Lambda.SdkClientException", - "Lambda.TooManyRequestsException" - ], - "IntervalSeconds": 2, - "MaxAttempts": 6, - "BackoffRate": 2 - } - ], - "Next": "WorkflowSucceeded" - } + }, + "Retry": [ + { + "ErrorEquals": [ + "Lambda.ServiceException", + "Lambda.AWSLambdaException", + "Lambda.SdkClientException", + "Lambda.TooManyRequestsException" + ], + "IntervalSeconds": 2, + "MaxAttempts": 6, + "BackoffRate": 2 + } + ], + "End": "true" + } }, "ResultPath": "$.payload.pobit", "Catch": [ @@ -502,10 +501,9 @@ "MaxAttempts": 1 } ], - "Next": "SaveCNMMessage" } + "Next": "WorkflowSucceeded" }, - "WorkflowSucceeded": { "Type": "Succeed" }, From 1c833cac4ffebdc890f83466d9eb1189ca161207 Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Tue, 4 Jun 2024 10:37:14 -0700 Subject: [PATCH 49/66] fix sm --- terraform/state_machine_definition.tpl | 42 +++++++++++++------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/terraform/state_machine_definition.tpl b/terraform/state_machine_definition.tpl index d245e76..76d5281 100644 --- a/terraform/state_machine_definition.tpl +++ b/terraform/state_machine_definition.tpl @@ -481,27 +481,27 @@ ], "End": "true" } - }, - "ResultPath": "$.payload.pobit", - "Catch": [ - { - "ErrorEquals": [ - "States.ALL" - ], - "ResultPath": "$.exception", - "Next": "WorkflowFailed" - } - ], - "Retry": [ - { - "ErrorEquals": [ - "States.ALL" - ], - "IntervalSeconds": 2, - "MaxAttempts": 1 - } - ], - } + } + }, + "ResultPath": "$.payload.pobit", + "Catch": [ + { + "ErrorEquals": [ + "States.ALL" + ], + "ResultPath": "$.exception", + "Next": "WorkflowFailed" + } + ], + "Retry": [ + { + "ErrorEquals": [ + "States.ALL" + ], + "IntervalSeconds": 2, + "MaxAttempts": 1 + } + ], "Next": "WorkflowSucceeded" }, "WorkflowSucceeded": { From 45c04356c06ba5d19df068d6e448fa45cbc5499e Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Tue, 4 Jun 2024 10:54:43 -0700 Subject: [PATCH 50/66] fix boolean --- terraform/state_machine_definition.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/state_machine_definition.tpl b/terraform/state_machine_definition.tpl index 76d5281..a8bdbeb 100644 --- a/terraform/state_machine_definition.tpl +++ b/terraform/state_machine_definition.tpl @@ -479,7 +479,7 @@ "BackoffRate": 2 } ], - "End": "true" + "End": true } } }, From e05dd3fb8c3b907e2b1c832687c860af8d20d718 Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Tue, 4 Jun 2024 12:39:08 -0700 Subject: [PATCH 51/66] debugging cnm input --- bignbit/save_cnm_message.py | 4 ++-- bignbit/send_to_gitc.py | 6 +----- terraform/state_machine_definition.tpl | 3 ++- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/bignbit/save_cnm_message.py b/bignbit/save_cnm_message.py index 4be1865..d1056cd 100644 --- a/bignbit/save_cnm_message.py +++ b/bignbit/save_cnm_message.py @@ -31,7 +31,7 @@ def process(self): """ pobit_audit_bucket = self.config['pobit_audit_bucket'] print(pobit_audit_bucket) - print(self.input.get('cnm_sent')) + print(self.input) collection_name = self.config['collection'] granule_ur = self.config['granule_ur'] @@ -39,7 +39,7 @@ def process(self): print(collection_name) print(granule_ur) - cnm_content = self.input + cnm_content = self.input['cnm'] print(cnm_content) print(cnm_content) diff --git a/bignbit/send_to_gitc.py b/bignbit/send_to_gitc.py index f45407a..4e3f42e 100644 --- a/bignbit/send_to_gitc.py +++ b/bignbit/send_to_gitc.py @@ -51,13 +51,9 @@ def process(self): image_set = ImageSet(**self.input['image_set']) gitc_id = image_set.name - response_payload = {} - response_payload = self.input.copy() - cnm_message = notify_gitc(image_set, cmr_provider, gitc_id, collection_name) - response_payload['cnm_sent'] = cnm_message - return response_payload + return cnm_message def notify_gitc(image_set: ImageSet, cmr_provider: str, gitc_id: str, collection_name: str): diff --git a/terraform/state_machine_definition.tpl b/terraform/state_machine_definition.tpl index a8bdbeb..fb069c1 100644 --- a/terraform/state_machine_definition.tpl +++ b/terraform/state_machine_definition.tpl @@ -448,6 +448,7 @@ "Type": "Task", "Resource": "arn:aws:states:::lambda:invoke", "TimeoutSeconds": 86400, + "ResultPath": "$.cnm", "Next": "SaveCNMMessage" }, "SaveCNMMessage": { @@ -461,7 +462,7 @@ "granule_ur": "{$.granule_ur}", "pobit_audit_bucket": "${PobitAuditBucket}", "cumulus_message": { - "input": "{$}" + "input": "{$.payload}" } } } From 1dd0b629931ff47abb067a569fc52773e3019bb5 Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Tue, 4 Jun 2024 13:01:24 -0700 Subject: [PATCH 52/66] fix input --- terraform/state_machine_definition.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/state_machine_definition.tpl b/terraform/state_machine_definition.tpl index fb069c1..220464c 100644 --- a/terraform/state_machine_definition.tpl +++ b/terraform/state_machine_definition.tpl @@ -462,7 +462,7 @@ "granule_ur": "{$.granule_ur}", "pobit_audit_bucket": "${PobitAuditBucket}", "cumulus_message": { - "input": "{$.payload}" + "input": "{$.cnm}" } } } From 0f30ca9a284d7f0bd37b93eb059143eaccadd16c Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Tue, 4 Jun 2024 13:50:33 -0700 Subject: [PATCH 53/66] change input --- terraform/state_machine_definition.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/state_machine_definition.tpl b/terraform/state_machine_definition.tpl index 220464c..2f4d46e 100644 --- a/terraform/state_machine_definition.tpl +++ b/terraform/state_machine_definition.tpl @@ -462,7 +462,7 @@ "granule_ur": "{$.granule_ur}", "pobit_audit_bucket": "${PobitAuditBucket}", "cumulus_message": { - "input": "{$.cnm}" + "input": "{$.event}" } } } From 27f9615e760d21b1e47edb0c218050a9e113ea0c Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Tue, 4 Jun 2024 14:08:03 -0700 Subject: [PATCH 54/66] debug input --- bignbit/save_cnm_message.py | 2 +- terraform/state_machine_definition.tpl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bignbit/save_cnm_message.py b/bignbit/save_cnm_message.py index d1056cd..148eb62 100644 --- a/bignbit/save_cnm_message.py +++ b/bignbit/save_cnm_message.py @@ -39,7 +39,7 @@ def process(self): print(collection_name) print(granule_ur) - cnm_content = self.input['cnm'] + cnm_content = self.input['cma']['event']['cnm'] print(cnm_content) print(cnm_content) diff --git a/terraform/state_machine_definition.tpl b/terraform/state_machine_definition.tpl index 2f4d46e..580a08f 100644 --- a/terraform/state_machine_definition.tpl +++ b/terraform/state_machine_definition.tpl @@ -462,7 +462,7 @@ "granule_ur": "{$.granule_ur}", "pobit_audit_bucket": "${PobitAuditBucket}", "cumulus_message": { - "input": "{$.event}" + "input": "{$}" } } } From bcc81e4290ddf9285e5ba6a64677a138dbbc324f Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Tue, 4 Jun 2024 14:40:12 -0700 Subject: [PATCH 55/66] add cnm as config parameter --- bignbit/save_cnm_message.py | 3 ++- terraform/state_machine_definition.tpl | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/bignbit/save_cnm_message.py b/bignbit/save_cnm_message.py index 148eb62..714b3f5 100644 --- a/bignbit/save_cnm_message.py +++ b/bignbit/save_cnm_message.py @@ -39,7 +39,8 @@ def process(self): print(collection_name) print(granule_ur) - cnm_content = self.input['cma']['event']['cnm'] + cnm_content = self.config['cnm'] + #cnm_content = self.input['cnm'] print(cnm_content) print(cnm_content) diff --git a/terraform/state_machine_definition.tpl b/terraform/state_machine_definition.tpl index 580a08f..547a558 100644 --- a/terraform/state_machine_definition.tpl +++ b/terraform/state_machine_definition.tpl @@ -460,6 +460,7 @@ "task_config": { "collection": "{$.collection_name}", "granule_ur": "{$.granule_ur}", + "cnm": "{$.cnm}", "pobit_audit_bucket": "${PobitAuditBucket}", "cumulus_message": { "input": "{$}" From 741a438c50ab2b99c20a715fddd4d8dcb0be708e Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Tue, 4 Jun 2024 14:46:04 -0700 Subject: [PATCH 56/66] lint --- bignbit/save_cnm_message.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bignbit/save_cnm_message.py b/bignbit/save_cnm_message.py index 714b3f5..1e645ee 100644 --- a/bignbit/save_cnm_message.py +++ b/bignbit/save_cnm_message.py @@ -40,7 +40,7 @@ def process(self): print(granule_ur) cnm_content = self.config['cnm'] - #cnm_content = self.input['cnm'] + print(cnm_content) print(cnm_content) From 9b551e9d09ae7a920aa70622ef782b978516ef09 Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Tue, 4 Jun 2024 15:22:45 -0700 Subject: [PATCH 57/66] fix cnm payload --- terraform/state_machine_definition.tpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/state_machine_definition.tpl b/terraform/state_machine_definition.tpl index 547a558..98a86be 100644 --- a/terraform/state_machine_definition.tpl +++ b/terraform/state_machine_definition.tpl @@ -460,7 +460,7 @@ "task_config": { "collection": "{$.collection_name}", "granule_ur": "{$.granule_ur}", - "cnm": "{$.cnm}", + "cnm": "{$.cnm.Payload.payload}", "pobit_audit_bucket": "${PobitAuditBucket}", "cumulus_message": { "input": "{$}" From 04be71047c499e0ece07001e081e778803d286c4 Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Tue, 4 Jun 2024 15:46:19 -0700 Subject: [PATCH 58/66] remove debugging statements --- bignbit/save_cnm_message.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/bignbit/save_cnm_message.py b/bignbit/save_cnm_message.py index 1e645ee..b3d551c 100644 --- a/bignbit/save_cnm_message.py +++ b/bignbit/save_cnm_message.py @@ -30,21 +30,12 @@ def process(self): """ pobit_audit_bucket = self.config['pobit_audit_bucket'] - print(pobit_audit_bucket) - print(self.input) collection_name = self.config['collection'] granule_ur = self.config['granule_ur'] - print(collection_name) - print(granule_ur) - cnm_content = self.config['cnm'] - print(cnm_content) - - print(cnm_content) - print(cnm_content['submissionTime']) cnm_key_name = collection_name + "/" + granule_ur + "." + cnm_content['submissionTime'] + "." + "cnm.json" upload_cnm(pobit_audit_bucket, cnm_key_name, cnm_content) From 9fb5af6b620154ff3b51ce7f66d890abd8155505 Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Tue, 4 Jun 2024 16:00:04 -0700 Subject: [PATCH 59/66] add prefix to cnm path and fix gitc response --- bignbit/handle_gitc_response.py | 2 +- bignbit/save_cnm_message.py | 3 ++- terraform/state_machine_definition.tpl | 1 + 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/bignbit/handle_gitc_response.py b/bignbit/handle_gitc_response.py index 90dfcef..ce9a79c 100644 --- a/bignbit/handle_gitc_response.py +++ b/bignbit/handle_gitc_response.py @@ -40,7 +40,7 @@ def handler(event, _): umm_json = utils.get_umm_json(granule_concept_id, cmr_env) granule_ur = umm_json['GranuleUR'] - cnm_key_name = collection_name + "/" + granule_ur + "." + message_body['submission_time'] + "." + "cnm-r.json" + cnm_key_name = os.environ['POBIT_AUDIT_PATH_NAME'] + "/" + collection_name + "/" + granule_ur + "." + message_body['submissionTime'] + "." + "cnm-r.json" utils.upload_cnm(os.environ['POBIT_AUDIT_BUCKET_NAME'], cnm_key_name, json.dumps(message_body)) diff --git a/bignbit/save_cnm_message.py b/bignbit/save_cnm_message.py index b3d551c..9e4f8f4 100644 --- a/bignbit/save_cnm_message.py +++ b/bignbit/save_cnm_message.py @@ -30,13 +30,14 @@ def process(self): """ pobit_audit_bucket = self.config['pobit_audit_bucket'] + pobit_audit_path = self.config['pobit_audit_path'] collection_name = self.config['collection'] granule_ur = self.config['granule_ur'] cnm_content = self.config['cnm'] - cnm_key_name = collection_name + "/" + granule_ur + "." + cnm_content['submissionTime'] + "." + "cnm.json" + cnm_key_name = pobit_audit_path + "/" + collection_name + "/" + granule_ur + "." + cnm_content['submissionTime'] + "." + "cnm.json" upload_cnm(pobit_audit_bucket, cnm_key_name, cnm_content) diff --git a/terraform/state_machine_definition.tpl b/terraform/state_machine_definition.tpl index 98a86be..64510aa 100644 --- a/terraform/state_machine_definition.tpl +++ b/terraform/state_machine_definition.tpl @@ -462,6 +462,7 @@ "granule_ur": "{$.granule_ur}", "cnm": "{$.cnm.Payload.payload}", "pobit_audit_bucket": "${PobitAuditBucket}", + "pobit_audit_path": "${PobitAuditPath}", "cumulus_message": { "input": "{$}" } From 323b5678579e072ab26690fab336b5bacce21ebb Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Tue, 4 Jun 2024 16:17:42 -0700 Subject: [PATCH 60/66] increase handle gitc response timeout --- terraform/lambda_functions.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/lambda_functions.tf b/terraform/lambda_functions.tf index b603976..5d62060 100644 --- a/terraform/lambda_functions.tf +++ b/terraform/lambda_functions.tf @@ -432,7 +432,7 @@ resource "aws_lambda_function" "handle_gitc_response" { } function_name = "${local.lambda_resources_name}-handle_gitc_response-lambda" role = var.lambda_role.arn - timeout = 5 + timeout = 15 memory_size = 128 environment { From 0871338af3400b8efd034215649e289924ed12f9 Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Tue, 4 Jun 2024 16:32:38 -0700 Subject: [PATCH 61/66] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d2c9010..eea1736 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added ### Deprecated ### Removed +- [issues/7](https://github.com/podaac/bignbit/issues/15): Remove the wait for GITC response ### Fixed ### Security From 0543496be1d8a8455f16175859e2222938af20d2 Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Tue, 4 Jun 2024 16:47:54 -0700 Subject: [PATCH 62/66] add original shortname as cnm prefix --- bignbit/handle_gitc_response.py | 2 +- bignbit/send_to_gitc.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/bignbit/handle_gitc_response.py b/bignbit/handle_gitc_response.py index ce9a79c..f9b85c5 100644 --- a/bignbit/handle_gitc_response.py +++ b/bignbit/handle_gitc_response.py @@ -33,7 +33,7 @@ def handler(event, _): for message in event["Records"]: message_body = loads(message["body"]) gitc_id = message_body["identifier"] - collection_name = message_body["collection"] + collection_name = message_body["shortname"] cmr_env = os.environ['CMR_ENVIRONMENT'] granule_concept_id = gitc_id[-19:] diff --git a/bignbit/send_to_gitc.py b/bignbit/send_to_gitc.py index 4e3f42e..cc47749 100644 --- a/bignbit/send_to_gitc.py +++ b/bignbit/send_to_gitc.py @@ -129,6 +129,7 @@ def construct_cnm(image_set: ImageSet, cmr_provider: str, gitc_id: str, collecti "version": "1.5.1", "duplicationid": image_set.name, "collection": new_collection, + "shortname": collection_name, "submissionTime": submission_time, "identifier": gitc_id, "product": product, From 267f59ccddfda2eb374c4982d812ce8c8b6346f3 Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Tue, 4 Jun 2024 17:28:03 -0700 Subject: [PATCH 63/66] revert shortname and change collection ref to save cnm & cnm-r in same location --- bignbit/handle_gitc_response.py | 2 +- bignbit/save_cnm_message.py | 2 +- bignbit/send_to_gitc.py | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/bignbit/handle_gitc_response.py b/bignbit/handle_gitc_response.py index f9b85c5..ce9a79c 100644 --- a/bignbit/handle_gitc_response.py +++ b/bignbit/handle_gitc_response.py @@ -33,7 +33,7 @@ def handler(event, _): for message in event["Records"]: message_body = loads(message["body"]) gitc_id = message_body["identifier"] - collection_name = message_body["shortname"] + collection_name = message_body["collection"] cmr_env = os.environ['CMR_ENVIRONMENT'] granule_concept_id = gitc_id[-19:] diff --git a/bignbit/save_cnm_message.py b/bignbit/save_cnm_message.py index 9e4f8f4..c926992 100644 --- a/bignbit/save_cnm_message.py +++ b/bignbit/save_cnm_message.py @@ -32,10 +32,10 @@ def process(self): pobit_audit_bucket = self.config['pobit_audit_bucket'] pobit_audit_path = self.config['pobit_audit_path'] - collection_name = self.config['collection'] granule_ur = self.config['granule_ur'] cnm_content = self.config['cnm'] + collection_name = cnm_content['collection'] cnm_key_name = pobit_audit_path + "/" + collection_name + "/" + granule_ur + "." + cnm_content['submissionTime'] + "." + "cnm.json" diff --git a/bignbit/send_to_gitc.py b/bignbit/send_to_gitc.py index cc47749..4e3f42e 100644 --- a/bignbit/send_to_gitc.py +++ b/bignbit/send_to_gitc.py @@ -129,7 +129,6 @@ def construct_cnm(image_set: ImageSet, cmr_provider: str, gitc_id: str, collecti "version": "1.5.1", "duplicationid": image_set.name, "collection": new_collection, - "shortname": collection_name, "submissionTime": submission_time, "identifier": gitc_id, "product": product, From abe609bf12c935b9cac6c62492f1931085598b99 Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Wed, 5 Jun 2024 08:44:45 -0700 Subject: [PATCH 64/66] remove save cma lambda no longer used --- bignbit/save_cma_message.py | 54 ------------------------------------- 1 file changed, 54 deletions(-) delete mode 100644 bignbit/save_cma_message.py diff --git a/bignbit/save_cma_message.py b/bignbit/save_cma_message.py deleted file mode 100644 index ce9dc8a..0000000 --- a/bignbit/save_cma_message.py +++ /dev/null @@ -1,54 +0,0 @@ -"""lambda function that stores the CMA message into a s3 bucket""" -import json -import logging - -import boto3 - - -def upload_cma(pobit_audit_bucket: str, cma_key_name: str, cma_content: dict): - """ - Upload CMA message into a s3 bucket - - Parameters - ---------- - pobit_audit_bucket: str - Bucket name containing where CMA should be uploaded - - cma_key_name: str - Key to object location in bucket - - cma_content: dict - The CMA message to upload - - Returns - ------- - None - """ - s3_client = boto3.client('s3') - s3_client.put_object( - Body=json.dumps(cma_content, default=str).encode("utf-8"), - Bucket=pobit_audit_bucket, - Key=cma_key_name - ) - - -def lambda_handler(event, _context): - """handler that gets called by aws lambda - Parameters - ---------- - event: dictionary - event from a lambda call - context: dictionary - context from a lambda call - - """ - logger = logging.getLogger('save_cma_message') - - pobit_audit_bucket = event['pobit_audit_bucket'] - cma_key_name = event['cma_key_name'] - cma_content = event['cma_content'] - - gitc_id = cma_content['identifier'] - - logger.info("Uploading CMA message to S3 for id %s", gitc_id) - upload_cma(pobit_audit_bucket, cma_key_name, cma_content) From b5398f3ac14d74bc19f6f2657e4dbc30dc2debcb Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Wed, 5 Jun 2024 08:45:49 -0700 Subject: [PATCH 65/66] remove EDL env params from sendtogitc --- terraform/lambda_functions.tf | 2 -- 1 file changed, 2 deletions(-) diff --git a/terraform/lambda_functions.tf b/terraform/lambda_functions.tf index 581efc0..e788d8c 100644 --- a/terraform/lambda_functions.tf +++ b/terraform/lambda_functions.tf @@ -406,8 +406,6 @@ resource "aws_lambda_function" "send_to_gitc" { REGION = var.region GIBS_REGION = var.gibs_region GIBS_SQS_URL = "https://sqs.${var.gibs_region}.amazonaws.com/${var.gibs_account_id}/${var.gibs_queue_name}" - EDL_USER_SSM = var.edl_user_ssm - EDL_PASS_SSM = var.edl_pass_ssm } } From 04e13967e74d29df6655c162b27bdeab21f72c46 Mon Sep 17 00:00:00 2001 From: Victoria McDonald Date: Wed, 5 Jun 2024 08:49:50 -0700 Subject: [PATCH 66/66] change parsing of granule concept id --- bignbit/handle_gitc_response.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bignbit/handle_gitc_response.py b/bignbit/handle_gitc_response.py index ce9a79c..b31f50e 100644 --- a/bignbit/handle_gitc_response.py +++ b/bignbit/handle_gitc_response.py @@ -36,7 +36,7 @@ def handler(event, _): collection_name = message_body["collection"] cmr_env = os.environ['CMR_ENVIRONMENT'] - granule_concept_id = gitc_id[-19:] + granule_concept_id = gitc_id.rpartition('_')[-1] umm_json = utils.get_umm_json(granule_concept_id, cmr_env) granule_ur = umm_json['GranuleUR']