merge develop

.github/workflows/cicd-pipeline.yml

@@ -121,8 +121,14 @@ jobs:
           startsWith(github.ref, 'refs/heads/main') &&
           steps.manual.conclusion == 'skipped'
         id: release
+        env:
+          CURRENT_VERSION: ${{ steps.get-version.outputs.current_version }}
+        # Remove rc* from end of version string
+        # The ${string%%substring} syntax below deletes the longest match of $substring from back of $string.
         run: |
-          poetry version major
+          poetry version ${CURRENT_VERSION%%rc*}
+          echo "software_version=$(poetry version | awk '{print $2}')" >> $GITHUB_ENV
+          echo "venue=ops" >> $GITHUB_ENV
           echo "TARGET_ENV_UPPERCASE=OPS" >> $GITHUB_ENV
       - name: Get install version
         # Get the version of the software being installed and save it as an ENV var
@@ -272,6 +278,7 @@ jobs:
       THE_VERSION: ${{ needs.build.outputs.version }}
       CONTAINER_IMAGE_URI: ${{ needs.docker.outputs.container_image_uri }}
       TF_MODULE_ARTIFACT_NAME: ${{ needs.build.outputs.tf_module_artifact_name }}
+      TF_VAR_gibs_account_id: ${{ secrets.GIBS_ACCOUNT_ID }}
     if: |
       github.ref == 'refs/heads/develop' ||
       github.ref == 'refs/heads/main'    ||

CHANGELOG.md

@@ -6,24 +6,31 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Added
+### Deprecated 
+### Removed
+- [issues/7](https://github.com/podaac/bignbit/issues/15): Remove the wait for GITC response
+### Fixed
+### Security
 
 ### Added
 
 
 ### Changed
 - [issues/15](https://github.com/podaac/bignbit/issues/15): Change 'convertToPNG' choice to a generic send to harmony choice
+- [issues/16](https://github.com/podaac/bignbit/issues/16): Change apply opera treatment choice and lambda to be specific to HLS
 
 ## [0.1.1]
 
 ### Added 
-- Initial port from JPL GHE to public GitHub.com
 - [issues/2](https://github.com/podaac/bignbit/issues/2): Create github action pipeline to build artifacts
 - [issues/3](https://github.com/podaac/bignbit/issues/3): Update terraform mock deployment of cumulus module to services accounts
+- Initial port from JPL GHE to public GitHub.com
 
 ### Changed
 - [issues/10](https://github.com/podaac/bignbit/issues/10): Move combined big and pobit state machine into terraform module
 - [issues/6](https://github.com/podaac/bignbit/issues/6): BIG terraform failing in SWOT venues due to long lambda name
 
 
-[Unreleased]: https://github.com/olivierlacan/keep-a-changelog/compare/0.1.1...HEAD
-[0.1.1]: https://github.com/olivierlacan/keep-a-changelog/releases/tag/0.1.1
+[Unreleased]: https://github.com/podaac/bignbit/compare/0.1.1...HEAD
+[0.1.1]: https://github.com/podaac/bignbit/releases/tag/0.1.1

README.md

@@ -14,7 +14,7 @@ _Visual representation of the bignbit step function state machine:_
 ## MacOS
 
 1. Install miniconda (or conda) and [poetry](https://python-poetry.org/)
-2. Run `conda env create -f conda-environment.yml` to install GDAL
+2. Run `conda env create -f conda-environment.yaml` to install GDAL
 3. Activate the bignbit conda environment `conda activate bignbit`
 4. Install python package and dependencies `poetry install`
 5. Verify tests pass `poetry run pytest tests/`

bignbit/apply_opera_treatment.py → bignbit/apply_opera_hls_treatment.py

@@ -16,7 +16,7 @@
 
 from bignbit import utils
 
-CUMULUS_LOGGER = CumulusLogger('apply_opera_treatment')
+CUMULUS_LOGGER = CumulusLogger('apply_opera_hls_treatment')
 
 
 def load_mgrs_gibs_intersection():
@@ -61,7 +61,7 @@ def process(self) -> List[Dict]:
 
 def transform_images(cma_file_list: List[Dict], temp_dir: pathlib.Path, mgrs_grid_code: str) -> List[Dict]:
     """
-    Applies special OPERA processing to each input image. Each input image will result in multiple output transformed
+    Applies special OPERA HLS processing to each input image. Each input image will result in multiple output transformed
     images.
 
     Parameters
@@ -91,8 +91,8 @@ def transform_images(cma_file_list: List[Dict], temp_dir: pathlib.Path, mgrs_gri
         # Reproject and resample image to sub-tiles
         transformed_images_dirpath = temp_dir.joinpath(source_image_local_filepath.stem)
         transformed_images_dirpath.mkdir(parents=True)
-        transformed_images_filepaths = the_opera_treatment(source_image_local_filepath, transformed_images_dirpath,
-                                                           mgrs_grid_code)
+        transformed_images_filepaths = the_opera_hls_treatment(source_image_local_filepath, transformed_images_dirpath,
+                                                               mgrs_grid_code)
         CUMULUS_LOGGER.info(f'Created new images: {[str(t) for t in transformed_images_filepaths]}')
 
         # Create new file metadata for each new image
@@ -138,8 +138,8 @@ def get_file(bucket: str, key: str, local_filepath: pathlib.Path) -> pathlib.Pat
     return local_filepath
 
 
-def the_opera_treatment(source_image_filepath: pathlib.Path, working_dirpath: pathlib.Path,
-                        mgrs_grid_code: str) -> List[pathlib.Path]:
+def the_opera_hls_treatment(source_image_filepath: pathlib.Path, working_dirpath: pathlib.Path,
+                            mgrs_grid_code: str) -> List[pathlib.Path]:
     """
     What is the OPERA treatment? Well, it is special.
 

bignbit/build_image_sets.py

@@ -6,7 +6,7 @@
 from cumulus_logger import CumulusLogger
 from cumulus_process import Process
 
-from bignbit.image_set import from_big_output, IncompleteImageSet
+from bignbit.image_set import from_big_output, IncompleteImageSet, ImageSet
 
 CUMULUS_LOGGER = CumulusLogger('build_image_sets')
 
@@ -52,11 +52,18 @@ def process(self):
             del response_payload['big']
             response_payload['pobit'] = []
 
-            for image_set in image_sets:
+            for big_image_set in image_sets:
+                pobit_image_set = ImageSet(
+                    name=big_image_set.name + '_' + self.input['granules'][0]['cmrConceptId'],
+                    image=big_image_set.image,
+                    image_metadata=big_image_set.image_metadata,
+                    world_file=big_image_set.world_file)
+
                 response_payload['pobit'].append({
-                    'image_set': image_set._asdict(),
+                    'image_set': pobit_image_set._asdict(),
                     'cmr_provider': self.config.get('cmr_provider'),
                     'collection_name': self.config.get('collection').get('name'),
+                    'granule_ur': self.input['granules'][0]['granuleId']
                 })
 
         return response_payload

bignbit/get_granule_umm_json.py

@@ -4,7 +4,6 @@
 import logging
 import os
 
-import requests
 from cumulus_logger import CumulusLogger
 from cumulus_process import Process
 
@@ -32,39 +31,12 @@ def process(self):
 
         """
         cmr_environment = self.config['cmr_environment']
-        cmr_link = self.input['granules'][0]['cmrLink']
+        cmr_concept_id = self.input['granules'][0]['cmrConceptId']
 
-        self.input['granule_umm_json'] = download_umm_json(cmr_link, cmr_environment)
+        self.input['granule_umm_json'] = utils.get_umm_json(cmr_concept_id, cmr_environment)
         return self.input
 
 
-def download_umm_json(cmr_link: str, cmr_environment: str) -> dict:
-    """
-    Retrieve the umm-json document from the given cmr_link
-
-    Parameters
-    ----------
-    cmr_link: str
-      Link to the umm-g for downloading
-
-    cmr_environment: str
-      CMR environment used to retrieve user token
-
-    Returns
-    -------
-    dict
-      The umm-json document
-    """
-    edl_user, edl_pass = utils.get_edl_creds()
-    token = utils.get_cmr_user_token(edl_user, edl_pass, cmr_environment)
-
-    umm_json_response = requests.get(cmr_link, headers={'Authorization': f'Bearer {token}'}, timeout=10)
-    umm_json_response.raise_for_status()
-    umm_json = umm_json_response.json()
-
-    return umm_json
-
-
 def lambda_handler(event, context):
     """handler that gets called by aws lambda
     Parameters

bignbit/handle_gitc_response.py

@@ -3,8 +3,8 @@
 import logging
 import os
 from json import loads
-import boto3
-from botocore.exceptions import ClientError
+
+from bignbit import utils
 
 
 def handler(event, _):
@@ -32,13 +32,18 @@ def handler(event, _):
 
     for message in event["Records"]:
         message_body = loads(message["body"])
-        task_token = message_body["identifier"]
-        client = boto3.client('stepfunctions')
-        try:
-            client.send_task_success(taskToken=task_token, output=json.dumps(message_body))
-            logger.info("Step function triggered for task token %s", task_token)
-        except ClientError:
-            logger.warning("Error sending task success for messageId %s task token %s",
-                           message['messageId'], task_token,
-                           exc_info=True)
+        gitc_id = message_body["identifier"]
+        collection_name = message_body["collection"]
+        cmr_env = os.environ['CMR_ENVIRONMENT']
+
+        granule_concept_id = gitc_id.rpartition('_')[-1]
+        umm_json = utils.get_umm_json(granule_concept_id, cmr_env)
+        granule_ur = umm_json['GranuleUR']
+
+        cnm_key_name = os.environ['POBIT_AUDIT_PATH_NAME'] + "/" + collection_name + "/" + granule_ur + "." + message_body['submissionTime'] + "." + "cnm-r.json"
+
+        utils.upload_cnm(os.environ['POBIT_AUDIT_BUCKET_NAME'], cnm_key_name, json.dumps(message_body))
+
+        logging.debug('CNM-R uploaded to s3 audit bucket for id %s', gitc_id)
+
     return {"statusCode": 200, "body": "All good"}

bignbit/save_cma_message.py → bignbit/save_cnm_message.py

@@ -7,10 +7,10 @@
 from cumulus_logger import CumulusLogger
 from cumulus_process import Process
 
-CUMULUS_LOGGER = CumulusLogger('save_cma_message')
+CUMULUS_LOGGER = CumulusLogger('save_cmm_message')
 
 
-class CMA(Process):
+class CNM(Process):
     """
     A cumulus message adapter
     """
@@ -21,7 +21,7 @@ def __init__(self, *args, **kwargs):
 
     def process(self):
         """
-        Upload CMA message into a s3 bucket
+        Upload CNM message into a s3 bucket
 
         Returns
         -------
@@ -30,37 +30,44 @@ def process(self):
 
         """
         pobit_audit_bucket = self.config['pobit_audit_bucket']
-        cma_key_name = self.config['cma_key_name']
+        pobit_audit_path = self.config['pobit_audit_path']
 
-        upload_cma(pobit_audit_bucket, cma_key_name, self.input)
+        granule_ur = self.config['granule_ur']
+
+        cnm_content = self.config['cnm']
+        collection_name = cnm_content['collection']
+
+        cnm_key_name = pobit_audit_path + "/" + collection_name + "/" + granule_ur + "." + cnm_content['submissionTime'] + "." + "cnm.json"
+
+        upload_cnm(pobit_audit_bucket, cnm_key_name, cnm_content)
 
         return self.input
 
 
-def upload_cma(pobit_audit_bucket: str, cma_key_name: str, cma_content: dict):
+def upload_cnm(pobit_audit_bucket: str, cnm_key_name: str, cnm_content: dict):
     """
-    Upload CMA message into a s3 bucket
+    Upload CNM message into a s3 bucket
 
     Parameters
     ----------
     pobit_audit_bucket: str
-      Bucket name containing where CMA should be uploaded
+      Bucket name containing where CNM should be uploaded
 
-    cma_key_name: str
+    cnm_key_name: str
       Key to object location in bucket
 
-    cma_content: dict
-      The CMA message to upload
+    cnm_content: dict
+      The CNM message to upload
 
     Returns
     -------
     None
     """
     s3_client = boto3.client('s3')
     s3_client.put_object(
-        Body=json.dumps(cma_content, default=str).encode("utf-8"),
+        Body=json.dumps(cnm_content, default=str).encode("utf-8"),
         Bucket=pobit_audit_bucket,
-        Key=cma_key_name
+        Key=cnm_key_name
     )
 
 
@@ -75,7 +82,7 @@ def lambda_handler(event, context):
     Returns
     ----------
         dict
-            A CMA json message
+            A CNM json message
     """
     # pylint: disable=duplicate-code
     levels = {
@@ -91,8 +98,8 @@ def lambda_handler(event, context):
     CUMULUS_LOGGER.logger.level = levels.get(logging_level, 'info')
     CUMULUS_LOGGER.setMetadata(event, context)
 
-    return CMA.cumulus_handler(event, context=context)
+    return CNM.cumulus_handler(event, context=context)
 
 
 if __name__ == "__main__":
-    CMA()
+    CNM()