Merge pull request #206 from ASFHyP3/develop

RELEASE: change browse image identification to use tags

CHANGELOG.md

@@ -4,6 +4,15 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+
+## [0.3.5]
+### Added
+- `POST /jobs` now excepts custom job parameters when submitting jobs
+- `GET /jobs` now shows parameters job was run with
+
+### Changed
+- `get_files.py` now uses tags to identify file_type instead of path
+
 ## [0.3.4]
 ### Added
 - Name field to job parameter, set in the same way as description but with max length of 20

api/src/hyp3_api/openapi-spec.yml

@@ -211,6 +211,20 @@ components:
       properties:
         granule:
           $ref: "#/components/schemas/granule"
+        resolution:
+          $ref: "#/components/schemas/resolution"
+        radiometry:
+          $ref: "#/components/schemas/radiometry"
+        speckle_filter:
+          $ref: "#/components/schemas/speckle_filter"
+        scale:
+          $ref: "#/components/schemas/scale"
+        dem_matching:
+          $ref: "#/components/schemas/dem_matching"
+        include_dem:
+          $ref: "#/components/schemas/include_dem"
+        include_inc_map:
+          $ref: "#/components/schemas/include_inc_map"
 
     granule:
       description: The name of the Sentinel-1 granule to process.  Only SLC and GRDH products with beam mode IW are supported.
@@ -220,6 +234,56 @@ components:
       maxLength: 67
       example: S1A_IW_SLC__1SSV_20150621T120220_20150621T120232_006471_008934_72D8
 
+    resolution:
+      description: Desired output resolution in meters.
+      type: string
+      enum:
+        - "30.0"
+        - "10.0"
+
+
+    radiometry:
+      description: Backscatter coefficient normalization, either by ground area (sigma0) or illuminated area projected into the look direction (gamma0)
+      type: string
+      enum:
+        - gamma0
+        - sigma0
+
+    scale:
+      description: Output image scale
+      type: string
+      enum:
+        - power
+        - amplitude
+
+    speckle_filter:
+      description: Apply an enhanced lee speckle filter
+      type: string
+      enum:
+        - "false"
+        - "true"
+
+    dem_matching:
+      description: Attempt to match the DEMs otherwise use dead reconing
+      type: string
+      enum:
+        - "false"
+        - "true"
+
+    include_dem:
+      description: Include the DEM files in the product package
+      type: string
+      enum:
+        - "false"
+        - "true"
+
+    include_inc_map:
+      description: Include the incident angle map in the product package
+      type: string
+      enum:
+        - "false"
+        - "true"
+
     datetime:
       description: Date and time object formatted according to ISO 8601
       type: string

batch-cluster/cloudformation.yml

@@ -99,6 +99,12 @@ Resources:
         bucket-prefix: ""
         username: !Ref EDLUsername
         password: !Ref EDLPassword
+        resolution: 30.0
+        radiometry: gamma0
+        scale: power
+        dem_matching: false
+        include_dem: false
+        include_inc_map: false
       ContainerProperties:
         Image: !Ref RtcGammaImage
         Vcpus: 4
@@ -115,6 +121,18 @@ Resources:
           - Ref::bucket
           - --bucket-prefix
           - Ref::bucket-prefix
+          - --resolution
+          - Ref::resolution
+          - --radiometry
+          - Ref::radiometry
+          - --scale
+          - Ref::scale
+          - --dem-matching
+          - Ref::dem_matching
+          - --include-dem
+          - Ref::include_dem
+          - --include-inc-map
+          - Ref::include_inc_map
           - Ref::granule
       Timeout:
         AttemptDurationSeconds: 5400
@@ -137,6 +155,9 @@ Resources:
               - Effect: Allow
                 Action: s3:PutObject
                 Resource: !Sub "arn:aws:s3:::${ContentBucket}/*"
+              - Effect: Allow
+                Action: s3:PutObjectTagging
+                Resource: !Sub "arn:aws:s3:::${ContentBucket}/*"
 
   BatchServiceRole:
     Type: AWS::IAM::Role

cloudformation.yml

@@ -125,6 +125,9 @@ Resources:
       LifecycleConfiguration:
         Rules:
           - Status: Enabled
+            TagFilters:
+              - Key: file_type
+                Value: product
             ExpirationInDays: !Ref ProductLifetimeInDays
           - Status: Enabled
             AbortIncompleteMultipartUpload:

step-function/get-files/src/main.py

@@ -7,31 +7,40 @@
 S3_CLIENT = boto3.client('s3')
 
 
-def get_download_url(key):
-    bucket = environ['BUCKET']
+def get_download_url(bucket, key):
     region = environ['AWS_REGION']
     return f'https://{bucket}.s3.{region}.amazonaws.com/{key}'
 
 
-def get_expiration_time(key):
-    s3_object = S3_CLIENT.get_object(Bucket=environ['BUCKET'], Key=key)
+def get_expiration_time(bucket, key):
+    s3_object = S3_CLIENT.get_object(Bucket=bucket, Key=key)
     expiration_string = s3_object['Expiration'].split('"')[1]
     expiration_datetime = datetime.strptime(expiration_string, '%a, %d %b %Y %H:%M:%S %Z')
     return expiration_datetime.isoformat(timespec='seconds') + '+00:00'
 
 
+def get_object_file_type(bucket, key):
+    response = S3_CLIENT.get_object_tagging(Bucket=bucket, Key=key)
+    for tag in response['TagSet']:
+        if tag['Key'] == 'file_type':
+            return tag['Value']
+    return None
+
+
 def lambda_handler(event, context):
-    response = S3_CLIENT.list_objects_v2(Bucket=environ['BUCKET'], Prefix=event['job_id'])
+    bucket = environ['BUCKET']
+    response = S3_CLIENT.list_objects_v2(Bucket=bucket, Prefix=event['job_id'])
 
     files = []
     browse_images = []
     thumbnail_images = []
 
     for item in response['Contents']:
-        download_url = get_download_url(item['Key'])
-        if item['Key'].startswith(f'{event["job_id"]}/browse/'):
+        download_url = get_download_url(bucket, item['Key'])
+        file_type = get_object_file_type(bucket, item['Key'])
+        if file_type == 'browse':
             browse_images.append(download_url)
-        elif item['Key'].startswith(f'{event["job_id"]}/thumbnail/'):
+        elif file_type == 'thumbnail':
             thumbnail_images.append(download_url)
         else:
             file = {
@@ -42,7 +51,7 @@ def lambda_handler(event, context):
             files.append(file)
 
     return {
-        'expiration_time': get_expiration_time(response['Contents'][0]['Key']),
+        'expiration_time': get_expiration_time(bucket, response['Contents'][0]['Key']),
         'files': files,
         'browse_images': browse_images,
         'thumbnail_images': thumbnail_images,