Merge pull request #2080 from ASFHyP3/develop

Release v6.1.0

.github/workflows/deploy-enterprise.yml

@@ -53,13 +53,13 @@ jobs:
             domain: hyp3-tibet-jpl.asf.alaska.edu
             template_bucket: cf-templates-1or0efwqffkgd-us-west-2
             image_tag: latest
-            product_lifetime_in_days: 60
+            product_lifetime_in_days: 30
             default_credits_per_user: 0
             reset_credits_monthly: true
             job_files: job_spec/INSAR_ISCE.yml job_spec/INSAR_ISCE_TEST.yml
             instance_types: c6id.xlarge,c6id.2xlarge,c6id.4xlarge,c6id.8xlarge
-            default_max_vcpus: 0
-            expanded_max_vcpus: 0
+            default_max_vcpus: 10000
+            expanded_max_vcpus: 10000
             required_surplus: 0
             security_environment: JPL-public
             ami_id: /aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id

.gitignore

@@ -5,6 +5,7 @@ apps/api/src/hyp3_api/api-spec/job_parameters.yml
 apps/api/src/hyp3_api/job_validation_map.yml
 apps/step-function.json
 apps/**/*-cf.yml
+lib/dynamo/dynamo/*.json
 
 
 # Byte-compiled / optimized / DLL files

CHANGELOG.md

@@ -4,6 +4,15 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [6.1.0]
+
+### Added
+- Previously, the `job_parameters` field of the `job` object returned by the `/jobs` API endpoint only included parameters whose values were specified by the user. Now, the field also includes optional, unspecified parameters, along with their default values. This does not change how jobs are processed, but gives the user a complete report of what parameters were used to process their jobs.
+
+### Changed
+- Increased maximum vCPUs from 0 to 10,000 in the hyp3-tibet-jpl deployment.
+- Decreased product lifetime from 60 days to 30 days in the hyp3-tibet-jpl deployment.
+
 ## [6.0.0]
 
 HyP3's monthly quota system has been replaced by a credits system. Previously, HyP3 provided each user with a certain number of jobs per month. Now, each job costs a particular number of credits, and users spend credits when they submit jobs. This release assigns every job a cost of 1 credit, but future releases will assign a different credit cost to each job type. Additionally, the main production deployment (`https://hyp3-api.asf.alaska.edu`) resets each user's balance to 1,000 credits each month, effectively granting each user 1,000 jobs per month. Therefore, users should not notice any difference when ordering jobs via ASF's On Demand service at <https://search.asf.alaska.edu>.

apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2

@@ -170,68 +170,6 @@ components:
       items:
         $ref: "#/components/schemas/job"
 
-    search_parameters:
-      description: parameters used to find data from the ASF Search API
-      additionalProperties: false
-      type: object
-      required:
-        - start
-        - end
-      properties:
-        start:
-          $ref: "#/components/schemas/datetime"
-        end:
-          $ref: "#/components/schemas/datetime"
-        platform:
-          type: string
-          enum:
-            - S1
-            - SA
-            - SB
-        processingLevel:
-          type: string
-          enum:
-            - GRD_HD
-            - SLC
-        beamMode:
-          type: array
-          minItems: 1
-          items:
-            type: string
-            enum:
-              - IW
-        intersectsWith:
-          $ref: "#/components/schemas/intersectsWith"
-        frame:
-          type: array
-          minItems: 1
-          items:
-            type: integer
-            minimum: 1
-            maximum: 1360
-        relativeOrbit:
-          type: array
-          minItems: 1
-          items:
-            type: integer
-            minimum: 1
-            maximum: 175
-        polarization:
-          type: array
-          minItems: 1
-          items:
-            type: string
-            enum:
-              - VV
-              - VV+VH
-              - HH
-              - HH+HV
-        flightDirection:
-          type: string
-          enum:
-            - ASCENDING
-            - DESCENDING
-
     job:
       description: Contains information about a submitted job.
       type: object
@@ -299,12 +237,6 @@ components:
       format: date-time
       example: "2020-06-04T18:00:03+00:00"
 
-    intersectsWith:
-      description: Area-of-interest as a WKT string.
-      type: string
-      format: wkt
-      example: POINT(0 0)
-
     status_code:
       description: Status of a submitted job.
       type: string

apps/render_cf.py

@@ -35,6 +35,18 @@ def render_templates(job_types, security_environment, api_name):
         template_file.with_suffix('').write_text(output)
 
 
+def render_default_params_by_job_type(job_types: dict) -> None:
+    default_params_by_job_type = {
+        job_type: {
+            key: value['api_schema']['default'] for key, value in job_spec['parameters'].items()
+            if key not in job_spec['required_parameters'] and 'api_schema' in value
+        }
+        for job_type, job_spec in job_types.items()
+    }
+    with open(Path('lib') / 'dynamo' / 'dynamo' / 'default_params_by_job_type.json', 'w') as f:
+        json.dump(default_params_by_job_type, f, indent=2)
+
+
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument('-j', '--job-spec-files', required=True, nargs='+', type=Path)
@@ -50,6 +62,7 @@ def main():
         for task in job_spec['tasks']:
             task['name'] = job_type + '_' + task['name'] if task['name'] else job_type
 
+    render_default_params_by_job_type(job_types)
     render_templates(job_types, args.security_environment, args.api_name)
 
 

lib/dynamo/dynamo/jobs.py

@@ -1,5 +1,7 @@
+import json
 from datetime import datetime, timezone
 from os import environ
+from pathlib import Path
 from typing import List, Optional
 from uuid import uuid4
 
@@ -8,6 +10,13 @@
 import dynamo.user
 from dynamo.util import DYNAMODB_RESOURCE, convert_floats_to_decimals, format_time, get_request_time_expression
 
+default_params_file = Path(__file__).parent / 'default_params_by_job_type.json'
+if default_params_file.exists():
+    DEFAULT_PARAMS_BY_JOB_TYPE = json.loads(default_params_file.read_text())
+else:
+    # Allows mocking with unittest.mock.patch
+    DEFAULT_PARAMS_BY_JOB_TYPE = {}
+
 
 class InsufficientCreditsError(Exception):
     """Raised when trying to submit jobs whose total cost exceeds the user's remaining credits."""
@@ -73,16 +82,22 @@ def _prepare_job_for_database(
         priority = 0
     else:
         priority = min(int(remaining_credits - running_cost), 9999)
-    return {
+    prepared_job = {
         'job_id': str(uuid4()),
         'user_id': user_id,
         'status_code': 'PENDING',
         'execution_started': False,
         'request_time': request_time,
-        'credit_cost': _get_credit_cost(job),
         'priority': priority,
         **job,
     }
+    if 'job_type' in prepared_job:
+        prepared_job['job_parameters'] = {
+            **DEFAULT_PARAMS_BY_JOB_TYPE[prepared_job['job_type']],
+            **prepared_job.get('job_parameters', {})
+        }
+    prepared_job['credit_cost'] = _get_credit_cost(prepared_job)
+    return prepared_job
 
 
 def query_jobs(user, start=None, end=None, status_code=None, name=None, job_type=None, start_key=None):

lib/dynamo/setup.py

@@ -12,4 +12,6 @@
     python_requires='~=3.9',
 
     packages=find_packages(),
+
+    package_data={'dynamo': ['*.json']},
 )

requirements-all.txt

@@ -5,16 +5,16 @@
 -r requirements-apps-start-execution-worker.txt
 -r requirements-apps-disable-private-dns.txt
 -r requirements-apps-update-db.txt
-boto3==1.34.32
+boto3==1.34.40
 jinja2==3.1.3
-moto[dynamodb]==5.0.0
+moto[dynamodb]==5.0.1
 pytest==8.0.0
 PyYAML==6.0.1
 responses==0.24.1
 flake8==7.0.0
 flake8-import-order==0.18.2
 flake8-blind-except==0.2.1
 flake8-builtins==2.2.0
-setuptools==69.0.3
+setuptools==69.1.0
 openapi-spec-validator==0.7.1
-cfn-lint==0.85.0
+cfn-lint==0.85.1

requirements-apps-disable-private-dns.txt

@@ -1 +1 @@
-boto3==1.34.32
+boto3==1.34.40

requirements-apps-start-execution-manager.txt

@@ -1,3 +1,3 @@
-boto3==1.34.32
+boto3==1.34.40
 ./lib/dynamo/
 ./lib/lambda_logging/

requirements-apps-start-execution-worker.txt

@@ -1,2 +1,2 @@
-boto3==1.34.32
+boto3==1.34.40
 ./lib/lambda_logging/

tests/test_dynamo/test_jobs.py

@@ -211,6 +211,45 @@ def test_put_jobs(tables, monkeypatch):
     ]
 
 
+def test_put_jobs_default_params(tables):
+    default_params = {
+        'JOB_TYPE_A': {'a1': 'a1_default', 'a2': 'a2_default'},
+        'JOB_TYPE_B': {'b1': 'b1_default'},
+        'JOB_TYPE_C': {},
+    }
+    payload = [
+        {},
+        {'job_type': 'JOB_TYPE_A'},
+        {'job_type': 'JOB_TYPE_A', 'job_parameters': {}},
+        {'job_type': 'JOB_TYPE_A', 'job_parameters': {'a1': 'foo'}},
+        {'job_type': 'JOB_TYPE_A', 'job_parameters': {'a1': 'foo', 'a2': 'bar'}},
+        {'job_type': 'JOB_TYPE_B', 'job_parameters': {}},
+        {'job_type': 'JOB_TYPE_B', 'job_parameters': {'b1': 'foo'}},
+        {'job_type': 'JOB_TYPE_B', 'job_parameters': {'b1': 'foo', 'b2': 'bar'}},
+        {'job_type': 'JOB_TYPE_C'},
+        {'job_type': 'JOB_TYPE_C', 'job_parameters': {}},
+        {'job_type': 'JOB_TYPE_C', 'job_parameters': {'c1': 'foo'}},
+        {'job_parameters': {'n1': 'foo'}},
+    ]
+    with unittest.mock.patch('dynamo.jobs.DEFAULT_PARAMS_BY_JOB_TYPE', default_params):
+        jobs = dynamo.jobs.put_jobs('user1', payload)
+
+    assert 'job_parameters' not in jobs[0]
+    assert jobs[1]['job_parameters'] == {'a1': 'a1_default', 'a2': 'a2_default'}
+    assert jobs[2]['job_parameters'] == {'a1': 'a1_default', 'a2': 'a2_default'}
+    assert jobs[3]['job_parameters'] == {'a1': 'foo', 'a2': 'a2_default'}
+    assert jobs[4]['job_parameters'] == {'a1': 'foo', 'a2': 'bar'}
+    assert jobs[5]['job_parameters'] == {'b1': 'b1_default'}
+    assert jobs[6]['job_parameters'] == {'b1': 'foo'}
+    assert jobs[7]['job_parameters'] == {'b1': 'foo', 'b2': 'bar'}
+    assert jobs[8]['job_parameters'] == {}
+    assert jobs[9]['job_parameters'] == {}
+    assert jobs[10]['job_parameters'] == {'c1': 'foo'}
+    assert jobs[11]['job_parameters'] == {'n1': 'foo'}
+
+    assert tables.jobs_table.scan()['Items'] == jobs
+
+
 def test_put_jobs_user_exists(tables):
     tables.users_table.put_item(Item={'user_id': 'user1', 'remaining_credits': 5})